From 4b66f62e10f1b765dc6dc6baa5bbfe633234ec25 Mon Sep 17 00:00:00 2001 From: Yukim1 <121286183+zwhzzz0821@users.noreply.github.com> Date: Thu, 13 Aug 2026 21:29:03 +0800 Subject: [PATCH 001/267] [Refactor] Remove internal stage_configs_path plumbing (#5741) Signed-off-by: zwhzzz0821 <2831474076@qq.com> Signed-off-by: Hongsheng Liu Co-authored-by: Hongsheng Liu --- .../offline_inference/covo_audio/end2end.py | 6 +- .../offline_inference/dynin_omni/end2end.py | 6 +- .../hunyuan_image3/end2end.py | 19 +-- .../offline_inference/minicpmo/end2end.py | 6 - .../offline_inference/qwen3_omni/end2end.py | 4 +- .../offline_inference/step_audio2/end2end.py | 22 +-- .../text_to_image/text_to_image.py | 12 +- .../text_to_speech/fish_speech/end2end.py | 13 +- .../text_to_speech/glm_tts/end2end.py | 2 +- .../text_to_speech/qwen3_tts/end2end.py | 4 +- tests/config/test_config_factory.py | 52 +------ .../quantization/test_quantization_fp8.py | 2 +- .../test_bagel_mooncake_connector.py | 2 +- .../test_bagel_shared_memory_connector.py | 4 +- .../test_audiox_model_expansion.py | 2 +- .../offline_inference/test_bagel_expansion.py | 2 +- .../test_cosyvoice3_expansion.py | 2 +- .../test_ernie_image_expansion.py | 2 +- .../test_glm_tts_expansion.py | 2 +- .../test_hunyuanimage3_i2t_expansion.py | 2 +- .../test_longcat_video_avatar.py | 1 - .../test_mammoth_moda2_expansion.py | 2 +- .../offline_inference/test_minicpmo_4_5.py | 2 +- .../test_moss_tts_nano_expansion.py | 2 +- .../test_omnivoice_expansion.py | 6 +- .../test_sensenova_u1_expansion.py | 2 +- .../test_sensenova_u1_img2img_expansion.py | 6 +- .../test_sensenova_u1_text2img_expansion.py | 6 +- .../test_soulxsinger_expansion.py | 4 +- .../test_step_audio2_expansion.py | 2 +- .../test_voxtral_tts_expansion.py | 2 +- tests/engine/test_arg_utils.py | 139 ------------------ tests/engine/test_async_omni_engine_abort.py | 2 +- tests/engine/test_single_stage_mode.py | 4 +- tests/entrypoints/test_async_omni.py | 2 +- .../test_async_omni_diffusion_config.py | 6 +- tests/entrypoints/test_serve.py | 2 +- tests/entrypoints/test_utils.py | 16 +- tests/helpers/runtime.py | 13 +- tools/baselines/omni_hash_baseline.py | 2 +- vllm_omni/config/omni_config.py | 1 - vllm_omni/config/stage_config.py | 43 +----- vllm_omni/engine/arg_utils.py | 6 - vllm_omni/engine/async_omni_engine.py | 93 +----------- vllm_omni/entrypoints/async_omni.py | 5 +- vllm_omni/entrypoints/cli/serve.py | 2 - vllm_omni/entrypoints/openai/api_server.py | 4 +- vllm_omni/entrypoints/utils.py | 89 ++--------- 48 files changed, 93 insertions(+), 537 deletions(-) diff --git a/examples/offline_inference/covo_audio/end2end.py b/examples/offline_inference/covo_audio/end2end.py index 5b7ac5fef..b5050f62c 100644 --- a/examples/offline_inference/covo_audio/end2end.py +++ b/examples/offline_inference/covo_audio/end2end.py @@ -59,7 +59,7 @@ def main(args): omni = Omni( model=args.model_name, - stage_configs_path=args.stage_configs_path, + deploy_config=args.deploy_config, log_stats=args.log_stats, stage_init_timeout=args.stage_init_timeout, batch_timeout=args.batch_timeout, @@ -167,10 +167,10 @@ def parse_args(): help="Sampling rate for audio loading (default: 16000).", ) parser.add_argument( - "--stage-configs-path", + "--deploy-config", type=str, default=None, - help="Path to stage configs YAML file.", + help="Path to a deploy config YAML file.", ) parser.add_argument( "--log-stats", diff --git a/examples/offline_inference/dynin_omni/end2end.py b/examples/offline_inference/dynin_omni/end2end.py index 0cab46fee..74d33fe01 100644 --- a/examples/offline_inference/dynin_omni/end2end.py +++ b/examples/offline_inference/dynin_omni/end2end.py @@ -876,7 +876,7 @@ def parse_args(repo_root: Path) -> argparse.Namespace: parser.add_argument("--task", type=str, required=True, choices=TASK_CHOICES) parser.add_argument("--model", type=str, required=True, help="HF repo id or local model directory.") parser.add_argument( - "--stage-config-path", + "--deploy-config", type=str, default=str(repo_root / "vllm_omni/deploy/dynin_omni.yaml"), help="Path to deploy config yaml.", @@ -1394,8 +1394,8 @@ def main() -> None: from vllm_omni.entrypoints.omni import Omni - stage_config_path = str(Path(args.stage_config_path).expanduser()) - omni = Omni(model=model_source, stage_configs_path=stage_config_path, dtype=args.dtype) + deploy_config_path = str(Path(args.deploy_config).expanduser()) + omni = Omni(model=model_source, deploy_config=deploy_config_path, dtype=args.dtype) sampling_params_list = [ SamplingParams(max_tokens=int(args.max_tokens_per_stage), temperature=0.0, top_p=1.0, detokenize=False) for _ in range(omni.num_stages) diff --git a/examples/offline_inference/hunyuan_image3/end2end.py b/examples/offline_inference/hunyuan_image3/end2end.py index 9a2350156..fbe4289be 100644 --- a/examples/offline_inference/hunyuan_image3/end2end.py +++ b/examples/offline_inference/hunyuan_image3/end2end.py @@ -73,7 +73,6 @@ def parse_args(): ) parser.add_argument("--sys-type", type=str, default=None, help="Override system prompt type.") parser.add_argument("--deploy-config", type=str, default=None, help="Custom deploy YAML path.") - parser.add_argument("--stage-configs-path", type=str, default=None, help="Custom legacy stage config YAML path.") parser.add_argument( "--stream", action="store_true", @@ -146,13 +145,7 @@ def main(): else: bot_task = args.bot_task - if args.deploy_config is not None and args.stage_configs_path is not None: - raise ValueError("--deploy-config and --stage-configs-path are mutually exclusive.") - - deploy_config = args.deploy_config - stage_configs_path = args.stage_configs_path - if deploy_config is None and stage_configs_path is None: - deploy_config = _MODALITY_DEFAULT_DEPLOY_CONFIG[args.modality] + deploy_config = args.deploy_config or _MODALITY_DEFAULT_DEPLOY_CONFIG[args.modality] omni_kwargs = { "model": args.model, @@ -168,10 +161,7 @@ def main(): if additional_config is not None: omni_kwargs["additional_config"] = additional_config - if deploy_config is not None: - omni_kwargs["deploy_config"] = deploy_config - else: - omni_kwargs["stage_configs_path"] = stage_configs_path + omni_kwargs["deploy_config"] = deploy_config omni = Omni(**omni_kwargs) @@ -272,10 +262,7 @@ def main(): print(f" Modality: {args.modality}") print(f" Prompt task: {task}") print(f" Bot task: {bot_task}") - if deploy_config is not None: - print(f" Deploy config: {deploy_config}") - else: - print(f" Stage config: {stage_configs_path}") + print(f" Deploy config: {deploy_config}") print(f" Num stages: {omni.num_stages}") if args.modality in ("text2img", "img2img"): print(f" Inference steps: {args.steps}") diff --git a/examples/offline_inference/minicpmo/end2end.py b/examples/offline_inference/minicpmo/end2end.py index e9c9019c0..857f1f00f 100644 --- a/examples/offline_inference/minicpmo/end2end.py +++ b/examples/offline_inference/minicpmo/end2end.py @@ -452,12 +452,6 @@ def parse_args(): default=None, help="Path to a .txt file with one prompt per line (preferred).", ) - parser.add_argument( - "--stage-configs-path", - type=str, - default=None, - help="Path to a stage configs file (deprecated; prefer --deploy-config).", - ) parser.add_argument( "--deploy-config", type=str, diff --git a/examples/offline_inference/qwen3_omni/end2end.py b/examples/offline_inference/qwen3_omni/end2end.py index 9434b3058..faa200bc9 100644 --- a/examples/offline_inference/qwen3_omni/end2end.py +++ b/examples/offline_inference/qwen3_omni/end2end.py @@ -484,10 +484,10 @@ def parse_args(): help="Path to a .txt file with one prompt per line (preferred).", ) parser.add_argument( - "--stage-configs-path", + "--deploy-config", type=str, default=None, - help="Path to a stage configs file.", + help="Path to a deploy config YAML.", ) parser.add_argument( "--video-path", diff --git a/examples/offline_inference/step_audio2/end2end.py b/examples/offline_inference/step_audio2/end2end.py index f175db750..418a7f129 100644 --- a/examples/offline_inference/step_audio2/end2end.py +++ b/examples/offline_inference/step_audio2/end2end.py @@ -232,15 +232,8 @@ def main(args): raise ValueError(f"Unknown query type: {args.query_type}") # Initialize vLLM-Omni with Step-Audio2 - # Resolve deploy config path. Keep explicit stage_configs_path for legacy - # custom configs, but use bundled deploy configs by default. - deploy_config_path = None - stage_config_path = None - if args.stage_configs_path and args.deploy_config: - raise ValueError("--stage-configs-path and --deploy-config are mutually exclusive") - if args.stage_configs_path: - stage_config_path = args.stage_configs_path - elif args.deploy_config: + # Resolve an explicit deploy config or use the bundled task default. + if args.deploy_config: deploy_config_path = args.deploy_config else: configs_dir = Path(__file__).parent.parent.parent.parent / "vllm_omni/deploy" @@ -262,10 +255,7 @@ def main(args): "ray_address": args.ray_address, "trust_remote_code": True, } - if stage_config_path is not None: - omni_kwargs["stage_configs_path"] = stage_config_path - else: - omni_kwargs["deploy_config"] = deploy_config_path + omni_kwargs["deploy_config"] = deploy_config_path omni_llm = Omni(**omni_kwargs) @@ -418,12 +408,6 @@ def parse_args(): " - HuggingFace ID: stepfun-ai/Step-Audio-2-mini" ), ) - parser.add_argument( - "--stage-configs-path", - type=str, - default=None, - help="Path to a legacy stage config YAML file (default: use bundled deploy config)", - ) parser.add_argument( "--deploy-config", type=str, diff --git a/examples/offline_inference/text_to_image/text_to_image.py b/examples/offline_inference/text_to_image/text_to_image.py index 360c726fd..2d1f9823a 100644 --- a/examples/offline_inference/text_to_image/text_to_image.py +++ b/examples/offline_inference/text_to_image/text_to_image.py @@ -92,12 +92,6 @@ def parse_args() -> argparse.Namespace: "meituan-longcat/LongCat-Image, OvisAI/Ovis-Image, " "stabilityai/stable-diffusion-3.5-medium, Tongyi-MAI/Z-Image-Turbo and etc.", ) - parser.add_argument( - "--stage-configs-path", - type=str, - default=None, - help="[Deprecated] Path to a legacy stage_args-format YAML. Prefer --deploy-config.", - ) parser.add_argument( "--deploy-config", type=str, @@ -475,8 +469,6 @@ def main(): omni_kwargs["tensor_parallel_size"] = args.tensor_parallel_size if args.enforce_eager is not None: omni_kwargs["enforce_eager"] = args.enforce_eager - if args.stage_configs_path: - omni_kwargs["stage_configs_path"] = args.stage_configs_path if args.deploy_config: omni_kwargs["deploy_config"] = args.deploy_config if use_nextstep: @@ -515,8 +507,8 @@ def main(): print(f" Image size: {args.width}x{args.height}") if args.lora_path: print(f" LoRA: scale={args.lora_scale}") - if args.stage_configs_path: - print(f" stage-configs-path: {args.stage_configs_path}") + if args.deploy_config: + print(f" deploy-config: {args.deploy_config}") print(f"{'=' * 60}\n") # Build LoRA request when --lora-path is set diff --git a/examples/offline_inference/text_to_speech/fish_speech/end2end.py b/examples/offline_inference/text_to_speech/fish_speech/end2end.py index 18b4d79f8..8ce4664b9 100644 --- a/examples/offline_inference/text_to_speech/fish_speech/end2end.py +++ b/examples/offline_inference/text_to_speech/fish_speech/end2end.py @@ -38,7 +38,7 @@ from vllm_omni.utils.tracking_parser import TrackingArgumentParser logger = logging.getLogger(__name__) DEFAULT_MODEL = "fishaudio/s2-pro" -DEFAULT_STAGE_CONFIG = os.path.join( +DEFAULT_DEPLOY_CONFIG = os.path.join( os.path.dirname(__file__), "..", "..", @@ -152,10 +152,7 @@ def _build_omni_kwargs(args, model_name: str) -> dict: log_stats=args.log_stats, stage_init_timeout=args.stage_init_timeout, ) - if args.deploy_config: - omni_kwargs["deploy_config"] = args.deploy_config - else: - omni_kwargs["stage_configs_path"] = args.stage_configs_path or DEFAULT_STAGE_CONFIG + omni_kwargs["deploy_config"] = args.deploy_config or DEFAULT_DEPLOY_CONFIG return omni_kwargs @@ -278,12 +275,6 @@ def parse_args(): default=None, help="Deploy config YAML path (under vllm_omni/deploy/).", ) - parser.add_argument( - "--stage-configs-path", - type=str, - default=None, - help="Path to stage configs YAML (legacy, prefer --deploy-config).", - ) parser.add_argument( "--output-dir", default="output_audio", diff --git a/examples/offline_inference/text_to_speech/glm_tts/end2end.py b/examples/offline_inference/text_to_speech/glm_tts/end2end.py index 6624c2b35..3f66ed8c3 100644 --- a/examples/offline_inference/text_to_speech/glm_tts/end2end.py +++ b/examples/offline_inference/text_to_speech/glm_tts/end2end.py @@ -145,7 +145,7 @@ def main(args): omni = Omni( model=args.model, - stage_configs_path=deploy_config_path, + deploy_config=deploy_config_path, log_stats=args.log_stats, stage_init_timeout=args.stage_init_timeout, ) diff --git a/examples/offline_inference/text_to_speech/qwen3_tts/end2end.py b/examples/offline_inference/text_to_speech/qwen3_tts/end2end.py index 7c0ce125e..75f562af6 100644 --- a/examples/offline_inference/text_to_speech/qwen3_tts/end2end.py +++ b/examples/offline_inference/text_to_speech/qwen3_tts/end2end.py @@ -490,10 +490,10 @@ def parse_args(): help="Path to a .txt file with one prompt per line (preferred).", ) parser.add_argument( - "--stage-configs-path", + "--deploy-config", type=str, default=None, - help="Path to a stage configs file.", + help="Path to a deploy config YAML.", ) parser.add_argument( "--audio-path", diff --git a/tests/config/test_config_factory.py b/tests/config/test_config_factory.py index c3c3edbd1..9708c1c50 100644 --- a/tests/config/test_config_factory.py +++ b/tests/config/test_config_factory.py @@ -6,7 +6,7 @@ Unit tests for StageConfigFactory and related classes. import importlib import warnings -from dataclasses import dataclass, fields +from dataclasses import dataclass from pathlib import Path from unittest.mock import patch @@ -34,9 +34,8 @@ from vllm_omni.config.stage_config import ( load_deploy_config, merge_pipeline_deploy, pipeline_cfg_resolver, - strip_parent_engine_args, ) -from vllm_omni.engine.arg_utils import SHARED_FIELDS, EngineArgs, internal_blacklist_keys +from vllm_omni.engine.arg_utils import SHARED_FIELDS, internal_blacklist_keys pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -475,7 +474,7 @@ class TestStageConfigFactory: cli_overrides = { "gpu_memory_utilization": 0.9, "model": "some_model", # Internal - "stage_configs_path": "/path", # Internal + "deploy_config": "/path", # Internal "batch_timeout": 10, # Internal } @@ -483,7 +482,7 @@ class TestStageConfigFactory: assert overrides["gpu_memory_utilization"] == 0.9 assert "model" not in overrides - assert "stage_configs_path" not in overrides + assert "deploy_config" not in overrides assert "batch_timeout" not in overrides def test_per_stage_override_excludes_internal_keys(self): @@ -525,49 +524,6 @@ class TestStageResolutionHelpers: assert "model" not in overrides assert "parallel_config" not in overrides - def test_strip_parent_engine_args_reports_only_surprising_parent_overrides(self): - parent_fields = {f.name: f for f in fields(EngineArgs)} - filtered, overridden = strip_parent_engine_args( - { - "model": "some/model", - "stage_configs_path": "/tmp/stages.yaml", - "tensor_parallel_size": 4, - "worker_extension_cls": "some.Extension", - "custom_pipeline_args": {"pipeline_class": "demo.Pipeline"}, - }, - parent_fields=parent_fields, - keep_keys={"worker_extension_cls"}, - strip_keys={"stage_configs_path"}, - no_warn_keys={"model"}, - ) - - assert filtered == { - "worker_extension_cls": "some.Extension", - "custom_pipeline_args": {"pipeline_class": "demo.Pipeline"}, - } - assert overridden == ["tensor_parallel_size"] - - def test_strip_parent_engine_args_keeps_allowed_media_access_controls(self): - parent_fields = {f.name: f for f in fields(EngineArgs)} - filtered, overridden = strip_parent_engine_args( - { - "model": "some/model", - "stage_configs_path": "/tmp/stages.yaml", - "allowed_local_media_path": "/data/qwentts", - "allowed_media_domains": ["example.com"], - }, - parent_fields=parent_fields, - keep_keys={"allowed_local_media_path", "allowed_media_domains"}, - strip_keys={"stage_configs_path"}, - no_warn_keys={"model"}, - ) - - assert filtered == { - "allowed_local_media_path": "/data/qwentts", - "allowed_media_domains": ["example.com"], - } - assert overridden == [] - class TestPipelineDiscovery: """Tests for the central pipeline registry (``OMNI_PIPELINES``).""" diff --git a/tests/diffusion/quantization/test_quantization_fp8.py b/tests/diffusion/quantization/test_quantization_fp8.py index 617083398..0da5b6dd3 100644 --- a/tests/diffusion/quantization/test_quantization_fp8.py +++ b/tests/diffusion/quantization/test_quantization_fp8.py @@ -199,7 +199,7 @@ def _generate_bagel_image( config_path = get_deploy_config_path("ci/bagel.yaml") omni_kwargs: dict[str, Any] = { "model": "ByteDance-Seed/BAGEL-7B-MoT", - "stage_configs_path": config_path, + "deploy_config": config_path, "stage_init_timeout": 300, } if diffusion_quantization_config: diff --git a/tests/distributed/omni_connectors/test_bagel_mooncake_connector.py b/tests/distributed/omni_connectors/test_bagel_mooncake_connector.py index 57997905c..3b9fcbb64 100644 --- a/tests/distributed/omni_connectors/test_bagel_mooncake_connector.py +++ b/tests/distributed/omni_connectors/test_bagel_mooncake_connector.py @@ -314,7 +314,7 @@ def test_bagel_text2img_mooncake_connector(run_level): temp_config_file = _resolve_deploy_config(temp_config_file, run_level) with OmniRunner( "ByteDance-Seed/BAGEL-7B-MoT", - stage_configs_path=temp_config_file, + deploy_config=temp_config_file, stage_init_timeout=300, ) as runner: generated_image = _generate_bagel_image(runner.omni) diff --git a/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py b/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py index b97161335..72b56d3e9 100644 --- a/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py +++ b/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py @@ -251,7 +251,7 @@ def test_bagel_img2img_shared_memory_connector(run_level): config_path = _resolve_deploy_config(BAGEL_CI_DEPLOY, run_level) with OmniRunner( "ByteDance-Seed/BAGEL-7B-MoT", - stage_configs_path=config_path, + deploy_config=config_path, ) as runner: generated_image = _generate_bagel_img2img(runner.omni, input_image) if run_level == "advanced_model": @@ -266,7 +266,7 @@ def test_bagel_text2img_shared_memory_connector(run_level): config_path = _resolve_deploy_config(BAGEL_CI_DEPLOY, run_level) with OmniRunner( "ByteDance-Seed/BAGEL-7B-MoT", - stage_configs_path=config_path, + deploy_config=config_path, ) as runner: generated_image = _generate_bagel_text2img(runner.omni) if run_level == "advanced_model": diff --git a/tests/e2e/offline_inference/test_audiox_model_expansion.py b/tests/e2e/offline_inference/test_audiox_model_expansion.py index af944b144..5739cc463 100644 --- a/tests/e2e/offline_inference/test_audiox_model_expansion.py +++ b/tests/e2e/offline_inference/test_audiox_model_expansion.py @@ -17,7 +17,7 @@ from vllm_omni.platforms import current_omni_platform # Override in CI if needed: AUDIOX_TEST_MODEL= models = [os.environ.get("AUDIOX_TEST_MODEL", "zhangj1an/audiox_random")] -# (model, stage_configs_path, extra_omni_kwargs) for ``omni_runner`` indirect parametrize +# (model, deploy_config_path, extra_omni_kwargs) for ``omni_runner`` indirect parametrize _OMNI_RUNNER_PARAMS = [(m, None, {"model_class_name": "AudioXPipeline"}) for m in models] pytestmark = [ diff --git a/tests/e2e/offline_inference/test_bagel_expansion.py b/tests/e2e/offline_inference/test_bagel_expansion.py index 980fc633f..e8ac6c19f 100644 --- a/tests/e2e/offline_inference/test_bagel_expansion.py +++ b/tests/e2e/offline_inference/test_bagel_expansion.py @@ -43,7 +43,7 @@ MODEL = "ByteDance-Seed/BAGEL-7B-MoT" BAGEL_STAGE_CONFIG = get_deploy_config_path("ci/bagel.yaml") DEFAULT_PROMPT = "<|im_start|>A cute cat<|im_end|>" -# (model, stage_configs_path) for ``@pytest.mark.parametrize("omni_runner", ..., indirect=True)`` +# (model, deploy_config_path) for ``@pytest.mark.parametrize("omni_runner", ..., indirect=True)`` _OMNI_RUNNER_PARAM = (MODEL, BAGEL_STAGE_CONFIG) diff --git a/tests/e2e/offline_inference/test_cosyvoice3_expansion.py b/tests/e2e/offline_inference/test_cosyvoice3_expansion.py index fa1bdfead..7e57433ef 100644 --- a/tests/e2e/offline_inference/test_cosyvoice3_expansion.py +++ b/tests/e2e/offline_inference/test_cosyvoice3_expansion.py @@ -126,7 +126,7 @@ def _build_reference_inputs(prompt_audio: tuple[np.ndarray, int]) -> list[dict[s ] -# (model, stage_configs_path, extra_omni_kwargs) for ``@pytest.mark.parametrize("omni_runner", ..., indirect=True)`` +# (model, deploy_config_path, extra_omni_kwargs) for ``@pytest.mark.parametrize("omni_runner", ..., indirect=True)`` _cosy_deployment = get_deploy_config_path("cosyvoice3.yaml") _cosy_model_path = str(_resolve_model_dir()) _OMNI_RUNNER_PARAMS = [ diff --git a/tests/e2e/offline_inference/test_ernie_image_expansion.py b/tests/e2e/offline_inference/test_ernie_image_expansion.py index 19793ff00..a43db7d7e 100644 --- a/tests/e2e/offline_inference/test_ernie_image_expansion.py +++ b/tests/e2e/offline_inference/test_ernie_image_expansion.py @@ -17,7 +17,7 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams MODEL = "baidu/ERNIE-Image" -# (model, stage_configs_path, extra_omni_kwargs) for ``omni_runner`` indirect parametrize +# (model, deploy_config_path, extra_omni_kwargs) for ``omni_runner`` indirect parametrize _OMNI_RUNNER_PARAM = ( MODEL, None, diff --git a/tests/e2e/offline_inference/test_glm_tts_expansion.py b/tests/e2e/offline_inference/test_glm_tts_expansion.py index e54dd5480..2bde38e6b 100644 --- a/tests/e2e/offline_inference/test_glm_tts_expansion.py +++ b/tests/e2e/offline_inference/test_glm_tts_expansion.py @@ -96,7 +96,7 @@ def test_offline_voice_clone_zh(async_chunk: bool) -> None: with OmniRunner( MODEL, - stage_configs_path=_get_deploy_config(async_chunk=async_chunk), + deploy_config=_get_deploy_config(async_chunk=async_chunk), stage_init_timeout=600, trust_remote_code=True, ) as omni_runner: diff --git a/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py b/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py index 4bdb42c34..84a9291b0 100644 --- a/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py +++ b/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py @@ -52,7 +52,7 @@ pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] def omni() -> Generator[Omni, None, None]: with OmniRunner( MODEL_NAME, - stage_configs_path=str(STAGE_CONFIG_PATH), + deploy_config=str(STAGE_CONFIG_PATH), trust_remote_code=True, ) as runner: yield runner.omni diff --git a/tests/e2e/offline_inference/test_longcat_video_avatar.py b/tests/e2e/offline_inference/test_longcat_video_avatar.py index f0fa81eb3..404ad9485 100644 --- a/tests/e2e/offline_inference/test_longcat_video_avatar.py +++ b/tests/e2e/offline_inference/test_longcat_video_avatar.py @@ -84,7 +84,6 @@ def longcat_omni_runner() -> Generator[OmniRunner, None, None]: with OmniRunner( model, seed=42, - stage_configs_path=None, model_class_name="LongCatVideoAvatarPipeline", additional_config={ "model_type": "avatar-v1.5", diff --git a/tests/e2e/offline_inference/test_mammoth_moda2_expansion.py b/tests/e2e/offline_inference/test_mammoth_moda2_expansion.py index ec1545f6a..f570a6ea7 100644 --- a/tests/e2e/offline_inference/test_mammoth_moda2_expansion.py +++ b/tests/e2e/offline_inference/test_mammoth_moda2_expansion.py @@ -38,7 +38,7 @@ _AR_PATCH_SIZE = 16 MODEL_PATH = "bytedance-research/MammothModa2-Preview" T2I_DEPLOY_CONFIG = get_deploy_config_path("mammoth_moda2.yaml") -_OMNI_RUNNER_PARAM = (MODEL_PATH, None, {"deploy_config": T2I_DEPLOY_CONFIG}) +_OMNI_RUNNER_PARAM = (MODEL_PATH, T2I_DEPLOY_CONFIG) # Golden pixel reference file. Set UPDATE_GOLDEN=1 to regenerate. _GOLDEN_T2I_PATH = Path(__file__).parent / "fixtures" / "mammoth_moda2_t2i_golden.json" diff --git a/tests/e2e/offline_inference/test_minicpmo_4_5.py b/tests/e2e/offline_inference/test_minicpmo_4_5.py index 6725ce2aa..5a518a7f7 100644 --- a/tests/e2e/offline_inference/test_minicpmo_4_5.py +++ b/tests/e2e/offline_inference/test_minicpmo_4_5.py @@ -17,7 +17,7 @@ models = ["openbmb/MiniCPM-o-4_5"] _CI_DEPLOY = get_deploy_config_path("minicpmo_4_5.yaml") -test_params = [(model, None, {"deploy_config": _CI_DEPLOY, "trust_remote_code": True}) for model in models] +test_params = [(model, _CI_DEPLOY, {"trust_remote_code": True}) for model in models] def get_question(prompt_type: str = "text") -> str: diff --git a/tests/e2e/offline_inference/test_moss_tts_nano_expansion.py b/tests/e2e/offline_inference/test_moss_tts_nano_expansion.py index b894fe4ea..41e2229d3 100644 --- a/tests/e2e/offline_inference/test_moss_tts_nano_expansion.py +++ b/tests/e2e/offline_inference/test_moss_tts_nano_expansion.py @@ -25,7 +25,7 @@ from vllm_omni import Omni MODEL_NAME = "OpenMOSS-Team/MOSS-TTS-Nano" STAGE_CONFIG = get_deploy_config_path("moss_tts_nano.yaml") -# (model, stage_configs_path, extra_omni_kwargs) for ``omni_runner`` indirect parametrize +# (model, deploy_config_path, extra_omni_kwargs) for ``omni_runner`` indirect parametrize _OMNI_RUNNER_PARAM = ( MODEL_NAME, STAGE_CONFIG, diff --git a/tests/e2e/offline_inference/test_omnivoice_expansion.py b/tests/e2e/offline_inference/test_omnivoice_expansion.py index 5b4c7a935..dd65cbdcc 100644 --- a/tests/e2e/offline_inference/test_omnivoice_expansion.py +++ b/tests/e2e/offline_inference/test_omnivoice_expansion.py @@ -20,13 +20,11 @@ from tests.helpers.stage_config import get_deploy_config_path MODEL = "k2-fsa/OmniVoice" DEPLOY_CONFIG = get_deploy_config_path("omnivoice.yaml") -# OmniRunner tuple: model, legacy stage config path, extra Omni kwargs. -# The migrated test passes deploy_config through extra Omni kwargs. +# OmniRunner tuple: model, deploy config path, extra Omni kwargs. _OMNI_RUNNER_PARAM = ( MODEL, - None, + DEPLOY_CONFIG, { - "deploy_config": DEPLOY_CONFIG, "trust_remote_code": True, "log_stats": True, }, diff --git a/tests/e2e/offline_inference/test_sensenova_u1_expansion.py b/tests/e2e/offline_inference/test_sensenova_u1_expansion.py index 4f5c97d11..42510d7c6 100644 --- a/tests/e2e/offline_inference/test_sensenova_u1_expansion.py +++ b/tests/e2e/offline_inference/test_sensenova_u1_expansion.py @@ -131,7 +131,7 @@ def test_sensenova_u1_lora_scale_and_deactivation(tmp_path) -> None: """Validate LoRA effect, scale sensitivity, bounded perturbation, and clean deactivation.""" lora_request = _make_file_lora_request(tmp_path / "sensenova_lora") - with OmniRunner(MODEL, stage_configs_path=None) as runner: + with OmniRunner(MODEL) as runner: omni = runner.omni baseline = _generate(omni) img_1x = _generate(omni, lora_request, lora_scale=1.0) diff --git a/tests/e2e/offline_inference/test_sensenova_u1_img2img_expansion.py b/tests/e2e/offline_inference/test_sensenova_u1_img2img_expansion.py index 1ac76da28..c2739844c 100644 --- a/tests/e2e/offline_inference/test_sensenova_u1_img2img_expansion.py +++ b/tests/e2e/offline_inference/test_sensenova_u1_img2img_expansion.py @@ -164,10 +164,7 @@ def _generate_sensenova_u1_img2img( def test_sensenova_u1_img2img(run_level): """Test SenseNova-U1 img2img (single-stage diffusion, no deploy YAML).""" input_image = _load_input_image() - with OmniRunner( - "SenseNova/SenseNova-U1-8B-MoT", - stage_configs_path=None, - ) as runner: + with OmniRunner("SenseNova/SenseNova-U1-8B-MoT") as runner: generated_image = _generate_sensenova_u1_img2img(runner.omni, input_image) if run_level == "full_model" or run_level == "advanced_model": _validate_pixels(generated_image) @@ -182,7 +179,6 @@ def test_sensenova_u1_img2img_cache_dit(): input_image = _load_input_image() with OmniRunner( "SenseNova/SenseNova-U1-8B-MoT", - stage_configs_path=None, cache_backend="cache_dit", ) as runner: _generate_sensenova_u1_img2img(runner.omni, input_image) diff --git a/tests/e2e/offline_inference/test_sensenova_u1_text2img_expansion.py b/tests/e2e/offline_inference/test_sensenova_u1_text2img_expansion.py index 092da16b9..2e07e2076 100644 --- a/tests/e2e/offline_inference/test_sensenova_u1_text2img_expansion.py +++ b/tests/e2e/offline_inference/test_sensenova_u1_text2img_expansion.py @@ -151,10 +151,7 @@ def _generate_sensenova_u1_image( @hardware_test(res={"cuda": "H100"}) def test_sensenova_u1_text2img(run_level): """Test SenseNova-U1 text2img (single-stage diffusion, no deploy YAML).""" - with OmniRunner( - "SenseNova/SenseNova-U1-8B-MoT", - stage_configs_path=None, - ) as runner: + with OmniRunner("SenseNova/SenseNova-U1-8B-MoT") as runner: generated_image = _generate_sensenova_u1_image(runner.omni) if run_level == "full_model" or run_level == "advanced_model": _validate_pixels(generated_image) @@ -168,7 +165,6 @@ def test_sensenova_u1_text2img_cache_dit(): """Test SenseNova-U1 text2img with Cache-DiT enabled.""" with OmniRunner( "SenseNova/SenseNova-U1-8B-MoT", - stage_configs_path=None, cache_backend="cache_dit", ) as runner: _generate_sensenova_u1_image(runner.omni) diff --git a/tests/e2e/offline_inference/test_soulxsinger_expansion.py b/tests/e2e/offline_inference/test_soulxsinger_expansion.py index c3a6091d6..a008be526 100644 --- a/tests/e2e/offline_inference/test_soulxsinger_expansion.py +++ b/tests/e2e/offline_inference/test_soulxsinger_expansion.py @@ -159,7 +159,7 @@ def test_soulxsinger_multistage_from_audio( with OmniRunner( model, - stage_configs_path=get_deploy_config_path(deploy_yaml), + deploy_config=get_deploy_config_path(deploy_yaml), async_chunk=False, ) as runner: sampling = OmniDiffusionSamplingParams( @@ -212,7 +212,7 @@ def test_soulxsinger_svs_precomputed(soulx_weights: tuple[Path, Path, Path]) -> with OmniRunner( str(base_dir), - stage_configs_path=get_deploy_config_path("soulxsinger_svs.yaml"), + deploy_config=get_deploy_config_path("soulxsinger_svs.yaml"), async_chunk=False, ) as runner: sampling = OmniDiffusionSamplingParams( diff --git a/tests/e2e/offline_inference/test_step_audio2_expansion.py b/tests/e2e/offline_inference/test_step_audio2_expansion.py index 483396b23..3a1b60c94 100644 --- a/tests/e2e/offline_inference/test_step_audio2_expansion.py +++ b/tests/e2e/offline_inference/test_step_audio2_expansion.py @@ -19,7 +19,7 @@ from vllm_omni.outputs import OmniRequestOutput MODEL = "stepfun-ai/Step-Audio-2-mini" DEPLOY_CONFIG = get_deploy_config_path("step_audio2_ci.yaml") -TEST_PARAMS = [(MODEL, None, {"deploy_config": DEPLOY_CONFIG, "trust_remote_code": True})] +TEST_PARAMS = [(MODEL, DEPLOY_CONFIG, {"trust_remote_code": True})] pytestmark = [pytest.mark.slow, pytest.mark.tts] diff --git a/tests/e2e/offline_inference/test_voxtral_tts_expansion.py b/tests/e2e/offline_inference/test_voxtral_tts_expansion.py index 2441c46dc..dcff41cda 100644 --- a/tests/e2e/offline_inference/test_voxtral_tts_expansion.py +++ b/tests/e2e/offline_inference/test_voxtral_tts_expansion.py @@ -102,7 +102,7 @@ def test_voxtral_tts_offline_streaming(): async def _run(): async_omni = AsyncOmni( model=MODEL, - stage_configs_path=STAGE_CONFIG, + deploy_config=STAGE_CONFIG, stage_init_timeout=300, enforce_eager=True, ) diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index 055866bd5..ed1031b64 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -4,7 +4,6 @@ invariant to the specific attributes of vLLM config except in cases where we explicitly patch values that differ from vLLM. """ -import argparse import inspect import json import os @@ -20,7 +19,6 @@ from vllm.engine.arg_utils import EngineArgs from vllm_omni.config.model import OmniModelConfig from vllm_omni.engine.arg_utils import OmniEngineArgs -from vllm_omni.engine.async_omni_engine import AsyncOmniEngine from vllm_omni.engine.stage_init_utils import build_engine_args_dict pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -149,19 +147,6 @@ def test_qwen3_tts_startup_task_type_is_validated(): ) -def test_from_cli_args_picks_up_stage_configs_path(): - """from_cli_args should pick up stage_configs_path from namespace.""" - ns = argparse.Namespace( - model="facebook/opt-125m", - stage_configs_path="/some/path.yaml", - custom_pipeline_args=None, - ) - - args = OmniEngineArgs.from_cli_args(ns) - assert args.stage_configs_path == "/some/path.yaml" - assert args.custom_pipeline_args is None - - def test_qwen3_tts_code2wav_injects_max_position_embeddings(monkeypatch): """Ensure Code2Wav mirrors stage max_model_len into nested HF overrides. @@ -418,130 +403,6 @@ def test_non_override_ar_stage_inputs_embeds_size_matches_hidden_size(): assert omni_config.get_inputs_embeds_size() == omni_config.hf_text_config.hidden_size -def test_stage_configs_path_field(): - """OmniEngineArgs with stage_configs_path should construct without error.""" - args = OmniEngineArgs(stage_configs_path="/some/path.yaml") - assert args.stage_configs_path == "/some/path.yaml" - - -def test_strip_single_engine_args(): - """_strip_single_engine_args should remove EngineArgs fields but keep omni fields.""" - kwargs = { - # Parent EngineArgs fields — stripped unless explicitly allowlisted - "compilation_config": '{"cudagraph_mode": "FULL_AND_PIECEWISE"}', - "tensor_parallel_size": 4, - "gpu_memory_utilization": 0.9, - "model": "some/model", - # Parent field that should be kept (allowlisted) - "worker_extension_cls": "some.Extension", - # OmniEngineArgs-only / non-engine fields — should pass through - "stage_configs_path": "/path/to/yaml", - "custom_pipeline_args": {"pipeline_class": "my.Pipeline"}, - "mode": "text-to-image", - "lora_path": "/some/lora", - } - - filtered = AsyncOmniEngine._strip_single_engine_args(kwargs) - - # Stripped — parent EngineArgs fields - assert "compilation_config" not in filtered - assert filtered["tensor_parallel_size"] == 4 - assert "gpu_memory_utilization" not in filtered - assert "model" not in filtered - - # Stripped — orchestrator-level OmniEngineArgs field - assert "stage_configs_path" not in filtered - - # Kept - assert filtered["worker_extension_cls"] == "some.Extension" - assert filtered["custom_pipeline_args"] == {"pipeline_class": "my.Pipeline"} - assert filtered["mode"] == "text-to-image" - assert filtered["lora_path"] == "/some/lora" - - -def test_strip_single_engine_args_model_does_not_trigger_warning(mocker): - """model is always in kwargs (callers set it via from_cli_args/asdict), - so it should not cause the override warning by itself or appear in it.""" - mock_warn = mocker.patch("vllm_omni.engine.async_omni_engine.logger.warning") - - # Typical caller kwargs: model is always present, no other parent - # EngineArgs fields are explicitly overridden. - AsyncOmniEngine._strip_single_engine_args( - { - "model": "some/model", - "custom_pipeline_args": {"pipeline_class": "my.Pipeline"}, - } - ) - mock_warn.assert_not_called() - - # When there *are* genuinely surprising overrides alongside model, - # the warning should mention them but not model. Keep-listed fields such as - # tensor_parallel_size are intentionally passed through and should not warn. - AsyncOmniEngine._strip_single_engine_args( - { - "model": "some/model", - "compilation_config": '{"cudagraph_mode": "FULL_AND_PIECEWISE"}', - "tensor_parallel_size": 4, - "custom_pipeline_args": {"pipeline_class": "my.Pipeline"}, - } - ) - mock_warn.assert_called_once() - warned_args = mock_warn.call_args[0][-1] # the formatted arg list - assert "compilation_config" in warned_args - assert "tensor_parallel_size" not in warned_args - assert "model" not in warned_args - - -def test_new_format_deploy_yaml_does_not_strip_interleave_cli_overrides(tmp_path): - """New-format deploy YAMLs must not strip interleave_mm_strings / media_io_kwargs. - - Pytest historically passed ``--stage-configs-path`` for deploy YAMLs; stripping - those CLI flags dropped MiniCPM interleaved AV packing and cut Daily-Omni - accuracy from ~78% to ~70%. - """ - from vllm_omni.entrypoints.utils import is_new_format_deploy_config - - yaml_path = tmp_path / "minicpm_deploy.yaml" - yaml_path.write_text( - """ -pipeline: minicpmo_4_5 -async_chunk: true -stages: - - stage_id: 0 - devices: "0" -""".strip(), - encoding="utf-8", - ) - path = str(yaml_path) - assert is_new_format_deploy_config(path) - - kwargs = { - "stage_configs_path": path, - "trust_remote_code": True, - "interleave_mm_strings": True, - "media_io_kwargs": {"video": {"fps": 1, "num_frames": 128}}, - "gpu_memory_utilization": 0.55, - } - # Mirror AsyncOmniEngine._resolve_stage_configs strip decision. - if kwargs.get("stage_configs_path") is not None and not is_new_format_deploy_config(kwargs["stage_configs_path"]): - base = AsyncOmniEngine._strip_single_engine_args(dict(kwargs)) - else: - base = dict(kwargs) - assert base.get("interleave_mm_strings") is True - assert base.get("media_io_kwargs") == {"video": {"fps": 1, "num_frames": 128}} - # Parent EngineArgs fields also remain available for deploy merge. - assert base.get("gpu_memory_utilization") == 0.55 - - legacy = tmp_path / "legacy.yaml" - legacy.write_text("stage_args:\n - stage_id: 0\n", encoding="utf-8") - assert not is_new_format_deploy_config(str(legacy)) - legacy_kwargs = dict(kwargs, stage_configs_path=str(legacy)) - stripped = AsyncOmniEngine._strip_single_engine_args(legacy_kwargs) - assert "interleave_mm_strings" not in stripped - assert "media_io_kwargs" not in stripped - assert "gpu_memory_utilization" not in stripped - - # For https://github.com/vllm-project/vllm-omni/issues/3293 def test_tensor_parallel_size_none_is_handled(): """Ensure the tensor parallel size of None isn't forwarded.""" diff --git a/tests/engine/test_async_omni_engine_abort.py b/tests/engine/test_async_omni_engine_abort.py index 40f89df84..56b25704d 100644 --- a/tests/engine/test_async_omni_engine_abort.py +++ b/tests/engine/test_async_omni_engine_abort.py @@ -68,7 +68,7 @@ async def test_abort(): with ExitStack() as after: engine = AsyncOmni( model=model, - stage_configs_path=stage_config, + deploy_config=stage_config, ) after.callback(engine.shutdown) diff --git a/tests/engine/test_single_stage_mode.py b/tests/engine/test_single_stage_mode.py index bde8ba490..65ec024d5 100644 --- a/tests/engine/test_single_stage_mode.py +++ b/tests/engine/test_single_stage_mode.py @@ -428,7 +428,7 @@ class TestSingleStageModeDetection: assert engine.single_stage_mode is False assert engine._single_stage_id_filter is None - def test_stage_configs_path_loads_duplex_runtime_config(self, mocker: MockerFixture): + def test_deploy_config_loads_duplex_runtime_config(self, mocker: MockerFixture): duplex_session = DuplexSessionRuntimeConfig(max_sessions=2) get_pipeline_config = mocker.patch( "vllm_omni.engine.async_omni_engine.StageConfigFactory.get_pipeline_config", @@ -441,7 +441,7 @@ class TestSingleStageModeDetection: engine = self._make_engine_no_thread( mocker, - stage_configs_path="/fake/duplex.yaml", + deploy_config="/fake/duplex.yaml", ) get_pipeline_config.assert_called_once_with( diff --git a/tests/entrypoints/test_async_omni.py b/tests/entrypoints/test_async_omni.py index 6120d5ea4..a672c32e9 100644 --- a/tests/entrypoints/test_async_omni.py +++ b/tests/entrypoints/test_async_omni.py @@ -364,7 +364,7 @@ async def test_omni_generate_request_id(): Text modality only; asserts caller-visible ids are preserved across the multi-stage orchestrator path on H100. """ - engine = AsyncOmni(model=OMNI_MODEL, stage_configs_path=OMNI_STAGE_CONFIG) + engine = AsyncOmni(model=OMNI_MODEL, deploy_config=OMNI_STAGE_CONFIG) try: for req_id in _OMNI_REQ_IDS: async for output in engine.generate( diff --git a/tests/entrypoints/test_async_omni_diffusion_config.py b/tests/entrypoints/test_async_omni_diffusion_config.py index 2fa8c92bd..75a9de2a7 100644 --- a/tests/entrypoints/test_async_omni_diffusion_config.py +++ b/tests/entrypoints/test_async_omni_diffusion_config.py @@ -478,12 +478,11 @@ def test_resolve_stage_configs_injects_additional_config_into_diffusion_stage(mo ) engine = AsyncOmniEngine.__new__(AsyncOmniEngine) - engine._strip_single_engine_args = lambda kwargs: kwargs _, stage_configs = engine._resolve_stage_configs( "dummy-model", { - "stage_configs_path": "dummy.yaml", + "deploy_config": "dummy.yaml", "additional_config": {"torchair_graph_config": {"enabled": True}}, }, trust_remote_code=False, @@ -516,12 +515,11 @@ def test_resolve_stage_configs_injects_quantization_config_into_diffusion_stage( ) engine = AsyncOmniEngine.__new__(AsyncOmniEngine) - engine._strip_single_engine_args = lambda kwargs: kwargs _, stage_configs = engine._resolve_stage_configs( "dummy-model", { - "stage_configs_path": "dummy.yaml", + "deploy_config": "dummy.yaml", "quantization_config": {"method": "bitsandbytes"}, }, trust_remote_code=False, diff --git a/tests/entrypoints/test_serve.py b/tests/entrypoints/test_serve.py index 04a29edd1..6aa5d6855 100644 --- a/tests/entrypoints/test_serve.py +++ b/tests/entrypoints/test_serve.py @@ -193,7 +193,7 @@ def test_run_headless_parses_and_forwards_stage_overrides(mocker: MockerFixture) with pytest.raises(ValueError, match="No stage config found for stage_id=0"): run_headless(args) - assert captured["args"][1] is None + assert len(captured["args"]) == 2 assert captured["deploy_config_path"] == "/tmp/deploy.yaml" assert captured["stage_overrides"] == {"0": {"devices": "0,1"}, "1": {"devices": "2"}} assert captured["strategy_config_path"] == "/tmp/strategy.yaml" diff --git a/tests/entrypoints/test_utils.py b/tests/entrypoints/test_utils.py index c6d8e0354..a20c97a58 100644 --- a/tests/entrypoints/test_utils.py +++ b/tests/entrypoints/test_utils.py @@ -381,7 +381,6 @@ class TestLoadAndResolveStageConfigs: config_path, stage_configs, _ = load_and_resolve_stage_configs( model="black-forest-labs/FLUX.2-klein-4B", - stage_configs_path=None, kwargs=kwargs, trust_remote_code=False, default_stage_cfg_factory=lambda: AsyncOmniEngine._create_default_diffusion_stage_cfg(kwargs), @@ -390,9 +389,7 @@ class TestLoadAndResolveStageConfigs: assert len(stage_configs) == 1 assert "dtype" in stage_configs[0]["engine_args"] - def test_stage_configs_path_promotes_new_deploy_yaml_without_expanding_replicas( - self, tmp_path, mocker: MockerFixture - ): + def test_deploy_config_preserves_cli_overrides_and_replicas(self, tmp_path, mocker: MockerFixture): deploy_path = tmp_path / "qwen3_multi.yaml" deploy_path.write_text( 'stages:\n - stage_id: 0\n devices: "0"\n - stage_id: 1\n devices: "1,2,3"\n num_replicas: 3\n', @@ -413,18 +410,23 @@ class TestLoadAndResolveStageConfigs: "vllm_omni.entrypoints.utils.load_stage_configs_from_model", return_value=(returned_stage_configs, None), ) + cli_overrides = { + "interleave_mm_strings": True, + "media_io_kwargs": {"video": {"fps": 1, "num_frames": 128}}, + "gpu_memory_utilization": 0.55, + } config_path, stage_configs, _ = load_and_resolve_stage_configs( model="dummy-model", - stage_configs_path=str(deploy_path), - kwargs={}, + kwargs=cli_overrides, trust_remote_code=True, + deploy_config_path=str(deploy_path), ) load_stage_configs.assert_called_once_with( "dummy-model", trust_remote_code=True, - base_engine_args={}, + base_engine_args=cli_overrides, deploy_config_path=str(deploy_path), stage_overrides=None, strategy_config_path=None, diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index 82fcc308d..cfc8cc75d 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -2635,7 +2635,7 @@ class OmniRunner: # affects the test runner wrapper. init_timeout: int = 1800, log_stats: bool = False, - stage_configs_path: str | None = None, + deploy_config: str | None = None, **kwargs, ) -> None: startup_t0 = time.perf_counter() @@ -2653,7 +2653,7 @@ class OmniRunner: stage_init_timeout=stage_init_timeout, batch_timeout=batch_timeout, init_timeout=init_timeout, - stage_configs_path=stage_configs_path, + deploy_config=deploy_config, **kwargs, ) startup_s = time.perf_counter() - startup_t0 @@ -3179,12 +3179,7 @@ def iter_omni_server( print("OmniServer stopping...") else: if stage_config_path is not None: - from vllm_omni.entrypoints.utils import is_new_format_deploy_config - - if is_new_format_deploy_config(stage_config_path): - server_args += ["--deploy-config", stage_config_path] - else: - server_args += ["--stage-configs-path", stage_config_path] + server_args += ["--deploy-config", stage_config_path] with ( OmniServer( @@ -3237,7 +3232,7 @@ def iter_omni_runner( model = model_prefix + model if run_level == "core_model" and request.node.get_closest_marker("diffusion"): model = resolve_tiny_model_path(model) - with OmniRunner(model, seed=42, stage_configs_path=stage_config_path, **extra_omni_kwargs) as runner: + with OmniRunner(model, seed=42, deploy_config=stage_config_path, **extra_omni_kwargs) as runner: print("OmniRunner started successfully") yield runner print("OmniRunner stopping...") diff --git a/tools/baselines/omni_hash_baseline.py b/tools/baselines/omni_hash_baseline.py index 1bb93099b..8d8f5ff13 100644 --- a/tools/baselines/omni_hash_baseline.py +++ b/tools/baselines/omni_hash_baseline.py @@ -147,7 +147,7 @@ def _run_once(model: str, yaml_path: str, *, async_mode: bool) -> list[dict]: # (see tests/e2e/accuracy/qwen3_omni). Confirm the exact knob at bench time. omni = Omni( model, - stage_configs_path=yaml_path, + deploy_config=yaml_path, stage_init_timeout=1800, init_timeout=3600, async_chunk=async_mode, diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index 4b5c73a39..32ee2c826 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -53,7 +53,6 @@ _NON_STAGE_ENGINE_CLI_FIELDS = frozenset( "model", "omni", "output_modalities", - "stage_configs_path", "stage_id", "tokenizer", } diff --git a/vllm_omni/config/stage_config.py b/vllm_omni/config/stage_config.py index 496aea208..86a6fed1b 100644 --- a/vllm_omni/config/stage_config.py +++ b/vllm_omni/config/stage_config.py @@ -4,7 +4,6 @@ from __future__ import annotations -import dataclasses import functools import re import warnings @@ -56,7 +55,7 @@ def build_stage_runtime_overrides( ``internal_keys`` defaults to the union of ``arg_utils.internal_blacklist_keys()`` and ``arg_utils.SHARED_FIELDS`` so that neither orchestrator-only fields nor shared-pipeline fields - (``model`` / ``stage_configs_path`` / ``log_stats`` / ``stage_id``) leak + (``model`` / ``log_stats`` / ``stage_id``) leak into a stage's per-stage runtime overrides — the orchestrator sets those uniformly for every stage, they are not per-stage knobs. Callers can pass an explicit set for tests or specialized flows. @@ -90,46 +89,6 @@ def build_stage_runtime_overrides( return result -def strip_parent_engine_args( - kwargs: dict[str, Any], - *, - parent_fields: dict[str, dataclasses.Field], - keep_keys: set[str] | frozenset[str] = frozenset(), - strip_keys: set[str] | frozenset[str] = frozenset(), - no_warn_keys: set[str] | frozenset[str] = frozenset(), -) -> tuple[dict[str, Any], list[str]]: - """Strip parent ``EngineArgs`` fields before merging into stage YAML.""" - overridden: list[str] = [] - result: dict[str, Any] = {} - - for key, value in kwargs.items(): - if key in strip_keys: - continue - - if key not in parent_fields or key in keep_keys: - result[key] = value - continue - - field_def = parent_fields[key] - if field_def.default is not dataclasses.MISSING: - default = field_def.default - elif field_def.default_factory is not dataclasses.MISSING: - default = field_def.default_factory() - else: - default = dataclasses.MISSING - - if default is dataclasses.MISSING or value is None: - continue - - if dataclasses.is_dataclass(default) and not isinstance(default, type): - default = asdict(default) - - if value != default and key not in no_warn_keys: - overridden.append(key) - - return result, sorted(overridden) - - def _apply_diffusion_parallel_runtime_overrides( engine_args: dict[str, Any], runtime_overrides: dict[str, Any], diff --git a/vllm_omni/engine/arg_utils.py b/vllm_omni/engine/arg_utils.py index a3aa96f60..29c84a1c9 100644 --- a/vllm_omni/engine/arg_utils.py +++ b/vllm_omni/engine/arg_utils.py @@ -167,9 +167,6 @@ class OmniEngineArgs(EngineArgs): Required when single-stage mode is active. omni_master_port: TCP port for the OmniMasterServer registration socket. Required when single-stage mode is active. - stage_configs_path: Optional path to a JSON/YAML file containing - stage configurations for the multi-stage pipeline. If None, - stage configs are resolved from the model's default configuration. output_modalities: Optional list of output modality names to enable (e.g. ["text", "audio"]). If None, all modalities supported by the model are used. @@ -231,7 +228,6 @@ class OmniEngineArgs(EngineArgs): omni_dp_size_local: int = 1 omni_lb_policy: str = "random" omni_heartbeat_timeout: float = 30.0 - stage_configs_path: str | None = None output_modalities: list[str] | None = None log_stats: bool = False custom_pipeline_args: dict[str, Any] | None = None @@ -490,7 +486,6 @@ class OrchestratorArgs: ray_address: str | None = None # === Config Files === - stage_configs_path: str | None = None deploy_config: str | None = None stage_overrides: str | None = None # raw JSON string; parsed downstream # Optional composable-parallel strategy.yaml; orchestrator reads it, overlays @@ -582,7 +577,6 @@ SHARED_FIELDS: frozenset[str] = frozenset( "model", # orch: detect model_type; engine: load weights "stage_id", # orch: route (headless); engine: identity "log_stats", # both want the flag - "stage_configs_path", # orch: load legacy YAML; engine: may reference for validation "async_chunk", # orch: read from CLI, redistribute; engine: per-stage flag "tokenizer", # orch: detect model type; engine: tokenization } diff --git a/vllm_omni/engine/async_omni_engine.py b/vllm_omni/engine/async_omni_engine.py index aa284067b..a613ebc80 100644 --- a/vllm_omni/engine/async_omni_engine.py +++ b/vllm_omni/engine/async_omni_engine.py @@ -9,7 +9,6 @@ from __future__ import annotations import asyncio import concurrent.futures -import dataclasses import json import queue import threading @@ -24,7 +23,6 @@ import janus import torch from omegaconf import OmegaConf from vllm import envs as vllm_envs -from vllm.engine.arg_utils import EngineArgs from vllm.inputs import PromptType from vllm.logger import init_logger from vllm.v1.engine import EngineCoreRequest @@ -34,7 +32,6 @@ from vllm_omni.config.config_factory import StageConfigFactory, with_trust_remot from vllm_omni.config.stage_config import ( DuplexSessionRuntimeConfig, load_deploy_config, - strip_parent_engine_args, ) from vllm_omni.diffusion.data import DiffusionParallelConfig, parse_attention_config from vllm_omni.diffusion.diffusion_engine import supports_audio_output @@ -69,7 +66,6 @@ from vllm_omni.engine.stage_runtime import ( ) from vllm_omni.entrypoints.pd_utils import PDDisaggregationMixin from vllm_omni.entrypoints.utils import ( - is_new_format_deploy_config, load_and_resolve_stage_configs, parse_stage_overrides, ) @@ -85,39 +81,6 @@ if TYPE_CHECKING: _STARTUP_POLL_INTERVAL_S = 1.0 _REQUEST_QUEUE_MAXSIZE = 256 -# ============================================================================ -# Parent-EngineArgs field-routing contracts (consumed by -# AsyncOmniEngine._strip_parent_engine_args when ``stage_configs_path`` is set). -# ============================================================================ - -# Fields that must survive the "equal to default → strip" filter because -# diffusion stages need them even when equal to vllm's default value -# (e.g. colocate worker setup relies on worker_extension_cls being forwarded). -_PARENT_ARGS_KEEP: frozenset[str] = frozenset( - { - "worker_extension_cls", - "allowed_local_media_path", - "allowed_media_domains", - # Legacy stage-config YAMLs may intentionally leave parallel or - # distributed knobs unspecified at the stage level and rely on - # top-level CLI values to fill them in during the per-stage merge. - # Keep these fields so stages that omit them can inherit CLI values, - # while stages with explicit YAML values still win because the legacy - # stage-config loader prefers stage-local engine args. - "tensor_parallel_size", - } -) - -# Omni orchestrator-level fields consumed by ``_resolve_stage_configs`` that -# must never leak into per-stage EngineArgs (``stage_configs_path`` would -# trigger the ``create_model_config`` guard). -_PARENT_ARGS_STRIP: frozenset[str] = frozenset({"stage_configs_path"}) - - -# Fields always populated by callers (via ``from_cli_args`` / ``asdict``) so -# their presence as an override is never a surprise — suppress the -# "override ignored" warning for these. -_PARENT_ARGS_NO_WARN: frozenset[str] = frozenset({"model"}) class AsyncOmniEngine: @@ -210,10 +173,8 @@ class AsyncOmniEngine: self._omni_master_port, ) - # Stage resolution pops deploy_config, so get pipeline-wide settings - # beforehand. The stage CLI exposes the same deploy YAML through - # stage_configs_path. - deploy_config_path = kwargs.get("deploy_config") or kwargs.get("stage_configs_path") + # Stage resolution pops deploy_config, so get pipeline-wide settings beforehand. + deploy_config_path = kwargs.get("deploy_config") # ``trust_remote_code`` is tri-state (bool | None): ``None`` means "not # specified" so stage-config resolution can defer to the deploy yaml's # per-stage value (see ``with_trust_remote_code_override``). The @@ -1116,40 +1077,6 @@ class AsyncOmniEngine: default_stage_cfg[0]["engine_args"]["model_stage"] = "diffusion" return default_stage_cfg - @staticmethod - def _strip_single_engine_args(kwargs: dict[str, Any]) -> dict[str, Any]: - """Remove parent ``EngineArgs`` fields from *kwargs*. - - When ``stage_configs_path`` is set, per-stage engine args are defined - in the YAML. Top-level single-engine fields (``compilation_config``, - ``tensor_parallel_size``, …) must not leak into per-stage configs via - the ``base_engine_args`` merge in ``load_stage_configs_from_yaml`` — - they can cause type errors (e.g. ``compilation_config`` as a JSON - string rejected by ``VllmConfig``) or silently override YAML values. - - Logs a warning for any parent field whose value differs from the - dataclass default, so users know their explicit overrides are ignored. - See the module-level ``_PARENT_ARGS_*`` constants for the routing - contracts this method enforces. - """ - parent_fields: dict[str, dataclasses.Field] = {f.name: f for f in dataclasses.fields(EngineArgs)} - result, overridden = strip_parent_engine_args( - kwargs, - parent_fields=parent_fields, - keep_keys=_PARENT_ARGS_KEEP, - strip_keys=_PARENT_ARGS_STRIP, - no_warn_keys=_PARENT_ARGS_NO_WARN, - ) - - if overridden: - logger.warning( - "stage_configs_path is set — the following top-level engine " - "args are ignored (per-stage YAML takes precedence): %s", - ", ".join(sorted(overridden)), - ) - - return result - def _apply_strategy_lb_policy(self, derived: str | None, kwargs: dict[str, Any]) -> None: """Apply a strategy-derived ``omni_lb_policy`` to the engine. @@ -1188,7 +1115,6 @@ class AsyncOmniEngine: ) -> tuple[str, list[Any]]: """Resolve stage configs and inject defaults shared by orchestrator/headless.""" - stage_configs_path = kwargs.get("stage_configs_path", None) deploy_config_path = kwargs.pop("deploy_config", None) strategy_config_path = kwargs.pop("strategy_config", None) stage_overrides_json = kwargs.pop("stage_overrides", None) @@ -1196,26 +1122,15 @@ class AsyncOmniEngine: if explicit_stage_configs is not None: logger.warning( "`stage_configs` is not part of the public API. " - "Ignoring it and resolving stages from stage_configs_path/model factory." + "Ignoring it and resolving stages from deploy_config/model factory." ) - # Legacy ``stage_args`` YAMLs own per-stage EngineArgs completely, so - # strip parent CLI fields. New-format deploy YAMLs (also accepted via - # ``--stage-configs-path``) merge CLI overrides the same way as - # ``--deploy-config`` — do not strip, or flags like - # ``interleave_mm_strings`` / ``media_io_kwargs`` are silently dropped. - if stage_configs_path is not None and not is_new_format_deploy_config(stage_configs_path): - base_kwargs = self._strip_single_engine_args(kwargs) - else: - base_kwargs = kwargs - # Parse --stage-overrides JSON string if provided stage_overrides = parse_stage_overrides(stage_overrides_json) config_path, stage_configs, strategy_lb_policy = load_and_resolve_stage_configs( model, - stage_configs_path, - base_kwargs, + kwargs, trust_remote_code=trust_remote_code, default_stage_cfg_factory=lambda: self._create_default_diffusion_stage_cfg(kwargs), deploy_config_path=deploy_config_path, diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index f6899deaa..33905b254 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -123,9 +123,8 @@ class AsyncOmni(EngineClient, OmniBase): Args: model: Model name or path to load. **kwargs: Additional keyword arguments. - - stage_configs_path: Optional path to YAML file containing stage - configurations. If None, configurations are resolved from model - pipeline factory. + - deploy_config: Optional path to a deploy YAML. If None, + configurations are resolved from the model pipeline factory. - log_stats: Whether to enable statistics logging. - stage_init_timeout: Timeout for per-stage initialization. - init_timeout: Total timeout for orchestrator startup. diff --git a/vllm_omni/entrypoints/cli/serve.py b/vllm_omni/entrypoints/cli/serve.py index dfa222cd1..62ee10d94 100644 --- a/vllm_omni/entrypoints/cli/serve.py +++ b/vllm_omni/entrypoints/cli/serve.py @@ -945,8 +945,6 @@ def run_headless(args: TrackingNamespace) -> None: config_path, stage_configs, _ = load_and_resolve_stage_configs( model, - # The serve CLI no longer accepts legacy stage_args YAMLs. - None, args_dict, # store_true cannot express an explicit False: absent maps to None # ("not specified") so the deploy yaml's per-stage value applies. diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 0a8daa055..16114bcb6 100755 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -679,7 +679,7 @@ async def build_async_omni_from_stage_config( EngineClient instance (AsyncOmni) ready for use Note: - Stage configurations are loaded from args.stage_configs_path if provided, + Stage configurations are loaded from ``args.deploy_config`` when provided, otherwise from the model's default configuration. """ @@ -1156,7 +1156,7 @@ async def omni_init_app_state( state.openai_serving_duplex = None if state.openai_serving_chat is not None and should_enable_duplex_endpoint( state.stage_configs, - config_path=getattr(args, "stage_configs_path", None) or getattr(args, "deploy_config", None), + config_path=getattr(args, "deploy_config", None), ): from vllm_omni.experimental.fullduplex.openai.serving import OmniDuplexSessionHandler diff --git a/vllm_omni/entrypoints/utils.py b/vllm_omni/entrypoints/utils.py index ff0cd4f9e..07b45f374 100644 --- a/vllm_omni/entrypoints/utils.py +++ b/vllm_omni/entrypoints/utils.py @@ -6,7 +6,6 @@ from dataclasses import fields, is_dataclass from pathlib import Path from typing import Any, get_args, get_origin -import yaml from vllm.logger import init_logger from vllm.sampling_params import RequestOutputKind, SamplingParams from vllm.transformers_utils.config import get_config, get_hf_file_to_dict @@ -30,15 +29,6 @@ PROJECT_ROOT = Path(__file__).parent.parent.parent logger = init_logger(__name__) -def is_new_format_deploy_config(path: str | None) -> bool: - """Return True when *path* is a deploy YAML (``stages:`` without legacy ``stage_args:``).""" - if not path or not os.path.exists(path): - return False - with open(path, encoding="utf-8") as f: - peek = yaml.safe_load(f) or {} - return isinstance(peek, dict) and "stages" in peek and "stage_args" not in peek - - _DIFFUSERS_CLASS_TO_CONFIG: dict[str, str] = { "GlmImagePipeline": "glm_image", } @@ -606,7 +596,6 @@ def parse_stage_overrides(value: Any) -> dict[str, dict[str, Any]] | None: def load_and_resolve_stage_configs( model: str, - stage_configs_path: str | None, kwargs: dict | None, *, trust_remote_code: bool | None, @@ -615,17 +604,15 @@ def load_and_resolve_stage_configs( stage_overrides: dict[str, dict[str, Any]] | None = None, strategy_config_path: str | None = None, ) -> tuple[str, list, str | None]: - """Load stage configurations from model or YAML file with fallback to defaults. + """Load stage configurations from a deploy YAML or model defaults. Args: model: Model name or path - stage_configs_path: Optional path to legacy YAML (stage_args format) kwargs: Engine arguments to merge with stage configs trust_remote_code: Whether to trust remote code while resolving the model config. default_stage_cfg_factory: Optional callable that takes no args and returns default stage config list when no configs are found - deploy_config_path: Optional path to deploy YAML (new format). - Mutually exclusive with ``stage_configs_path``. + deploy_config_path: Optional path to a deploy YAML. stage_overrides: Per-stage overrides from ``--stage-overrides`` JSON. Keys are stage_id strings, values are dicts of overrides. strategy_config_path: Optional path to a composable-parallel @@ -636,65 +623,21 @@ def load_and_resolve_stage_configs( the strategy-derived pipeline-wide load-balancer policy (``None`` when no strategy set one), returned for the engine to apply. """ - if stage_configs_path is not None and deploy_config_path is not None: - raise ValueError( - "--stage-configs-path and --deploy-config are mutually exclusive: " - "they use different path resolution rules and loading paths. " - "Use --deploy-config for new-format YAMLs (preferred); " - "--stage-configs-path is kept only for the legacy `stage_args` format " - "and will be removed in a future release." - ) - if stage_configs_path is not None and deploy_config_path is None: - if not os.path.exists(stage_configs_path): - raise FileNotFoundError( - f"--stage-configs-path {stage_configs_path!r} does not exist. " - "Legacy `stage_configs/` yamls were replaced by `vllm_omni/deploy/.yaml`; " - "use --deploy-config. See docs/configuration/stage_configs.md." - ) - if is_new_format_deploy_config(stage_configs_path): - deploy_config_path = stage_configs_path - stage_configs_path = None + config_path = deploy_config_path if deploy_config_path is not None else resolve_model_config_path(model) + stage_configs, omni_lb_policy = load_stage_configs_from_model( + model, + trust_remote_code=trust_remote_code, + base_engine_args=kwargs, + deploy_config_path=deploy_config_path, + stage_overrides=stage_overrides, + strategy_config_path=strategy_config_path, + ) + if not stage_configs: + if default_stage_cfg_factory is not None: + default_stage_cfg = default_stage_cfg_factory() + stage_configs = create_config(_convert_dataclasses_to_dict(default_stage_cfg)) else: - logger.warning( - "--stage-configs-path is deprecated; migrate %r and use --deploy-config.", - stage_configs_path, - ) - - omni_lb_policy: str | None = None - if deploy_config_path is not None: - config_path = deploy_config_path - stage_configs, omni_lb_policy = load_stage_configs_from_model( - model, - trust_remote_code=trust_remote_code, - base_engine_args=kwargs, - deploy_config_path=deploy_config_path, - stage_overrides=stage_overrides, - strategy_config_path=strategy_config_path, - ) - if not stage_configs: - if default_stage_cfg_factory is not None: - default_stage_cfg = default_stage_cfg_factory() - stage_configs = create_config(_convert_dataclasses_to_dict(default_stage_cfg)) - else: - stage_configs = [] - elif stage_configs_path is None: - config_path = resolve_model_config_path(model) - stage_configs, omni_lb_policy = load_stage_configs_from_model( - model, - trust_remote_code=trust_remote_code, - base_engine_args=kwargs, - stage_overrides=stage_overrides, - strategy_config_path=strategy_config_path, - ) - if not stage_configs: - if default_stage_cfg_factory is not None: - default_stage_cfg = default_stage_cfg_factory() - stage_configs = create_config(_convert_dataclasses_to_dict(default_stage_cfg)) - else: - stage_configs = [] - else: - config_path = stage_configs_path - stage_configs = load_stage_configs_from_yaml(stage_configs_path, base_engine_args=kwargs) + stage_configs = [] stage_configs = filter_stages(config_path, stage_configs, kwargs) logger.debug(f"stage_configs: {stage_configs}") From 83d1f11449a0eaf2910a0a8e34c055df20e4f169 Mon Sep 17 00:00:00 2001 From: Mu GuanLin <1203789601@qq.com> Date: Fri, 14 Aug 2026 00:13:14 +0800 Subject: [PATCH 002/267] [Feat] LTX Standard Two-Stage Pipeline (#5500) Signed-off-by: mglyn <1203789601@qq.com> --- .buildkite/cuda/test-nightly.yml | 10 +- docs/models/supported_models.md | 3 +- docs/user_guide/quantization/fp8.md | 4 +- recipes/LTX/LTX-2.md | 118 +++-- .../models/ltx2/test_ltx2_latents.py | 43 +- .../models/ltx2/test_ltx2_parallel.py | 14 + .../models/ltx2/test_ltx2_phase_adapter.py | 483 ++++++++++++++++++ .../models/ltx2/test_ltx2_pipeline.py | 372 +++++++++++++- .../models/ltx2/test_ltx2_transformer.py | 93 +++- tests/e2e/accuracy/ltx/run_ltx_reference.py | 155 ++++-- .../ltx/test_ltx_official_similarity.py | 444 ++++++++++++---- tests/e2e/online_serving/test_ltx.py | 109 ++++ .../e2e/online_serving/test_ltx2_expansion.py | 91 ---- tests/model_extras/test_model_extras.py | 11 +- tests/model_tests/diffusion/test_alignment.py | 1 + vllm_omni/diffusion/models/ltx2/__init__.py | 6 +- .../models/ltx2/ltx2_adapter_parser.py | 156 ++++++ .../diffusion/models/ltx2/ltx2_components.py | 133 ++++- .../diffusion/models/ltx2/ltx2_denoise.py | 23 +- .../diffusion/models/ltx2/ltx2_guidance.py | 32 +- .../diffusion/models/ltx2/ltx2_latents.py | 50 +- .../models/ltx2/ltx2_phase_adapter.py | 430 ++++++++++++++++ .../diffusion/models/ltx2/ltx2_recipes.py | 50 ++ .../diffusion/models/ltx2/ltx2_request.py | 31 +- .../diffusion/models/ltx2/ltx2_runtime.py | 38 +- .../diffusion/models/ltx2/ltx2_transformer.py | 63 ++- .../models/ltx2/pipeline_ltx2_two_stage.py | 45 +- vllm_omni/diffusion/registry.py | 6 + vllm_omni/model_extras/registry.py | 1 + 29 files changed, 2644 insertions(+), 371 deletions(-) create mode 100644 tests/diffusion/models/ltx2/test_ltx2_phase_adapter.py create mode 100644 tests/e2e/online_serving/test_ltx.py delete mode 100644 tests/e2e/online_serving/test_ltx2_expansion.py create mode 100644 vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py create mode 100644 vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index d3246a98f..ea74847b9 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -456,7 +456,7 @@ steps: timeout_in_minutes: 90 commands: - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model" - - pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx2_expansion.py -m "full_model and cuda" --run-level "full_model" + - pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx.py -m "full_model and cuda" --run-level "full_model" mirror_hardwares: h100_2 - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test" @@ -477,10 +477,16 @@ steps: commands: - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model - - pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_one_stage_matches_official[ltx2_3_i2v]' -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/test_video_streaming_output_similarity.py -m full_model --run-level full_model mirror_hardwares: h100_2 + - label: ":full_moon: Diffusion X2V · LTX Accuracy Test" + timeout_in_minutes: 180 + commands: + - pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_matches_official[ltx2_3_i2v]' -m full_model --run-level full_model + - pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_matches_official[ltx23_two_stage_layer_fused_i2v]' -m full_model --run-level full_model + mirror_hardwares: h100_1 + - label: ":full_moon: Diffusion X2V · Perf Test" key: nightly-diffusion-x2v-performance timeout_in_minutes: 180 diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index b9686047a..73fdc55e4 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -42,7 +42,8 @@ th { | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | | `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | | — | | `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | — | -| `LTX2DistilledPipeline` | LTX-2 distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | | — | +| `LTX2TwoStagePipeline` | LTX-2 / LTX-2.3 ordinary two-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` + matching Lightricks LoRA and upsampler | ✅︎ | ✅︎ | | | — | +| `LTX2DistilledPipeline` | LTX-2 / LTX-2.3 full-distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` + `Lightricks/LTX-2.3` upsampler | ✅︎ | ✅︎ | | | — | | `LingBotVideoPipeline` | LingBot-Video dense and MoE T2I, T2V, TI2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | | — | | `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | ✅︎ | | | [Published](https://recipes.vllm.ai/MiniMaxAI/MiniMax-H3) | | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | diff --git a/docs/user_guide/quantization/fp8.md b/docs/user_guide/quantization/fp8.md index ddca3ea19..05473b15e 100644 --- a/docs/user_guide/quantization/fp8.md +++ b/docs/user_guide/quantization/fp8.md @@ -80,12 +80,14 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)]) ## Model Type Support -### Diffusion Model (Qwen-Image, Wan2.2) +### Diffusion Models | Model | HF models | Online | Pre-calibrated | Recommendation | `ignored_layers` | Text-Encoder quantization | |-------|-----------|:-------:|:------:|----------------|------------------|------------------| | Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes | Yes | Skip sensitive image-stream MLPs when quality regresses | `img_mlp` | | | Wan2.2 | Wan2.2 diffusion pipelines | Not validated | Not validated | Validate against BF16 before documenting as supported | TBD | | +| LTX-2 | `Lightricks/LTX-2`, `rootonchair/LTX-2-19b-distilled` | Yes | Not validated | Transformer only; use dynamic phase LoRA for ordinary two-stage | None | | +| LTX-2.3 | `diffusers/LTX-2.3-Diffusers`, `diffusers/LTX-2.3-Distilled-Diffusers` | Yes | Not validated | Transformer only; use dynamic phase LoRA for ordinary two-stage | None | | | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | Yes | Yes | All layers | None | ✅︎ | | FLUX.1 | `black-forest-labs/FLUX.1-dev`, `black-forest-labs/FLUX.1-schnell` | Yes | Yes | All layers | None | | | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | Yes | Not validated | All layers | None | ✅︎ | diff --git a/recipes/LTX/LTX-2.md b/recipes/LTX/LTX-2.md index 3142f9fd1..123880d09 100644 --- a/recipes/LTX/LTX-2.md +++ b/recipes/LTX/LTX-2.md @@ -4,11 +4,20 @@ ## Pipelines -| Model | Checkpoint | Task | `--model-class-name` | Batching | -|---|---|---|---|---| -| LTX-2 | `Lightricks/LTX-2` | One-stage T2V/I2V | `LTX2Pipeline` | Yes | -| LTX-2 distilled | `rootonchair/LTX-2-19b-distilled` | Two-stage T2V/I2V | `LTX2DistilledPipeline` | No | -| LTX-2.3 | `diffusers/LTX-2.3-Diffusers` | One-stage T2V/I2V | `LTX2Pipeline` | Yes | +| `--model-class-name` | Task | Required checkpoint repositories | +|---|---|---| +| `LTX2Pipeline` | LTX-2 one-stage T2V/I2V | `Lightricks/LTX-2` | +| `LTX2TwoStagePipeline` | LTX-2 ordinary two-stage T2V/I2V | `Lightricks/LTX-2` | +| `LTX2DistilledPipeline` | LTX-2 full-distilled two-stage T2V/I2V | `rootonchair/LTX-2-19b-distilled` | +| `LTX2Pipeline` | LTX-2.3 one-stage T2V/I2V | `diffusers/LTX-2.3-Diffusers` | +| `LTX2TwoStagePipeline` | LTX-2.3 ordinary two-stage T2V/I2V | `diffusers/LTX-2.3-Diffusers`
`Lightricks/LTX-2.3` | +| `LTX2DistilledPipeline` | LTX-2.3 full-distilled two-stage T2V/I2V | `diffusers/LTX-2.3-Distilled-Diffusers`
`Lightricks/LTX-2.3` | + +Repositories in the table are download units. A full pipeline repository +contains the Transformer, text encoder, connectors, VAEs, vocoder, scheduler, +and tokenizer; an additional repository supplies LoRA or upsampler sidecars. +The runtime first searches the model root for sidecars under their official +filenames, then downloads them from the matching Lightricks Hub repository. `LTX2Pipeline` is the unified one-stage entry. Checkpoint metadata selects the LTX-2 or LTX-2.3 profile; omitting an image selects T2V, while one initial @@ -16,10 +25,10 @@ image selects I2V. Both one-stage repositories declare this class, so `--model-class-name` is optional. LTX-2.3 requires the Diffusers checkpoint; the raw `Lightricks/LTX-2.3` safetensors repository is not directly loadable. -`LTX2DistilledPipeline` is the unified distilled entry: omit `image` for T2V -or provide one initial image for I2V. As in the official LTX pipelines, -distilled mode is selected explicitly rather than inferred from the checkpoint -name or metadata; always pass `--model-class-name LTX2DistilledPipeline`. +`LTX2TwoStagePipeline` samples the regular model at half resolution, upsamples, +then refines with the distilled LoRA. `LTX2DistilledPipeline` uses a fully +merged distilled Transformer in both stages. Both entries support T2V and I2V; +select their class explicitly. ## API Migration @@ -68,24 +77,28 @@ The model recipe also supplies the default negative prompt. Top-level API defaults `num_frames` to `1`, so set it explicitly; the offline LTX scripts default to `121`. -## Distilled Two-Stage Defaults +## Two-Stage Defaults -| Parameter | Value | -|---|---:| -| Final width × height | 1536 × 1024 | -| Stage 1 width × height | 768 × 512 | -| Frames / frame rate | 121 / 24 | -| Stage 1 / Stage 2 denoise steps | 8 / 3 | -| Guidance | Fixed positive-only | +| Parameter | Ordinary | Full-distilled | +|---|---:|---:| +| Final width × height | 1536 × 1024 | 1536 × 1024 | +| Stage 1 width × height | 768 × 512 | 768 × 512 | +| Frames / frame rate | 121 / 24 | 121 / 24 | +| Stage 1 / Stage 2 steps | 40 (LTX-2) or 30 (LTX-2.3) / 3 | 8 / 3 | +| Guidance | Stage 1 guided; Stage 2 positive-only | Fixed positive-only | -The dimensions passed to the API are the final output dimensions and must be -divisible by 64. Both one- and two-stage requests require `num_frames = 8k+1`. -The distilled recipe fixes both sigma schedules; `num_inference_steps` must be -`8` when supplied. +API dimensions are final dimensions and must be divisible by 64. All LTX +requests require `num_frames = 8k+1`. Ordinary Stage 1 uses the LTX-2 or +LTX-2.3 one-stage defaults shown above. Distilled schedules are fixed, so +`num_inference_steps`, when supplied, must be `8`. Both entries reject custom +sigmas and input latents. + +Ordinary two-stage uses layer-fused LoRA for an unquantized BF16 Transformer +and automatically switches to dynamic LoRA when quantization is enabled. ## Serving -Start either checkpoint: +Start either one-stage checkpoint: ```bash vllm serve Lightricks/LTX-2 --omni --stage-init-timeout 600 @@ -95,11 +108,19 @@ vllm serve Lightricks/LTX-2 --omni --stage-init-timeout 600 vllm serve diffusers/LTX-2.3-Diffusers --omni --stage-init-timeout 600 ``` -Start the distilled checkpoint with its two-stage entry: +Start a two-stage checkpoint with its explicit class: ```bash vllm serve rootonchair/LTX-2-19b-distilled --omni \ --model-class-name LTX2DistilledPipeline --stage-init-timeout 600 +# LTX-2.3 full-distilled; the v1.1 x2 upsampler is resolved when absent +vllm serve diffusers/LTX-2.3-Distilled-Diffusers --omni \ + --model-class-name LTX2DistilledPipeline --stage-init-timeout 600 +# Ordinary LTX-2.3 two-stage +vllm serve diffusers/LTX-2.3-Diffusers --omni \ + --model-class-name LTX2TwoStagePipeline \ + --enable-layerwise-offload \ + --stage-init-timeout 600 ``` The same server handles T2V and I2V. A T2V request using the selected recipe's @@ -135,8 +156,9 @@ together with `input_reference`. ## Guidance -LTX supports independent video/audio CFG, spatio-temporal guidance (STG), -cross-modality guidance, and rescaling: +One-stage and ordinary Stage 1 support independent video/audio CFG, +spatio-temporal guidance (STG), cross-modality guidance, and rescaling. +Distilled stages and ordinary Stage 2 are fixed positive-only. | Parameter | Default | Effect | Alias | |---|---:|---|---| @@ -172,9 +194,9 @@ complete guidance plan in parallel. Despite the legacy `cfg` name, it covers text CFG, STG, and cross-modality guidance passes; guidance rescaling is applied after all pass predictions are gathered. -The default one-stage recipe has four Transformer passes per denoise step: -`cond`, `uncond`, `ptb` (STG), and `mod` (cross-modality). The useful balanced -configurations are therefore: +The default one-stage recipe and ordinary Stage 1 have four Transformer passes +per denoise step: `cond`, `uncond`, `ptb` (STG), and `mod` +(cross-modality). The useful balanced configurations are therefore: | `--cfg-parallel-size` | Passes per rank | Guidance-slot utilization | Notes | |---:|---:|---:|---| @@ -195,9 +217,8 @@ vllm serve diffusers/LTX-2.3-Diffusers --omni \ ``` The total device count is the product of `cfg_parallel_size` and the other -configured parallel dimensions. Distilled pipelines use positive-only -guidance, so `cfg_parallel_size > 1` does not provide guidance speedup for -them. +configured parallel dimensions. Positive-only distilled phases do not benefit +from `cfg_parallel_size > 1`. ### Python API @@ -242,22 +263,22 @@ noted below. | `image` | image or batch, `None` | Direct value wins over request images; no image selects T2V. I2V accepts one image per prompt, and a batch cannot mix T2V/I2V. | | `prompt` | string or list, `None` | Positive-text fallback; request prompts win. Mutually exclusive with `prompt_embeds`. | | `negative_prompt` | string or list, `None` | Fallback after request values and before the recipe default; mutually exclusive with negative embeddings. | -| `height` | `int`, `None` | Request → direct value → recipe default; divisible by 32 one-stage or 64 distilled. | +| `height` | `int`, `None` | Request → direct value → recipe default; divisible by 32 one-stage or 64 two-stage. | | `width` | `int`, `None` | Same precedence and alignment as `height`. | | `num_frames` | `int`, `None` | Request → direct value → recipe default; must be `8k+1` and also determines audio duration with `frame_rate`. | | `frame_rate` | `float`, `None` | Request `frame_rate` → request `fps` → direct value → recipe default. | -| `num_inference_steps` | `int`, `None` | Request → direct value → recipe default; minimum 2 one-stage, fixed at 8 for distilled Stage 1. Custom one-stage `sigmas` determine actual steps. | +| `num_inference_steps` | `int`, `None` | Request → direct value → recipe default; minimum 2 one-stage and ordinary Stage 1, fixed at 8 for distilled Stage 1. Custom one-stage `sigmas` determine actual steps. | | `sigmas` | list of float, `None` | One-stage only. Request values win; every request in a fused batch must use the same schedule. | | `timesteps` | list of int, `None` | Compatibility slot; LTX accepts only `None`. Use `sigmas`. | -| `guidance_scale` | `float`, `None` | One-stage common video/audio CFG fallback; an explicit request value wins. Distilled two-stage uses fixed positive-only guidance. | +| `guidance_scale` | `float`, `None` | One-stage and ordinary Stage 1 common video/audio CFG fallback; an explicit request value wins. Distilled phases use fixed positive-only guidance. | | `guidance_rescale` | `float`, `None` | Accepts only `None` or `0.0`; use the modality rescale fields. | | `noise_scale` | `float`, `0.0` | Compatibility slot; LTX accepts only `0.0`. | | `num_videos_per_prompt` | `int`, `1` | Output-count fallback; positive request `num_outputs_per_prompt` wins. | | `generator` | generator or list, `None` | Explicit RNG; otherwise request generators/seeds are collated. Lists must match the effective output batch. | -| `latents` | tensor, `None` | Request tensors win. One-stage accepts packed `[B, S, C]` and validated unpacked video layouts. | -| `audio_latents` | tensor, `None` | One-stage initial audio latents; request tensors win and are collated. | +| `latents` | tensor, `None` | One-stage only. Request tensors win; packed `[B, S, C]` and validated unpacked video layouts are accepted. | +| `audio_latents` | tensor, `None` | One-stage only. Request tensors win and are collated. | | `prompt_embeds` | tensor, `None` | Precomputed positive conditioning; requires `prompt_attention_mask` and cannot accompany `prompt`. | -| `negative_prompt_embeds` | tensor, `None` | One-stage precomputed negative conditioning; requires its mask and cannot accompany `negative_prompt`. | +| `negative_prompt_embeds` | tensor, `None` | One-stage and ordinary Stage 1 conditioning; requires its mask and cannot accompany `negative_prompt`. | | `prompt_attention_mask` | tensor, `None` | Mask for positive embeddings; request `prompt_attention_mask` or `attention_mask` wins. | | `negative_prompt_attention_mask` | tensor, `None` | Mask for negative embeddings; request negative mask fields win. | | `decode_timestep` | float or list, `0.0` | Video-VAE decode timestep; request value wins. Lists may match 1, prompt batch, or output batch. | @@ -273,13 +294,13 @@ request prompt payload; LTX guidance fields live in sampling `extra_args`. ### Recipe-Specific Request Capabilities -| Override | One-stage | Distilled two-stage | -|---|---|---| -| Guidance | Supported | Fixed positive-only | -| Negative prompt/embeddings | Supported | Rejected | -| `num_inference_steps` | Supported | Fixed at 8 for Stage 1; Stage 2 uses 3 | -| Custom `sigmas` | Supported | Rejected; both phases use fixed schedules | -| Video/audio latents | Supported | Rejected | +| Override | One-stage | Ordinary two-stage | Distilled two-stage | +|---|---|---|---| +| Guidance | Supported | Stage 1 only; Stage 2 is positive-only | Fixed positive-only | +| Negative prompt/embeddings | Supported | Supported by Stage 1 | Rejected | +| `num_inference_steps` | Supported | Controls Stage 1; Stage 2 uses 3 | Fixed at 8 for Stage 1; Stage 2 uses 3 | +| Custom `sigmas` | Supported | Rejected | Rejected; both phases use fixed schedules | +| Video/audio latents | Supported | Rejected | Rejected | These capability checks apply equally to direct `forward` keywords and values in `OmniDiffusionSamplingParams`; unsupported values fail instead of @@ -309,6 +330,10 @@ bundled offline CLI do not currently expose `sigmas`. `--max-num-seqs 1`. - `--cfg-parallel-size` shards the complete LTX guidance plan, including STG, modality guidance, and rescale-compatible prediction gathering. +- Sequence parallelism may pad audio latents. Pure Ulysses masks the padding; + Ring cannot, so audio length must be SP-divisible. Use `ring_degree=1` or a + divisible request shape. +- Cache-DiT is one-stage only; multi-stage configurations are rejected. ## Operational Notes @@ -323,14 +348,15 @@ bundled offline CLI do not currently expose `sigmas`. request parameter. - For benchmarks, use `tests/dfx/perf/tests/test_ltx2_vllm_omni.json` with `tests/dfx/perf/scripts/run_diffusion_benchmark.py`. -- Current two-stage support is limited to the distilled checkpoint; official - LTX two-stage/HQ execution remains out of scope. +- Ordinary and full-distilled two-stage T2V/I2V are supported; HQ execution + remains out of scope. ## References - - - +- - [Online video generation](../../docs/user_guide/examples/online_serving/text_to_video.md) - [Diffusion execution modes](../../docs/user_guide/diffusion/execution_modes.md) - [T2V offline example](../../examples/offline_inference/text_to_video/text_to_video.md) diff --git a/tests/diffusion/models/ltx2/test_ltx2_latents.py b/tests/diffusion/models/ltx2/test_ltx2_latents.py index 36f7e9687..cad5c300d 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_latents.py +++ b/tests/diffusion/models/ltx2/test_ltx2_latents.py @@ -23,7 +23,7 @@ def _make_pipeline(pipeline_cls, sequence_parallel_size: int = 1): return pipeline -def test_prepare_video_latents_samples_directly_in_packed_token_space(): +def test_prepare_video_latents_matches_official_values_and_token_major_layout(): pipeline = _make_pipeline(LTX2Pipeline) pipeline.vae_spatial_compression_ratio = 8 pipeline.vae_temporal_compression_ratio = 8 @@ -44,6 +44,7 @@ def test_prepare_video_latents_samples_directly_in_packed_token_space(): ) torch.testing.assert_close(actual, expected, rtol=0, atol=0) + assert actual.stride()[1:] == (1, actual.shape[1]) def test_prepare_audio_latents_samples_directly_in_packed_token_space(): @@ -81,6 +82,42 @@ def test_prepare_audio_latents_pads_generated_dummy_length_for_sp(): assert original_num_frames == 1 assert padded_num_frames == 2 assert latents.shape == (1, 2, 128) + torch.testing.assert_close(latents[:, 1:], torch.zeros_like(latents[:, 1:])) + + +def test_prepare_audio_latents_request_rng_is_invariant_to_sp_padding(): + pipeline_sp1 = _make_pipeline(LTX2Pipeline, sequence_parallel_size=1) + pipeline_sp4 = _make_pipeline(LTX2Pipeline, sequence_parallel_size=4) + generator_sp1 = torch.Generator().manual_seed(42) + generator_sp4 = torch.Generator().manual_seed(42) + + latents_sp1, _, _ = pipeline_sp1.prepare_audio_latents( + batch_size=1, + num_channels_latents=2, + num_mel_bins=8, + audio_latent_length=3, + dtype=torch.float32, + device=torch.device("cpu"), + generator=generator_sp1, + ) + latents_sp4, _, _ = pipeline_sp4.prepare_audio_latents( + batch_size=1, + num_channels_latents=2, + num_mel_bins=8, + audio_latent_length=3, + dtype=torch.float32, + device=torch.device("cpu"), + generator=generator_sp4, + ) + + torch.testing.assert_close(latents_sp4[:, :3], latents_sp1, rtol=0, atol=0) + torch.testing.assert_close(latents_sp4[:, 3:], torch.zeros_like(latents_sp4[:, 3:])) + torch.testing.assert_close( + torch.randn(8, generator=generator_sp4), + torch.randn(8, generator=generator_sp1), + rtol=0, + atol=0, + ) def test_prepare_audio_latents_pads_packed_sequence_dim_for_provided_latents(): @@ -137,7 +174,9 @@ def test_prepare_audio_latents_accepts_already_padded_4d_latents_for_sp(): assert original_num_frames == 10 assert padded_num_frames == 12 assert padded.shape == (1, 12, 8) - torch.testing.assert_close(padded, latent_ops.pack_audio_latents(latents)) + packed = latent_ops.pack_audio_latents(latents) + torch.testing.assert_close(padded[:, :10], packed[:, :10]) + torch.testing.assert_close(padded[:, 10:], torch.zeros_like(padded[:, 10:])) def test_resolve_audio_latent_length_preserves_legacy_4d_shape_inference(): diff --git a/tests/diffusion/models/ltx2/test_ltx2_parallel.py b/tests/diffusion/models/ltx2/test_ltx2_parallel.py index 8d8c22e79..95ca6e868 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_parallel.py +++ b/tests/diffusion/models/ltx2/test_ltx2_parallel.py @@ -12,6 +12,19 @@ import torch pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +def test_velocity_from_x0_uses_official_host_scalar_sigma(mocker): + from vllm_omni.diffusion.models.ltx2.ltx2_guidance import velocity_from_x0 + + sigma = mocker.MagicMock() + sigma.to.return_value.item.return_value = 0.5 + + actual = velocity_from_x0(torch.tensor([1.0]), torch.tensor([0.0]), sigma) + + sigma.to.assert_called_once_with(torch.float32) + sigma.to.return_value.item.assert_called_once_with() + torch.testing.assert_close(actual, torch.tensor([2.0])) + + class TestCFGParallelHelpers: """Test LTX-2.3 CFG helper math without loading model weights.""" @@ -199,6 +212,7 @@ class TestCFGParallelHelpers: ), attention_kwargs=None, audio_scheduler=SimpleNamespace(sigmas=torch.stack([audio_sigma])), + original_audio_num_frames=state.audio.shape[1], ) actual_video, actual_audio = executor.predict_parallel_guidance( diff --git a/tests/diffusion/models/ltx2/test_ltx2_phase_adapter.py b/tests/diffusion/models/ltx2/test_ltx2_phase_adapter.py new file mode 100644 index 000000000..4b1d18cb5 --- /dev/null +++ b/tests/diffusion/models/ltx2/test_ltx2_phase_adapter.py @@ -0,0 +1,483 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Unit tests for the fixed LTX refinement-phase adapter.""" + +from types import SimpleNamespace + +import pytest +import torch +from safetensors.torch import save_file + +from vllm_omni.diffusion.models.ltx2 import ltx2_phase_adapter +from vllm_omni.diffusion.models.ltx2.ltx2_adapter_parser import ( + AdapterTarget, + parse_ltx_adapter, +) +from vllm_omni.diffusion.models.ltx2.ltx2_components import LTX23_TWO_STAGE_COMPONENT_PROFILE +from vllm_omni.diffusion.models.ltx2.ltx2_phase_adapter import LTXPhaseAdapterRuntime +from vllm_omni.diffusion.models.ltx2.ltx2_recipes import ( + LTX2_TWO_STAGE_RECIPE, + LTX_DISTILLED_ADAPTER_SLOT, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _target( + lora_a, + lora_b, + *, + source_module="proj", + module="proj", + target_slice=None, +): + return AdapterTarget( + source_module=source_module, + module=module, + slice=target_slice, + a_key="a", + b_key="b", + a_shape=tuple(lora_a.shape), + b_shape=tuple(lora_b.shape), + ) + + +def _wrap( + layer, + lora_a, + lora_b, + *, + source_module="proj", + module="proj", + target_slice=None, + layer_fused=False, +): + target = _target( + lora_a, + lora_b, + source_module=source_module, + module=module, + target_slice=target_slice, + ) + wrapper = ltx2_phase_adapter._PhaseAdapterLinear( + layer, + [target], + adapter_dtype=lora_a.dtype, + layer_fused=layer_fused, + ) + wrapper.load_target(target, lora_a, lora_b) + wrapper.set_enabled(True) + return wrapper + + +class _LoRAAttention(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.to_qkv = torch.nn.Identity() + + +class _LoRATransformer(torch.nn.Module): + stacked_params_mapping = ((".attn1.to_qkv", ".attn1.to_q", "q"),) + + def __init__(self) -> None: + super().__init__() + self.proj_in = torch.nn.Linear(4, 6, bias=False) + block = torch.nn.Module() + block.attn1 = _LoRAAttention() + self.transformer_blocks = torch.nn.ModuleList([block]) + + +def test_ltx_adapter_parser_maps_official_and_packed_module_names(tmp_path): + adapter_path = tmp_path / "distilled.safetensors" + save_file( + { + "diffusion_model.text_embedding_projection.aggregate_embed.lora_A.weight": torch.randn(2, 4), + "diffusion_model.patchify_proj.lora_A.weight": torch.randn(2, 4), + "diffusion_model.patchify_proj.lora_B.weight": torch.randn(6, 2), + "diffusion_model.transformer_blocks.0.attn1.to_q.lora_A.weight": torch.randn(2, 4), + "diffusion_model.transformer_blocks.0.attn1.to_q.lora_B.weight": torch.randn(4, 2), + }, + adapter_path, + ) + + manifest = parse_ltx_adapter(_LoRATransformer(), str(adapter_path)) + assert {(target.source_module, target.module, target.slice) for target in manifest.targets} == { + ("proj_in", "proj_in", None), + ("transformer_blocks.0.attn1.to_q", "transformer_blocks.0.attn1.to_qkv", "q"), + } + + +def test_ltx_adapter_parser_rejects_unpaired_b_tensor(tmp_path): + adapter_path = tmp_path / "distilled.safetensors" + save_file( + {"diffusion_model.patchify_proj.lora_B.weight": torch.randn(6, 2)}, + adapter_path, + ) + + with pytest.raises(ValueError, match="Missing paired LoRA A tensors"): + parse_ltx_adapter(_LoRATransformer(), str(adapter_path)) + + +def _pipeline(*, dtype, quantization_config): + return SimpleNamespace( + pipeline_recipe=LTX2_TWO_STAGE_RECIPE, + component_profile=LTX23_TWO_STAGE_COMPONENT_PROFILE, + transformer=object(), + od_config=SimpleNamespace( + model="unused", + lora_path=None, + dtype=dtype, + quantization_config=quantization_config, + ), + ) + + +@pytest.mark.parametrize( + ("dtype", "quantization_config", "layer_fused"), + [ + (torch.float32, object(), False), + (torch.bfloat16, None, True), + ], +) +def test_ltx_phase_adapter_selects_execution_from_quantization( + monkeypatch, + dtype, + quantization_config, + layer_fused, +): + pipeline = _pipeline(dtype=dtype, quantization_config=quantization_config) + manifest = object() + captured = {} + + def resolve_artifact(*_args, **kwargs): + assert not kwargs + return "adapter" + + def parse(transformer, path): + assert transformer is pipeline.transformer + assert path == "adapter" + return manifest + + class Runtime: + def __init__(self, transformer, parsed_manifest, **kwargs): + captured.update(transformer=transformer, manifest=parsed_manifest, **kwargs) + + monkeypatch.setattr(ltx2_phase_adapter, "resolve_ltx_artifact", resolve_artifact) + monkeypatch.setattr(ltx2_phase_adapter, "parse_ltx_adapter", parse) + monkeypatch.setattr(ltx2_phase_adapter, "LTXPhaseAdapterRuntime", Runtime) + + runtime = ltx2_phase_adapter.build_ltx_phase_adapter(pipeline) + + assert isinstance(runtime, Runtime) + assert captured == { + "transformer": pipeline.transformer, + "manifest": manifest, + "dtype": dtype, + "layer_fused": layer_fused, + } + + +def test_ltx_phase_adapter_rejects_non_bfloat16_layer_fused(monkeypatch): + pipeline = _pipeline(dtype=torch.float16, quantization_config=None) + monkeypatch.setattr(ltx2_phase_adapter, "resolve_ltx_artifact", lambda *_args, **_kwargs: "adapter") + + with pytest.raises(ValueError, match="bfloat16"): + ltx2_phase_adapter.build_ltx_phase_adapter(pipeline) + + +def test_ltx_phase_adapter_keeps_one_transformer_and_switches_a_fixed_slot(tmp_path): + class TinyTransformer(torch.nn.Module): + stacked_params_mapping = () + + def __init__(self) -> None: + super().__init__() + self.proj = torch.nn.Linear(2, 2, bias=False) + + adapter_path = tmp_path / "distilled.safetensors" + save_file( + { + "diffusion_model.proj.lora_A.weight": torch.tensor([[1.0, 2.0]]), + "diffusion_model.proj.lora_B.weight": torch.tensor([[3.0], [4.0]]), + }, + adapter_path, + ) + transformer = TinyTransformer() + runtime = LTXPhaseAdapterRuntime( + transformer, + parse_ltx_adapter(transformer, str(adapter_path)), + dtype=torch.float32, + ) + + assert runtime.base_parameter_name("proj.weight") == "proj.base_layer.weight" + assert {name for name, _ in transformer.named_parameters()} == {"proj.base_layer.weight"} + with torch.no_grad(): + transformer.proj.base_layer.weight.copy_(torch.eye(2)) + runtime.finalize() + + x = torch.tensor([[1.0, 1.0]]) + runtime.activate(None) + torch.testing.assert_close(transformer.proj(x), torch.tensor([[1.0, 1.0]])) + runtime.activate(LTX_DISTILLED_ADAPTER_SLOT) + torch.testing.assert_close(transformer.proj(x), torch.tensor([[10.0, 13.0]])) + + +def test_ltx_layer_fused_adapter_matches_official_bf16_weight_fusion(): + layer = torch.nn.Linear(4, 3, bias=True, dtype=torch.bfloat16) + base_weight = torch.tensor( + [ + [0.5, -0.25, 0.125, 0.75], + [-0.375, 0.625, 0.25, -0.5], + [0.875, -0.125, -0.75, 0.375], + ], + dtype=torch.bfloat16, + ) + lora_a = torch.tensor( + [ + [0.1015625, -0.203125, 0.3046875, -0.40625], + [0.5078125, -0.609375, 0.7109375, -0.8125], + ], + dtype=torch.bfloat16, + ) + lora_b = torch.tensor( + [[0.9140625, -0.1171875], [-0.21875, 0.3203125], [0.421875, -0.5234375]], + dtype=torch.bfloat16, + ) + with torch.no_grad(): + layer.weight.copy_(base_weight) + layer.bias.copy_(torch.tensor([0.125, -0.25, 0.5], dtype=torch.bfloat16)) + wrapper = _wrap(layer, lora_a, lora_b, layer_fused=True) + + expected_weight = torch.matmul(lora_b, lora_a) + expected_weight.add_(base_weight) + x = torch.tensor([[0.6015625, -0.703125, 0.8046875, -0.90625]], dtype=torch.bfloat16) + expected = torch.nn.functional.linear(x, expected_weight, layer.bias) + + assert torch.equal(wrapper(x), expected) + assert torch.equal(layer.weight, base_weight) + + +def test_ltx_layer_fused_row_parallel_uses_one_collective(monkeypatch): + from vllm.model_executor.layers.linear import UnquantizedLinearMethod + + class FakeRow(torch.nn.Module): + input_size = 4 + output_size = 2 + input_size_per_partition = 2 + input_is_parallel = True + tp_rank = 0 + tp_size = 2 + reduce_results = True + skip_bias_add = False + return_bias = False + + def __init__(self): + super().__init__() + self.weight = torch.nn.Parameter(torch.eye(2, dtype=torch.bfloat16)) + self.bias = torch.nn.Parameter(torch.tensor([0.25, -0.5], dtype=torch.bfloat16)) + self.quant_method = UnquantizedLinearMethod() + + monkeypatch.setattr(ltx2_phase_adapter, "RowParallelLinear", FakeRow) + reductions = [] + + def all_reduce(value): + reductions.append(value.clone()) + return value + 1 + + monkeypatch.setattr(ltx2_phase_adapter, "tensor_model_parallel_all_reduce", all_reduce) + layer = FakeRow() + lora_a = torch.tensor([[0.5, -0.25, 0.75, -0.125]], dtype=torch.bfloat16) + lora_b = torch.tensor([[0.25], [-0.5]], dtype=torch.bfloat16) + wrapper = _wrap(layer, lora_a, lora_b, layer_fused=True) + + expected_weight = torch.matmul(lora_b, lora_a[:, :2]) + expected_weight.add_(layer.weight) + x = torch.tensor([[1.0, 2.0]], dtype=torch.bfloat16) + expected_local = torch.nn.functional.linear(x, expected_weight, layer.bias) + + actual = wrapper(x) + assert len(reductions) == 1 + assert torch.equal(reductions[0], expected_local) + assert torch.equal(actual, expected_local + 1) + + +def test_ltx_layer_fused_column_parallel_uses_local_weight_shard(monkeypatch): + from vllm.model_executor.layers.linear import UnquantizedLinearMethod + + class FakeColumn(torch.nn.Module): + input_size = 3 + output_size = 4 + output_size_per_partition = 2 + tp_rank = 1 + tp_size = 2 + gather_output = False + skip_bias_add = False + return_bias = False + + def __init__(self): + super().__init__() + self.weight = torch.nn.Parameter( + torch.tensor([[0.5, -0.25, 0.125], [-0.375, 0.625, 0.25]], dtype=torch.bfloat16) + ) + self.bias = torch.nn.Parameter(torch.tensor([0.25, -0.5], dtype=torch.bfloat16)) + self.quant_method = UnquantizedLinearMethod() + + monkeypatch.setattr(ltx2_phase_adapter, "ColumnParallelLinear", FakeColumn) + layer = FakeColumn() + base_weight = layer.weight.detach().clone() + lora_a = torch.tensor([[0.5, -0.25, 0.75]], dtype=torch.bfloat16) + lora_b = torch.tensor([[0.25], [-0.5], [0.75], [-1.0]], dtype=torch.bfloat16) + wrapper = _wrap(layer, lora_a, lora_b, layer_fused=True) + + expected_weight = torch.matmul(lora_b[2:], lora_a) + expected_weight.add_(base_weight) + x = torch.tensor([[1.0, -0.5, 0.25]], dtype=torch.bfloat16) + expected = torch.nn.functional.linear(x, expected_weight, layer.bias) + + assert torch.equal(wrapper(x), expected) + assert torch.equal(layer.weight, base_weight) + + +def test_ltx_layer_fused_packed_qkv_updates_only_the_target_slice(monkeypatch): + from vllm.model_executor.layers.linear import UnquantizedLinearMethod + + class FakeColumn(torch.nn.Module): + pass + + class FakeQKV(FakeColumn): + input_size = 4 + head_size = 2 + v_head_size = 2 + total_num_heads = 2 + total_num_kv_heads = 2 + num_heads = 1 + num_kv_heads = 1 + num_kv_head_replicas = 1 + tp_rank = 1 + tp_size = 2 + gather_output = False + skip_bias_add = True + return_bias = True + + def __init__(self): + super().__init__() + self.weight = torch.nn.Parameter( + torch.arange(24, dtype=torch.float32).reshape(6, 4).to(torch.bfloat16) / 16 + ) + self.bias = torch.nn.Parameter(torch.arange(6, dtype=torch.float32).to(torch.bfloat16) / 8) + self.quant_method = UnquantizedLinearMethod() + + monkeypatch.setattr(ltx2_phase_adapter, "ColumnParallelLinear", FakeColumn) + monkeypatch.setattr(ltx2_phase_adapter, "QKVParallelLinear", FakeQKV) + layer = FakeQKV() + base_weight = layer.weight.detach().clone() + lora_a = torch.tensor([[0.5, -0.25, 0.75, -0.125]], dtype=torch.bfloat16) + lora_b = torch.tensor([[0.25], [-0.5], [0.75], [-1.0]], dtype=torch.bfloat16) + wrapper = _wrap( + layer, + lora_a, + lora_b, + source_module="to_k", + module="to_qkv", + target_slice="k", + layer_fused=True, + ) + + expected_weight = base_weight.clone() + expected_k = torch.matmul(lora_b[2:], lora_a) + expected_k.add_(base_weight[2:4]) + expected_weight[2:4].copy_(expected_k) + x = torch.tensor([[1.0, -0.5, 0.25, -0.125]], dtype=torch.bfloat16) + expected_output = torch.nn.functional.linear(x, expected_weight) + + actual_output, returned_bias = wrapper(x) + assert torch.equal(actual_output, expected_output) + assert returned_bias is layer.bias + assert torch.equal(layer.weight, base_weight) + + +@pytest.mark.parametrize( + ("layer_factory", "target_slice", "expected_a", "expected_b", "expected_output"), + [ + ( + lambda: type( + "FakeColumn", + (torch.nn.Module,), + { + "input_size": 4, + "output_size": 4, + "output_size_per_partition": 2, + "tp_rank": 1, + "gather_output": False, + "forward": lambda self, x: x.new_zeros((*x.shape[:-1], 2)), + }, + )(), + None, + torch.tensor([[1.0, 10.0, 100.0, 1000.0]]), + torch.tensor([[1.0], [2.0], [3.0], [4.0]]), + torch.tensor([[12963.0, 17284.0]]), + ), + ( + lambda: type( + "FakeRow", + (torch.nn.Module,), + { + "input_size": 4, + "output_size": 2, + "input_size_per_partition": 2, + "tp_rank": 1, + "reduce_results": False, + "forward": lambda self, x: x.new_zeros((*x.shape[:-1], 2)), + }, + )(), + None, + torch.tensor([[1.0, 10.0, 100.0, 1000.0]]), + torch.tensor([[2.0], [3.0]]), + torch.tensor([[8600.0, 12900.0]]), + ), + ( + lambda: type( + "FakeQKV", + (torch.nn.Module,), + { + "input_size": 4, + "head_size": 2, + "v_head_size": 2, + "total_num_heads": 2, + "total_num_kv_heads": 2, + "num_heads": 1, + "num_kv_heads": 1, + "num_kv_head_replicas": 1, + "tp_rank": 1, + "forward": lambda self, x: x.new_zeros((*x.shape[:-1], 6)), + }, + )(), + "k", + torch.tensor([[1.0, 10.0, 100.0, 1000.0]]), + torch.tensor([[1.0], [2.0], [3.0], [4.0]]), + torch.tensor([[0.0, 0.0, 12963.0, 17284.0, 0.0, 0.0]]), + ), + ], +) +def test_ltx_phase_adapter_uses_local_tp_lora_slices( + monkeypatch, + layer_factory, + target_slice, + expected_a, + expected_b, + expected_output, +): + layer = layer_factory() + layer.register_buffer("device_anchor", torch.empty(0)) + layer_type = type(layer) + if target_slice == "k": + monkeypatch.setattr(ltx2_phase_adapter, "QKVParallelLinear", layer_type) + elif "Column" in layer_type.__name__: + monkeypatch.setattr(ltx2_phase_adapter, "ColumnParallelLinear", layer_type) + else: + monkeypatch.setattr(ltx2_phase_adapter, "RowParallelLinear", layer_type) + + wrapper = _wrap(layer, expected_a, expected_b, target_slice=target_slice) + torch.testing.assert_close(wrapper(torch.tensor([[1.0, 2.0, 3.0, 4.0]])), expected_output) + expected_local_a_width = getattr(layer, "input_size_per_partition", layer.input_size) + assert wrapper.pieces[0].lora_a.shape[-1] == expected_local_a_width diff --git a/tests/diffusion/models/ltx2/test_ltx2_pipeline.py b/tests/diffusion/models/ltx2/test_ltx2_pipeline.py index d70820f41..2f3ae522e 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_pipeline.py +++ b/tests/diffusion/models/ltx2/test_ltx2_pipeline.py @@ -13,13 +13,21 @@ import pytest import torch from vllm_omni.diffusion.data import DiffusionOutput -from vllm_omni.diffusion.models.ltx2 import ltx2_components +from vllm_omni.diffusion.models.ltx2 import ( + ltx2_components, + ltx2_transformer, +) from vllm_omni.diffusion.models.ltx2.ltx2_components import ( LTX2_COMPONENT_PROFILE, LTX2_DISTILLED_COMPONENT_PROFILE, + LTX2_TWO_STAGE_COMPONENT_PROFILE, LTX23_COMPONENT_PROFILE, + LTX23_DISTILLED_COMPONENT_PROFILE, + LTX23_TWO_STAGE_COMPONENT_PROFILE, _install_connector_attention, detect_ltx_model_version, + resolve_ltx_artifact, + resolve_ltx_component_profile, ) from vllm_omni.diffusion.models.ltx2.ltx2_conditioning import LTXI2VConditioningMixin from vllm_omni.diffusion.models.ltx2.ltx2_denoise import ( @@ -33,6 +41,7 @@ from vllm_omni.diffusion.models.ltx2.ltx2_guidance import ( LTXGuidancePlan, LTXGuidanceSpec, LTXModalityGuidance, + _repeat_batch, build_perturbation_kwargs, combine_guided_x0, ) @@ -40,18 +49,29 @@ from vllm_omni.diffusion.models.ltx2.ltx2_latents import LTXAVState from vllm_omni.diffusion.models.ltx2.ltx2_recipes import ( LTX2_DISTILLED_TWO_STAGE_RECIPE, LTX2_ONE_STAGE_RECIPE, + LTX2_TWO_STAGE_RECIPE, + LTX23_DISTILLED_TWO_STAGE_RECIPE, LTX23_ONE_STAGE_RECIPE, + LTX23_TWO_STAGE_RECIPE, LTX_DEFAULT_NEGATIVE_PROMPT, LTX_POSITIVE_ONLY_RECIPE, + resolve_ltx_pipeline_recipe, +) +from vllm_omni.diffusion.models.ltx2.ltx2_request import ( + LTXRequestInputs, + validate_ltx_checkpoint, + validate_pipeline_request, ) -from vllm_omni.diffusion.models.ltx2.ltx2_request import LTXRequestInputs, validate_pipeline_request from vllm_omni.diffusion.models.ltx2.ltx2_runtime import LTXRuntime from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import ( LTX2I2VDMD2Pipeline, LTX2Pipeline, LTX2T2VDMD2Pipeline, ) -from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_two_stage import LTX2DistilledPipeline +from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_two_stage import ( + LTX2DistilledPipeline, + LTX2TwoStagePipeline, +) pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -104,10 +124,22 @@ def test_ltx_public_entries_share_runtime_and_keep_recipe_boundaries(): ) assert issubclass(LTX2Pipeline, LTXRuntime) + assert issubclass(LTX2TwoStagePipeline, LTXRuntime) assert issubclass(LTX2DistilledPipeline, LTXRuntime) assert LTX2Pipeline.component_profile is LTX2_COMPONENT_PROFILE assert LTX2Pipeline.pipeline_recipe is LTX2_ONE_STAGE_RECIPE + assert LTX2TwoStagePipeline.component_profile is LTX2_TWO_STAGE_COMPONENT_PROFILE + assert LTX2TwoStagePipeline.pipeline_recipe is LTX2_TWO_STAGE_RECIPE assert LTX2DistilledPipeline.component_profile is LTX2_DISTILLED_COMPONENT_PROFILE + for pipeline_cls, profile in ( + (LTX2Pipeline, LTX2_COMPONENT_PROFILE), + (LTX2TwoStagePipeline, LTX2_TWO_STAGE_COMPONENT_PROFILE), + (LTX2DistilledPipeline, LTX2_DISTILLED_COMPONENT_PROFILE), + ): + assert pipeline_cls._dit_modules == list(profile.dit_modules) + assert pipeline_cls._encoder_modules == list(profile.encoder_modules) + assert pipeline_cls._vae_modules == list(profile.vae_modules) + assert pipeline_cls._resident_modules == list(profile.resident_modules) assert LTX2_COMPONENT_PROFILE.video_vae_cls is DistributedAutoencoderKLLTX2Video assert LTX23_COMPONENT_PROFILE.video_vae_cls is DistributedAutoencoderKLLTX2Video @@ -123,6 +155,72 @@ def test_ltx_checkpoint_version_detection_uses_metadata(tmp_path): assert detect_ltx_model_version(str(tmp_path)) == "2" +def test_ltx_artifact_falls_back_to_unpinned_hub_download(tmp_path, monkeypatch): + filename = "ltx-sidecar.safetensors" + calls = [] + + def fake_download(**kwargs): + calls.append(kwargs) + return "/cache/ltx-sidecar.safetensors" + + monkeypatch.setattr(ltx2_components, "hf_hub_download", fake_download) + + assert ( + resolve_ltx_artifact(str(tmp_path / "model"), "Lightricks/test", filename) == "/cache/ltx-sidecar.safetensors" + ) + assert calls == [{"repo_id": "Lightricks/test", "filename": filename}] + + +def test_ltx_artifact_prefers_model_root(tmp_path, monkeypatch): + filename = "ltx-sidecar.safetensors" + expected = tmp_path / filename + expected.write_bytes(b"sidecar") + monkeypatch.setattr(ltx2_components, "hf_hub_download", lambda **_kwargs: pytest.fail("unexpected Hub lookup")) + + assert resolve_ltx_artifact(str(tmp_path), "Lightricks/test", filename) == str(expected) + + +@pytest.mark.parametrize( + ("expected_kind", "scheduler_config"), + [ + ("regular", {"use_dynamic_shifting": True, "shift_terminal": 0.1}), + ("distilled", {"use_dynamic_shifting": False, "shift_terminal": None}), + ], +) +def test_ltx_checkpoint_validation_accepts_matching_scheduler_metadata(expected_kind, scheduler_config): + validate_ltx_checkpoint( + scheduler_config, + expected_kind=expected_kind, + pipeline_name="TestLTXPipeline", + ) + + +@pytest.mark.parametrize( + ("expected_kind", "scheduler_config", "error"), + [ + ("regular", {"use_dynamic_shifting": False, "shift_terminal": None}, "regular non-distilled"), + ("distilled", {"use_dynamic_shifting": True, "shift_terminal": 0.1}, "merged distilled"), + ], +) +def test_ltx_checkpoint_validation_rejects_mismatched_scheduler_metadata(expected_kind, scheduler_config, error): + with pytest.raises(ValueError, match=error): + validate_ltx_checkpoint( + scheduler_config, + expected_kind=expected_kind, + pipeline_name="TestLTXPipeline", + ) + + +def test_ltx_rejects_advanced_uaa_before_component_initialization(): + od_config = SimpleNamespace( + model="unused", + parallel_config=SimpleNamespace(ulysses_mode="advanced_uaa"), + ) + + with pytest.raises(ValueError, match="does not support ulysses_mode='advanced_uaa'"): + LTX2Pipeline(od_config=od_config) + + def test_ltx23_checkpoint_selects_version_specific_one_stage_profile(tmp_path, monkeypatch): from vllm_omni.diffusion.models.ltx2 import ltx2_runtime @@ -144,16 +242,104 @@ def test_ltx23_checkpoint_selects_version_specific_one_stage_profile(tmp_path, m assert pipe.reports_stage_durations -def test_ltx_one_stage_entry_exposes_batch_image_and_distributed_decode_contracts(): +def test_ltx_multistage_rejects_cache_dit_before_component_initialization(tmp_path, monkeypatch): + from vllm_omni.diffusion.models.ltx2 import ltx2_runtime + + (tmp_path / "model_index.json").write_text(json.dumps({"vocoder": ["ltx2", "LTX2Vocoder"]})) + initialized = False + + def stub_components(*_args, **_kwargs): + nonlocal initialized + initialized = True + + monkeypatch.setattr(ltx2_runtime, "initialize_pipeline_components", stub_components) + + with pytest.raises(ValueError, match="only one-stage LTX recipes"): + LTX2TwoStagePipeline( + od_config=SimpleNamespace( + model=str(tmp_path), + cache_backend="cache_dit", + ) + ) + + assert not initialized + + +def test_ltx23_checkpoint_selects_version_specific_two_stage_profiles(tmp_path, monkeypatch): + from vllm_omni.diffusion.models.ltx2 import ltx2_runtime + + (tmp_path / "model_index.json").write_text(json.dumps({"vocoder": ["ltx2", "LTX2VocoderWithBWE"]})) + + def stub_components(pipe, od_config): + pipe.od_config = od_config + pipe.vae_spatial_compression_ratio = 32 + + monkeypatch.setattr(ltx2_runtime, "initialize_pipeline_components", stub_components) + monkeypatch.setattr(LTXRuntime, "setup_diffusion_pipeline_profiler", lambda *_args, **_kwargs: None) + monkeypatch.setattr(ltx2_runtime, "build_ltx_phase_adapter", lambda *_args: SimpleNamespace()) + pipe = LTX2TwoStagePipeline( + od_config=SimpleNamespace( + model=str(tmp_path), + enable_diffusion_pipeline_profiler=False, + lora_path=None, + model_config={}, + ) + ) + + assert pipe.model_version == "2.3" + assert pipe.component_profile is LTX23_TWO_STAGE_COMPONENT_PROFILE + assert pipe.pipeline_recipe is LTX23_TWO_STAGE_RECIPE + + +def test_ltx23_checkpoint_selects_full_distilled_profile_and_recipe(tmp_path, monkeypatch): + from vllm_omni.diffusion.models.ltx2 import ltx2_runtime + + (tmp_path / "model_index.json").write_text(json.dumps({"vocoder": ["ltx2", "LTX2VocoderWithBWE"]})) + + def stub_components(pipe, od_config): + pipe.od_config = od_config + pipe.vae_spatial_compression_ratio = 32 + + monkeypatch.setattr(ltx2_runtime, "initialize_pipeline_components", stub_components) + monkeypatch.setattr(LTXRuntime, "setup_diffusion_pipeline_profiler", lambda *_args, **_kwargs: None) + + pipe = LTX2DistilledPipeline( + od_config=SimpleNamespace( + model=str(tmp_path), + enable_diffusion_pipeline_profiler=False, + ) + ) + + assert pipe.model_version == "2.3" + assert pipe.component_profile is LTX23_DISTILLED_COMPONENT_PROFILE + assert pipe.pipeline_recipe is LTX23_DISTILLED_TWO_STAGE_RECIPE + assert pipe._phase_adapter is None + assert pipe.preserve_sp_padded_audio_duration + assert pipe.reports_stage_durations + + +def test_ltx_entries_expose_batch_image_and_distributed_decode_contracts(): assert LTX2Pipeline.supports_request_batch assert LTX2Pipeline.support_image_input assert LTX2Pipeline.unified_text_image_entry assert LTX2Pipeline.distributed_video_decode + assert not LTX2TwoStagePipeline.supports_request_batch + assert LTX2TwoStagePipeline.support_image_input + assert LTX2TwoStagePipeline.unified_text_image_entry assert not LTX2DistilledPipeline.supports_request_batch assert LTX2DistilledPipeline.support_image_input assert LTX2DistilledPipeline.unified_text_image_entry +def test_ltx_two_stage_resolves_request_embedded_images(): + pipe = object.__new__(LTX2TwoStagePipeline) + request_inputs = SimpleNamespace(latents=None) + image = torch.zeros(3, 8, 8) + req = SimpleNamespace(prompts=[{"prompt": "image conditioned", "multi_modal_data": {"image": image}}]) + + assert pipe._resolve_request_image(req, None, request_inputs) is image + + def test_ltx_dmd2_entries_retain_their_task_boundaries(): assert not LTX2T2VDMD2Pipeline.support_image_input assert LTX2I2VDMD2Pipeline.support_image_input @@ -330,6 +516,15 @@ def test_ltx_guidance_uses_official_close_to_disabled_semantics(): assert LTXGuidancePlan.build(LTXGuidanceSpec(video=guidance)).names == ("cond",) +def test_ltx_positive_only_guidance_preserves_token_major_layout(): + tensor = torch.arange(24).view(1, 3, 8).transpose(1, 2).contiguous().transpose(1, 2) + + repeated = _repeat_batch(tensor, 1) + + assert repeated is tensor + assert repeated.stride() == tensor.stride() + + def test_ltx_guidance_rescale_is_invariant_to_request_batching(): cond = torch.tensor( [ @@ -380,6 +575,32 @@ def test_ltx_guidance_rescale_handles_zero_variance_prediction(): torch.testing.assert_close(actual, torch.zeros_like(cond)) +def test_ltx_audio_guidance_rescale_ignores_sp_padding_tokens(): + logical_cond = torch.tensor([[[1.0, 2.0], [3.0, 5.0]]]) + logical_uncond = torch.tensor([[[0.5, -1.0], [2.0, 1.0]]]) + guidance = LTXModalityGuidance(cfg_scale=3.0, rescale_scale=1.0) + expected = combine_guided_x0( + cond=logical_cond, + uncond_text=logical_uncond, + uncond_perturbed=0.0, + uncond_modality=0.0, + guidance=guidance, + ) + padded_cond = torch.cat([logical_cond, torch.tensor([[[1000.0, -1000.0]]])], dim=1) + padded_uncond = torch.cat([logical_uncond, torch.tensor([[[-500.0, 500.0]]])], dim=1) + + actual = combine_guided_x0( + cond=padded_cond, + uncond_text=padded_uncond, + uncond_perturbed=0.0, + uncond_modality=0.0, + guidance=guidance, + rescale_token_count=2, + ) + + torch.testing.assert_close(actual[:, :2], expected) + + def test_ltx_guidance_builds_per_sample_stg_and_modality_masks(): plan = LTXGuidancePlan.build(LTX23_ONE_STAGE_RECIPE.request_guidance) perturbations = build_perturbation_kwargs(plan, batch_size=2, reference=torch.ones(8, 1, 1)) @@ -547,11 +768,19 @@ def test_ltx_connector_attention_dispatches_through_omni_kernel(has_learned_regi def test_ltx2_distilled_pipeline_shares_recipe_runtime(): assert issubclass(LTX2DistilledPipeline, LTXI2VConditioningMixin) - assert issubclass(LTX2DistilledPipeline, LTX2Pipeline) + assert not issubclass(LTX2DistilledPipeline, LTX2Pipeline) assert LTX2DistilledPipeline._run_recipe is LTXRuntime._run_recipe assert "_forward_request" not in LTX2DistilledPipeline.__dict__ +def test_ltx_two_stage_entry_is_a_thin_pipeline_variant(): + assert "_run_recipe" not in LTX2TwoStagePipeline.__dict__ + assert "_forward_request" not in LTX2TwoStagePipeline.__dict__ + assert "load_weights" not in LTX2TwoStagePipeline.__dict__ + assert issubclass(LTX2TwoStagePipeline, LTXI2VConditioningMixin) + assert not issubclass(LTX2TwoStagePipeline, LTX2Pipeline) + + def test_ltx_variants_share_denoise_loop_and_i2v_conditioning(): assert "_denoise_loop" not in LTX2Pipeline.__dict__ assert issubclass(LTX2Pipeline, LTXI2VConditioningMixin) @@ -594,7 +823,8 @@ def test_denoise_executor_owns_progress_and_interrupt(): torch.testing.assert_close(state.audio, torch.tensor(11.0)) -def test_ltx2_distilled_two_stage_executes_declarative_phase_plan(): +@pytest.mark.parametrize("pipeline_cls", [LTX2TwoStagePipeline, LTX2DistilledPipeline]) +def test_ltx_two_stage_executes_declarative_i2v_phase_plan(pipeline_cls): request_inputs = LTXRequestInputs( prompt="prompt", negative_prompt="", @@ -671,12 +901,15 @@ def test_ltx2_distilled_two_stage_executes_declarative_phase_plan(): return latents.repeat_interleave(2, dim=-2).repeat_interleave(2, dim=-1) prompt_context_sentinel = prompt_context - pipeline = object.__new__(LTX2DistilledPipeline) + pipeline = object.__new__(pipeline_cls) torch.nn.Module.__init__(pipeline) pipeline.device = torch.device("cpu") pipeline.vae_spatial_compression_ratio = 32 pipeline.vae_temporal_compression_ratio = 8 pipeline.latent_upsampler = FakeUpsampler() + activated_slots = [] + if pipeline_cls is LTX2TwoStagePipeline: + pipeline._phase_adapter = SimpleNamespace(activate=activated_slots.append) object.__setattr__(pipeline, "_resolve_request_inputs", resolve_request_inputs) object.__setattr__(pipeline, "run_phase", run_phase) object.__setattr__(pipeline, "decode_phase", decode_phase) @@ -686,14 +919,18 @@ def test_ltx2_distilled_two_stage_executes_declarative_phase_plan(): assert len(phase_calls) == 2 assert phase_calls[1][2] is prompt_context_sentinel + if pipeline_cls is LTX2TwoStagePipeline: + assert activated_slots == [None, "ltx_distilled"] torch.testing.assert_close(output.output[0], torch.full((1, 128, 1, 2, 2), 3.0)) - torch.testing.assert_close(output.output[1], torch.full((1, 8, 1, 2), 4.0)) + expected_audio = 2.0 if pipeline_cls is LTX2TwoStagePipeline else 4.0 + torch.testing.assert_close(output.output[1], torch.full((1, 8, 1, 2), expected_audio)) -def test_ltx2_distilled_stage2_reapplies_final_resolution_i2v_conditioning(): +@pytest.mark.parametrize("pipeline_cls", [LTX2TwoStagePipeline, LTX2DistilledPipeline]) +def test_ltx_two_stage_stage2_reapplies_final_resolution_i2v_conditioning(pipeline_cls): from vllm_omni.diffusion.models.ltx2 import ltx2_latents - pipeline = object.__new__(LTX2DistilledPipeline) + pipeline = object.__new__(pipeline_cls) torch.nn.Module.__init__(pipeline) pipeline.vae_spatial_compression_ratio = 32 pipeline.vae_temporal_compression_ratio = 8 @@ -750,6 +987,18 @@ def test_ltx2_distilled_two_stage_recipe_is_fixed_positive_only(): assert (LTX2_DISTILLED_TWO_STAGE_RECIPE.height, LTX2_DISTILLED_TWO_STAGE_RECIPE.width) == (1024, 1536) +def test_ltx23_full_distilled_route_uses_merged_weights_without_lora(): + assert resolve_ltx_component_profile("distilled_two_stage", "2.3") is LTX23_DISTILLED_COMPONENT_PROFILE + assert resolve_ltx_pipeline_recipe("distilled_two_stage", "2.3") is LTX23_DISTILLED_TWO_STAGE_RECIPE + assert LTX23_DISTILLED_TWO_STAGE_RECIPE is LTX2_DISTILLED_TWO_STAGE_RECIPE + assert all(phase.adapter_slot is None for phase in LTX23_DISTILLED_TWO_STAGE_RECIPE.phases) + assert LTX23_DISTILLED_COMPONENT_PROFILE.distilled_lora_filename is None + assert LTX23_DISTILLED_COMPONENT_PROFILE.vocoder_cls is LTX23_COMPONENT_PROFILE.vocoder_cls + assert LTX23_DISTILLED_COMPONENT_PROFILE.latent_upsampler_filename == ( + "ltx-2.3-spatial-upscaler-x2-1.1.safetensors" + ) + + @pytest.mark.parametrize( ("request_inputs", "error"), [ @@ -958,6 +1207,98 @@ def test_ltx_custom_sigmas_bypass_scheduler_shifting(): torch.testing.assert_close(timesteps, expected[:-1] * scheduler.config.num_train_timesteps) +@pytest.mark.parametrize( + ("recipe", "steps", "stg_block"), + [ + (LTX2_TWO_STAGE_RECIPE, 40, 29), + (LTX23_TWO_STAGE_RECIPE, 30, 28), + ], +) +def test_ltx_official_two_stage_recipes_only_vary_by_model_defaults(recipe, steps, stg_block): + stage1, stage2 = recipe.phases + assert (recipe.width, recipe.height) == (1536, 1024) + assert recipe.num_inference_steps == steps + assert stage1.spatial_downscale == 2 + assert stage1.adapter_slot is None + assert stage1.guidance.video.stg_blocks == (stg_block,) + assert stage1.guidance.audio.stg_blocks == (stg_block,) + assert stage2.input_transform == "spatial_upsample" + assert stage2.adapter_slot == "ltx_distilled" + assert stage2.guidance == LTXGuidanceSpec.positive_only() + assert stage2.num_inference_steps == 3 + assert stage2.sigmas == (0.909375, 0.725, 0.421875, 0.0) + # Official Stage 2 refines video only; its audio result is discarded. + assert (recipe.video_output_phase, recipe.audio_output_phase) == (1, 0) + + +def test_ltx_two_stage_entries_select_the_official_adapter_slots(): + phase_switches = [] + ordinary_pipeline = object.__new__(LTX2TwoStagePipeline) + ordinary_pipeline._phase_adapter = SimpleNamespace(activate=phase_switches.append) + for phase in LTX2_TWO_STAGE_RECIPE.phases: + ordinary_pipeline._enter_phase(phase) + assert phase_switches == [None, "ltx_distilled"] + + distilled_pipeline = object.__new__(LTX2DistilledPipeline) + for phase in LTX2_DISTILLED_TWO_STAGE_RECIPE.phases: + distilled_pipeline._enter_phase(phase) + + +def test_ltx23_gate_projection_declares_its_global_column_parallel_layout(monkeypatch): + class FakeColumn(torch.nn.Module): + def __init__( + self, + input_size, + output_size, + *, + bias, + gather_output, + return_bias, + quant_config, + prefix, + ): + super().__init__() + self.input_size = input_size + self.output_size = output_size + self.bias_enabled = bias + self.gather_output = gather_output + self.return_bias = return_bias + self.quant_config = quant_config + self.prefix = prefix + + class FakeRow(torch.nn.Module): + def __init__(self, *args, **kwargs): + super().__init__() + + class FakeAttention(torch.nn.Module): + def __init__(self, **kwargs): + super().__init__() + + monkeypatch.setattr(ltx2_transformer, "ColumnParallelLinear", FakeColumn) + monkeypatch.setattr(ltx2_transformer, "RowParallelLinear", FakeRow) + monkeypatch.setattr(ltx2_transformer, "_LTX2ParallelAttention", FakeAttention) + monkeypatch.setattr(ltx2_transformer, "get_tensor_model_parallel_world_size", lambda: 2) + + attention = ltx2_transformer.LTX2Attention( + query_dim=16, + heads=4, + dim_head=4, + qk_norm="rms_norm", + apply_gated_attention=True, + pack_qkv=False, + prefix="transformer_blocks.0.attn1", + ) + + gate = attention.to_gate_logits + assert isinstance(gate, FakeColumn) + assert gate.input_size == 16 + assert gate.output_size == 4 + assert gate.gather_output is False + assert gate.return_bias is False + assert gate.quant_config is None + assert gate.prefix == "transformer_blocks.0.attn1.to_gate_logits" + + class TestLTXRequestParsing: def test_request_resolves_official_and_explicit_negative_prompts_per_sample(self): from vllm_omni.diffusion.request import OmniDiffusionRequest @@ -1432,6 +1773,7 @@ class TestPipelineComponents: def test_distilled_profile_keeps_upsampler_in_managed_resident_components(self): assert LTX2_DISTILLED_COMPONENT_PROFILE.resident_modules == ("vocoder", "latent_upsampler") + assert LTX23_DISTILLED_COMPONENT_PROFILE.resident_modules == ("vocoder", "latent_upsampler") class TestLTX23DecodeConditioning: @@ -1477,6 +1819,11 @@ class TestRegistryIntegration: from vllm_omni.diffusion.registry import _DIFFUSION_MODELS assert _DIFFUSION_MODELS["LTX2Pipeline"] == ("ltx2", "pipeline_ltx2", "LTX2Pipeline") + assert _DIFFUSION_MODELS["LTX2TwoStagePipeline"] == ( + "ltx2", + "pipeline_ltx2_two_stage", + "LTX2TwoStagePipeline", + ) assert _DIFFUSION_MODELS["LTX2DistilledPipeline"] == ( "ltx2", "pipeline_ltx2_two_stage", @@ -1488,6 +1835,9 @@ class TestRegistryIntegration: "LTX23ImageToVideoPipeline", "LTX2TwoStagesPipeline", "LTX2ImageToVideoTwoStagesPipeline", + "LTX2ImageToVideoTwoStagePipeline", + "LTX23TwoStagePipeline", + "LTX23ImageToVideoTwoStagePipeline", } assert removed_entries.isdisjoint(_DIFFUSION_MODELS) @@ -1497,6 +1847,7 @@ class TestRegistryIntegration: expected = [ "LTX2Pipeline", + "LTX2TwoStagePipeline", "LTX2DistilledPipeline", "LTX2T2VDMD2Pipeline", "LTX2I2VDMD2Pipeline", @@ -1509,6 +1860,7 @@ class TestRegistryIntegration: "model_class_name", [ "LTX2Pipeline", + "LTX2TwoStagePipeline", "LTX2DistilledPipeline", "LTX2T2VDMD2Pipeline", "LTX2I2VDMD2Pipeline", diff --git a/tests/diffusion/models/ltx2/test_ltx2_transformer.py b/tests/diffusion/models/ltx2/test_ltx2_transformer.py index 73e10463e..f74b2991d 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_transformer.py +++ b/tests/diffusion/models/ltx2/test_ltx2_transformer.py @@ -1,13 +1,20 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace + import pytest +import torch from torch import nn +from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig -from vllm_omni.diffusion.models.ltx2.ltx2_transformer import LTX2VideoTransformer3DModel, _make_rms_norm - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +from vllm_omni.diffusion.models.ltx2.ltx2_transformer import ( + LTX2AudioVideoAttnProcessor, + LTX2VideoTransformer3DModel, + _LTX2ParallelAttention, + _make_rms_norm, +) pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] @@ -66,3 +73,83 @@ def test_ltx_sp_plan_shards_video_and_audio_timesteps_together(rope_type): assert video_timestep.expected_dims == audio_timestep.expected_dims == 2 assert not video_timestep.split_output assert not audio_timestep.split_output + + +def test_ltx_sp_keeps_cross_attention_key_padding_mask(monkeypatch): + monkeypatch.setattr(LTX2AudioVideoAttnProcessor, "_is_sp_enabled", staticmethod(lambda: True)) + processor = LTX2AudioVideoAttnProcessor() + hidden_states = torch.zeros(1, 3, 4) + encoder_hidden_states = torch.zeros(1, 4, 4) + key_padding_mask = torch.tensor([[True, True, True, False]]) + + actual = processor._prepare_attention_mask( + SimpleNamespace(), + hidden_states, + encoder_hidden_states, + key_padding_mask, + batch_size=1, + sequence_length=4, + ) + + torch.testing.assert_close(actual, key_padding_mask[:, None, None, :]) + + +def _fake_ltx_parallel_attention(): + calls = [] + + class Backend: + @staticmethod + def get_name(): + return "FLASH_ATTN" + + @staticmethod + def supports_attention_mask(): + return True + + def run(name): + def forward(query, _key, _value, _metadata): + calls.append(name) + return torch.zeros_like(query) + + return forward + + layer = object.__new__(_LTX2ParallelAttention) + nn.Module.__init__(layer) + layer.attn_backend = Backend + layer.attention = SimpleNamespace(forward=run("native")) + layer.sdpa_fallback = SimpleNamespace(forward=run("sdpa")) + layer.backend_pref = "FLASH_ATTN" + return layer, calls + + +@pytest.mark.parametrize(("query_length", "key_length"), [(4, 4), (3, 4)]) +def test_ltx_masked_cross_attention_uses_sdpa(query_length, key_length): + layer, calls = _fake_ltx_parallel_attention() + query = torch.zeros(1, query_length, 2, 4, dtype=torch.bfloat16) + key = value = torch.zeros(1, key_length, 2, 4, dtype=torch.bfloat16) + key_padding_mask = torch.tensor([[True] * (key_length - 1) + [False]]) + metadata = AttentionMetadata(attn_mask=key_padding_mask[:, None, None, :]) + + layer._run_local_attention(query, key, value, metadata) + + assert calls == ["sdpa"] + + +def test_ltx_masked_self_attention_uses_sdpa(): + layer, calls = _fake_ltx_parallel_attention() + query = key = value = torch.zeros(1, 4, 2, 4, dtype=torch.bfloat16) + key_padding_mask = torch.tensor([[True, True, True, False]]) + metadata = AttentionMetadata(attn_mask=key_padding_mask[:, None, None, :]) + + layer._run_local_attention(query, key, value, metadata) + + assert calls == ["sdpa"] + + +def test_ltx_unmasked_cross_attention_keeps_native_backend(): + layer, calls = _fake_ltx_parallel_attention() + query = key = value = torch.zeros(1, 4, 2, 4, dtype=torch.bfloat16) + + layer._run_local_attention(query, key, value, None) + + assert calls == ["native"] diff --git a/tests/e2e/accuracy/ltx/run_ltx_reference.py b/tests/e2e/accuracy/ltx/run_ltx_reference.py index a4a180047..3ab794662 100644 --- a/tests/e2e/accuracy/ltx/run_ltx_reference.py +++ b/tests/e2e/accuracy/ltx/run_ltx_reference.py @@ -20,6 +20,11 @@ from PIL import Image def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--backend", choices=("official", "omni"), required=True) + parser.add_argument( + "--pipeline-kind", + choices=("one_stage", "distilled", "two_stage"), + default="one_stage", + ) parser.add_argument("--request", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--model") @@ -27,6 +32,8 @@ def _parse_args() -> argparse.Namespace: parser.add_argument("--official-root", type=Path) parser.add_argument("--checkpoint", type=Path) parser.add_argument("--gemma-root", type=Path) + parser.add_argument("--spatial-upsampler", type=Path) + parser.add_argument("--distilled-lora", type=Path) parser.add_argument("--enable-layerwise-offload", action="store_true") return parser.parse_args() @@ -91,9 +98,13 @@ def _configure_official_sdpa(pipeline: Any) -> None: attention=attention, masked_attention=attention, ) - owners_and_attributes = ( - (pipeline.stage, "_transformer_builder"), + owners_and_attributes = [ (pipeline.prompt_encoder, "_embeddings_processor_builder"), + ] + owners_and_attributes.extend( + (stage, "_transformer_builder") + for name in ("stage", "stage_1", "stage_2") + if (stage := getattr(pipeline, name, None)) is not None ) for owner, attribute in owners_and_attributes: builder = getattr(owner, attribute) @@ -104,6 +115,12 @@ def _configure_official_sdpa(pipeline: Any) -> None: ) +def _require_path(path: Path | None, description: str) -> str: + if path is None or not path.is_file(): + raise ValueError(f"Official {description} is required and must exist: {path}") + return str(path) + + @torch.inference_mode() def _run_official(args: argparse.Namespace, request: dict[str, Any]) -> None: if args.official_root is None or args.checkpoint is None or args.gemma_root is None: @@ -113,16 +130,54 @@ def _run_official(args: argparse.Namespace, request: dict[str, Any]) -> None: torch.backends.cuda.enable_cudnn_sdp(False) from ltx_core.components.guiders import MultiModalGuiderParams + from ltx_core.loader import LTXV_LORA_COMFY_RENAMING_MAP, LoraPathStrengthAndSDOps + from ltx_pipelines.distilled import DistilledPipeline from ltx_pipelines.ti2vid_one_stage import TI2VidOneStagePipeline + from ltx_pipelines.ti2vid_two_stages import TI2VidTwoStagesPipeline from ltx_pipelines.utils.args import ImageConditioningInput from ltx_pipelines.utils.types import OffloadMode - pipeline = TI2VidOneStagePipeline( - checkpoint_path=str(args.checkpoint), - gemma_root=str(args.gemma_root), - loras=(), - offload_mode=OffloadMode.CPU if args.enable_layerwise_offload else OffloadMode.NONE, - ) + if request.get("pipeline_kind", "one_stage") != args.pipeline_kind: + raise ValueError( + f"Request pipeline kind {request.get('pipeline_kind')!r} does not match {args.pipeline_kind!r}" + ) + checkpoint = _require_path(args.checkpoint, "checkpoint") + gemma_root = str(args.gemma_root) + if not Path(gemma_root).is_dir(): + raise ValueError(f"Official Gemma root is required and must be a directory: {gemma_root}") + offload_mode = OffloadMode.CPU if args.enable_layerwise_offload else OffloadMode.NONE + pipeline: Any + if args.pipeline_kind == "one_stage": + pipeline = TI2VidOneStagePipeline( + checkpoint_path=checkpoint, + gemma_root=gemma_root, + loras=(), + offload_mode=offload_mode, + ) + elif args.pipeline_kind == "distilled": + pipeline = DistilledPipeline( + distilled_checkpoint_path=checkpoint, + spatial_upsampler_path=_require_path(args.spatial_upsampler, "spatial upsampler"), + gemma_root=gemma_root, + loras=(), + offload_mode=offload_mode, + ) + else: + distilled_lora = [ + LoraPathStrengthAndSDOps( + path=_require_path(args.distilled_lora, "distilled LoRA"), + strength=1.0, + sd_ops=LTXV_LORA_COMFY_RENAMING_MAP, + ) + ] + pipeline = TI2VidTwoStagesPipeline( + checkpoint_path=checkpoint, + distilled_lora=distilled_lora, + spatial_upsampler_path=_require_path(args.spatial_upsampler, "spatial upsampler"), + gemma_root=gemma_root, + loras=(), + offload_mode=offload_mode, + ) _configure_official_sdpa(pipeline) image_path = request.get("image") images = ( @@ -140,34 +195,45 @@ def _run_official(args: argparse.Namespace, request: dict[str, Any]) -> None: ) ] ) - video, audio = pipeline( - prompt=request["prompt"], - negative_prompt=request["negative_prompt"], - seed=request["seed"], - height=request["height"], - width=request["width"], - num_frames=request["num_frames"], - frame_rate=request["fps"], - num_inference_steps=request["num_inference_steps"], - video_guider_params=MultiModalGuiderParams( - cfg_scale=request["video_cfg_scale"], - stg_scale=request["video_stg_scale"], - rescale_scale=request["video_rescale_scale"], - modality_scale=request["video_modality_scale"], - skip_step=0, - stg_blocks=request["video_stg_blocks"], - ), - audio_guider_params=MultiModalGuiderParams( - cfg_scale=request["audio_cfg_scale"], - stg_scale=request["audio_stg_scale"], - rescale_scale=request["audio_rescale_scale"], - modality_scale=request["audio_modality_scale"], - skip_step=0, - stg_blocks=request["audio_stg_blocks"], - ), - images=images, - max_batch_size=4, - ) + if args.pipeline_kind == "distilled": + video, audio = pipeline( + prompt=request["prompt"], + seed=request["seed"], + height=request["height"], + width=request["width"], + num_frames=request["num_frames"], + frame_rate=request["fps"], + images=images, + ) + else: + video, audio = pipeline( + prompt=request["prompt"], + negative_prompt=request["negative_prompt"], + seed=request["seed"], + height=request["height"], + width=request["width"], + num_frames=request["num_frames"], + frame_rate=request["fps"], + num_inference_steps=request["num_inference_steps"], + video_guider_params=MultiModalGuiderParams( + cfg_scale=request["video_cfg_scale"], + stg_scale=request["video_stg_scale"], + rescale_scale=request["video_rescale_scale"], + modality_scale=request["video_modality_scale"], + skip_step=0, + stg_blocks=request["video_stg_blocks"], + ), + audio_guider_params=MultiModalGuiderParams( + cfg_scale=request["audio_cfg_scale"], + stg_scale=request["audio_stg_scale"], + rescale_scale=request["audio_rescale_scale"], + modality_scale=request["audio_modality_scale"], + skip_step=0, + stg_blocks=request["audio_stg_blocks"], + ), + images=images, + max_batch_size=4, + ) video_tensor = torch.cat([chunk.detach().cpu() for chunk in video], dim=0) _save_outputs( args.output_dir, @@ -179,6 +245,7 @@ def _run_official(args: argparse.Namespace, request: dict[str, Any]) -> None: "attention_backend": "torch_sdpa", "official_revision": os.environ.get("VLLM_TEST_LTX_OFFICIAL_REVISION"), "checkpoint": str(args.checkpoint), + "pipeline_kind": args.pipeline_kind, }, ) @@ -253,6 +320,10 @@ def _canonical_video(video: Any) -> torch.Tensor: def _run_omni(args: argparse.Namespace, request: dict[str, Any]) -> None: if args.model is None or args.model_class_name is None: raise ValueError("Omni backend requires --model and --model-class-name") + if request.get("pipeline_kind", "one_stage") != args.pipeline_kind: + raise ValueError( + f"Request pipeline kind {request.get('pipeline_kind')!r} does not match {args.pipeline_kind!r}" + ) attention_config = {"default": {"backend": "TORCH_SDPA"}} @@ -273,7 +344,8 @@ def _run_omni(args: argparse.Namespace, request: dict[str, Any]) -> None: parallel_config=DiffusionParallelConfig(), ) try: - model_class_name = get_model_class_name(omni) + detected_model_class_name = get_model_class_name(omni) + model_class_name = args.model_class_name sampling_params = OmniDiffusionSamplingParams( height=request["height"], width=request["width"], @@ -289,10 +361,9 @@ def _run_omni(args: argparse.Namespace, request: dict[str, Any]) -> None: key: value for key, value in request.items() if key.startswith("video_") or key.startswith("audio_") } apply_declared_extra_args(sampling_params, get_extra_body_params(model_class_name), guidance) - prompt: dict[str, Any] = { - "prompt": request["prompt"], - "negative_prompt": request["negative_prompt"], - } + prompt: dict[str, Any] = {"prompt": request["prompt"]} + if request.get("negative_prompt"): + prompt["negative_prompt"] = request["negative_prompt"] image_path = request.get("image") if image_path is not None: with Image.open(str(image_path)) as source_image: @@ -312,6 +383,8 @@ def _run_omni(args: argparse.Namespace, request: dict[str, Any]) -> None: "attention_backend": "torch_sdpa", "model": args.model, "model_class_name": model_class_name, + "model_config_class_name": detected_model_class_name, + "pipeline_kind": args.pipeline_kind, }, ) finally: diff --git a/tests/e2e/accuracy/ltx/test_ltx_official_similarity.py b/tests/e2e/accuracy/ltx/test_ltx_official_similarity.py index 647ece7a7..8a3be4a99 100644 --- a/tests/e2e/accuracy/ltx/test_ltx_official_similarity.py +++ b/tests/e2e/accuracy/ltx/test_ltx_official_similarity.py @@ -3,10 +3,9 @@ """E2E accuracy guard against a pinned Lightricks LTX pipeline revision. -The comparison runs both runtimes through PyTorch SDPA and uses -``max_batch_size=4`` in the official reference to match Omni's fused guidance -batch. Video and audio guidance use the official non-HQ one-stage defaults; -only the generation shape and step count are reduced for CI runtime. +The original reduced one-stage guards remain unchanged. Four complementary +default-shape cases cover both model versions, both two-stage weight families, +and T2V/I2V without expanding every combination into a duplicate golden. """ from __future__ import annotations @@ -17,8 +16,9 @@ import shutil import subprocess import sys import time -from dataclasses import dataclass +from dataclasses import asdict, dataclass, replace from pathlib import Path +from typing import Literal import numpy as np import pytest @@ -53,74 +53,260 @@ NEGATIVE_PROMPT = ( # Both runtimes use PyTorch SDPA with the current Torch dispatch defaults. ATTENTION_BACKEND = "torch_sdpa" -VIDEO_SSIM_MEAN_THRESHOLD = 0.95 -VIDEO_SSIM_MIN_THRESHOLD = 0.90 -VIDEO_PSNR_MEAN_THRESHOLD = 30.0 -AUDIO_RELATIVE_L2_THRESHOLD = 0.2 -AUDIO_COSINE_THRESHOLD = 0.95 + + +@dataclass(frozen=True) +class LTXArtifact: + repo_id: str + filename: str + revision: str + env: str + repo_type: Literal["model", "dataset"] = "model" + + +@dataclass(frozen=True) +class LTXAccuracyThresholds: + video_ssim_mean: float + video_ssim_min: float + video_psnr_mean_db: float + # Zero disables gating for waveform-sensitive audio that is not expected + # to align with the official output. + audio_relative_l2: float + audio_cosine_similarity: float + + +STRICT_THRESHOLDS = LTXAccuracyThresholds( + video_ssim_mean=0.95, + video_ssim_min=0.90, + video_psnr_mean_db=30.0, + audio_relative_l2=0.20, + audio_cosine_similarity=0.95, +) @dataclass(frozen=True) class LTXAccuracyCase: name: str + pipeline_kind: Literal["one_stage", "distilled", "two_stage"] model_id: str model_revision: str model_env: str model_class_name: str - checkpoint_repo: str - checkpoint_filename: str - checkpoint_revision: str - checkpoint_env: str - stg_block: int + checkpoint: LTXArtifact + width: int + height: int + num_frames: int + num_inference_steps: int + seed: int + stg_block: int | None + thresholds: LTXAccuracyThresholds prompt: str = PROMPT - image_repo: str | None = None - image_filename: str | None = None - image_revision: str | None = None + gemma_model_id: str | None = None + gemma_model_revision: str | None = None + gemma_model_env: str | None = None + image: LTXArtifact | None = None + spatial_upsampler: LTXArtifact | None = None + distilled_lora: LTXArtifact | None = None + # Official uses block streaming; Omni uses layerwise offload. Both preserve + # bf16 arithmetic while keeping full-resolution two-stage cases on one H100. + enable_layerwise_offload: bool = False -CASES = ( +LTX2_REVISION = "47da56e2ad66ce4125a9922b4a8826bf407f9d0a" +LTX23_REVISION = "4229404625088d21c4f112eb640fb04a0900ee25" +LTX2_CHECKPOINT = LTXArtifact( + repo_id="Lightricks/LTX-2", + filename="ltx-2-19b-dev.safetensors", + revision=LTX2_REVISION, + env="VLLM_TEST_LTX2_OFFICIAL_CHECKPOINT", +) +LTX2_DISTILLED_CHECKPOINT = replace( + LTX2_CHECKPOINT, + filename="ltx-2-19b-distilled.safetensors", + env="VLLM_TEST_LTX2_DISTILLED_OFFICIAL_CHECKPOINT", +) +LTX2_UPSAMPLER = replace( + LTX2_CHECKPOINT, + filename="ltx-2-spatial-upscaler-x2-1.0.safetensors", + env="VLLM_TEST_LTX2_UPSAMPLER", +) +LTX2_DISTILLED_LORA = replace( + LTX2_CHECKPOINT, + filename="ltx-2-19b-distilled-lora-384.safetensors", + env="VLLM_TEST_LTX2_DISTILLED_LORA", +) +LTX23_CHECKPOINT = LTXArtifact( + repo_id="Lightricks/LTX-2.3", + filename="ltx-2.3-22b-dev.safetensors", + revision=LTX23_REVISION, + env="VLLM_TEST_LTX23_OFFICIAL_CHECKPOINT", +) +LTX23_DISTILLED_CHECKPOINT = replace( + LTX23_CHECKPOINT, + # The pinned Diffusers checkpoint contains the original merged distilled + # Transformer. Keep the reference Transformer on the same version; the + # spatial upsampler is versioned independently and remains on 1.1 below. + filename="ltx-2.3-22b-distilled.safetensors", + env="VLLM_TEST_LTX23_DISTILLED_OFFICIAL_CHECKPOINT", +) +LTX23_UPSAMPLER = replace( + LTX23_CHECKPOINT, + filename="ltx-2.3-spatial-upscaler-x2-1.1.safetensors", + env="VLLM_TEST_LTX23_UPSAMPLER", +) +LTX23_DISTILLED_LORA = replace( + LTX23_CHECKPOINT, + filename="ltx-2.3-22b-distilled-lora-384-1.1.safetensors", + env="VLLM_TEST_LTX23_DISTILLED_LORA", +) +I2V_IMAGE = LTXArtifact( + repo_id="huggingface/documentation-images", + filename="diffusers/svd/rocket.png", + revision="645d8364f0c7a101180b364811b5a11a362e4010", + env="VLLM_TEST_LTX_I2V_IMAGE", + repo_type="dataset", +) + + +LEGACY_CASES = ( LTXAccuracyCase( name="ltx2", + pipeline_kind="one_stage", model_id="Lightricks/LTX-2", - model_revision="47da56e2ad66ce4125a9922b4a8826bf407f9d0a", + model_revision=LTX2_REVISION, model_env="VLLM_TEST_LTX2_MODEL", model_class_name="LTX2Pipeline", - checkpoint_repo="Lightricks/LTX-2", - checkpoint_filename="ltx-2-19b-dev.safetensors", - checkpoint_revision="47da56e2ad66ce4125a9922b4a8826bf407f9d0a", - checkpoint_env="VLLM_TEST_LTX2_OFFICIAL_CHECKPOINT", + checkpoint=LTX2_CHECKPOINT, + width=512, + height=384, + num_frames=25, + num_inference_steps=20, + seed=42, stg_block=29, + thresholds=STRICT_THRESHOLDS, ), LTXAccuracyCase( name="ltx2_3", + pipeline_kind="one_stage", model_id="diffusers/LTX-2.3-Diffusers", model_revision="8eee8edcf067e838b843f926ec4d4cc9b2be1aaf", model_env="VLLM_TEST_LTX23_MODEL", model_class_name="LTX2Pipeline", - checkpoint_repo="Lightricks/LTX-2.3", - checkpoint_filename="ltx-2.3-22b-dev.safetensors", - checkpoint_revision="4229404625088d21c4f112eb640fb04a0900ee25", - checkpoint_env="VLLM_TEST_LTX23_OFFICIAL_CHECKPOINT", + checkpoint=LTX23_CHECKPOINT, + width=512, + height=384, + num_frames=25, + num_inference_steps=20, + seed=42, stg_block=28, + thresholds=STRICT_THRESHOLDS, ), LTXAccuracyCase( name="ltx2_3_i2v", + pipeline_kind="one_stage", model_id="diffusers/LTX-2.3-Diffusers", model_revision="8eee8edcf067e838b843f926ec4d4cc9b2be1aaf", model_env="VLLM_TEST_LTX23_MODEL", model_class_name="LTX2Pipeline", - checkpoint_repo="Lightricks/LTX-2.3", - checkpoint_filename="ltx-2.3-22b-dev.safetensors", - checkpoint_revision="4229404625088d21c4f112eb640fb04a0900ee25", - checkpoint_env="VLLM_TEST_LTX23_OFFICIAL_CHECKPOINT", + checkpoint=LTX23_CHECKPOINT, + width=512, + height=384, + num_frames=25, + num_inference_steps=20, + seed=42, stg_block=28, - image_repo="huggingface/documentation-images", - image_filename="diffusers/svd/rocket.png", - image_revision="645d8364f0c7a101180b364811b5a11a362e4010", + thresholds=STRICT_THRESHOLDS, + image=I2V_IMAGE, ), ) +TWO_STAGE_CASES = ( + LTXAccuracyCase( + name="ltx2_distilled_t2v", + pipeline_kind="distilled", + model_id="rootonchair/LTX-2-19b-distilled", + model_revision="388e2846f54aae51687498ffb6b27c7c2c9ce9e5", + model_env="VLLM_TEST_LTX2_DISTILLED_MODEL", + model_class_name="LTX2DistilledPipeline", + checkpoint=LTX2_DISTILLED_CHECKPOINT, + spatial_upsampler=LTX2_UPSAMPLER, + width=1536, + height=1024, + num_frames=121, + num_inference_steps=8, + seed=10, + stg_block=None, + gemma_model_id="Lightricks/LTX-2", + gemma_model_revision=LTX2_REVISION, + gemma_model_env="VLLM_TEST_LTX2_MODEL", + thresholds=STRICT_THRESHOLDS, + ), + LTXAccuracyCase( + name="ltx23_distilled_i2v", + pipeline_kind="distilled", + model_id="diffusers/LTX-2.3-Distilled-Diffusers", + model_revision="432e0d3c2d1769aaa4d295f9243f7062bf6b47ee", + model_env="VLLM_TEST_LTX23_DISTILLED_MODEL", + model_class_name="LTX2DistilledPipeline", + checkpoint=LTX23_DISTILLED_CHECKPOINT, + spatial_upsampler=LTX23_UPSAMPLER, + width=1536, + height=1024, + num_frames=121, + num_inference_steps=8, + seed=10, + stg_block=None, + gemma_model_id="diffusers/LTX-2.3-Diffusers", + gemma_model_revision="8eee8edcf067e838b843f926ec4d4cc9b2be1aaf", + gemma_model_env="VLLM_TEST_LTX23_MODEL", + thresholds=STRICT_THRESHOLDS, + image=I2V_IMAGE, + ), + LTXAccuracyCase( + name="ltx2_two_stage_layer_fused_t2v", + pipeline_kind="two_stage", + model_id="Lightricks/LTX-2", + model_revision=LTX2_REVISION, + model_env="VLLM_TEST_LTX2_MODEL", + model_class_name="LTX2TwoStagePipeline", + checkpoint=LTX2_CHECKPOINT, + spatial_upsampler=LTX2_UPSAMPLER, + distilled_lora=LTX2_DISTILLED_LORA, + width=1536, + height=1024, + num_frames=121, + num_inference_steps=40, + seed=10, + stg_block=29, + enable_layerwise_offload=True, + thresholds=STRICT_THRESHOLDS, + ), + LTXAccuracyCase( + name="ltx23_two_stage_layer_fused_i2v", + pipeline_kind="two_stage", + model_id="diffusers/LTX-2.3-Diffusers", + model_revision="8eee8edcf067e838b843f926ec4d4cc9b2be1aaf", + model_env="VLLM_TEST_LTX23_MODEL", + model_class_name="LTX2TwoStagePipeline", + checkpoint=LTX23_CHECKPOINT, + spatial_upsampler=LTX23_UPSAMPLER, + distilled_lora=LTX23_DISTILLED_LORA, + width=1536, + height=1024, + num_frames=121, + num_inference_steps=30, + seed=10, + stg_block=28, + enable_layerwise_offload=True, + thresholds=STRICT_THRESHOLDS, + image=I2V_IMAGE, + ), +) + +CASES = (*LEGACY_CASES, *TWO_STAGE_CASES) + + def _run(command: list[str], *, env: dict[str, str], timeout: int = 1800) -> None: start = time.perf_counter() subprocess.run(command, env=env, timeout=timeout, check=True) @@ -209,6 +395,7 @@ def _resolve_model(case: LTXAccuracyCase) -> Path: "model_index.json", "audio_vae/*", "connectors/*", + "latent_upsampler/*", "processor/*", "scheduler/*", "text_encoder/config.json", @@ -223,74 +410,82 @@ def _resolve_model(case: LTXAccuracyCase) -> Path: ) -def _resolve_gemma_root(model: Path) -> Path: +def _resolve_gemma_root(case: LTXAccuracyCase, model: Path) -> Path: configured_root = os.environ.get("VLLM_TEST_LTX_GEMMA_ROOT") if configured_root: root = Path(configured_root) assert root.is_dir(), f"Gemma root not found: {root}" return root + if case.gemma_model_id is not None: + configured_model = os.environ.get(case.gemma_model_env or "") + if configured_model and Path(configured_model).is_dir(): + return Path(configured_model) + return Path( + snapshot_download( + repo_id=case.gemma_model_id, + revision=case.gemma_model_revision, + allow_patterns=[ + "processor/*", + "text_encoder/*", + "tokenizer/*", + ], + ) + ) return model -def _resolve_checkpoint(case: LTXAccuracyCase, model: Path) -> Path: - configured_checkpoint = os.environ.get(case.checkpoint_env) - if configured_checkpoint: - checkpoint = Path(configured_checkpoint) - assert checkpoint.is_file(), f"Official checkpoint not found: {checkpoint}" - return checkpoint - model_checkpoint = model / case.checkpoint_filename - if model_checkpoint.is_file(): - return model_checkpoint +def _resolve_artifact(artifact: LTXArtifact, model: Path | None = None) -> Path: + configured_path = os.environ.get(artifact.env) + if configured_path: + path = Path(configured_path) + assert path.is_file(), f"Configured LTX artifact not found: {path}" + return path + if model is not None: + model_path = model / artifact.filename + if model_path.is_file(): + return model_path return Path( hf_hub_download( - repo_id=case.checkpoint_repo, - filename=case.checkpoint_filename, - revision=case.checkpoint_revision, + repo_id=artifact.repo_id, + repo_type=None if artifact.repo_type == "model" else artifact.repo_type, + filename=artifact.filename, + revision=artifact.revision, ) ) def _resolve_image(case: LTXAccuracyCase) -> Path | None: - if case.image_filename is None: + if case.image is None: return None - if case.image_repo is None or case.image_revision is None: - raise ValueError(f"Incomplete image source for LTX accuracy case {case.name!r}.") - configured_image = os.environ.get("VLLM_TEST_LTX_I2V_IMAGE") - if configured_image: - image = Path(configured_image) - assert image.is_file(), f"LTX I2V conditioning image not found: {image}" - return image - return Path( - hf_hub_download( - repo_id=case.image_repo, - repo_type="dataset", - filename=case.image_filename, - revision=case.image_revision, - ) - ) + return _resolve_artifact(case.image) def _request(case: LTXAccuracyCase, image: Path | None) -> dict[str, object]: request: dict[str, object] = { + "pipeline_kind": case.pipeline_kind, "prompt": case.prompt, - "negative_prompt": NEGATIVE_PROMPT, - "width": 512, - "height": 384, - "num_frames": 25, + "negative_prompt": "" if case.pipeline_kind == "distilled" else NEGATIVE_PROMPT, + "width": case.width, + "height": case.height, + "num_frames": case.num_frames, "fps": 24, - "num_inference_steps": 20, - "seed": 42, - "video_cfg_scale": 3.0, - "audio_cfg_scale": 7.0, - "video_stg_scale": 1.0, - "audio_stg_scale": 1.0, - "video_modality_scale": 3.0, - "audio_modality_scale": 3.0, - "video_rescale_scale": 0.7, - "audio_rescale_scale": 0.7, - "video_stg_blocks": [case.stg_block], - "audio_stg_blocks": [case.stg_block], + "num_inference_steps": case.num_inference_steps, + "seed": case.seed, } + if case.pipeline_kind != "distilled": + assert case.stg_block is not None + request.update( + video_cfg_scale=3.0, + audio_cfg_scale=7.0, + video_stg_scale=1.0, + audio_stg_scale=1.0, + video_modality_scale=3.0, + audio_modality_scale=3.0, + video_rescale_scale=0.7, + audio_rescale_scale=0.7, + video_stg_blocks=[case.stg_block], + audio_stg_blocks=[case.stg_block], + ) if image is not None: request["image"] = str(image.resolve()) return request @@ -299,20 +494,30 @@ def _request(case: LTXAccuracyCase, image: Path | None) -> dict[str, object]: def _video_metrics(reference: np.ndarray, prediction: np.ndarray) -> dict[str, float]: assert reference.shape == prediction.shape assert reference.ndim == 4 and reference.shape[-1] == 3 + device = torch.device("cpu") + ssim = StructuralSimilarityIndexMeasure(data_range=1.0).to(device) + psnr = PeakSignalNoiseRatio(data_range=1.0).to(device) ssim_scores: list[float] = [] psnr_scores: list[float] = [] - for reference_frame, prediction_frame in zip(reference, prediction, strict=True): - reference_tensor = torch.from_numpy(reference_frame).permute(2, 0, 1).unsqueeze(0) - prediction_tensor = torch.from_numpy(prediction_frame).permute(2, 0, 1).unsqueeze(0) - ssim_scores.append(float(StructuralSimilarityIndexMeasure(data_range=1.0)(prediction_tensor, reference_tensor))) - psnr_scores.append(float(PeakSignalNoiseRatio(data_range=1.0)(prediction_tensor, reference_tensor))) - difference = np.abs(reference.astype(np.float64) - prediction.astype(np.float64)) + max_abs = 0.0 + absolute_error_sum = 0.0 + with torch.inference_mode(): + for reference_frame, prediction_frame in zip(reference, prediction, strict=True): + reference_tensor = torch.from_numpy(reference_frame).permute(2, 0, 1).unsqueeze(0).to(device) + prediction_tensor = torch.from_numpy(prediction_frame).permute(2, 0, 1).unsqueeze(0).to(device) + ssim_scores.append(float(ssim(prediction_tensor, reference_tensor))) + psnr_scores.append(float(psnr(prediction_tensor, reference_tensor))) + ssim.reset() + psnr.reset() + difference = np.abs(reference_frame.astype(np.float64) - prediction_frame.astype(np.float64)) + max_abs = max(max_abs, float(difference.max())) + absolute_error_sum += float(difference.sum()) return { "ssim_mean": float(np.mean(ssim_scores)), "ssim_min": float(np.min(ssim_scores)), "psnr_mean_db": float(np.mean(psnr_scores)), - "max_abs": float(difference.max()), - "mean_abs": float(difference.mean()), + "max_abs": max_abs, + "mean_abs": absolute_error_sum / reference.size, } @@ -343,13 +548,19 @@ def _audio_metrics(reference: np.ndarray, prediction: np.ndarray) -> dict[str, f @pytest.mark.benchmark @pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=1) -def test_ltx_one_stage_matches_official(case: LTXAccuracyCase, accuracy_artifact_root: Path) -> None: - """Compare official and Omni raw AV outputs from the same E2E request.""" +def test_ltx_matches_official(case: LTXAccuracyCase, accuracy_artifact_root: Path) -> None: + """Compare official and Omni raw AV outputs from the same single-H100 request.""" + configured_artifact_root = os.environ.get("VLLM_TEST_LTX_ARTIFACT_ROOT") + if configured_artifact_root: + accuracy_artifact_root = Path(configured_artifact_root) + accuracy_artifact_root.mkdir(parents=True, exist_ok=True) output_root = reset_artifact_dir(accuracy_artifact_root / "ltx_official" / case.name) official_root, official_revision = _official_source(accuracy_artifact_root / "ltx_official") model = _resolve_model(case) - gemma_root = _resolve_gemma_root(model) - checkpoint = _resolve_checkpoint(case, model) + gemma_root = _resolve_gemma_root(case, model) + checkpoint = _resolve_artifact(case.checkpoint, model) + spatial_upsampler = _resolve_artifact(case.spatial_upsampler, model) if case.spatial_upsampler is not None else None + distilled_lora = _resolve_artifact(case.distilled_lora, model) if case.distilled_lora is not None else None image = _resolve_image(case) request_path = output_root / "request.json" request_path.write_text(json.dumps(_request(case, image), indent=2) + "\n") @@ -359,8 +570,13 @@ def test_ltx_one_stage_matches_official(case: LTXAccuracyCase, accuracy_artifact str(runner), "--request", str(request_path), + "--pipeline-kind", + case.pipeline_kind, ] - if os.environ.get("VLLM_TEST_LTX_ENABLE_LAYERWISE_OFFLOAD", "").lower() in {"1", "true", "yes", "on"}: + enable_layerwise_offload = case.enable_layerwise_offload or os.environ.get( + "VLLM_TEST_LTX_ENABLE_LAYERWISE_OFFLOAD", "" + ).lower() in {"1", "true", "yes", "on"} + if enable_layerwise_offload: runner_args.append("--enable-layerwise-offload") env = os.environ.copy() env["VLLM_TEST_LTX_OFFICIAL_REVISION"] = official_revision @@ -372,10 +588,9 @@ def test_ltx_one_stage_matches_official(case: LTXAccuracyCase, accuracy_artifact ) official_output = output_root / "official" - _run( - _official_runner_prefix() - + runner_args - + [ + official_args = _official_runner_prefix() + runner_args + official_args.extend( + [ "--backend", "official", "--output-dir", @@ -386,12 +601,16 @@ def test_ltx_one_stage_matches_official(case: LTXAccuracyCase, accuracy_artifact str(checkpoint), "--gemma-root", str(gemma_root), - ], - env=env, + ] ) + if spatial_upsampler is not None: + official_args.extend(["--spatial-upsampler", str(spatial_upsampler)]) + if distilled_lora is not None: + official_args.extend(["--distilled-lora", str(distilled_lora)]) + _run(official_args, env=env) omni_output = output_root / "omni" - _run( + omni_args = ( [sys.executable] + runner_args + [ @@ -403,9 +622,9 @@ def test_ltx_one_stage_matches_official(case: LTXAccuracyCase, accuracy_artifact str(model), "--model-class-name", case.model_class_name, - ], - env=env, + ] ) + _run(omni_args, env=env) official_metadata = json.loads((official_output / "metadata.json").read_text()) omni_metadata = json.loads((omni_output / "metadata.json").read_text()) @@ -423,18 +642,27 @@ def test_ltx_one_stage_matches_official(case: LTXAccuracyCase, accuracy_artifact result = { "case": case.name, "task": "i2v" if image is not None else "t2v", + "pipeline_kind": case.pipeline_kind, + "model_class_name": case.model_class_name, "attention_backend": ATTENTION_BACKEND, "official_revision": official_revision, "model_revision": case.model_revision, - "checkpoint_revision": case.checkpoint_revision, + "checkpoint_revision": case.checkpoint.revision, + "spatial_upsampler_revision": (case.spatial_upsampler.revision if case.spatial_upsampler is not None else None), + "distilled_lora_revision": case.distilled_lora.revision if case.distilled_lora is not None else None, + "enable_layerwise_offload": enable_layerwise_offload, + "thresholds": asdict(case.thresholds), + "request": _request(case, image), "video": video_metrics, "audio": audio_metrics, } (output_root / "metrics.json").write_text(json.dumps(result, indent=2) + "\n") print(json.dumps(result, indent=2)) - assert video_metrics["ssim_mean"] >= VIDEO_SSIM_MEAN_THRESHOLD - assert video_metrics["ssim_min"] >= VIDEO_SSIM_MIN_THRESHOLD - assert video_metrics["psnr_mean_db"] >= VIDEO_PSNR_MEAN_THRESHOLD - assert audio_metrics["relative_l2"] <= AUDIO_RELATIVE_L2_THRESHOLD - assert audio_metrics["cosine_similarity"] >= AUDIO_COSINE_THRESHOLD + assert video_metrics["ssim_mean"] >= case.thresholds.video_ssim_mean + assert video_metrics["ssim_min"] >= case.thresholds.video_ssim_min + assert video_metrics["psnr_mean_db"] >= case.thresholds.video_psnr_mean_db + if case.thresholds.audio_relative_l2 > 0: + assert audio_metrics["relative_l2"] <= case.thresholds.audio_relative_l2 + if case.thresholds.audio_cosine_similarity > 0: + assert audio_metrics["cosine_similarity"] >= case.thresholds.audio_cosine_similarity diff --git a/tests/e2e/online_serving/test_ltx.py b/tests/e2e/online_serving/test_ltx.py new file mode 100644 index 000000000..43df5df2d --- /dev/null +++ b/tests/e2e/online_serving/test_ltx.py @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Online-serving smoke tests for the public LTX-2 pipeline entries.""" + +import os + +import pytest + +from tests.helpers.mark import hardware_marks +from tests.helpers.media import generate_synthetic_image +from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler + +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + +DISTILLED_MODEL = os.getenv("VLLM_TEST_LTX23_DISTILLED_MODEL", "diffusers/LTX-2.3-Distilled-Diffusers") +BASE_MODEL = os.getenv("VLLM_TEST_LTX23_MODEL", "diffusers/LTX-2.3-Diffusers") +PROMPT = "A serene lake at sunset with mountains in the background." +SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"}) +HSDP_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2) + + +def _server( + model: str, + model_class_name: str, + *extra_args: str, +) -> OmniServerParams: + return OmniServerParams( + model=model, + server_args=[ + "--enforce-eager", + *extra_args, + "--model-class-name", + model_class_name, + ], + ) + + +def _cases(): + return [ + pytest.param( + _server(BASE_MODEL, "LTX2Pipeline"), + 3, + id="one_stage", + marks=SINGLE_CARD_MARKS, + ), + pytest.param( + _server(DISTILLED_MODEL, "LTX2DistilledPipeline"), + 8, + id="distilled", + marks=SINGLE_CARD_MARKS, + ), + pytest.param( + _server( + DISTILLED_MODEL, + "LTX2DistilledPipeline", + "--use-hsdp", + "--hsdp-shard-size", + "2", + ), + 8, + id="distilled_hsdp2", + marks=HSDP_MARKS, + ), + pytest.param( + _server( + BASE_MODEL, + "LTX2TwoStagePipeline", + "--enable-layerwise-offload", + ), + 3, + id="two_stage", + marks=SINGLE_CARD_MARKS, + ), + ] + + +@pytest.mark.parametrize( + ("omni_server", "num_inference_steps"), + _cases(), + indirect=["omni_server"], +) +def test_ltx_pipeline_entries( + omni_server: OmniServer, + num_inference_steps: int, + openai_client: OpenAIClientHandler, + subtests, +): + for task in ("t2v", "i2v"): + with subtests.test(task=task): + form_data = { + "prompt": PROMPT, + "model": omni_server.model, + "height": 128, + "width": 128, + "num_frames": 9, + "fps": 8, + "num_inference_steps": num_inference_steps, + "seed": 42, + } + request_config = { + "model": omni_server.model, + "form_data": form_data, + } + if task == "i2v": + request_config["image_reference"] = ( + f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" + ) + + openai_client.send_video_diffusion_request(request_config) diff --git a/tests/e2e/online_serving/test_ltx2_expansion.py b/tests/e2e/online_serving/test_ltx2_expansion.py deleted file mode 100644 index 4d0eb6c90..000000000 --- a/tests/e2e/online_serving/test_ltx2_expansion.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Expansion tests for LTX-2 two-stage pipelines. - -Coverage: -- HSDP with LTX2DistilledPipeline (text-to-video and image-to-video) -""" - -import os - -import pytest - -from tests.helpers.mark import hardware_marks -from tests.helpers.media import generate_synthetic_image -from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler - -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] - -MODEL = os.getenv("VLLM_TEST_LTX2_MODEL", "rootonchair/LTX-2-19b-distilled") -PROMPT = "A serene lake at sunset with mountains in the background." -PARALLEL_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2) -HSDP_ARGS = ["--use-hsdp", "--hsdp-shard-size", "2"] - - -def _cases(): - cases = [] - - cases.append( - pytest.param( - OmniServerParams( - model=MODEL, - server_args=[ - *HSDP_ARGS, - "--model-class-name", - "LTX2DistilledPipeline", - ], - ), - False, - id="t2v_hsdp", - marks=PARALLEL_MARKS, - ) - ) - - cases.append( - pytest.param( - OmniServerParams( - model=MODEL, - server_args=[ - *HSDP_ARGS, - "--model-class-name", - "LTX2DistilledPipeline", - ], - ), - True, - id="i2v_hsdp", - marks=PARALLEL_MARKS, - ) - ) - - return cases - - -@pytest.mark.parametrize(("omni_server", "is_i2v"), _cases(), indirect=["omni_server"]) -def test_ltx2_two_stage_hsdp( - omni_server: OmniServer, - is_i2v: bool, - openai_client: OpenAIClientHandler, -): - # Keep CI small while exercising the fixed 8+3-step distilled recipe. - # Height and width describe the final output; Stage 1 runs at half size. - form_data = { - "prompt": PROMPT, - "model": omni_server.model, - "height": 128, - "width": 128, - "num_frames": 9, - "fps": 8, - "num_inference_steps": 8, - "seed": 42, - } - - request_config = { - "model": omni_server.model, - "form_data": form_data, - } - - if is_i2v: - request_config["image_reference"] = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" - - openai_client.send_video_diffusion_request(request_config) diff --git a/tests/model_extras/test_model_extras.py b/tests/model_extras/test_model_extras.py index d57671adc..557fb5f6c 100644 --- a/tests/model_extras/test_model_extras.py +++ b/tests/model_extras/test_model_extras.py @@ -233,10 +233,13 @@ def test_ltx_extra_registry_declares_official_guidance_params() -> None: } ) - assert get_extra_body_params("LTX2Pipeline") == expected - assert get_extra_output_params("LTX2Pipeline") == frozenset() - assert get_extra_body_params("LTX2DistilledPipeline") == expected - assert get_extra_output_params("LTX2DistilledPipeline") == frozenset() + for pipeline_name in ( + "LTX2Pipeline", + "LTX2TwoStagePipeline", + "LTX2DistilledPipeline", + ): + assert get_extra_body_params(pipeline_name) == expected + assert get_extra_output_params(pipeline_name) == frozenset() @pytest.mark.core_model diff --git a/tests/model_tests/diffusion/test_alignment.py b/tests/model_tests/diffusion/test_alignment.py index be064f603..f5d6c18c4 100644 --- a/tests/model_tests/diffusion/test_alignment.py +++ b/tests/model_tests/diffusion/test_alignment.py @@ -27,6 +27,7 @@ EXCLUDED_MODELS = [ "OvisImagePipeline", "WanPipeline", "WanVACEPipeline", + "LTX2TwoStagePipeline", "LTX2DistilledPipeline", "LTX2T2VDMD2Pipeline", "LTX2I2VDMD2Pipeline", diff --git a/vllm_omni/diffusion/models/ltx2/__init__.py b/vllm_omni/diffusion/models/ltx2/__init__.py index 300295bd8..20ada6adc 100644 --- a/vllm_omni/diffusion/models/ltx2/__init__.py +++ b/vllm_omni/diffusion/models/ltx2/__init__.py @@ -12,12 +12,16 @@ from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import ( LTX2Pipeline, LTX2T2VDMD2Pipeline, ) -from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_two_stage import LTX2DistilledPipeline +from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_two_stage import ( + LTX2DistilledPipeline, + LTX2TwoStagePipeline, +) __all__ = [ "LTX2Pipeline", "LTX2T2VDMD2Pipeline", "LTX2I2VDMD2Pipeline", + "LTX2TwoStagePipeline", "LTX2DistilledPipeline", "get_ltx2_post_process_func", "load_transformer_config", diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py b/vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py new file mode 100644 index 000000000..d8228370e --- /dev/null +++ b/vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Parse official LTX distilled adapters without materializing their tensors. + +The official LTX adapters are raw safetensors files rather than PEFT +checkpoints. This module is deliberately LTX-specific: it translates the +official names into the locally constructed Diffusers transformer names and +describes the resulting adapter targets. Applying those targets is the job +of :mod:`ltx2_phase_adapter`. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path + +import torch +import torch.nn as nn +from safetensors import safe_open + +_IGNORED_OFFICIAL_LORA_MODULES = {"text_embedding_projection.aggregate_embed"} +_OFFICIAL_TO_DIFFUSERS_PREFIXES = ( + ("audio_prompt_adaln_single.", "audio_prompt_adaln."), + ("prompt_adaln_single.", "prompt_adaln."), + ("audio_adaln_single.", "audio_time_embed."), + ("adaln_single.", "time_embed."), + ("audio_patchify_proj", "audio_proj_in"), + ("patchify_proj", "proj_in"), + ("av_ca_video_scale_shift_adaln_single.", "av_cross_attn_video_scale_shift."), + ("av_ca_audio_scale_shift_adaln_single.", "av_cross_attn_audio_scale_shift."), + ("av_ca_a2v_gate_adaln_single.", "av_cross_attn_video_a2v_gate."), + ("av_ca_v2a_gate_adaln_single.", "av_cross_attn_audio_v2a_gate."), +) + + +@dataclass(frozen=True) +class AdapterTarget: + """One A/B pair mapped onto a local transformer linear layer.""" + + # Keep the original Diffusers module name for diagnostics after LTX packs + # Q/K/V into the runtime module path. + source_module: str + module: str + slice: str | None + a_key: str + b_key: str + a_shape: tuple[int, ...] + b_shape: tuple[int, ...] + + +@dataclass(frozen=True) +class AdapterManifest: + """Tensor-free description of the fixed refinement adapter.""" + + path: str + targets: tuple[AdapterTarget, ...] + + +def _to_diffusers_module_name(name: str) -> str: + for official_prefix, diffusers_prefix in _OFFICIAL_TO_DIFFUSERS_PREFIXES: + if name.startswith(official_prefix): + return diffusers_prefix + name[len(official_prefix) :] + return name + + +def _resolve_lora_target(transformer: nn.Module, module_name: str) -> tuple[str, str | int | None] | None: + modules = dict(transformer.named_modules()) + if module_name in modules: + return module_name, None + for packed_suffix, sub_suffix, shard_id in getattr(transformer, "stacked_params_mapping", ()): + if sub_suffix in module_name: + packed_name = module_name.replace(sub_suffix, packed_suffix) + if packed_name in modules: + return packed_name, shard_id + return None + + +def parse_ltx_adapter(transformer: nn.Module, path: str) -> AdapterManifest: + """Translate one official raw LTX adapter into local Transformer targets.""" + path = str(Path(path)) + targets: list[AdapterTarget] = [] + unresolved: list[str] = [] + seen: set[tuple[str, str | int | None]] = set() + + with safe_open(path, framework="pt", device="cpu") as handle: + tensor_names = set(handle.keys()) + official_name_set = { + tensor_name.removeprefix("diffusion_model.").rsplit(".lora_", 1)[0] + for tensor_name in tensor_names + if tensor_name.endswith(".lora_A.weight") + } + official_b_names = { + tensor_name.removeprefix("diffusion_model.").rsplit(".lora_", 1)[0] + for tensor_name in tensor_names + if tensor_name.endswith(".lora_B.weight") + } + missing_a = sorted(official_b_names - official_name_set - _IGNORED_OFFICIAL_LORA_MODULES) + if missing_a: + raise ValueError(f"Missing paired LoRA A tensors for {missing_a} in {path}.") + + official_names = sorted(official_name_set) + for official_name in official_names: + if official_name in _IGNORED_OFFICIAL_LORA_MODULES: + continue + key_prefix = f"diffusion_model.{official_name}" + key_a = f"{key_prefix}.lora_A.weight" + key_b = f"{key_prefix}.lora_B.weight" + if key_b not in tensor_names: + raise ValueError(f"Missing paired LoRA tensor {key_b!r} in {path}.") + + a_shape = tuple(handle.get_slice(key_a).get_shape()) + b_shape = tuple(handle.get_slice(key_b).get_shape()) + if len(a_shape) != 2 or len(b_shape) != 2 or b_shape[1] != a_shape[0]: + raise ValueError(f"Invalid LoRA pair for {official_name}: A={a_shape}, B={b_shape}.") + + source_module = _to_diffusers_module_name(official_name) + resolved = _resolve_lora_target(transformer, source_module) + if resolved is None: + unresolved.append(official_name) + continue + module, shard_id = resolved + target_id = (module, shard_id) + if target_id in seen: + raise ValueError(f"Official LTX LoRA contains duplicate target {module!r}, slice {shard_id!r}.") + seen.add(target_id) + targets.append( + AdapterTarget( + source_module=source_module, + module=module, + slice=shard_id, + a_key=key_a, + b_key=key_b, + a_shape=a_shape, + b_shape=b_shape, + ) + ) + + if unresolved: + raise ValueError(f"Official LTX LoRA contains unmapped modules: {unresolved}.") + if not targets: + raise ValueError(f"No Transformer LoRA tensors were found in {path}.") + return AdapterManifest(path=path, targets=tuple(targets)) + + +def iter_adapter_tensors(manifest: AdapterManifest) -> Iterator[tuple[AdapterTarget, torch.Tensor, torch.Tensor]]: + """Stream adapter pairs in manifest order without a full checkpoint copy.""" + + with safe_open(manifest.path, framework="pt", device="cpu") as handle: + for target in manifest.targets: + yield ( + target, + handle.get_tensor(target.a_key), + handle.get_tensor(target.b_key), + ) diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_components.py b/vllm_omni/diffusion/models/ltx2/ltx2_components.py index 54c866eb7..7f2f31a60 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_components.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_components.py @@ -9,7 +9,8 @@ import inspect import json import logging import os -from dataclasses import dataclass +from dataclasses import dataclass, replace +from pathlib import Path from typing import TYPE_CHECKING, Any import torch @@ -19,6 +20,8 @@ from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel from diffusers.pipelines.ltx2.vocoder import LTX2Vocoder from diffusers.video_processor import VideoProcessor from huggingface_hub import hf_hub_download +from safetensors import safe_open +from safetensors.torch import load_file from transformers import AutoTokenizer, Gemma3ForConditionalGeneration from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata @@ -32,6 +35,7 @@ from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery if TYPE_CHECKING: from vllm.model_executor.layers.quantization.base_config import QuantizationConfig +from .ltx2_request import LTXCheckpointKind, validate_ltx_checkpoint from .ltx2_transformer import ( LTX2VideoTransformer3DModel, apply_interleaved_rotary_emb, @@ -70,6 +74,9 @@ class LTXComponentProfile: video_vae_cls: type = AutoencoderKLLTX2Video vocoder_cls: type = LTX2Vocoder vocoder_fallback_cls: type | None = None + artifact_repo_id: str | None = None + latent_upsampler_filename: str | None = None + distilled_lora_filename: str | None = None LTX2_COMPONENT_PROFILE = LTXComponentProfile( @@ -101,16 +108,96 @@ LTX2_DISTILLED_COMPONENT_PROFILE = LTXComponentProfile( video_vae_cls=DistributedAutoencoderKLLTX2Video, ) +LTX23_DISTILLED_COMPONENT_PROFILE = replace( + LTX23_COMPONENT_PROFILE, + name="ltx2_3_distilled", + resident_modules=(*LTX23_COMPONENT_PROFILE.resident_modules, "latent_upsampler"), + artifact_repo_id="Lightricks/LTX-2.3", + latent_upsampler_filename="ltx-2.3-spatial-upscaler-x2-1.1.safetensors", +) + +LTX2_TWO_STAGE_COMPONENT_PROFILE = replace( + LTX2_COMPONENT_PROFILE, + name="ltx2_two_stage", + resident_modules=(*LTX2_COMPONENT_PROFILE.resident_modules, "latent_upsampler"), + artifact_repo_id="Lightricks/LTX-2", + latent_upsampler_filename="ltx-2-spatial-upscaler-x2-1.0.safetensors", + distilled_lora_filename="ltx-2-19b-distilled-lora-384.safetensors", +) + +LTX23_TWO_STAGE_COMPONENT_PROFILE = replace( + LTX23_COMPONENT_PROFILE, + name="ltx2_3_two_stage", + resident_modules=(*LTX23_COMPONENT_PROFILE.resident_modules, "latent_upsampler"), + artifact_repo_id="Lightricks/LTX-2.3", + latent_upsampler_filename="ltx-2.3-spatial-upscaler-x2-1.1.safetensors", + distilled_lora_filename="ltx-2.3-22b-distilled-lora-384-1.1.safetensors", +) + _COMPONENT_PROFILES: dict[tuple[str, str], LTXComponentProfile] = { ("one_stage", "2"): LTX2_COMPONENT_PROFILE, ("one_stage", "2.3"): LTX23_COMPONENT_PROFILE, + ("two_stage", "2"): LTX2_TWO_STAGE_COMPONENT_PROFILE, + ("two_stage", "2.3"): LTX23_TWO_STAGE_COMPONENT_PROFILE, ("distilled_two_stage", "2"): LTX2_DISTILLED_COMPONENT_PROFILE, + ("distilled_two_stage", "2.3"): LTX23_DISTILLED_COMPONENT_PROFILE, ("dmd2", "2"): LTX2_COMPONENT_PROFILE, ("dmd2", "2.3"): LTX23_COMPONENT_PROFILE, } +def resolve_ltx_artifact( + model: str, + repo_id: str, + filename: str, +) -> str: + """Resolve an official LTX sidecar from the model root or its Hub repository.""" + candidate = Path(model) / filename + if candidate.is_file(): + return str(candidate) + + try: + return hf_hub_download(repo_id=repo_id, filename=filename) + except Exception as exc: + raise FileNotFoundError( + f"Unable to resolve LTX artifact {filename!r}. Searched {candidate}; " + f"place the file in the model root or make {repo_id} available." + ) from exc + + +def _load_ltx_latent_upsampler_single_file(path: str, dtype: torch.dtype) -> LTX2LatentUpsamplerModel: + """Load an official single-file upsampler into the Diffusers module.""" + with safe_open(path, framework="pt", device="cpu") as handle: + metadata = handle.metadata() or {} + raw_config = json.loads(metadata.get("config", "{}")) + config = { + "in_channels": raw_config.get("in_channels", 128), + "mid_channels": raw_config.get("mid_channels", 1024), + "num_blocks_per_stage": raw_config.get("num_blocks_per_stage", 4), + "dims": raw_config.get("dims", 3), + "spatial_upsample": raw_config.get("spatial_upsample", True), + "temporal_upsample": raw_config.get("temporal_upsample", False), + "rational_spatial_scale": raw_config.get("spatial_scale", 2.0), + "use_rational_resampler": raw_config.get("rational_resampler", False), + } + with torch.device("cpu"): + upsampler = LTX2LatentUpsamplerModel(**config).to(dtype=dtype) + + state_dict = load_file(path, device="cpu") + if "upsampler.0.weight" in state_dict and hasattr(upsampler.upsampler, "conv"): + state_dict["upsampler.conv.weight"] = state_dict.pop("upsampler.0.weight") + state_dict["upsampler.conv.bias"] = state_dict.pop("upsampler.0.bias") + missing, unexpected = upsampler.load_state_dict(state_dict, strict=False) + unresolved_missing = set(missing) - {"upsampler.blur_down.kernel"} + if unresolved_missing or unexpected: + raise ValueError( + f"Invalid LTX latent upsampler {path}: missing={sorted(unresolved_missing)}, " + f"unexpected={sorted(unexpected)}." + ) + return upsampler + + def resolve_ltx_component_profile(pipeline_kind: str, model_version: str) -> LTXComponentProfile: """Resolve component construction independently from execution recipes.""" try: @@ -392,16 +479,34 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None: ) if "latent_upsampler" in profile.resident_modules: - # BlurDownsample constructs an integer kernel that must be initialized - # on CPU; component placement is handled uniformly after construction. - with torch.device("cpu"): - pipeline.latent_upsampler = _load_component( - LTX2LatentUpsamplerModel, + upsampler_config = os.path.join(model, "latent_upsampler", "config.json") + if os.path.isfile(upsampler_config) or not local_files_only: + try: + pipeline.latent_upsampler = _load_component( + LTX2LatentUpsamplerModel, + model, + "latent_upsampler", + local_files_only=local_files_only, + dtype=dtype, + ) + except (OSError, ValueError): + if profile.latent_upsampler_filename is None or profile.artifact_repo_id is None: + raise + upsampler_path = resolve_ltx_artifact( + model, + profile.artifact_repo_id, + profile.latent_upsampler_filename, + ) + pipeline.latent_upsampler = _load_ltx_latent_upsampler_single_file(upsampler_path, dtype) + else: + if profile.latent_upsampler_filename is None or profile.artifact_repo_id is None: + raise FileNotFoundError(f"LTX latent upsampler component not found under {model}.") + upsampler_path = resolve_ltx_artifact( model, - "latent_upsampler", - local_files_only=local_files_only, - dtype=dtype, + profile.artifact_repo_id, + profile.latent_upsampler_filename, ) + pipeline.latent_upsampler = _load_ltx_latent_upsampler_single_file(upsampler_path, dtype) transformer_config = load_transformer_config(model, "transformer", local_files_only) quant_config = getattr(od_config, "quantization_config", None) @@ -412,6 +517,16 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None: subfolder="scheduler", local_files_only=local_files_only, ) + expected_checkpoint_kind: LTXCheckpointKind | None = None + if pipeline.pipeline_kind == "two_stage": + expected_checkpoint_kind = "regular" + elif pipeline.pipeline_kind == "distilled_two_stage": + expected_checkpoint_kind = "distilled" + validate_ltx_checkpoint( + pipeline.scheduler.config, + expected_kind=expected_checkpoint_kind, + pipeline_name=type(pipeline).__name__, + ) pipeline.vae_spatial_compression_ratio = pipeline.vae.spatial_compression_ratio pipeline.vae_temporal_compression_ratio = pipeline.vae.temporal_compression_ratio diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py b/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py index 4a149be85..dd55f3953 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py @@ -62,13 +62,14 @@ class LTXDenoiseContext: audio_latents: torch.Tensor video_coords: torch.Tensor audio_coords: torch.Tensor + audio_attention_mask: torch.Tensor | None = None conditioning_mask: torch.Tensor | None = None conditioning_mask_for_model: torch.Tensor | None = None @dataclass class LTXPhaseResult: - """Denoised, unpacked AV latents and the context used to produce them.""" + """Denoised AV latents and the context used to produce them.""" forward_context: LTXForwardContext video: torch.Tensor @@ -284,7 +285,8 @@ def prepare_rope_coords_stage( latents: torch.Tensor, audio_latents: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: - video_coords = pipeline.transformer.rope.prepare_video_coords( + transformer = pipeline.transformer + video_coords = transformer.rope.prepare_video_coords( latents.shape[0], forward_ctx.latent_num_frames, forward_ctx.latent_height, @@ -292,7 +294,7 @@ def prepare_rope_coords_stage( latents.device, fps=forward_ctx.request_inputs.frame_rate, ) - audio_coords = pipeline.transformer.audio_rope.prepare_audio_coords( + audio_coords = transformer.audio_rope.prepare_audio_coords( audio_latents.shape[0], forward_ctx.padded_audio_num_frames, audio_latents.device, @@ -331,6 +333,7 @@ def build_transformer_kwargs( # with a learned register, making all output context tokens valid. "encoder_attention_mask": None, "audio_encoder_attention_mask": None, + "audio_attention_mask": denoise_ctx.audio_attention_mask, "num_frames": forward_ctx.latent_num_frames, "height": forward_ctx.latent_height, "width": forward_ctx.latent_width, @@ -442,11 +445,25 @@ class LTXPhaseExecutor: video_audio_step_adapter=video_audio_step_adapter, ) video_coords, audio_coords = prepare_rope_coords_stage(pipeline, forward_ctx, latents, audio_latents) + ring_degree = getattr(pipeline.od_config.parallel_config, "ring_degree", 1) or 1 + if padded_audio_num_frames > original_audio_num_frames and ring_degree > 1: + raise ValueError( + "LTX audio padding requires an attention mask, which Ring sequence parallelism does not support. " + "Use Ulysses-only SP or choose a request whose audio latent length is divisible by the SP size." + ) denoise_ctx = LTXDenoiseContext( latents=latents, audio_latents=audio_latents, video_coords=video_coords, audio_coords=audio_coords, + audio_attention_mask=( + torch.arange(padded_audio_num_frames, device=audio_latents.device) + .lt(original_audio_num_frames) + .unsqueeze(0) + .expand(audio_latents.shape[0], -1) + if padded_audio_num_frames > original_audio_num_frames + else None + ), conditioning_mask=conditioning_mask, ) denoise_ctx = pipeline._prepare_denoise_context_for_guidance(forward_ctx, denoise_ctx) diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py b/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py index f35f5b998..bc84e7c32 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py @@ -126,7 +126,10 @@ def x0_from_velocity(sample: torch.Tensor, velocity: torch.Tensor, sigma: torch. def velocity_from_x0(sample: torch.Tensor, x0: torch.Tensor, sigma: torch.Tensor) -> torch.Tensor: """Convert x0 back to velocity using the official fp32 arithmetic order.""" - sigma = sigma.to(torch.float32) + # Official LTX converts the scalar scheduler sigma to a host value before + # division. A CUDA scalar tensor selects different kernel arithmetic and + # can cross bf16 rounding boundaries late in the denoise trajectory. + sigma = sigma.to(torch.float32).item() return ((sample.to(torch.float32) - x0.to(torch.float32)) / sigma).to(sample.dtype) @@ -148,6 +151,7 @@ def combine_guided_x0( uncond_perturbed: torch.Tensor | float, uncond_modality: torch.Tensor | float, guidance: LTXModalityGuidance, + rescale_token_count: int | None = None, ) -> torch.Tensor: dtype = cond.dtype cond = cond.float() @@ -163,9 +167,18 @@ def combine_guided_x0( # uses its full-tensor standard deviation. In Omni, dimension 0 may # instead contain independently scheduled requests. Reduce each item # separately so its result cannot depend on unrelated co-batched work. - reduce_dims = tuple(range(1, pred.ndim)) - cond_std = cond.std(dim=reduce_dims, keepdim=True) - pred_std = pred.std(dim=reduce_dims, keepdim=True) + stats_cond = cond + stats_pred = pred + if rescale_token_count is not None: + if cond.ndim < 2 or not 0 < rescale_token_count <= cond.shape[1]: + raise ValueError( + f"Guidance rescale token count must be in [1, {cond.shape[1]}], got {rescale_token_count}." + ) + stats_cond = cond[:, :rescale_token_count] + stats_pred = pred[:, :rescale_token_count] + reduce_dims = tuple(range(1, stats_pred.ndim)) + cond_std = stats_cond.std(dim=reduce_dims, keepdim=True) + pred_std = stats_pred.std(dim=reduce_dims, keepdim=True) eps = torch.finfo(pred.dtype).eps factor = cond_std / pred_std.clamp_min(eps) factor = torch.where(pred_std > eps, factor, torch.ones_like(factor)) @@ -175,6 +188,8 @@ def combine_guided_x0( def _repeat_batch(tensor: torch.Tensor, repeats: int) -> torch.Tensor: + if repeats == 1: + return tensor return tensor.repeat((repeats,) + (1,) * (tensor.ndim - 1)) @@ -278,6 +293,11 @@ class LTXGuidanceExecutor: if model_pass_count > 1: denoise_ctx.video_coords = _repeat_batch(denoise_ctx.video_coords, model_pass_count) denoise_ctx.audio_coords = _repeat_batch(denoise_ctx.audio_coords, model_pass_count) + if denoise_ctx.audio_attention_mask is not None: + denoise_ctx.audio_attention_mask = _repeat_batch( + denoise_ctx.audio_attention_mask, + model_pass_count, + ) return denoise_ctx @staticmethod @@ -322,6 +342,7 @@ class LTXGuidanceExecutor: guidance: LTXModalityGuidance, *, model_sigma: torch.Tensor | None = None, + rescale_token_count: int | None = None, ) -> torch.Tensor: model_sigma = sigma if model_sigma is None else model_sigma # Official guidance reduces contiguous BSC tensors. The fused @@ -334,6 +355,7 @@ class LTXGuidanceExecutor: uncond_perturbed=x0.get("ptb", 0.0), uncond_modality=x0.get("mod", 0.0), guidance=guidance, + rescale_token_count=rescale_token_count, ) return velocity_from_x0(sample, guided, sigma) @@ -434,6 +456,7 @@ class LTXGuidanceExecutor: audio_splits, forward_ctx.audio_scheduler.sigmas[index], plan.spec.audio, + rescale_token_count=forward_ctx.original_audio_num_frames, ), ) @@ -509,6 +532,7 @@ class LTXGuidanceExecutor: audio_splits, forward_ctx.audio_scheduler.sigmas[index], plan.spec.audio, + rescale_token_count=forward_ctx.original_audio_num_frames, ), ) diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_latents.py b/vllm_omni/diffusion/models/ltx2/ltx2_latents.py index bb0155e79..11929d331 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_latents.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_latents.py @@ -117,7 +117,9 @@ def create_noised_state( device=latents.device, dtype=latents.dtype, ) - return noise_scale * noise + (1 - noise_scale) * latents + if isinstance(noise_scale, torch.Tensor): + noise_scale = noise_scale.float() + return torch.lerp(latents.float(), noise.float(), noise_scale).to(latents.dtype) def pack_audio_latents(latents: torch.Tensor) -> torch.Tensor: @@ -135,6 +137,16 @@ def unpad_audio_latents(latents: torch.Tensor, num_frames: int) -> torch.Tensor: return latents[:, :num_frames] +def clear_audio_padding(latents: torch.Tensor, num_frames: int) -> torch.Tensor: + """Keep Omni's SP-only audio padding outside the sampler state.""" + if not 0 < num_frames <= latents.shape[1]: + raise ValueError(f"Audio frame count must be in [1, {latents.shape[1]}], got {num_frames}.") + if num_frames == latents.shape[1]: + return latents + padding = latents.new_zeros(latents.shape[0], latents.shape[1] - num_frames, latents.shape[2]) + return torch.cat([latents[:, :num_frames], padding], dim=1) + + def get_sp_padded_audio_latent_length(audio_latent_length: int, sp_size: int) -> int: if sp_size > 1: audio_latent_length += (sp_size - (audio_latent_length % sp_size)) % sp_size @@ -205,7 +217,12 @@ def prepare_video_latents( f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" f" size of {batch_size}. Make sure the batch size matches the length of the generators." ) - return randn_tensor(shape, generator=generator, device=device, dtype=dtype) + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + # Official LTX samples the 5D video latent first and then patchifies it, + # leaving token-major storage behind the [B, tokens, channels] view. The + # logical values are unchanged, but preserving that layout is important: + # CUDA selects a different bf16 GEMM path for a contiguous token tensor. + return latents.transpose(1, 2).contiguous().transpose(1, 2) def prepare_audio_latents( @@ -226,6 +243,16 @@ def prepare_audio_latents( sp_size = getattr(pipeline.od_config.parallel_config, "sequence_parallel_size", 1) or 1 padded_latent_length = get_sp_padded_audio_latent_length(original_latent_length, int(sp_size)) + def pad_logical_latents(logical_latents: torch.Tensor) -> torch.Tensor: + if padded_latent_length == original_latent_length: + return logical_latents + padding = logical_latents.new_zeros( + logical_latents.shape[0], + padded_latent_length - original_latent_length, + logical_latents.shape[2], + ) + return torch.cat([logical_latents, padding], dim=1) + if latents is not None: if latents.ndim == 4: latents = pack_audio_latents(latents) @@ -237,29 +264,24 @@ def prepare_audio_latents( pipeline.audio_vae.latents_mean, pipeline.audio_vae.latents_std, ) - latents = create_noised_state(latents, noise_scale, generator) if latents.shape[1] not in {original_latent_length, padded_latent_length}: raise ValueError( "Provided `audio_latents` has incompatible audio frame count " f"{latents.shape[1]}; expected {original_latent_length} or {padded_latent_length}." ) - if latents.shape[1] == original_latent_length and padded_latent_length > original_latent_length: - padding = torch.zeros( - latents.shape[0], - padded_latent_length - original_latent_length, - latents.shape[2], - dtype=latents.dtype, - device=latents.device, - ) - latents = torch.cat([latents, padding], dim=1) + # Padding is an Omni implementation detail, not part of the official + # audio state. Draw request RNG only for logical tokens so later phases + # remain seed-invariant across SP degrees, then append deterministic 0s. + latents = create_noised_state(latents[:, :original_latent_length], noise_scale, generator) + latents = pad_logical_latents(latents) return latents.to(device=device, dtype=dtype), original_latent_length, padded_latent_length - shape = (batch_size, padded_latent_length, num_channels_latents * latent_mel_bins) + shape = (batch_size, original_latent_length, num_channels_latents * latent_mel_bins) if isinstance(generator, list) and len(generator) != batch_size: raise ValueError( f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" f" size of {batch_size}. Make sure the batch size matches the length of the generators." ) - latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + latents = pad_logical_latents(randn_tensor(shape, generator=generator, device=device, dtype=dtype)) return latents, original_latent_length, padded_latent_length diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py b/vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py new file mode 100644 index 000000000..a205fce6b --- /dev/null +++ b/vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py @@ -0,0 +1,430 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Construction and execution of a fixed LTX refinement-phase adapter. + +Checkpoint key parsing remains in :mod:`ltx2_adapter_parser`; this module +resolves the declared artifact and makes the existing Transformer linears +adapter-capable while retaining their quantization and tensor-parallel paths. +""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Any + +import torch +import torch.nn as nn +import torch.nn.functional as F +import vllm.envs as envs +from vllm.distributed import ( + split_tensor_along_last_dim, + tensor_model_parallel_all_reduce, +) +from vllm.logger import init_logger +from vllm.model_executor.layers.batch_invariant import linear_batch_invariant +from vllm.model_executor.layers.linear import ( + ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear, + UnquantizedLinearMethod, +) +from vllm.model_executor.layers.utils import dispatch_unquantized_gemm +from vllm.platforms import current_platform + +from .ltx2_adapter_parser import AdapterManifest, AdapterTarget, iter_adapter_tensors, parse_ltx_adapter +from .ltx2_components import resolve_ltx_artifact +from .ltx2_recipes import LTX_DISTILLED_ADAPTER_SLOT + +logger = init_logger(__name__) + + +@dataclass(frozen=True) +class _AdapterLayout: + """Rank-local storage and execution layout for one LoRA pair.""" + + kind: str + input_size: int + output_size: int + a_input_start: int + a_input_size: int + b_output_start: int + b_output_size: int + output_start: int + reduce_results: bool = False + + +def _module_device(module: nn.Module) -> torch.device: + for tensor in (*module.parameters(), *module.buffers()): + if tensor.device.type != "meta": + return tensor.device + raise RuntimeError(f"Cannot materialize an LTX phase adapter for meta module {module._get_name()}.") + + +def _build_layout(layer: nn.Module, target: AdapterTarget) -> _AdapterLayout: + if isinstance(layer, QKVParallelLinear): + if target.slice not in {"q", "k", "v"}: + raise ValueError( + f"LTX packed QKV target {target.module!r} requires a q/k/v adapter slice, got {target.slice!r}." + ) + q_local = layer.num_heads * layer.head_size + k_local = layer.num_kv_heads * layer.head_size + v_local = layer.num_kv_heads * layer.v_head_size + if target.slice == "q": + output_size = layer.total_num_heads * layer.head_size + b_output_start = layer.tp_rank * q_local + b_output_size = q_local + output_start = 0 + elif target.slice == "k": + output_size = layer.total_num_kv_heads * layer.head_size + shard_rank = layer.tp_rank // layer.num_kv_head_replicas + b_output_start = shard_rank * k_local + b_output_size = k_local + output_start = q_local + else: + output_size = layer.total_num_kv_heads * layer.v_head_size + shard_rank = layer.tp_rank // layer.num_kv_head_replicas + b_output_start = shard_rank * v_local + b_output_size = v_local + output_start = q_local + k_local + return _AdapterLayout( + kind="qkv", + input_size=layer.input_size, + output_size=output_size, + a_input_start=0, + a_input_size=layer.input_size, + b_output_start=b_output_start, + b_output_size=b_output_size, + output_start=output_start, + ) + + if isinstance(layer, ColumnParallelLinear): + if layer.gather_output: + raise ValueError( + f"LTX phase adapter does not support gathered ColumnParallelLinear target {target.module!r}." + ) + return _AdapterLayout( + kind="column", + input_size=layer.input_size, + output_size=layer.output_size, + a_input_start=0, + a_input_size=layer.input_size, + b_output_start=layer.tp_rank * layer.output_size_per_partition, + b_output_size=layer.output_size_per_partition, + output_start=0, + ) + + if isinstance(layer, RowParallelLinear): + return _AdapterLayout( + kind="row", + input_size=layer.input_size, + output_size=layer.output_size, + a_input_start=layer.tp_rank * layer.input_size_per_partition, + a_input_size=layer.input_size_per_partition, + b_output_start=0, + b_output_size=layer.output_size, + output_start=0, + reduce_results=layer.reduce_results, + ) + + if isinstance(layer, nn.Linear): + return _AdapterLayout( + kind="replicated", + input_size=layer.in_features, + output_size=layer.out_features, + a_input_start=0, + a_input_size=layer.in_features, + b_output_start=0, + b_output_size=layer.out_features, + output_start=0, + ) + + raise TypeError(f"LTX phase adapter target {target.module!r} has unsupported layer type {type(layer).__name__}.") + + +class _AdapterPiece(nn.Module): + """One rank-local A/B pair stored as non-persistent buffers.""" + + def __init__(self, target: AdapterTarget, layout: _AdapterLayout) -> None: + super().__init__() + self.target = target + self.layout = layout + # The fixed graph is installed before base checkpoint loading. The + # empty buffers are replaced only after the base layer's quant/TP layout + # has settled and before offload/compile snapshots are taken. + self.register_buffer("lora_a", torch.empty(0), persistent=False) + self.register_buffer("lora_b", torch.empty(0), persistent=False) + + def load(self, lora_a: torch.Tensor, lora_b: torch.Tensor, *, device: torch.device, dtype: torch.dtype) -> None: + if tuple(lora_a.shape) != self.target.a_shape or tuple(lora_b.shape) != self.target.b_shape: + raise ValueError( + f"LTX adapter source shape changed for {self.target.source_module!r}: " + f"A={tuple(lora_a.shape)}, B={tuple(lora_b.shape)}." + ) + if lora_a.ndim != 2 or lora_b.ndim != 2: + raise ValueError(f"LTX adapter target {self.target.source_module!r} must use rank-2 A/B tensors.") + rank = self.target.a_shape[0] + if lora_a.shape[1] != self.layout.input_size or lora_b.shape[0] != self.layout.output_size: + raise ValueError( + f"LTX adapter shape does not match {self.target.module!r}: " + f"expected A=({rank}, {self.layout.input_size}), " + f"B=({self.layout.output_size}, {rank}); " + f"got A={tuple(lora_a.shape)}, B={tuple(lora_b.shape)}." + ) + + local_a = lora_a.narrow(1, self.layout.a_input_start, self.layout.a_input_size) + local_b = lora_b.narrow(0, self.layout.b_output_start, self.layout.b_output_size) + self.lora_a = local_a.to(device=device, dtype=dtype) + self.lora_b = local_b.to(device=device, dtype=dtype) + + def delta(self, input_: torch.Tensor) -> torch.Tensor: + if self.lora_a.numel() == 0 or self.lora_b.numel() == 0: + raise RuntimeError(f"LTX phase adapter target {self.target.module!r} was not materialized.") + if self.layout.kind == "row" and input_.shape[-1] != self.layout.a_input_size: + input_ = input_.narrow(-1, self.layout.a_input_start, self.layout.a_input_size) + input_ = input_.to(dtype=self.lora_a.dtype) + delta = F.linear(F.linear(input_, self.lora_a), self.lora_b) + if self.layout.kind == "row" and self.layout.reduce_results: + delta = tensor_model_parallel_all_reduce(delta) + return delta + + def weight_delta(self) -> torch.Tensor: + """Materialize the rank-local BF16 weight delta used by official fusion.""" + if self.lora_a.numel() == 0 or self.lora_b.numel() == 0: + raise RuntimeError(f"LTX phase adapter target {self.target.module!r} was not materialized.") + return torch.matmul(self.lora_b, self.lora_a) + + +def _apply_unquantized_gemm_with_weight( + layer: nn.Module, + input_: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor | None, +) -> torch.Tensor: + if envs.VLLM_BATCH_INVARIANT and current_platform.is_cuda_alike(): + return linear_batch_invariant(input_, weight, bias) + return dispatch_unquantized_gemm()(layer, input_, weight, bias) + + +def _forward_with_weight( + layer: nn.Module, + input_: torch.Tensor, + weight: torch.Tensor, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor | None]: + """Run an unquantized base layer with an ephemeral fused weight.""" + if isinstance(layer, nn.Linear): + return F.linear(input_, weight, layer.bias) + + quant_method = getattr(layer, "quant_method", None) + if not isinstance(quant_method, UnquantizedLinearMethod): + raise ValueError("LTX layer-fused phase LoRA requires unquantized Transformer linears.") + + if isinstance(layer, RowParallelLinear): + if layer.input_is_parallel: + input_parallel = input_ + else: + input_parallel = split_tensor_along_last_dim(input_, num_partitions=layer.tp_size)[ + layer.tp_rank + ].contiguous() + bias = None if layer.tp_rank > 0 or layer.skip_bias_add else layer.bias + output_parallel = _apply_unquantized_gemm_with_weight(layer, input_parallel, weight, bias) + if layer.reduce_results and layer.tp_size > 1: + output = tensor_model_parallel_all_reduce(output_parallel) + else: + output = output_parallel + elif isinstance(layer, ColumnParallelLinear): + bias = layer.bias if not layer.skip_bias_add else None + output = _apply_unquantized_gemm_with_weight(layer, input_, weight, bias) + else: + raise TypeError(f"Unsupported LTX layer-fused linear type {type(layer).__name__}.") + + if not layer.return_bias: + return output + return output, layer.bias if layer.skip_bias_add else None + + +class _PhaseAdapterLinear(nn.Module): + """Delegate the base linear unchanged and add an enabled adapter slot.""" + + def __init__( + self, + base_layer: nn.Module, + targets: Iterable[AdapterTarget], + *, + adapter_dtype: torch.dtype, + layer_fused: bool = False, + ) -> None: + super().__init__() + self.base_layer = base_layer + self.adapter_dtype = adapter_dtype + if layer_fused and not ( + isinstance(base_layer, nn.Linear) + or isinstance(getattr(base_layer, "quant_method", None), UnquantizedLinearMethod) + ): + raise ValueError("LTX layer-fused phase LoRA requires unquantized Transformer linears.") + self.layer_fused = layer_fused + pieces = [_AdapterPiece(target, _build_layout(base_layer, target)) for target in targets] + self.pieces = nn.ModuleList(pieces) + self._pieces_by_target = {piece.target: piece for piece in pieces} + self._enabled = False + + def load_target(self, target: AdapterTarget, lora_a: torch.Tensor, lora_b: torch.Tensor) -> None: + try: + piece = self._pieces_by_target[target] + except KeyError as exc: + raise ValueError(f"Adapter target {target.module!r} is not installed on this layer.") from exc + piece.load(lora_a, lora_b, device=_module_device(self.base_layer), dtype=self.adapter_dtype) + + def set_enabled(self, enabled: bool) -> None: + self._enabled = enabled + + def _add_adapter_delta(self, input_: torch.Tensor, output: torch.Tensor) -> torch.Tensor: + if not self._enabled: + return output + if output.requires_grad: + output = output.clone() + for piece in self.pieces: + delta = piece.delta(input_).to(dtype=output.dtype) + start = piece.layout.output_start + output[..., start : start + piece.layout.b_output_size].add_(delta) + return output + + @torch.no_grad() + def _fused_weight(self) -> torch.Tensor: + base_weight = self.base_layer.weight + fused_weight = base_weight.clone() + for piece in self.pieces: + start = piece.layout.output_start + base_slice = base_weight.narrow(0, start, piece.layout.b_output_size) + delta = piece.weight_delta() + if delta.shape != base_slice.shape: + raise ValueError( + f"LTX layer-fused delta shape {tuple(delta.shape)} does not match base weight slice " + f"{tuple(base_slice.shape)} for {piece.target.module!r}." + ) + # Match official fusion exactly: materialize B@A in the adapter + # dtype, then add the base tensor in-place before the main GEMM. + delta.add_(base_slice) + fused_weight.narrow(0, start, piece.layout.b_output_size).copy_(delta) + return fused_weight + + def forward(self, input_: torch.Tensor): + if not self._enabled: + return self.base_layer(input_) + if self.layer_fused: + return _forward_with_weight(self.base_layer, input_, self._fused_weight()) + output = self.base_layer(input_) + if isinstance(output, tuple): + return (self._add_adapter_delta(input_, output[0]), *output[1:]) + return self._add_adapter_delta(input_, output) + + +class LTXPhaseAdapterRuntime: + """Install and operate one fixed phase adapter on an existing transformer.""" + + def __init__( + self, + transformer: nn.Module, + manifest: AdapterManifest, + *, + dtype: torch.dtype, + layer_fused: bool = False, + ) -> None: + self.transformer = transformer + self.manifest = manifest + self._wrappers: dict[str, _PhaseAdapterLinear] = {} + self._target_names: tuple[str, ...] = () + self._materialized = False + grouped: dict[str, list[AdapterTarget]] = defaultdict(list) + for target in manifest.targets: + grouped[target.module].append(target) + + original_layers = {name: self.transformer.get_submodule(name) for name in grouped} + for name, targets in grouped.items(): + wrapper = _PhaseAdapterLinear( + original_layers[name], + targets, + adapter_dtype=dtype, + layer_fused=layer_fused, + ) + parent_name, _, child_name = name.rpartition(".") + parent = self.transformer.get_submodule(parent_name) if parent_name else self.transformer + if child_name not in parent._modules: + raise ValueError(f"LTX phase adapter target {name!r} is no longer a direct module child.") + parent._modules[child_name] = wrapper + self._wrappers[name] = wrapper + + self._target_names = tuple(sorted(self._wrappers, key=len, reverse=True)) + # LTX's custom loader receives the original checkpoint names. Keep the + # adaptation local to that loader rather than changing AutoWeightsLoader. + self.transformer._phase_adapter_parameter_name = self.base_parameter_name + logger.info( + "Installed %d LTX %s phase-adapter linears", + len(self._wrappers), + "layer-fused" if layer_fused else "dynamic", + ) + + def base_parameter_name(self, name: str) -> str: + for target_name in self._target_names: + prefix = f"{target_name}." + if name.startswith(prefix): + return f"{target_name}.base_layer.{name[len(prefix) :]}" + return name + + def finalize(self) -> None: + """Stream rank-local A/B buffers after base weight processing completes.""" + if self._materialized: + return + for target, lora_a, lora_b in iter_adapter_tensors(self.manifest): + self._wrappers[target.module].load_target(target, lora_a, lora_b) + self._materialized = True + logger.info("Materialized %d LTX phase-adapter tensor pairs", len(self.manifest.targets)) + + def activate(self, adapter_slot: str | None) -> None: + if adapter_slot is not None: + if adapter_slot != LTX_DISTILLED_ADAPTER_SLOT: + raise ValueError(f"Unknown LTX phase adapter slot {adapter_slot!r}.") + if not self._materialized: + raise RuntimeError("LTX phase adapter data must be materialized before activation.") + for wrapper in self._wrappers.values(): + wrapper.set_enabled(adapter_slot is not None) + + +def build_ltx_phase_adapter(pipeline: Any) -> LTXPhaseAdapterRuntime | None: + """Build the fixed adapter required by a multi-phase LTX recipe.""" + adapter_slots = {phase.adapter_slot for phase in pipeline.pipeline_recipe.phases if phase.adapter_slot is not None} + if not adapter_slots: + return None + if adapter_slots != {LTX_DISTILLED_ADAPTER_SLOT}: + raise ValueError(f"Unsupported LTX phase adapter slots: {sorted(adapter_slots)}.") + + profile = pipeline.component_profile + if profile.distilled_lora_filename is None or profile.artifact_repo_id is None: + raise ValueError(f"{profile.name} does not declare the required distilled adapter artifact.") + if getattr(pipeline.od_config, "lora_path", None) is not None: + raise ValueError( + f"{pipeline.__class__.__name__} reserves LoRA execution for its phase adapter; " + "request or static LoRA composition is not supported yet." + ) + + quantization_config = getattr(pipeline.od_config, "quantization_config", None) + layer_fused = quantization_config is None + adapter_path = resolve_ltx_artifact( + pipeline.od_config.model, + profile.artifact_repo_id, + profile.distilled_lora_filename, + ) + + if layer_fused and pipeline.od_config.dtype is not torch.bfloat16: + raise ValueError("LTX layer-fused phase LoRA requires a bfloat16 pipeline dtype.") + + manifest = parse_ltx_adapter(pipeline.transformer, adapter_path) + runtime = LTXPhaseAdapterRuntime( + pipeline.transformer, + manifest, + dtype=pipeline.od_config.dtype, + layer_fused=layer_fused, + ) + return runtime diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_recipes.py b/vllm_omni/diffusion/models/ltx2/ltx2_recipes.py index a70f349ac..bb42bc0e9 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_recipes.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_recipes.py @@ -24,6 +24,7 @@ LTX_DEFAULT_NEGATIVE_PROMPT = ( LTX_DISTILLED_SIGMAS = (1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875, 0.0) LTX_STAGE_2_DISTILLED_SIGMAS = (0.909375, 0.725, 0.421875, 0.0) +LTX_DISTILLED_ADAPTER_SLOT = "ltx_distilled" @dataclass(frozen=True) @@ -36,6 +37,7 @@ class LTXPhaseRecipe: sigmas: tuple[float, ...] | None = None noise_scale: float = 0.0 input_transform: Literal["initial", "spatial_upsample"] = "initial" + adapter_slot: str | None = None allow_guidance_override: bool = True use_official_sigma_schedule: bool = True @@ -157,11 +159,59 @@ LTX2_DISTILLED_TWO_STAGE_RECIPE = LTXPipelineRecipe( fixed_num_inference_steps=True, ) +# LTX-2.3 full-distilled uses the same official fixed 8 + 3 sigma schedules +# and request contract. Its distinct component profile selects the 22B merged +# distilled Transformer, BWE vocoder, and matching spatial upsampler; no +# distilled LoRA is loaded for either phase. +LTX23_DISTILLED_TWO_STAGE_RECIPE = LTX2_DISTILLED_TWO_STAGE_RECIPE + + +def _official_two_stage_recipe(one_stage_recipe: LTXPipelineRecipe) -> LTXPipelineRecipe: + return LTXPipelineRecipe( + height=one_stage_recipe.height * 2, + width=one_stage_recipe.width * 2, + num_frames=one_stage_recipe.num_frames, + frame_rate=one_stage_recipe.frame_rate, + num_inference_steps=one_stage_recipe.num_inference_steps, + negative_prompt=one_stage_recipe.negative_prompt, + phases=( + LTXPhaseRecipe( + name="generate_lowres", + guidance=one_stage_recipe.request_guidance, + spatial_downscale=2, + noise_scale=1.0, + ), + LTXPhaseRecipe( + name="refine", + guidance=LTXGuidanceSpec.positive_only(), + sigmas=LTX_STAGE_2_DISTILLED_SIGMAS, + noise_scale=LTX_STAGE_2_DISTILLED_SIGMAS[0], + input_transform="spatial_upsample", + adapter_slot=LTX_DISTILLED_ADAPTER_SLOT, + allow_guidance_override=False, + use_official_sigma_schedule=False, + ), + ), + video_output_phase=1, + # The official second stage refines video only and deliberately + # discards its audio result. Decode the full-context Stage-1 audio. + audio_output_phase=0, + allow_request_sigmas=False, + allow_request_latents=False, + ) + + +LTX2_TWO_STAGE_RECIPE = _official_two_stage_recipe(LTX2_ONE_STAGE_RECIPE) +LTX23_TWO_STAGE_RECIPE = _official_two_stage_recipe(LTX23_ONE_STAGE_RECIPE) + _PIPELINE_RECIPES: dict[tuple[str, str], LTXPipelineRecipe] = { ("one_stage", "2"): LTX2_ONE_STAGE_RECIPE, ("one_stage", "2.3"): LTX23_ONE_STAGE_RECIPE, + ("two_stage", "2"): LTX2_TWO_STAGE_RECIPE, + ("two_stage", "2.3"): LTX23_TWO_STAGE_RECIPE, ("distilled_two_stage", "2"): LTX2_DISTILLED_TWO_STAGE_RECIPE, + ("distilled_two_stage", "2.3"): LTX23_DISTILLED_TWO_STAGE_RECIPE, ("dmd2", "2"): LTX_POSITIVE_ONLY_RECIPE, ("dmd2", "2.3"): LTX_POSITIVE_ONLY_RECIPE, } diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_request.py b/vllm_omni/diffusion/models/ltx2/ltx2_request.py index e2eef19df..91d62943a 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_request.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_request.py @@ -5,8 +5,9 @@ from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass, replace -from typing import Any +from typing import Any, Literal import torch @@ -49,6 +50,34 @@ class LTXRequestInputs: return self.guidance.video.cfg_scale +LTXCheckpointKind = Literal["distilled", "regular"] + + +def validate_ltx_checkpoint( + scheduler_config: Mapping[str, Any], + *, + expected_kind: LTXCheckpointKind | None, + pipeline_name: str, +) -> None: + """Ensure scheduler metadata matches the checkpoint required by a profile.""" + if expected_kind is None: + return + + if expected_kind == "distilled": + if scheduler_config.get("use_dynamic_shifting", True) or scheduler_config.get("shift_terminal") is not None: + raise ValueError( + f"{pipeline_name} requires a merged distilled LTX2 checkpoint; " + "the loaded scheduler metadata describes a non-distilled checkpoint." + ) + return + + if not scheduler_config.get("use_dynamic_shifting", False) or scheduler_config.get("shift_terminal") is None: + raise ValueError( + f"{pipeline_name} requires a regular non-distilled LTX checkpoint; " + "the loaded scheduler metadata describes a distilled checkpoint." + ) + + def validate_pipeline_request( request_inputs: LTXRequestInputs, *, diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_runtime.py b/vllm_omni/diffusion/models/ltx2/ltx2_runtime.py index 4b74ebcb8..cb4130259 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_runtime.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_runtime.py @@ -47,6 +47,7 @@ from .ltx2_guidance import ( LTXGuidanceExecutor, LTXGuidancePlan, ) +from .ltx2_phase_adapter import LTXPhaseAdapterRuntime, build_ltx_phase_adapter from .ltx2_recipes import ( LTXPhaseRecipe, LTXPipelineRecipe, @@ -137,9 +138,21 @@ class LTXRuntime( def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = "") -> None: del prefix + parallel_config = getattr(od_config, "parallel_config", None) + if getattr(parallel_config, "ulysses_mode", "strict") == "advanced_uaa": + raise ValueError( + f"{self.__class__.__name__} does not support ulysses_mode='advanced_uaa'. " + "Use the default ulysses_mode='strict' for LTX sequence parallelism." + ) self.model_version = detect_ltx_model_version(od_config.model) self.component_profile = resolve_ltx_component_profile(self.pipeline_kind, self.model_version) self.pipeline_recipe = resolve_ltx_pipeline_recipe(self.pipeline_kind, self.model_version) + if getattr(od_config, "cache_backend", "none") == "cache_dit" and len(self.pipeline_recipe.phases) > 1: + raise ValueError( + f"{self.__class__.__name__} does not support cache_backend='cache_dit'. " + "Cache-DiT currently supports only one-stage LTX recipes; multi-stage recipes require " + "phase-aware cache enable and refresh." + ) self._dit_modules = list(self.component_profile.dit_modules) self._encoder_modules = list(self.component_profile.encoder_modules) self._vae_modules = list(self.component_profile.vae_modules) @@ -150,6 +163,7 @@ class LTXRuntime( super().__init__() self._guidance_plan = LTXGuidancePlan.build(self.pipeline_recipe.request_guidance) initialize_pipeline_components(self, od_config) + self._phase_adapter: LTXPhaseAdapterRuntime | None = build_ltx_phase_adapter(self) self.setup_diffusion_pipeline_profiler( enable_diffusion_pipeline_profiler=self.od_config.enable_diffusion_pipeline_profiler ) @@ -205,6 +219,8 @@ class LTXRuntime( max_sequence_length=max_sequence_length, ) image = self._resolve_request_image(req, image, request_inputs) + if image is not None and not self.support_image_input: + raise ValueError(f"{self.__class__.__name__} does not support `image` input.") request_sigmas = self._resolve_request_sigmas(req, sigmas) validate_pipeline_request( request_inputs, @@ -214,6 +230,13 @@ class LTXRuntime( pipeline_name=self.__class__.__name__, request_sigmas=request_sigmas, ) + phase_adapter = getattr(self, "_phase_adapter", None) + if phase_adapter is not None and any( + getattr(sampling, "lora_request", None) is not None for sampling in req.sampling_params_list + ): + raise ValueError( + f"{self.__class__.__name__} cannot compose a request LoRA with its internal phase adapter." + ) return self._run_recipe(req, request_inputs, request_sigmas=request_sigmas, image=image) def _run_recipe( @@ -258,8 +281,20 @@ class LTXRuntime( return self.decode_phase(output_phase) def _enter_phase(self, phase: LTXPhaseRecipe) -> None: - """Hook for a future phase-weight strategy.""" self._active_phase_name = phase.name + phase_adapter = getattr(self, "_phase_adapter", None) + if phase_adapter is None: + if phase.adapter_slot is not None: + raise RuntimeError(f"LTX phase {phase.name!r} requires adapter slot {phase.adapter_slot!r}.") + return + phase_adapter.activate(phase.adapter_slot) + + def eval(self): + result = super().eval() + phase_adapter = getattr(self, "_phase_adapter", None) + if phase_adapter is not None: + phase_adapter.finalize() + return result def prepare_latents( self, @@ -692,6 +727,7 @@ class LTXRuntime( noise_pred_audio, timestep, ) + audio = latent_ops.clear_audio_padding(audio, forward_ctx.original_audio_num_frames) return latent_ops.LTXAVState(video=video, audio=audio) def _unpack_and_denormalize_stage( diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py b/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py index 95724bd19..a59655cd3 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py @@ -355,6 +355,19 @@ def to_ltx_padding_mask(attention_mask: torch.Tensor) -> torch.Tensor: return attention_mask +class _LTX2ParallelAttention(Attention): + """Preserve LTX SP collectives while applying model-specific padding masks.""" + + def _run_local_attention(self, query, key, value, attn_metadata): + attention_mask = attn_metadata.attn_mask if attn_metadata is not None else None + if attention_mask is not None: + # LTX masks describe K/V padding. Keep this semantic local to LTX: + # Flash varlen applies a 2D mask to both Q and K, while SP uses a + # broadcast 4D key mask that only SDPA consumes correctly. + return self.sdpa_fallback.forward(query, key, value, attn_metadata) + return super()._run_local_attention(query, key, value, attn_metadata) + + class LTX2AudioVideoAttnProcessor: r""" Processor for implementing attention (SDPA is used by default if you're using PyTorch 2.0) for the LTX-2.0 model. @@ -396,11 +409,10 @@ class LTX2AudioVideoAttnProcessor: return None if self._is_sp_enabled(): - # In SP, Ulysses expects a 2D padding mask that matches query length. - # For cross-attention, encoder sequence length != query length, so drop the mask. - if encoder_hidden_states is not None and encoder_hidden_states.shape[1] != hidden_states.shape[1]: - return None - return to_ltx_padding_mask(attention_mask) + # Keep the full key mask replicated while strict Ulysses reshards + # Q/K/V. The broadcast Q dimension also works when cross-attention + # has different Q and K lengths, without changing generic Ulysses. + return to_ltx_padding_mask(attention_mask)[:, None, None, :] attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1]) @@ -696,7 +708,7 @@ class LTX2Attention(torch.nn.Module): torch.nn.Dropout(dropout) if dropout > 0 else torch.nn.Identity(), ] ) - self.attn = Attention( + self.attn = _LTX2ParallelAttention( num_heads=self.query_num_heads, head_size=dim_head, num_kv_heads=self.kv_num_heads, @@ -709,7 +721,15 @@ class LTX2Attention(torch.nn.Module): # LTX-2.3: per-head gated attention # leave unquantized for this linear if apply_gated_attention: - self.to_gate_logits = nn.Linear(query_dim, self.query_num_heads, bias=True) + self.to_gate_logits = ColumnParallelLinear( + query_dim, + heads, + bias=True, + gather_output=False, + return_bias=False, + quant_config=None, + prefix=f"{prefix}.to_gate_logits" if prefix else "to_gate_logits", + ) else: self.to_gate_logits = None @@ -1498,6 +1518,14 @@ class LTX2VideoTransformer3DModel(nn.Module): _layerwise_offload_blocks_attrs = ["transformer_blocks"] _hsdp_shard_conditions = [is_transformer_block_module] _sp_plan: dict[str, Any] | None = None + stacked_params_mapping = ( + (".attn1.to_qkv", ".attn1.to_q", "q"), + (".attn1.to_qkv", ".attn1.to_k", "k"), + (".attn1.to_qkv", ".attn1.to_v", "v"), + (".audio_attn1.to_qkv", ".audio_attn1.to_q", "q"), + (".audio_attn1.to_qkv", ".audio_attn1.to_k", "k"), + (".audio_attn1.to_qkv", ".audio_attn1.to_v", "v"), + ) packed_modules_mapping = { "to_qkv": ["to_q", "to_k", "to_v"], } @@ -1828,6 +1856,7 @@ class LTX2VideoTransformer3DModel(nn.Module): audio_sigma: torch.Tensor | None = None, encoder_attention_mask: torch.Tensor | None = None, audio_encoder_attention_mask: torch.Tensor | None = None, + audio_attention_mask: torch.Tensor | None = None, num_frames: int | None = None, height: int | None = None, width: int | None = None, @@ -1861,6 +1890,8 @@ class LTX2VideoTransformer3DModel(nn.Module): Optional multiplicative text attention mask of shape `(batch_size, text_seq_len)`. audio_encoder_attention_mask (`torch.Tensor`, *optional*): Optional multiplicative text attention mask of shape `(batch_size, text_seq_len)` for audio modeling. + audio_attention_mask (`torch.Tensor`, *optional*): + Optional audio-token key padding mask shared by audio self-attention and audio-to-video attention. num_frames (`int`, *optional*): The number of latent video frames. Used if calculating the video coordinates for RoPE. height (`int`, *optional*): @@ -2044,6 +2075,8 @@ class LTX2VideoTransformer3DModel(nn.Module): "ca_audio_rotary_emb": audio_cross_attn_rotary_emb, "encoder_attention_mask": encoder_attention_mask, "audio_encoder_attention_mask": audio_encoder_attention_mask, + "audio_self_attention_mask": audio_attention_mask, + "a2v_cross_attention_mask": audio_attention_mask, "video_self_attention_perturbation_mask": perturbation_mask_for("video_self_attention", block_idx), "audio_self_attention_perturbation_mask": perturbation_mask_for("audio_self_attention", block_idx), "a2v_cross_attention_perturbation_mask": perturbation_mask_for("a2v_cross_attention", block_idx), @@ -2097,16 +2130,8 @@ class LTX2VideoTransformer3DModel(nn.Module): Returns: Set of parameter names that were successfully loaded. """ - stacked_params_mapping = [ - (".attn1.to_qkv", ".attn1.to_q", "q"), - (".attn1.to_qkv", ".attn1.to_k", "k"), - (".attn1.to_qkv", ".attn1.to_v", "v"), - (".audio_attn1.to_qkv", ".audio_attn1.to_q", "q"), - (".audio_attn1.to_qkv", ".audio_attn1.to_k", "k"), - (".audio_attn1.to_qkv", ".audio_attn1.to_v", "v"), - ] - params_dict = dict(self.named_parameters()) + phase_adapter_parameter_name = getattr(self, "_phase_adapter_parameter_name", None) tp_size = get_tensor_model_parallel_world_size() tp_rank = get_tensor_model_parallel_rank() if tp_size > 1 else 0 loaded_params: set[str] = set() @@ -2127,10 +2152,12 @@ class LTX2VideoTransformer3DModel(nn.Module): return weight for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: + for param_name, weight_name, shard_id in self.stacked_params_mapping: if weight_name not in name: continue packed_name = name.replace(weight_name, param_name) + if callable(phase_adapter_parameter_name): + packed_name = phase_adapter_parameter_name(packed_name) if packed_name not in params_dict: continue name = packed_name @@ -2139,6 +2166,8 @@ class LTX2VideoTransformer3DModel(nn.Module): weight_loader(param, loaded_weight, shard_id) break else: + if callable(phase_adapter_parameter_name): + name = phase_adapter_parameter_name(name) if name not in params_dict: logger.warning( "Skipping transformer weight %s -- not found in model " diff --git a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py index 0e5bbc558..3cb6b84f1 100644 --- a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py +++ b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py @@ -1,27 +1,48 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Distilled two-stage entry points for the LTX model family.""" +"""Two-stage entry points for the LTX model family.""" -from vllm_omni.diffusion.data import OmniDiffusionConfig +from typing import ClassVar -from .ltx2_components import LTX2_DISTILLED_COMPONENT_PROFILE +from .ltx2_components import ( + LTX2_DISTILLED_COMPONENT_PROFILE, + LTX2_TWO_STAGE_COMPONENT_PROFILE, +) from .ltx2_components import ( get_ltx2_post_process_func as get_ltx2_post_process_func, # noqa: F401 ) -from .ltx2_recipes import LTX2_DISTILLED_TWO_STAGE_RECIPE -from .pipeline_ltx2 import LTX2Pipeline +from .ltx2_conditioning import LTXI2VConditioningMixin +from .ltx2_recipes import ( + LTX2_DISTILLED_TWO_STAGE_RECIPE, + LTX2_TWO_STAGE_RECIPE, +) +from .ltx2_runtime import LTXRuntime -class LTX2DistilledPipeline(LTX2Pipeline): - """Unified LTX-2 distilled two-stage T2V/I2V entry.""" +class _LTX2TwoStageBase(LTXI2VConditioningMixin, LTXRuntime): + """Static discovery contract shared by the two-stage public entries.""" - pipeline_kind = "distilled_two_stage" - component_profile = LTX2_DISTILLED_COMPONENT_PROFILE - pipeline_recipe = LTX2_DISTILLED_TWO_STAGE_RECIPE + _dit_modules: ClassVar[list[str]] = list(LTX2_TWO_STAGE_COMPONENT_PROFILE.dit_modules) + _encoder_modules: ClassVar[list[str]] = list(LTX2_TWO_STAGE_COMPONENT_PROFILE.encoder_modules) + _vae_modules: ClassVar[list[str]] = list(LTX2_TWO_STAGE_COMPONENT_PROFILE.vae_modules) + _resident_modules: ClassVar[list[str]] = list(LTX2_TWO_STAGE_COMPONENT_PROFILE.resident_modules) supports_request_batch = False support_image_input = True unified_text_image_entry = True - def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""): - super().__init__(od_config=od_config, prefix=prefix) + +class LTX2TwoStagePipeline(_LTX2TwoStageBase): + """Regular checkpoint with low-resolution generation and LoRA refinement.""" + + pipeline_kind = "two_stage" + component_profile = LTX2_TWO_STAGE_COMPONENT_PROFILE + pipeline_recipe = LTX2_TWO_STAGE_RECIPE + + +class LTX2DistilledPipeline(_LTX2TwoStageBase): + """Unified LTX-2/LTX-2.3 full-distilled two-stage T2V/I2V entry.""" + + pipeline_kind = "distilled_two_stage" + component_profile = LTX2_DISTILLED_COMPONENT_PROFILE + pipeline_recipe = LTX2_DISTILLED_TWO_STAGE_RECIPE diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index 75ef7391a..0e814886b 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -71,6 +71,11 @@ _DIFFUSION_MODELS = { "pipeline_ltx2", "LTX2Pipeline", ), + "LTX2TwoStagePipeline": ( + "ltx2", + "pipeline_ltx2_two_stage", + "LTX2TwoStagePipeline", + ), "LTX2DistilledPipeline": ( "ltx2", "pipeline_ltx2_two_stage", @@ -516,6 +521,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = { "WanPipeline": "get_wan22_post_process_func", "WanVACEPipeline": "get_wan22_vace_post_process_func", "LTX2Pipeline": "get_ltx2_post_process_func", + "LTX2TwoStagePipeline": "get_ltx2_post_process_func", "LTX2DistilledPipeline": "get_ltx2_post_process_func", "LTX2T2VDMD2Pipeline": "get_ltx2_post_process_func", "LTX2I2VDMD2Pipeline": "get_ltx2_post_process_func", diff --git a/vllm_omni/model_extras/registry.py b/vllm_omni/model_extras/registry.py index 8d7ee1e29..bc923ea86 100644 --- a/vllm_omni/model_extras/registry.py +++ b/vllm_omni/model_extras/registry.py @@ -205,6 +205,7 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = { } for model_class_name in ( "LTX2Pipeline", + "LTX2TwoStagePipeline", "LTX2DistilledPipeline", ) }, From dbc0dd6d85272aab507c78b35f837d8651807087 Mon Sep 17 00:00:00 2001 From: Sy03 <1370724210@qq.com> Date: Thu, 13 Aug 2026 10:00:50 -0700 Subject: [PATCH 003/267] [Bugfix][Higgs-Audio-V3] Disable XQA decode (#6068) Signed-off-by: Sy03 <1370724210@qq.com> Signed-off-by: Sysy <1370724210@qq.com> --- tests/config/test_omni_config.py | 1 + tests/engine/test_stage_engine_args.py | 2 + .../higgs_audio_v3/test_higgs_audio_v3.py | 8 ++- vllm_omni/config/omni_config.py | 2 + vllm_omni/deploy/README_higgs_audio_v3.md | 59 +++++++++++++++++++ vllm_omni/deploy/higgs_multimodal_qwen3.yaml | 5 ++ ...iggs_multimodal_qwen3_high_throughput.yaml | 5 ++ .../higgs_multimodal_qwen3_low_latency.yaml | 5 ++ 8 files changed, 86 insertions(+), 1 deletion(-) diff --git a/tests/config/test_omni_config.py b/tests/config/test_omni_config.py index 1fb32239d..cc08ed397 100644 --- a/tests/config/test_omni_config.py +++ b/tests/config/test_omni_config.py @@ -471,6 +471,7 @@ def test_sub_config_fields_match_structured_scopes(): "trust_remote_code", "dtype", "attention_backend", + "attention_config", "moe_backend", "hf_overrides", "limit_mm_per_prompt", diff --git a/tests/engine/test_stage_engine_args.py b/tests/engine/test_stage_engine_args.py index 75660cbc7..55ae43603 100644 --- a/tests/engine/test_stage_engine_args.py +++ b/tests/engine/test_stage_engine_args.py @@ -139,6 +139,7 @@ def _engine_arg_inputs(tmp_path: Path) -> tuple[PipelineConfig, DeployConfig, st engine_extras={ "model": str(stage_model), "attention_backend": "FLASHINFER", + "attention_config": {"use_trtllm_attention": False}, "hf_overrides": {"rope_scaling": {"factor": 2}}, "limit_mm_per_prompt": {"audio": 1}, "logits_processors": ["test.custom.LogitsProcessor"], @@ -229,6 +230,7 @@ def test_typed_llm_engine_args_preserve_legacy_adapter_behavior(tmp_path): assert thinker_args["omni_kv_config"] == {"need_send_cache": True} assert thinker_args["worker_cls"] == "test.custom.Worker" assert thinker_args["attention_backend"] == "FLASHINFER" + assert thinker_args["attention_config"] == {"use_trtllm_attention": False} assert thinker_args["trust_remote_code"] is True assert thinker_args["dtype"] == "float16" assert thinker_args["distributed_executor_backend"] == "mp" diff --git a/tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py b/tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py index 6d81f300a..77a32ad7b 100755 --- a/tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py +++ b/tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py @@ -1326,7 +1326,9 @@ class TestRegistry: assert "higgs_multimodal_qwen3" in OMNI_PIPELINES - def test_deploy_yaml_exists(self): + def test_deploy_yaml_uses_native_flashinfer(self): + from vllm_omni.config.stage_config import load_deploy_config + for name in ( "higgs_multimodal_qwen3.yaml", "higgs_multimodal_qwen3_high_throughput.yaml", @@ -1334,6 +1336,10 @@ class TestRegistry: ): yaml_path = get_deploy_config_path(name) assert os.path.isfile(yaml_path), f"Deploy YAML not found at {yaml_path}" + deploy_config = load_deploy_config(yaml_path) + stage_0 = next(stage for stage in deploy_config.stages if stage.stage_id == 0) + assert stage_0.engine_extras["attention_backend"] == "FLASHINFER" + assert stage_0.engine_extras["attention_config"] == {"use_trtllm_attention": False} # ---- AC-3: Prompt Builder ---- diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index 32ee2c826..8c4a933e2 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -90,6 +90,7 @@ class _ModelEngineOverrides(TypedDict, total=False): trust_remote_code: bool dtype: Any attention_backend: Any + attention_config: Any moe_backend: str hf_overrides: Any limit_mm_per_prompt: dict[str, Any] @@ -321,6 +322,7 @@ class OmniStageModelConfig: trust_remote_code: bool = False dtype: Any = "auto" attention_backend: Any = None + attention_config: Any = None moe_backend: str = "auto" hf_overrides: Any = None limit_mm_per_prompt: dict[str, Any] | None = None diff --git a/vllm_omni/deploy/README_higgs_audio_v3.md b/vllm_omni/deploy/README_higgs_audio_v3.md index ea1363069..44f7f75c7 100644 --- a/vllm_omni/deploy/README_higgs_audio_v3.md +++ b/vllm_omni/deploy/README_higgs_audio_v3.md @@ -44,6 +44,65 @@ vllm-omni serve bosonai/higgs-audio-v3-tts-4b \ --deploy-config vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml ``` +## Why Stage 0 Pins Native FlashInfer + +The three Stage 0 profiles set `attention_backend: FLASHINFER` and also set: + +```yaml +attention_config: + use_trtllm_attention: false +``` + +`FLASHINFER` names the overall vLLM attention backend. Starting with vLLM +0.25, that backend can internally route Hopper decode to the TRT-LLM XQA +kernel. The XQA eligibility check answers whether the kernel can run, not +whether it is the fastest kernel for the current workload. In particular, the +decode builder can select XQA statically when all of the following hold: + +- the explicit `use_trtllm_attention` override is not `false`; +- the device is Hopper/SM90 and the XQA cubin is available; and +- the query-head count is divisible by the KV-head count. + +Higgs has 32 query heads and 8 KV heads, so it passes the head-layout check. +The static gate does not include a model dtype, KV-cache dtype, context-length, +batch-shape, or measured-performance comparison. This matters because kernel +support is not a performance guarantee. + +Higgs also has an unusual output shape that must not be confused with its +attention shape. One autoregressive audio step advances one token per request. +The resulting hidden row is then projected into eight codebook distributions +with shape `[1, 8, codebook_size]`. The eight codebooks therefore do not turn +the attention query into `q_len=8`; at concurrency one, XQA still receives a +single BF16 query row. At this small per-step shape, any XQA advantage for +other KV dtypes, batch sizes, or context lengths does not necessarily apply. +The difference is paid in every transformer layer and every generated audio +step, so even a small per-call disadvantage accumulates in TTFP and RTF. + +The upstream XQA integration also showed that the result is dtype-sensitive: +its FP8 benchmark favored XQA, while its BF16 benchmark reported 21.16 ms mean +TPOT for XQA versus 19.47 ms for FA3. That is not a direct native-FlashInfer +comparison, but it demonstrates why SM90 compatibility alone is insufficient +to choose the fastest BF16 kernel. See the +[vLLM XQA benchmark](https://github.com/vllm-project/vllm/pull/43232#issuecomment-4501176335). + +The end-to-end H800 A/B in +[issue #5584](https://github.com/vllm-project/vllm-omni/issues/5584#issuecomment-5188700218) +isolated the routing decision on the same vLLM 0.25 stack: + +| Decode path | TTFP (ms) | RTF | +|---|---:|---:| +| XQA (automatic default) | 175.76 | 0.3405 | +| Native FlashInfer (`use_trtllm_attention: false`) | 166.39 | 0.3233 | +| vLLM 0.24 reference | 166.68 | 0.3209 | + +This proves that XQA selection causes the observed local regression and that +native FlashInfer recovers the pre-v0.25 range. It does not by itself identify +one defective XQA instruction or memory transaction. Establishing that deeper +kernel-level cause requires an attention-only microbenchmark and GPU profiler +trace for the exact Higgs batch/context distribution. Until vLLM has a +workload-aware selector or XQA is faster for this BF16 shape, the model-level +pin is the deterministic choice. + ## Notes - Stage 1 remains `enforce_eager: true` in both profiles. diff --git a/vllm_omni/deploy/higgs_multimodal_qwen3.yaml b/vllm_omni/deploy/higgs_multimodal_qwen3.yaml index da299415b..ae4e373cb 100644 --- a/vllm_omni/deploy/higgs_multimodal_qwen3.yaml +++ b/vllm_omni/deploy/higgs_multimodal_qwen3.yaml @@ -38,6 +38,11 @@ stages: gpu_memory_utilization: 0.6 enforce_eager: true attention_backend: FLASHINFER + # Eight codebooks share one attention row, so decode remains BF16 q_len=1 + # per request. vLLM's SM90 XQA capability gate does not benchmark this + # shape; keep the faster native path (README_higgs_audio_v3.md, #5584). + attention_config: + use_trtllm_attention: false trust_remote_code: true # Stage-0 prefix caching is on by default. The talker's HS opt-out and # deferred ``codes.audio`` mm-output write (see higgs_audio_v3_talker.py) diff --git a/vllm_omni/deploy/higgs_multimodal_qwen3_high_throughput.yaml b/vllm_omni/deploy/higgs_multimodal_qwen3_high_throughput.yaml index 8a751483c..bb37bdb4c 100644 --- a/vllm_omni/deploy/higgs_multimodal_qwen3_high_throughput.yaml +++ b/vllm_omni/deploy/higgs_multimodal_qwen3_high_throughput.yaml @@ -37,6 +37,11 @@ stages: gpu_memory_utilization: 0.6 enforce_eager: true attention_backend: FLASHINFER + # Eight codebooks share one attention row, so decode remains BF16 q_len=1 + # per request. vLLM's SM90 XQA capability gate does not benchmark this + # shape; keep the faster native path (README_higgs_audio_v3.md, #5584). + attention_config: + use_trtllm_attention: false trust_remote_code: true # Stage-0 prefix caching is on by default. The talker's HS opt-out and # deferred ``codes.audio`` mm-output write (see higgs_audio_v3_talker.py) diff --git a/vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml b/vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml index 0d6ae8218..13c6961e6 100644 --- a/vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml +++ b/vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml @@ -43,6 +43,11 @@ stages: gpu_memory_utilization: 0.6 enforce_eager: false attention_backend: FLASHINFER + # Eight codebooks share one attention row, so decode remains BF16 q_len=1 + # per request. vLLM's SM90 XQA capability gate does not benchmark this + # shape; keep the faster native path (README_higgs_audio_v3.md, #5584). + attention_config: + use_trtllm_attention: false trust_remote_code: true enable_prefix_caching: true async_scheduling: false From 90a08c0edaa15eda9f0b7bc740fd226703b77470 Mon Sep 17 00:00:00 2001 From: "Lucas(Ruijie)" <39816436+Moore-Z@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:58:33 -0600 Subject: [PATCH 004/267] =?UTF-8?q?[Model][TTS][WIP]=20Add=20dots.tts=20(r?= =?UTF-8?q?ednote-hilab)=20=E2=80=94=20continuous-AR=2048kHz=20TTS=20(#476?= =?UTF-8?q?5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Moore-Z --- docs/models/supported_models.md | 1 + .../text_to_speech/README.md | 26 + .../text_to_speech/dots_tts/end2end.py | 125 ++ recipes/rednote-hilab/dots.tts.md | 152 ++ .../models/dots_tts/__init__.py | 2 + .../models/dots_tts/test_talker.py | 137 ++ vllm_omni/config/pipeline_registry.py | 2 + vllm_omni/deploy/dots_tts.yaml | 28 + .../models/dots_tts/__init__.py | 8 + .../models/dots_tts/dots_tts_dit.py | 457 ++++++ .../models/dots_tts/dots_tts_patch_encoder.py | 355 ++++ .../models/dots_tts/dots_tts_prompt.py | 70 + .../dots_tts/dots_tts_speaker_encoder.py | 732 +++++++++ .../models/dots_tts/dots_tts_talker.py | 1426 ++++++++++++++++ .../models/dots_tts/dots_tts_vocoder.py | 1434 +++++++++++++++++ .../models/dots_tts/pipeline.py | 46 + vllm_omni/model_executor/models/registry.py | 6 + .../transformers_utils/configs/__init__.py | 3 + .../transformers_utils/configs/dots_tts.py | 102 ++ 19 files changed, 5112 insertions(+) create mode 100644 examples/offline_inference/text_to_speech/dots_tts/end2end.py create mode 100644 recipes/rednote-hilab/dots.tts.md create mode 100644 tests/model_executor/models/dots_tts/__init__.py create mode 100644 tests/model_executor/models/dots_tts/test_talker.py create mode 100644 vllm_omni/deploy/dots_tts.yaml create mode 100644 vllm_omni/model_executor/models/dots_tts/__init__.py create mode 100644 vllm_omni/model_executor/models/dots_tts/dots_tts_dit.py create mode 100644 vllm_omni/model_executor/models/dots_tts/dots_tts_patch_encoder.py create mode 100644 vllm_omni/model_executor/models/dots_tts/dots_tts_prompt.py create mode 100644 vllm_omni/model_executor/models/dots_tts/dots_tts_speaker_encoder.py create mode 100644 vllm_omni/model_executor/models/dots_tts/dots_tts_talker.py create mode 100644 vllm_omni/model_executor/models/dots_tts/dots_tts_vocoder.py create mode 100644 vllm_omni/model_executor/models/dots_tts/pipeline.py create mode 100644 vllm_omni/transformers_utils/configs/dots_tts.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 73fdc55e4..995afc28d 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -59,6 +59,7 @@ th { | `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ | — | | `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | | — | | `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | | — | +| `DotsTTSForConditionalGeneration` | dots.tts | `rednote-hilab/dots.tts-soar` | ✅︎ | | | | — | | `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | — | | `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | — | | `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | diff --git a/examples/offline_inference/text_to_speech/README.md b/examples/offline_inference/text_to_speech/README.md index d246f9e45..64e9b4d42 100644 --- a/examples/offline_inference/text_to_speech/README.md +++ b/examples/offline_inference/text_to_speech/README.md @@ -23,6 +23,7 @@ list of supported architectures across all modalities, see | OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz | | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants | 24 kHz | | VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | ✓ (online) | continuation | 48 kHz | +| dots.tts | `rednote-hilab/dots.tts-soar` | single (native AR) | — (not wired yet) | — | — | 48 kHz | | IndexTTS-2 | `IndexTeam/IndexTTS-2` | 2 (AR talker + S2Mel DiT + BigVGAN) | ✓ (required) | — | emotion control (`--emo-audio`, `--emo-text`, `--emo-vector`) | 22.05 kHz | | IndexTTS-2.5 | native `checkpoints/` bundle | 2 (AR talker + EnhancedCodec + S2Mel DiT + BigVGAN) | ✓ (required) | — | multilingual (`--lang`) + emotion control | 22.05 kHz | | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets | 24 kHz | @@ -430,6 +431,31 @@ Streaming is exposed through the online OpenAI Speech API (`stream=true`). See [ --- +## dots.tts + +Single-stage native AR TTS at 48 kHz (rednote-hilab). Pipeline: `Qwen2.5-1.5B base LM → DiT (10-step Euler flow matching) → patch_encoder AR loopback → AudioVAE (streaming decode)`. Same "vLLM-native base LM + side-path computation" pattern as VoxCPM2, with a plain Qwen2 backbone instead of MiniCPM4 and no FSQ / residual LM stage. + +### Quick start +```bash +python examples/offline_inference/text_to_speech/dots_tts/end2end.py \ + --model rednote-hilab/dots.tts-soar \ + --text "Hello, this is a test of dots TTS running on vLLM Omni." +``` + +### Voice cloning +Not wired in this release — generation is zero-shot only. The CAM++ x-vector speaker encoder weights load, but `end2end.py` has no `--ref-audio`/`--ref-text` flags yet. + +### Streaming +The AudioVAE decoder has an internal streaming path (`init_stream_state` / `stream_step` / `stream_flush`) used to avoid boundary artifacts between 160 ms patches, but it is not yet exposed through an online serving endpoint or example. + +### Notes +- Output: 48 kHz mono WAV. +- Deploy config: `vllm_omni/deploy/dots_tts.yaml` (auto-loaded by HF `model_type`). +- Checkpoints: `rednote-hilab/dots.tts-soar` is the validated default. `dots.tts-base` shares the same architecture but is unvalidated in this repo. `dots.tts-mf` (MeanFlow, 2-4 step) is not supported yet. +- Known limitation: no CUDA graph capture and no batched side-path computation yet, so concurrent requests do not currently scale (each request's DiT Euler steps run serially). See `recipes/rednote-hilab/dots.tts.md` for details and the roadmap. + +--- + ## IndexTTS-2 and IndexTTS-2.5 Both versions use a 2-stage TTS pipeline and produce 22.05 kHz mono audio. Stage 0 is a GPT AR talker. IndexTTS-2 Stage 1 uses RepCodec semantic embeddings, GPT latent, S2Mel CFM/DiT, and BigVGAN. IndexTTS-2.5 replaces RepCodec with EnhancedCodec and uses the official code-only path (`use_gpt_latent=false`). Every request requires reference audio for zero-shot voice cloning. diff --git a/examples/offline_inference/text_to_speech/dots_tts/end2end.py b/examples/offline_inference/text_to_speech/dots_tts/end2end.py new file mode 100644 index 000000000..8705f03f3 --- /dev/null +++ b/examples/offline_inference/text_to_speech/dots_tts/end2end.py @@ -0,0 +1,125 @@ +"""Offline dots.tts inference example (single-stage native AR pipeline). + +Uses the single-stage native AR config (vllm_omni/deploy/dots_tts.yaml). +Reference checkpoint: rednote-hilab/dots.tts-soar. +""" + +from __future__ import annotations + +import os +import time +from pathlib import Path + +import soundfile as sf +import torch + +from vllm_omni import Omni +from vllm_omni.utils.tracking_parser import TrackingArgumentParser + +REPO_ROOT = Path(__file__).resolve().parents[4] +SAMPLE_RATE = 48_000 + + +def parse_args(): + parser = TrackingArgumentParser(description="Offline dots.tts native AR inference") + parser.add_argument( + "--model", + type=str, + default="rednote-hilab/dots.tts-soar", + help="dots.tts model path or HuggingFace repo ID.", + ) + parser.add_argument( + "--text", + type=str, + default="Hello, this is a test of dots TTS running on vLLM Omni.", + help="Text to synthesize.", + ) + parser.add_argument( + "--output-dir", + type=str, + default="output_audio", + help="Directory for output WAV files.", + ) + parser.add_argument( + "--deploy-config", + type=str, + default=None, + help="Override the deploy config path. If unset, auto-loads " + "vllm_omni/deploy/dots_tts.yaml based on the HF model_type.", + ) + return parser.parse_args() + + +def extract_audio(multimodal_output: dict) -> torch.Tensor: + """Extract the final complete audio tensor from multimodal output. + + The output processor concatenates per-step delta tensors under + ``model_outputs``. Falls back to ``audio`` for backwards compat. + """ + audio = multimodal_output.get("model_outputs") + if audio is None: + audio = multimodal_output.get("audio") + if audio is None: + raise ValueError(f"No audio key in multimodal_output: {list(multimodal_output.keys())}") + + if isinstance(audio, list): + valid = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio if a is not None] + if not valid: + raise ValueError("Audio list is empty or all elements are None.") + return torch.cat(valid, dim=0) if len(valid) > 1 else valid[0] + + return torch.as_tensor(audio).float().cpu().reshape(-1) + + +def main(): + args = parse_args() + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + engine = Omni( + model=args.model, + deploy_config=args.deploy_config, + ) + + from transformers import AutoTokenizer + + from vllm_omni.model_executor.models.dots_tts.dots_tts_prompt import ( + build_dots_tts_prompt, + ) + + tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True) + + prompt = build_dots_tts_prompt( + tokenizer=tokenizer, + text=args.text, + ) + + print(f"Model : {args.model}") + print(f"Text : {args.text}") + print(f"Output dir : {output_dir}") + print(f"Prompt len : {len(prompt['prompt_token_ids'])} tokens") + + t_start = time.perf_counter() + outputs = engine.generate([prompt]) + elapsed = time.perf_counter() - t_start + + request_output = outputs[0] + mm = request_output.outputs[0].multimodal_output + audio = extract_audio(mm) + + duration = audio.numel() / SAMPLE_RATE + rtf = elapsed / duration if duration > 0 else float("inf") + + output_path = output_dir / "output.wav" + sf.write(str(output_path), audio.numpy(), SAMPLE_RATE, format="WAV") + + print(f"Saved : {output_path}") + print(f"Duration : {duration:.2f}s") + print(f"Inference : {elapsed:.2f}s") + print(f"RTF : {rtf:.3f}") + + +if __name__ == "__main__": + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + main() diff --git a/recipes/rednote-hilab/dots.tts.md b/recipes/rednote-hilab/dots.tts.md new file mode 100644 index 000000000..d5c7868cb --- /dev/null +++ b/recipes/rednote-hilab/dots.tts.md @@ -0,0 +1,152 @@ +# dots.tts + +> Offline continuous-AR TTS at 48 kHz (rednote-hilab) + +## Summary + +- Vendor: rednote-hilab +- Model: `rednote-hilab/dots.tts-soar` +- Task: Text-to-speech, zero-shot synthesis only +- Mode: Offline end-to-end example only (not yet wired for online serving — + see [Known limitations](#known-limitations)) +- Maintainer: Community + +## When to use this recipe + +Use this recipe as a known-good starting point for running +`rednote-hilab/dots.tts-soar` offline on vLLM-Omni on consumer-class GPUs. +dots.tts is a ~1.7B-parameter continuous-AR TTS model (Qwen2.5-1.5B base LM ++ 344M DiT flow-matching head + 180M AudioVAE) that emits 48 kHz mono audio. +It follows the same "vLLM-native base LM + side-path computation" pattern as +VoxCPM2 — single-stage pipeline +`Qwen2.5-1.5B base LM → DiT (10-step Euler flow matching) → patch_encoder AR +loopback → AudioVAE (streaming decode)` — with a plain Qwen2 backbone +instead of MiniCPM4, and no FSQ / residual-LM stage. + +This is an early integration (see [Known limitations](#known-limitations) +below before depending on it for anything beyond offline experimentation). + +## References + +- Offline end-to-end script: + [`examples/offline_inference/text_to_speech/dots_tts/end2end.py`](../../examples/offline_inference/text_to_speech/dots_tts/end2end.py) +- Example guide: + [`examples/offline_inference/text_to_speech/README.md`](../../examples/offline_inference/text_to_speech/README.md#dotstts) +- Default deploy config: + [`vllm_omni/deploy/dots_tts.yaml`](../../vllm_omni/deploy/dots_tts.yaml) +- Talker / pipeline source: + [`vllm_omni/model_executor/models/dots_tts/`](../../vllm_omni/model_executor/models/dots_tts/) +- Upstream: [rednote-hilab/dots.tts](https://github.com/rednote-hilab/dots.tts) + +## Hardware Support + +This recipe documents one tested 16 GB consumer-GPU configuration. Other +vendor sections (ROCm, NPU) and larger-VRAM configurations are welcome as +community validation lands. + +## GPU + +### 1 x RTX 5080 16GB (Single GPU, Minimum Recommended) + +dots.tts (~1.7B params across the base LM + DiT + AudioVAE + CAM++ +speaker encoder, bfloat16) fits comfortably on a single 16 GB GPU. The +bundled default config at +[`vllm_omni/deploy/dots_tts.yaml`](../../vllm_omni/deploy/dots_tts.yaml) +(`gpu_memory_utilization: 0.8`, `max_num_seqs: 4`, `enforce_eager: true`, +`enable_prefix_caching: false`) loads cleanly with ~5.1 GiB for model +weights and ~0.3 GiB peak activation; the remainder of the configured +budget is available for KV cache. Total resident footprint at idle is +roughly **7-8 GiB / 16 GB** — the only tight spot in the full CUDA-Graph +roadmap would be step 8's graph capture (not implemented yet; this +release runs `enforce_eager: true`, so it doesn't apply today). + +#### Environment + +- OS: Linux (WSL2) +- Python: 3.12 +- Driver / runtime: NVIDIA driver 595.95 +- torch: 2.11.0+cu130 +- vLLM: 0.26.0 +- vLLM-Omni: 0.22.1.dev (current `main`) + +#### Command + +```bash +python examples/offline_inference/text_to_speech/dots_tts/end2end.py \ + --model rednote-hilab/dots.tts-soar \ + --text "Hello, this is a test of dots TTS running on vLLM Omni." +``` + +The deploy config at +[`vllm_omni/deploy/dots_tts.yaml`](../../vllm_omni/deploy/dots_tts.yaml) +is loaded automatically by the model registry (HF `model_type=dots_tts`). +Pass `--deploy-config ` to override. + +#### Verification + +**T1 — offline zero-shot synthesis**: + +```bash +python examples/offline_inference/text_to_speech/dots_tts/end2end.py \ + --model rednote-hilab/dots.tts-soar \ + --text "Hello, this is a test of dots TTS running on vLLM Omni." +``` + +Observed: `output_audio/output.wav`, 3.52 s @ 48 kHz mono. Single +one-shot process (init → one `generate()` → exit), so the reported +numbers include engine init and first-request warmup, not just steady-state +per-step throughput — same caveat as VoxCPM2's recipe. `Inference: 5.52s`, +`RTF: 1.569`. + +Weight-loading breakdown from the same run (all tensors matched, no +missing/extra keys): 951/951 AudioVAE, 244/244 DiT, 270/270 +patch_encoder, 198 Qwen2, 938/938 CAM++ speaker encoder. + +Whisper transcription of the output matched the input text with no +dropped leading word (confirms the streaming-vocoder patch-boundary fix +described in [Known limitations](#known-limitations)). + +#### Notes + +- Output: 48 kHz mono WAV. +- Checkpoints: `rednote-hilab/dots.tts-soar` is the validated default + used throughout this recipe. `rednote-hilab/dots.tts-base` shares the + same architecture but is unvalidated in this repo. `rednote-hilab/dots.tts-mf` + (MeanFlow, 2-4 step) is not supported — see below. +- `enforce_eager: true` and `enable_prefix_caching: false` in the deploy + config are load-bearing, not just conservative defaults: with prefix + caching enabled, vLLM-Omni's prefix-cache multimodal-output merge path + does not preserve the `sparse_audio` marker this model relies on to + route audio output correctly, and generation silently truncates to a + single ~160 ms patch. Do not override `enable_prefix_caching` for this + model until that framework-level gap is fixed. + +## Known limitations + +- **No online serving yet.** dots.tts is not registered in + `vllm_omni/entrypoints/openai/tts_adapters/` (the framework other TTS + models in this repo use for `/v1/audio/speech`), so `vllm serve + rednote-hilab/dots.tts-soar --omni` will start but the OpenAI-compatible + speech endpoint does not know how to build request params for it yet. + Offline `Omni()` (as used in `end2end.py`) is the only supported path + today. +- **Voice cloning is not wired.** The CAM++ x-vector speaker encoder + weights load and are exercised by `load_weights()`, but nothing in the + prompt builder consumes a reference audio yet — generation is zero-shot + only. `end2end.py` has no `--ref-audio`/`--ref-text` flags. +- **`dots.tts-mf` (MeanFlow, 2-4 step) checkpoint is not supported.** Only + the fixed 10-step Euler DiT sampler used by `dots.tts-soar` / + `dots.tts-base` is implemented. +- **No CUDA graph capture.** The talker runs fully eager. voxcpm2's three + captured graphs (base LM decode, CFM solver, VAE decode) have no + dots.tts equivalent yet. +- **Concurrent requests do not scale.** Each request's 10-step DiT Euler + integration runs serially in the side path (no cross-request batching, + unlike voxcpm2's `enable_batched_cfm`). A community review of this + integration measured no throughput gain at `c=4` concurrent requests + versus `c=1`. +- **`SamplingParams.seed` does not control audio-generation randomness.** + The DiT's flow-matching noise is deterministically derived per-request + from a fixed internal seed (reproducible run-to-run, matching + voxcpm2's `deterministic_cfm_seed` convention) rather than from the + caller-supplied `seed` field — the same limitation voxcpm2 has today. diff --git a/tests/model_executor/models/dots_tts/__init__.py b/tests/model_executor/models/dots_tts/__init__.py new file mode 100644 index 000000000..208f01a7c --- /dev/null +++ b/tests/model_executor/models/dots_tts/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/model_executor/models/dots_tts/test_talker.py b/tests/model_executor/models/dots_tts/test_talker.py new file mode 100644 index 000000000..ed3c6bcaa --- /dev/null +++ b/tests/model_executor/models/dots_tts/test_talker.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression tests for the dots.tts talker's stop-signal pairing and +per-request state lifecycle. + +Pure-CPU tests on a bare talker (``__new__`` + the handful of attributes +the methods under test read) — no weights, no GPU, no engine. Mirrors +tests/model_executor/models/voxcpm2/test_talker_state_eviction.py. +""" + +from __future__ import annotations + +import functools +from types import SimpleNamespace + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +torch = pytest.importorskip("torch") + +_VOCAB_SIZE = 151672 + + +@functools.lru_cache(maxsize=1) +def _dots_tts_talker_mod(): + """Defer talker import (pulls vLLM model_executor) until first use.""" + from vllm_omni.model_executor.models.dots_tts.dots_tts_talker import ( + DotsTTSForConditionalGeneration, + _RequestState, + ) + + return DotsTTSForConditionalGeneration, _RequestState + + +def _make_bare_talker(): + DotsTTSForConditionalGeneration, _ = _dots_tts_talker_mod() + talker = DotsTTSForConditionalGeneration.__new__(DotsTTSForConditionalGeneration) + talker.config = SimpleNamespace(vocab_size=_VOCAB_SIZE) + talker._active_states = {} + talker._pending_requests = [] + talker._results_queue = [] + talker._audio_queue = [] + talker._deferred_cleanup_ids = set() + talker._beta_trace = False + return talker + + +def _add_state(talker, req_id: str, *, is_stopping: bool = False): + _, _RequestState = _dots_tts_talker_mod() + state = _RequestState(request_id=req_id) + state.prefill_completed = True + state.is_stopping = is_stopping + talker._active_states[req_id] = state + return state + + +def _hidden(bsz: int) -> torch.Tensor: + return torch.zeros(bsz, 1536, dtype=torch.float32) + + +class TestComputeLogitsPairing: + """compute_logits pairs _results_queue entries with batch rows + positionally, so every request must push exactly one entry per step — + including prefills, which push a (req_id, None) placeholder.""" + + def test_mixed_batch_prefill_placeholder_keeps_rows_aligned(self) -> None: + talker = _make_bare_talker() + _add_state(talker, "prefill-req") + decode_state = _add_state(talker, "decode-req", is_stopping=True) + decode_state.precomputed_stop_logits = torch.tensor([[0.1, 0.9]]) + + # Row 0 is the prefill (None placeholder), row 1 the stopping decode. + talker._results_queue = [ + ("prefill-req", None), + ("decode-req", torch.tensor([[0.1, 0.9]])), + ] + logits = talker.compute_logits(_hidden(2)) + + assert logits.shape == (2, _VOCAB_SIZE) + # Prefill row: forced continue. + assert logits[0, 0].item() == 1.0 + assert logits[0, 1].item() == float("-inf") + # Stopping decode row: stop slot wins. + assert logits[1, 1].item() == 1.0 + assert logits[1, 0].item() == 0.0 + # Everything else stays masked. + assert logits[0, 2].item() == float("-inf") + assert logits[1, 2].item() == float("-inf") + # Queue drained. + assert talker._results_queue == [] + + def test_below_threshold_stop_logits_force_continue(self) -> None: + """Raw (continue, stop) softmax must not reach the sampler: a + prob_stop in (0.5, 0.8] would flip greedy argmax even though + is_stopping stayed False (the 0.8-threshold contract).""" + talker = _make_bare_talker() + _add_state(talker, "req", is_stopping=False) + talker._results_queue = [("req", torch.tensor([[0.3, 0.7]]))] + + logits = talker.compute_logits(_hidden(1)) + + assert logits[0, 0].item() == 1.0 + assert logits[0, 1].item() == float("-inf") + + def test_empty_queue_defaults_to_all_continue(self) -> None: + talker = _make_bare_talker() + logits = talker.compute_logits(_hidden(3)) + + assert torch.all(logits[:, 0] == 1.0) + assert torch.all(logits[:, 1] == float("-inf")) + + +class TestStateEviction: + """on_requests_finished fires BEFORE forward(), so eviction must be + deferred to _flush_deferred_cleanup at the end of the step — the + finishing request's audio still drains through the current forward.""" + + def test_finished_request_survives_until_flush(self) -> None: + talker = _make_bare_talker() + _add_state(talker, "req-a") + _add_state(talker, "req-b") + + talker.on_requests_finished(["req-a"]) + assert "req-a" in talker._active_states # not evicted yet + + talker._flush_deferred_cleanup() + assert "req-a" not in talker._active_states + assert "req-b" in talker._active_states + assert talker._deferred_cleanup_ids == set() + + def test_unknown_request_id_is_ignored(self) -> None: + talker = _make_bare_talker() + talker.on_requests_finished(["never-seen"]) + talker._flush_deferred_cleanup() + assert talker._active_states == {} diff --git a/vllm_omni/config/pipeline_registry.py b/vllm_omni/config/pipeline_registry.py index f02a3578e..f06259eaa 100644 --- a/vllm_omni/config/pipeline_registry.py +++ b/vllm_omni/config/pipeline_registry.py @@ -47,6 +47,7 @@ from vllm_omni.model_executor.models.bagel.pipeline import ( ) from vllm_omni.model_executor.models.cosyvoice3.pipeline import COSYVOICE3_PIPELINE from vllm_omni.model_executor.models.covo_audio.pipeline import COVO_AUDIO_PIPELINE +from vllm_omni.model_executor.models.dots_tts.pipeline import DOTS_TTS_PIPELINE from vllm_omni.model_executor.models.dreamzero.pipeline import DREAMZERO_PIPELINE from vllm_omni.model_executor.models.dynin_omni.pipeline import DYNIN_OMNI_PIPELINE from vllm_omni.model_executor.models.fish_speech.pipeline import FISH_SPEECH_PIPELINE @@ -144,6 +145,7 @@ OMNI_PIPELINES: dict[str, PipelineConfig | PipelineResolverFunc] = { "hunyuan_video_15": HUNYUAN_VIDEO_15_PIPELINE, "wan2_2_ti2v": WAN2_2_TI2V_PIPELINE, "voxcpm2": VOXCPM2_PIPELINE, + "dots_tts": DOTS_TTS_PIPELINE, "cosyvoice3": COSYVOICE3_PIPELINE, "audex_tts": AUDEX_TTS_PIPELINE, "audex_tta": AUDEX_TTA_PIPELINE, diff --git a/vllm_omni/deploy/dots_tts.yaml b/vllm_omni/deploy/dots_tts.yaml new file mode 100644 index 000000000..acd28de52 --- /dev/null +++ b/vllm_omni/deploy/dots_tts.yaml @@ -0,0 +1,28 @@ +# dots.tts deployment config (single-stage AR talker). +# +# Conservative first-integration settings: eager execution, no CUDA +# Graphs, no batched side-path optimizations yet — see voxcpm2.yaml for +# the runtime knobs a mature deployment of this model family grows. +# +# Verified locally on RTX 5080 16GB. + +async_chunk: false +dtype: bfloat16 + +stages: + - stage_id: 0 + max_num_seqs: 4 + gpu_memory_utilization: 0.8 + enforce_eager: true + trust_remote_code: true + enable_prefix_caching: false + async_scheduling: true + max_num_batched_tokens: 4096 + max_model_len: 4096 + devices: "0" + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 4096 + repetition_penalty: 1.0 diff --git a/vllm_omni/model_executor/models/dots_tts/__init__.py b/vllm_omni/model_executor/models/dots_tts/__init__.py new file mode 100644 index 000000000..fb6738a29 --- /dev/null +++ b/vllm_omni/model_executor/models/dots_tts/__init__.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# NOTE: Do not import model classes in this file. Importing any +# submodule in this package triggers __init__.py execution, and +# both the model registry and pipeline registry import submodules +# directly — heavy imports here would be loaded as a side effect +# even though nothing depends on these re-exports. diff --git a/vllm_omni/model_executor/models/dots_tts/dots_tts_dit.py b/vllm_omni/model_executor/models/dots_tts/dots_tts_dit.py new file mode 100644 index 000000000..086ac109a --- /dev/null +++ b/vllm_omni/model_executor/models/dots_tts/dots_tts_dit.py @@ -0,0 +1,457 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 The vLLM-Omni team. +# Copyright (c) rednote-hilab. All rights reserved. +# Adapted from: +# https://github.com/rednote-hilab/dots.tts/tree/a393d2e/src/dots_tts/modules/backbone +# +# Merged from two upstream files (preserved as-is, inference behaviour +# unchanged; only internal imports were folded inline): +# - modules/backbone/layers.py -> Dropout, Mlp, rotate_half, +# apply_rotary_pos_emb, RotaryEmbedding, +# MultiHeadAttention +# (Conv1d / ConvTranspose1d are NOT +# re-vendored here — they already live in +# dots_tts_vocoder.py) +# - modules/backbone/dit.py -> modulate, TimestepEmbedder, FinalLayer, +# DiTBlock, DiT +# +# Per-section upstream attributions are preserved at each section header. +# Nothing in this file diverges from upstream behaviour. + +from __future__ import annotations + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange + +# ============================================================================ +# Building blocks (adapted from modules/backbone/layers.py) +# ---------------------------------------------------------------------------- +# Conv1d / ConvTranspose1d from layers.py are already vendored in +# dots_tts_vocoder.py (they are used by the BigVGAN-style AudioVAE). The +# pieces below are the ones consumed by the DiT flow-matching head. +# ============================================================================ + + +class Dropout(nn.Module): + def __init__(self, p: float = 0.5, inplace: bool = False, force_drop: bool = False, **_kwargs): + super().__init__() + if p < 0.0 or p > 1.0: + raise ValueError(f"dropout probability has to be between 0 and 1, but got {p}") + self.p = p + self.inplace = inplace + self.force_drop = force_drop + + def forward(self, x, **_kwargs): + return F.dropout( + x, + p=self.p, + training=True if self.force_drop else self.training, + inplace=self.inplace, + ) + + +class Mlp(nn.Module): + def __init__( + self, + hidden_size, + ffn_hidden_size=4096, + act_layer=nn.GELU, + dropout=0.0, + **_kwargs, + ): + super().__init__() + self.fc1 = nn.Linear(hidden_size, ffn_hidden_size) + self.act = act_layer() + self.fc2 = nn.Linear(ffn_hidden_size, hidden_size) + self.drop = Dropout(dropout) + + def forward(self, x, _mask=None): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + return self.drop(x) + + +def rotate_half(x): + x1, x2 = x.chunk(2, dim=-1) + return torch.cat((-x2, x1), dim=-1) + + +@torch.autocast(enabled=False, device_type="cuda") +def apply_rotary_pos_emb(pos, t): + if pos.dim() == 3: + pos = pos.unsqueeze(1) + return t * pos.cos() + rotate_half(t) * pos.sin() + + +class RotaryEmbedding(nn.Module): + def __init__(self, dim, theta=50000): + super().__init__() + self.register_buffer( + "inv_freq", + 1.0 / (theta ** (torch.arange(0, dim, 2).float() / dim)), + persistent=False, + ) + self._theta = float(theta) + + def _apply(self, fn): + inv_freq = self.inv_freq + super()._apply(fn) + self.inv_freq = inv_freq.to(device=self.inv_freq.device, dtype=torch.float32) + return self + + @torch.autocast(enabled=False, device_type="cuda") + def forward(self, t): + inv_freq = self.inv_freq + if inv_freq.device != t.device: + raise RuntimeError(f"RotaryEmbedding buffer device mismatch: inv_freq={inv_freq.device} input={t.device}.") + t = t.to(dtype=inv_freq.dtype) + if t.dim() == 1: + freqs = torch.einsum("i , j -> i j", t, inv_freq) + else: + freqs = torch.einsum("bi, j -> bij", t, inv_freq) + return torch.cat((freqs, freqs), dim=-1) + + +class MultiHeadAttention(nn.Module): + """Multi-head attention""" + + def __init__( + self, + hidden_size: int, + num_heads: int = 8, + qkv_bias: bool = False, + qk_norm: bool = False, + attn_drop: float = 0.0, + dropout: float = 0.0, + norm_layer: str = "LayerNorm", + rotary_bias: bool = False, + rotary_theta: float | None = 50000, + **_kwargs, + ): + super().__init__() + assert hidden_size % num_heads == 0, "hidden_size should be divisible by num_heads" + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + self.scale = self.head_dim**-0.5 + self.rotary_bias = rotary_bias + + self.q_proj = nn.Linear(hidden_size, hidden_size, bias=qkv_bias) + self.k_proj = nn.Linear(hidden_size, hidden_size, bias=qkv_bias) + self.v_proj = nn.Linear(hidden_size, hidden_size, bias=qkv_bias) + + norm_layer = getattr(nn, norm_layer) + self.q_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity() + self.k_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity() + + self.attn_drop = Dropout(attn_drop) + self.o_proj = nn.Linear(hidden_size, hidden_size) + self.o_dropout = Dropout(dropout) + + if self.rotary_bias: + self.rotary = RotaryEmbedding(self.head_dim, theta=rotary_theta) + + def forward(self, q, k=None, v=None, mask=None, pos_ids=None, **_kwargs): + k = k or q + v = v or q + B, L, _ = q.shape + _, S, _ = v.shape + if mask is not None: + if mask.ndim == 2: # [B, L] + assert L == S + mask = rearrange(mask, "b j -> b 1 1 j") + mask = mask.expand(-1, self.num_heads, L, -1) + elif mask.ndim == 3: # [B, L, S] + assert mask.size(1) == L and mask.size(2) == S + mask = mask.unsqueeze(1).expand(-1, self.num_heads, -1, -1) + + q, k, v = self.q_proj(q), self.k_proj(k), self.v_proj(v) + q = rearrange(q, "b n (h d) -> b h n d", h=self.num_heads) + k = rearrange(k, "b n (h d) -> b h n d", h=self.num_heads) + v = rearrange(v, "b n (h d) -> b h n d", h=self.num_heads) + q, k = self.q_norm(q), self.k_norm(k) + + # Apply rotary + if self.rotary_bias: + if L == S: + if pos_ids is None: + rotary_emb = self.rotary(torch.arange(L, device=q.device)) + else: + rotary_emb = self.rotary(pos_ids) + q, k = (apply_rotary_pos_emb(rotary_emb, tensor) for tensor in (q, k)) + else: + q_rotary_emb = self.rotary(torch.arange(L, device=q.device)) + k_rotary_emb = self.rotary(torch.arange(S, device=k.device)) + q = apply_rotary_pos_emb(q_rotary_emb, q) + k = apply_rotary_pos_emb(k_rotary_emb, k) + + attn_bias = torch.zeros(B, self.num_heads, L, S, dtype=q.dtype, device=q.device) + + if mask is not None: + attn_bias.masked_fill_(mask.logical_not(), float("-inf")) + + out = F.scaled_dot_product_attention( + q, + k, + v, + attn_mask=attn_bias, + dropout_p=self.attn_drop.p if self.training else 0.0, + ) + + out = rearrange(out, "b h n d -> b n (h d)") + return self.o_dropout(self.o_proj(out)) + + def decode_step(self, x, *, cache, positions: torch.Tensor): + if x.size(1) <= 0: + raise ValueError("MultiHeadAttention.decode_step expects a non-empty input.") + if positions.ndim != 1 or positions.size(0) != x.size(1): + raise ValueError("MultiHeadAttention.decode_step positions must match the decode block length.") + + q = self.q_proj(x) + k = self.k_proj(x) + v = self.v_proj(x) + + q = rearrange(q, "b n (h d) -> b h n d", h=self.num_heads) + k = rearrange(k, "b n (h d) -> b h n d", h=self.num_heads) + v = rearrange(v, "b n (h d) -> b h n d", h=self.num_heads) + q, k = self.q_norm(q), self.k_norm(k) + block_len = q.size(2) + + if self.rotary_bias: + rotary_emb = self.rotary(positions) + q = apply_rotary_pos_emb(rotary_emb, q) + k = apply_rotary_pos_emb(rotary_emb, k) + + cached_k, cached_v = cache + cached_k.index_copy_(2, positions, k) + cached_v.index_copy_(2, positions, v) + + cache_capacity = cached_k.size(2) + key_positions = torch.arange( + cache_capacity, + device=x.device, + dtype=torch.long, + ).unsqueeze(0) + query_positions = positions.unsqueeze(1) + causal_mask = key_positions <= query_positions + valid_mask = key_positions <= positions[-1] + attn_bias = torch.zeros( + q.size(0), + self.num_heads, + block_len, + cache_capacity, + dtype=q.dtype, + device=q.device, + ) + attn_bias.masked_fill_( + (causal_mask & valid_mask).unsqueeze(0).unsqueeze(0).logical_not(), + float("-inf"), + ) + + out = F.scaled_dot_product_attention( + q, + cached_k, + cached_v, + attn_mask=attn_bias, + dropout_p=self.attn_drop.p if self.training else 0.0, + ) + out = rearrange(out, "b h n d -> b n (h d)") + return self.o_dropout(self.o_proj(out)), cache + + +# ============================================================================ +# DiT flow-matching head (adapted from modules/backbone/dit.py) +# ---------------------------------------------------------------------------- +# DiT.__init__ takes a ``transformer_config`` object exposing ``.to_dict()``, +# ``.hidden_size`` and ``.num_layers``. Upstream constructs it from a pydantic +# config class higher up the stack; vLLM-Omni callers can pass anything that +# satisfies the same minimal interface (e.g. a stdlib @dataclass with a +# ``to_dict`` method or SimpleNamespace). See dots_tts_talker.py for the +# concrete soar-checkpoint config factory. +# ============================================================================ + + +def modulate(x, shift, scale, **_kwargs): + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) + + +class TimestepEmbedder(nn.Module): + def __init__(self, hidden_size, frequency_embedding_size=256): + super().__init__() + self.mlp = nn.Sequential( + nn.Linear(frequency_embedding_size, hidden_size, bias=True), + nn.SiLU(), + nn.Linear(hidden_size, hidden_size, bias=True), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t, dim, max_period=10000): + half = dim // 2 + freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half).to( + device=t.device + ) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t): + t_freq = self.timestep_embedding(t, self.frequency_embedding_size) + return self.mlp(t_freq) + + +class FinalLayer(nn.Module): + def __init__(self, hidden_size, output_size): + super().__init__() + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), + nn.Linear(hidden_size, 2 * hidden_size, bias=True), + ) + self.norm = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-5) + self.linear = nn.Linear(hidden_size, output_size, bias=True) + + def forward(self, x, c, **_kwargs): + shift, scale = self.adaLN_modulation(c).chunk(2, dim=1) + x = modulate(self.norm(x), shift, scale) + return self.linear(x) + + +class DiTBlock(nn.Module): + def __init__( + self, + attention: nn.Module, + ffn: nn.Module, + hidden_size: int = 1024, + modulation: bool = False, + eps: float = 1e-5, + **_kwargs, + ): + super().__init__() + self.norm1 = nn.LayerNorm(hidden_size, elementwise_affine=not modulation, eps=eps) + self.norm2 = nn.LayerNorm(hidden_size, elementwise_affine=not modulation, eps=eps) + self.attn = attention + self.ffn = ffn + self.modulation = modulation + if modulation: + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), + nn.Linear(hidden_size, 6 * hidden_size, bias=True), + ) + + def forward(self, x, condition=None, mask=None, **kwargs): + if condition is None: + assert not self.modulation, "Without global condition, must set modulation to False" + else: + assert self.modulation, "With global condition, must set modulation to True" + shift_attn, scale_attn, gate_attn, shift_ffn, scale_ffn, gate_ffn = self.adaLN_modulation(condition).chunk( + 6, dim=1 + ) + + if condition is not None: + pack_indices = kwargs.get("pack_indices") + if pack_indices is not None: + gate_attn = gate_attn[pack_indices] + gate_ffn = gate_ffn[pack_indices] + else: + gate_attn = gate_attn.unsqueeze(1) + gate_ffn = gate_ffn.unsqueeze(1) + + if condition is not None: + x = x + gate_attn * self.attn( + modulate(self.norm1(x), shift_attn, scale_attn, **kwargs), + mask=mask, + **kwargs, + ) + else: + x = x + self.attn(self.norm1(x), mask=mask, **kwargs) + + if condition is not None: + x = x + gate_ffn * self.ffn(modulate(self.norm2(x), shift_ffn, scale_ffn, **kwargs)) + else: + x = x + self.ffn(self.norm2(x), mask=mask) + return x + + +class DiT(nn.Module): + def __init__( + self, + in_dim, + out_dim, + transformer_config, + *, + mode: str = "flow_matching", + ): + super().__init__() + if mode not in {"flow_matching", "meanflow"}: + raise ValueError(f"DiT mode must be 'flow_matching' or 'meanflow', got {mode!r}.") + + transformer_kwargs = transformer_config.to_dict() + model_dim = transformer_config.hidden_size + self.mode = mode + self.num_layers = transformer_config.num_layers + + self.input_layer = nn.Linear(in_dim, model_dim) + self.time_embedder = TimestepEmbedder(model_dim) + if mode == "meanflow": + self.duration_embedder = TimestepEmbedder(model_dim) + + self.blocks = nn.ModuleList() + for i in range(self.num_layers): + attn_block = MultiHeadAttention(**transformer_kwargs, name=f"layer_{i}") + ffn_block = Mlp(act_layer=lambda: nn.GELU(approximate="tanh"), **transformer_kwargs) + self.blocks.append(DiTBlock(attention=attn_block, ffn=ffn_block, **transformer_kwargs)) + + self.output_layer = FinalLayer(model_dim, out_dim) + self.initialize_weights() + + def initialize_weights(self): + def _basic_init(module): + if isinstance(module, nn.Linear): + torch.nn.init.xavier_uniform_(module.weight) + if module.bias is not None: + nn.init.constant_(module.bias, 0) + + self.apply(_basic_init) + + nn.init.normal_(self.time_embedder.mlp[0].weight, std=0.02) + nn.init.normal_(self.time_embedder.mlp[2].weight, std=0.02) + + for block in self.blocks: + if hasattr(block, "adaLN_modulation"): + nn.init.constant_(block.adaLN_modulation[-1].weight, 0) + nn.init.constant_(block.adaLN_modulation[-1].bias, 0) + + nn.init.constant_(self.output_layer.adaLN_modulation[-1].weight, 0) + nn.init.constant_(self.output_layer.adaLN_modulation[-1].bias, 0) + nn.init.constant_(self.output_layer.linear.weight, 0) + nn.init.constant_(self.output_layer.linear.bias, 0) + + def forward( + self, + x, + timesteps, + duration: torch.Tensor | None = None, + mask=None, + attn_mask=None, + g_cond: torch.Tensor | None = None, + **kwargs, + ): + t = self.time_embedder(timesteps) + c = t + duration_embedder = getattr(self, "duration_embedder", None) + if duration_embedder is not None and duration is not None: + c = c + duration_embedder(duration) + if g_cond is not None: + c = c + g_cond + + x = self.input_layer(x) + for block in self.blocks: + x = block(x, c, mask=attn_mask, **kwargs) + return self.output_layer(x, c, **kwargs) diff --git a/vllm_omni/model_executor/models/dots_tts/dots_tts_patch_encoder.py b/vllm_omni/model_executor/models/dots_tts/dots_tts_patch_encoder.py new file mode 100644 index 000000000..39d1a4973 --- /dev/null +++ b/vllm_omni/model_executor/models/dots_tts/dots_tts_patch_encoder.py @@ -0,0 +1,355 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 The vLLM-Omni team. +# Copyright (c) rednote-hilab. All rights reserved. +# Adapted from: +# https://github.com/rednote-hilab/dots.tts/tree/a393d2e/src/dots_tts/modules/backbone +# +# Single upstream source (preserved as-is, inference behaviour unchanged; +# only the internal `from dots_tts.modules.backbone.layers import ...` line +# was folded to point at our sibling vendor files): +# - modules/backbone/semantic_encoder.py -> SemanticEncoderDecodeState, +# TransformerEncoderLayer, +# SuperviseEncoder, +# VAESemanticEncoder +# +# Building blocks (Conv1d / Mlp / MultiHeadAttention) are imported from the +# sibling vendor files where they already live: +# - Conv1d <- dots_tts_vocoder.py +# - Mlp <- dots_tts_dit.py +# - MultiHeadAttention <- dots_tts_dit.py +# +# Nothing else in this file diverges from upstream behaviour. + +from __future__ import annotations + +from dataclasses import dataclass + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange + +from vllm_omni.model_executor.models.dots_tts.dots_tts_dit import ( + Mlp, + MultiHeadAttention, +) +from vllm_omni.model_executor.models.dots_tts.dots_tts_vocoder import Conv1d + + +@dataclass +class SemanticEncoderDecodeState: + conv_tail: torch.Tensor + layer_caches: tuple[tuple[torch.Tensor, torch.Tensor], ...] + seq_len: int + + +class TransformerEncoderLayer(nn.Module): + def __init__( + self, + hidden_size, + num_heads=16, + ffn_hidden_size=4096, + attn_dropout=0.0, + ffn_dropout=0.0, + norm_layer="LayerNorm", + **kwargs, + ): + super().__init__() + self.attn = MultiHeadAttention( + hidden_size, + num_heads, + attn_drop=attn_dropout, + norm_layer=norm_layer, + **kwargs, + ) + norm_cls = getattr(nn, norm_layer) + self.attn_norm = norm_cls(hidden_size) + self.ffn = Mlp(hidden_size, ffn_hidden_size, dropout=ffn_dropout, act_layer=nn.SiLU) + self.ffn_norm = norm_cls(hidden_size) + self.hidden_size = hidden_size + + def _build_causal_mask(self, T: int, device): # noqa: N803 + return torch.tril(torch.ones(T, T, dtype=torch.bool, device=device)) + + def _build_padding_mask(self, x_lens, max_len: int, device): + B = x_lens.size(0) + positions = torch.arange(max_len, device=device).unsqueeze(0).expand(B, -1) + return positions < x_lens.unsqueeze(1) + + def _fuse_attn_mask(self, causal_mask, padding_mask): + if causal_mask is None and padding_mask is None: + return None + if causal_mask is None: + row = padding_mask.unsqueeze(2) + col = padding_mask.unsqueeze(1) + return row & col + if padding_mask is None: + return causal_mask.unsqueeze(0) + + _B, _T = padding_mask.shape + causal = causal_mask.unsqueeze(0) + row = padding_mask.unsqueeze(2) + col = padding_mask.unsqueeze(1) + pad_2d = row & col + return causal & pad_2d + + def forward( + self, + x, + x_lens=None, + causal=True, + ): + _B, T, C = x.shape + assert self.hidden_size == C + device = x.device + + causal_mask = self._build_causal_mask(T, device) if causal else None + if x_lens is not None: + padding_mask = self._build_padding_mask(x_lens, T, device) + else: + padding_mask = None + fused_mask = self._fuse_attn_mask(causal_mask, padding_mask) + + h = self.attn_norm(x) + h = self.attn( + q=h, + mask=fused_mask, + ) + x = x + h + + h = self.ffn_norm(x) + h = self.ffn(h) + return x + h + + def decode_step( + self, + x, + *, + cache: tuple[torch.Tensor, torch.Tensor], + positions: torch.Tensor, + ): + if x.size(1) <= 0: + raise ValueError("TransformerEncoderLayer.decode_step expects a non-empty input.") + + h = self.attn_norm(x) + h, cache = self.attn.decode_step(h, cache=cache, positions=positions) + x = x + h + + h = self.ffn_norm(x) + h = self.ffn(h) + return x + h, cache + + +class SuperviseEncoder(nn.Module): + def __init__(self, config): + super().__init__() + self.hidden_size = config.get("hidden_size", 1024) + self.layers = nn.ModuleList( + [ + TransformerEncoderLayer( + hidden_size=self.hidden_size, + num_heads=config.get("num_heads", 16), + ffn_hidden_size=config.get("ffn_hidden_size", 4096), + norm_layer=config.get("norm_layer", "LayerNorm"), + ) + for _ in range(config.get("num_layers", 6)) + ] + ) + self.causal = config.get("causal", False) + + def forward(self, x, x_lens=None): + batch_size, seq_len, _ = x.shape + if x_lens is None: + x_lens = torch.full((batch_size,), seq_len, device=x.device, dtype=torch.long) + for layer in self.layers: + x = layer(x, x_lens=x_lens, causal=self.causal) + return x + + def init_decode_state( + self, + *, + batch_size: int, + max_seq_len: int, + device: torch.device, + dtype: torch.dtype, + ): + layer_caches = [] + for layer in self.layers: + cache_shape = ( + batch_size, + layer.attn.num_heads, + max_seq_len, + layer.attn.head_dim, + ) + layer_caches.append( + ( + torch.zeros(cache_shape, dtype=dtype, device=device), + torch.zeros(cache_shape, dtype=dtype, device=device), + ) + ) + return tuple(layer_caches) + + def reset_decode_state( + self, + layer_caches: tuple[tuple[torch.Tensor, torch.Tensor], ...], + ) -> None: + if len(layer_caches) != len(self.layers): + raise ValueError("Layer cache count does not match encoder depth.") + for key_cache, value_cache in layer_caches: + key_cache.zero_() + value_cache.zero_() + + def decode_step(self, x, *, layer_caches, positions: torch.Tensor): + if len(layer_caches) != len(self.layers): + raise ValueError("Layer cache count does not match encoder depth.") + + for layer, cache in zip(self.layers, layer_caches, strict=True): + x, _ = layer.decode_step(x, cache=cache, positions=positions) + return x + + +class VAESemanticEncoder(nn.Module): + def __init__(self, in_dim, out_dim, config): + super().__init__() + in_ds_rate = 2 + self.patch_size = int(config.patch_size) + self.in_ds_rate = in_ds_rate + self.ds_proj = Conv1d(in_dim, in_dim, kernel_size=in_ds_rate, stride=in_ds_rate, causal=True) + self.in_proj = nn.Linear(in_dim, config.PatchEncoder.hidden_size) + self.encoder = SuperviseEncoder(config.PatchEncoder) + self.out_ds_rate = self.patch_size // in_ds_rate + self.out_proj = nn.Linear(config.PatchEncoder.hidden_size * self.out_ds_rate, out_dim) + + def forward(self, x, x_lens=None): + x = self._downsample(x) + x = self.in_proj(x) + z = self.encoder(x, x_lens=x_lens) + return self._project_embeddings(z) + + def init_decode_state( + self, + *, + max_audio_patch_count: int, + batch_size: int, + device: torch.device, + dtype: torch.dtype, + ) -> SemanticEncoderDecodeState: + return SemanticEncoderDecodeState( + conv_tail=torch.zeros( + (batch_size, self.ds_proj.in_channels, self.ds_proj.left_padding), + dtype=dtype, + device=device, + ), + layer_caches=self.encoder.init_decode_state( + batch_size=batch_size, + max_seq_len=max_audio_patch_count * self.out_ds_rate, + device=device, + dtype=dtype, + ), + seq_len=0, + ) + + def reset_decode_state(self, state: SemanticEncoderDecodeState) -> None: + state.conv_tail.zero_() + self.encoder.reset_decode_state(state.layer_caches) + state.seq_len = 0 + + def prefill( + self, + x, + state: SemanticEncoderDecodeState, + ) -> tuple[torch.Tensor, SemanticEncoderDecodeState]: + if x.ndim != 3: + raise ValueError(f"VAESemanticEncoder.prefill expects rank-3 input, got {tuple(x.shape)}.") + if x.size(1) % self.patch_size != 0: + raise ValueError(f"Prompt latent length {x.size(1)} must be divisible by patch_size={self.patch_size}.") + + if x.size(1) == 0: + return ( + x.new_zeros((x.size(0), 0, self.out_proj.out_features)), + state, + ) + if state.conv_tail.size(0) != x.size(0): + raise ValueError("VAESemanticEncoder.prefill batch size does not match decode state.") + + step_inputs = self.in_proj(self._downsample(x)) + expected_token_count = (x.size(1) // self.patch_size) * self.out_ds_rate + if step_inputs.size(1) != expected_token_count: + raise RuntimeError( + "Patch encoder prefill produced an unexpected token count: " + f"expected={expected_token_count} actual={step_inputs.size(1)}." + ) + + current_seq_len = state.seq_len + next_seq_len = current_seq_len + step_inputs.size(1) + cache_capacity = state.layer_caches[0][0].size(2) + if next_seq_len > cache_capacity: + raise ValueError( + "Patch encoder prefill exceeds decode-state capacity: " + f"required={next_seq_len} capacity={cache_capacity}." + ) + + positions = torch.arange(step_inputs.size(1), device=x.device, dtype=torch.long) + current_seq_len + encoded = self.encoder.decode_step( + step_inputs, + layer_caches=state.layer_caches, + positions=positions, + ) + embedding = self._project_embeddings(encoded) + raw = x.transpose(1, 2) + state.conv_tail.copy_(raw[..., -self.ds_proj.left_padding :]) + state.seq_len = next_seq_len + return embedding, state + + def decode_patch( + self, + latent_patch, + conv_tail: torch.Tensor, + layer_caches: tuple[tuple[torch.Tensor, torch.Tensor], ...], + positions: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + if latent_patch.ndim != 3: + raise ValueError(f"VAESemanticEncoder.decode_patch expects rank-3 input, got {tuple(latent_patch.shape)}.") + if latent_patch.size(1) != self.patch_size: + raise ValueError(f"decode_patch expects patch length {self.patch_size}, got {latent_patch.size(1)}.") + if positions.ndim != 1 or positions.size(0) != self.out_ds_rate: + raise ValueError("decode_patch positions must be a rank-1 tensor matching out_ds_rate.") + + step_inputs, conv_tail = self._downsample_step( + latent_patch, + conv_tail=conv_tail, + ) + if step_inputs.size(1) != self.out_ds_rate: + raise RuntimeError(f"Downsample step produced {step_inputs.size(1)} tokens, expected {self.out_ds_rate}.") + + encoded = self.encoder.decode_step( + step_inputs, + layer_caches=layer_caches, + positions=positions, + ) + embedding = self._project_embeddings(encoded) + return embedding, conv_tail + + def _downsample(self, x): + return self.ds_proj(x.transpose(1, 2)).transpose(1, 2) + + def _project_embeddings(self, z): + if self.out_ds_rate > 1: + z = rearrange(z, "b (s d) h -> b s (d h)", d=self.out_ds_rate) + return self.out_proj(z) + + def _downsample_step(self, latent_patch, *, conv_tail): + raw = latent_patch.transpose(1, 2) + conv_input = torch.cat([conv_tail, raw], dim=-1) + + projected = F.conv1d( + conv_input, + self.ds_proj.weight, + self.ds_proj.bias, + stride=self.ds_proj.stride[0], + padding=0, + dilation=self.ds_proj.dilation[0], + groups=self.ds_proj.groups, + ).transpose(1, 2) + new_conv_tail = raw[..., -self.ds_proj.left_padding :] + return self.in_proj(projected), new_conv_tail diff --git a/vllm_omni/model_executor/models/dots_tts/dots_tts_prompt.py b/vllm_omni/model_executor/models/dots_tts/dots_tts_prompt.py new file mode 100644 index 000000000..a2c4ac9f9 --- /dev/null +++ b/vllm_omni/model_executor/models/dots_tts/dots_tts_prompt.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""dots.tts prompt builder for vllm-omni's Omni engine + serving layer. + +Mirrors upstream ``build_generation_schedule`` (rednote-hilab/dots.tts @ +a393d2e data/pipelines/tokenizing.py:169) for the +no-prompt-audio + audio-placeholder path — text-only zero-shot TTS. + +V1 returns real Qwen2 token IDs (not placeholders ``[1] * N`` like +voxcpm2's builder). Reason: our talker's preprocess() runs +``self.model.embed_tokens(input_ids)`` directly, so real IDs flow +through naturally. voxcpm2 uses placeholders because its prefill mixes +reference-audio embeddings (no token IDs); dots.tts V1 zero-shot is +purely text + audio_gen_start, all real IDs. + +Voice clone (reference audio + speaker x-vector) is intentionally not +wired here — V1 is zero-shot only. Add reference_audio / +voice_profile / prompt_text branches in a follow-up (mirror voxcpm2 +``build_voxcpm2_prompt``). +""" + +from __future__ import annotations + +from typing import Any + +from vllm.inputs import tokens_input + +# Upstream rednote-hilab/dots.tts @ a393d2e +# data/pipelines/tts_pipeline.py:17-18 +_TTS_TEXT_PREFIX = "[文本]" +_TTS_AUDIO_PREFIX = "[文本对应语音]" +_AUDIO_GEN_START_TOKEN = "<|audio_gen_start|>" + + +def build_dots_tts_prompt( + tokenizer: Any, + text: str, +) -> dict[str, Any]: + """Build a dots.tts prefill prompt dict for ``Omni.generate()``. + + Returns ``{"prompt_token_ids": [...], "additional_information": {...}}`` + where ``prompt_token_ids`` is the real Qwen2 token sequence: + + [text_tokens of "[文本][文本对应语音]", audio_gen_start_id] + + talker's preprocess() embeds these into ``hidden_states`` for the AR + base LM. Decode then drives via ``state.curr_embed_for_next`` (DiT + → patch_encoder AR loopback). + """ + if not isinstance(text, str) or not text.strip(): + raise ValueError(f"text must be a non-empty string, got {text!r}") + + prompt_text = f"{_TTS_TEXT_PREFIX}{text}{_TTS_AUDIO_PREFIX}" + text_ids = tokenizer.encode(prompt_text, add_special_tokens=False) + + audio_gen_start_id = tokenizer.convert_tokens_to_ids(_AUDIO_GEN_START_TOKEN) + if not isinstance(audio_gen_start_id, int) or audio_gen_start_id == tokenizer.unk_token_id: + raise ValueError( + f"Tokenizer does not know {_AUDIO_GEN_START_TOKEN!r} " + f"(got id={audio_gen_start_id!r}). Did you load the " + f"dots.tts tokenizer (with added_tokens.json)?" + ) + prompt_token_ids = list(text_ids) + [audio_gen_start_id] + + prompt = tokens_input(prompt_token_ids=prompt_token_ids) + prompt["additional_information"] = { + # Reserved for future voice clone (ref_audio / speaker x-vector). + # V1 zero-shot leaves this empty; preprocess() ignores it. + } + return prompt diff --git a/vllm_omni/model_executor/models/dots_tts/dots_tts_speaker_encoder.py b/vllm_omni/model_executor/models/dots_tts/dots_tts_speaker_encoder.py new file mode 100644 index 000000000..09961d0ab --- /dev/null +++ b/vllm_omni/model_executor/models/dots_tts/dots_tts_speaker_encoder.py @@ -0,0 +1,732 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 The vLLM-Omni team. +# Copyright (c) rednote-hilab. All rights reserved. +# Portions Copyright 3D-Speaker +# (https://github.com/alibaba-damo-academy/3D-Speaker), Apache-2.0. +# Adapted from: +# https://github.com/rednote-hilab/dots.tts/tree/a393d2e/src/dots_tts/modules/speaker +# (whose CAM++ implementation is itself adapted from 3D-Speaker). +# +# Merged from four upstream files (preserved as-is, inference behaviour +# unchanged; only internal cross-module imports were folded inline): +# - modules/speaker/encoder.py -> SpeakerXVectorFeatures +# - modules/speaker/fbank.py -> fbank constants + extract_speaker_fbank +# - modules/speaker/campplus.py -> FCM, CAMPPlus +# - modules/speaker/campplus_layers.py -> get_nonlinear, statistics_pooling, +# StatsPool, TDNNLayer, CAMLayer, +# CAMDenseTDNNLayer, CAMDenseTDNNBlock, +# TransitLayer, DenseLayer, BasicResBlock +# +# Plus two tiny helpers folded from utils/audio.py (used by +# extract_speaker_fbank): +# - high_quality_resample, extract_fbank +# +# Nothing in this file diverges from upstream behaviour. + +from __future__ import annotations + +import math +import random +from collections import OrderedDict + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as cp +import torchaudio +import torchaudio.compliance.kaldi as Kaldi +import torchaudio.functional as AF +from torch.nn.utils.rnn import pad_sequence + +# ============================================================================ +# Constants (from modules/speaker/fbank.py) +# ============================================================================ + +_SPEAKER_FBANK_SAMPLE_RATE = 16000 +_SPEAKER_FBANK_N_MELS = 80 +_SPEAKER_FBANK_MEAN_NORM = True +_SPEAKER_FBANK_DITHER = 0.0 + + +# ============================================================================ +# Audio helpers (folded from utils/audio.py — used by extract_speaker_fbank) +# ============================================================================ + + +def high_quality_resample(x, orig_sr, target_sr): + return AF.resample( + x, + orig_freq=orig_sr, + new_freq=target_sr, + lowpass_filter_width=64, + rolloff=0.95, + resampling_method="sinc_interp_kaiser", + ) + + +def extract_fbank( + waveform: torch.Tensor, + *, + sample_rate: int, + n_mels: int, + dither: float = 0.0, + mean_norm: bool = False, +) -> torch.Tensor: + if waveform.ndim == 1: + feature_input = waveform.unsqueeze(0) + elif waveform.ndim == 2: + feature_input = waveform if waveform.size(0) == 1 else waveform[0:1, :] + else: + raise ValueError(f"FBank expects a 1D or 2D waveform, got shape {tuple(waveform.shape)}.") + features = Kaldi.fbank( + feature_input, + num_mel_bins=n_mels, + sample_frequency=sample_rate, + dither=dither, + ) + if mean_norm: + features = features - features.mean(dim=0, keepdim=True) + return features + + +# ============================================================================ +# Speaker fbank entry (from modules/speaker/fbank.py) +# ============================================================================ + + +def extract_speaker_fbank( + waveform: torch.Tensor, + *, + sample_rate: int, +) -> torch.Tensor: + feature_input = waveform + if sample_rate != _SPEAKER_FBANK_SAMPLE_RATE: + feature_input = high_quality_resample( + waveform, + orig_sr=sample_rate, + target_sr=_SPEAKER_FBANK_SAMPLE_RATE, + ) + return extract_fbank( + feature_input, + sample_rate=_SPEAKER_FBANK_SAMPLE_RATE, + n_mels=_SPEAKER_FBANK_N_MELS, + dither=_SPEAKER_FBANK_DITHER, + mean_norm=_SPEAKER_FBANK_MEAN_NORM, + ) + + +# ============================================================================ +# CAM++ building blocks (from modules/speaker/campplus_layers.py) +# ---------------------------------------------------------------------------- +# 3D-Speaker, Apache-2.0. +# ============================================================================ + + +def get_nonlinear(config_str, channels): + nonlinear = nn.Sequential() + for name in config_str.split("-"): + if name == "relu": + nonlinear.add_module("relu", nn.ReLU(inplace=True)) + elif name == "prelu": + nonlinear.add_module("prelu", nn.PReLU(channels)) + elif name == "batchnorm": + nonlinear.add_module("batchnorm", nn.BatchNorm1d(channels)) + elif name == "batchnorm_": + nonlinear.add_module("batchnorm", nn.BatchNorm1d(channels, affine=False)) + else: + raise ValueError(f"Unexpected module ({name}).") + return nonlinear + + +def statistics_pooling(x, dim=-1, keepdim=False, unbiased=True, _eps=1e-2): + mean = x.mean(dim=dim) + std = x.std(dim=dim, unbiased=unbiased) + stats = torch.cat([mean, std], dim=-1) + if keepdim: + stats = stats.unsqueeze(dim=dim) + return stats + + +class StatsPool(nn.Module): + def forward(self, x): + return statistics_pooling(x) + + +class TDNNLayer(nn.Module): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + bias=False, + config_str="batchnorm-relu", + ): + super().__init__() + if padding < 0: + assert kernel_size % 2 == 1, f"Expect equal paddings, but got even kernel size ({kernel_size})" + padding = (kernel_size - 1) // 2 * dilation + self.linear = nn.Conv1d( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + ) + self.nonlinear = get_nonlinear(config_str, out_channels) + + def forward(self, x): + x = self.linear(x) + return self.nonlinear(x) + + +class CAMLayer(nn.Module): + def __init__( + self, + bn_channels, + out_channels, + kernel_size, + stride, + padding, + dilation, + bias, + reduction=2, + ): + super().__init__() + self.linear_local = nn.Conv1d( + bn_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + ) + self.linear1 = nn.Conv1d(bn_channels, bn_channels // reduction, 1) + self.relu = nn.ReLU(inplace=True) + self.linear2 = nn.Conv1d(bn_channels // reduction, out_channels, 1) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + y = self.linear_local(x) + context = x.mean(-1, keepdim=True) + self.seg_pooling(x) + context = self.relu(self.linear1(context)) + m = self.sigmoid(self.linear2(context)) + return y * m + + def seg_pooling(self, x, seg_len=100, stype="avg"): + if stype == "avg": + seg = F.avg_pool1d(x, kernel_size=seg_len, stride=seg_len, ceil_mode=True) + elif stype == "max": + seg = F.max_pool1d(x, kernel_size=seg_len, stride=seg_len, ceil_mode=True) + else: + raise ValueError("Wrong segment pooling type.") + shape = seg.shape + seg = seg.unsqueeze(-1).expand(*shape, seg_len).reshape(*shape[:-1], -1) + return seg[..., : x.shape[-1]] + + +class CAMDenseTDNNLayer(nn.Module): + def __init__( + self, + in_channels, + out_channels, + bn_channels, + kernel_size, + stride=1, + dilation=1, + bias=False, + config_str="batchnorm-relu", + memory_efficient=False, + ): + super().__init__() + assert kernel_size % 2 == 1, f"Expect equal paddings, but got even kernel size ({kernel_size})" + padding = (kernel_size - 1) // 2 * dilation + self.memory_efficient = memory_efficient + self.nonlinear1 = get_nonlinear(config_str, in_channels) + self.linear1 = nn.Conv1d(in_channels, bn_channels, 1, bias=False) + self.nonlinear2 = get_nonlinear(config_str, bn_channels) + self.cam_layer = CAMLayer( + bn_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + ) + + def bn_function(self, x): + return self.linear1(self.nonlinear1(x)) + + def forward(self, x): + if self.training and self.memory_efficient: + x = cp.checkpoint(self.bn_function, x) + else: + x = self.bn_function(x) + return self.cam_layer(self.nonlinear2(x)) + + +class CAMDenseTDNNBlock(nn.ModuleList): + def __init__( + self, + num_layers, + in_channels, + out_channels, + bn_channels, + kernel_size, + stride=1, + dilation=1, + bias=False, + config_str="batchnorm-relu", + memory_efficient=False, + ): + super().__init__() + for i in range(num_layers): + layer = CAMDenseTDNNLayer( + in_channels=in_channels + i * out_channels, + out_channels=out_channels, + bn_channels=bn_channels, + kernel_size=kernel_size, + stride=stride, + dilation=dilation, + bias=bias, + config_str=config_str, + memory_efficient=memory_efficient, + ) + self.add_module(f"tdnnd{i + 1}", layer) + + def forward(self, x): + for layer in self: + x = torch.cat([x, layer(x)], dim=1) + return x + + +class TransitLayer(nn.Module): + def __init__(self, in_channels, out_channels, bias=True, config_str="batchnorm-relu"): + super().__init__() + self.nonlinear = get_nonlinear(config_str, in_channels) + self.linear = nn.Conv1d(in_channels, out_channels, 1, bias=bias) + + def forward(self, x): + x = self.nonlinear(x) + return self.linear(x) + + +class DenseLayer(nn.Module): + def __init__(self, in_channels, out_channels, bias=False, config_str="batchnorm-relu"): + super().__init__() + self.linear = nn.Conv1d(in_channels, out_channels, 1, bias=bias) + self.nonlinear = get_nonlinear(config_str, out_channels) + + def forward(self, x): + if len(x.shape) == 2: + x = self.linear(x.unsqueeze(dim=-1)).squeeze(dim=-1) + else: + x = self.linear(x) + return self.nonlinear(x) + + +class BasicResBlock(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super().__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=(stride, 1), padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d( + in_planes, + self.expansion * planes, + kernel_size=1, + stride=(stride, 1), + bias=False, + ), + nn.BatchNorm2d(self.expansion * planes), + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + out += self.shortcut(x) + return F.relu(out) + + +# ============================================================================ +# CAM++ model (from modules/speaker/campplus.py) +# ---------------------------------------------------------------------------- +# 3D-Speaker, Apache-2.0. +# ============================================================================ + + +class FCM(nn.Module): + def __init__( + self, + block=BasicResBlock, + num_blocks=(2, 2), + m_channels=32, + feat_dim=_SPEAKER_FBANK_N_MELS, + ): + super().__init__() + self.in_planes = m_channels + self.conv1 = nn.Conv2d(1, m_channels, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(m_channels) + + self.layer1 = self._make_layer(block, m_channels, num_blocks[0], stride=2) + self.layer2 = self._make_layer(block, m_channels, num_blocks[1], stride=2) + + self.conv2 = nn.Conv2d(m_channels, m_channels, kernel_size=3, stride=(2, 1), padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(m_channels) + self.out_channels = m_channels * (feat_dim // 8) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + x = x.unsqueeze(1) + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + out = F.relu(self.bn2(self.conv2(out))) + + shape = out.shape + return out.reshape(shape[0], shape[1] * shape[2], shape[3]) + + +class CAMPPlus(nn.Module): + _TDNN_KERNEL_SIZE = 5 + _TDNN_STRIDE = 2 + _TDNN_PADDING = 2 + + def __init__( + self, + feat_dim=_SPEAKER_FBANK_N_MELS, + embedding_size=512, + growth_rate=32, + bn_size=4, + init_channels=128, + config_str="batchnorm-relu", + memory_efficient=True, + ): + super().__init__() + + self.head = FCM(feat_dim=feat_dim) + channels = self.head.out_channels + + self.xvector = nn.Sequential( + OrderedDict( + [ + ( + "tdnn", + TDNNLayer( + channels, + init_channels, + self._TDNN_KERNEL_SIZE, + stride=self._TDNN_STRIDE, + dilation=1, + padding=-1, + config_str=config_str, + ), + ), + ] + ) + ) + channels = init_channels + for i, (num_layers, kernel_size, dilation) in enumerate(zip((12, 24, 16), (3, 3, 3), (1, 2, 2), strict=True)): + block = CAMDenseTDNNBlock( + num_layers=num_layers, + in_channels=channels, + out_channels=growth_rate, + bn_channels=bn_size * growth_rate, + kernel_size=kernel_size, + dilation=dilation, + config_str=config_str, + memory_efficient=memory_efficient, + ) + self.xvector.add_module(f"block{i + 1}", block) + channels = channels + num_layers * growth_rate + self.xvector.add_module( + f"transit{i + 1}", + TransitLayer(channels, channels // 2, bias=False, config_str=config_str), + ) + channels //= 2 + + self.xvector.add_module("out_nonlinear", get_nonlinear(config_str, channels)) + + self.xvector.add_module("stats", StatsPool()) + self.xvector.add_module("dense", DenseLayer(channels * 2, embedding_size, config_str="batchnorm_")) + + for m in self.modules(): + if isinstance(m, (nn.Conv1d, nn.Linear)): + nn.init.kaiming_normal_(m.weight.data) + if m.bias is not None: + nn.init.zeros_(m.bias) + + @staticmethod + def _conv_output_lengths(lengths, kernel_size, stride=1, padding=0, dilation=1): + return ( + torch.div( + lengths + 2 * padding - dilation * (kernel_size - 1) - 1, + stride, + rounding_mode="floor", + ) + + 1 + ) + + @staticmethod + def _make_length_mask(lengths, max_len, device): + lengths = lengths.to(device=device, dtype=torch.long).clamp(min=0, max=max_len) + return torch.arange(max_len, device=device).unsqueeze(0) < lengths.unsqueeze(1) + + def _masked_stats_pooling(self, x, lengths, unbiased=True, eps=1e-2): + lengths = lengths.to(device=x.device, dtype=torch.long).clamp(min=1, max=x.size(-1)) + mask = self._make_length_mask(lengths, x.size(-1), x.device).unsqueeze(1) + mask = mask.to(dtype=x.dtype) + + denom = lengths.to(dtype=x.dtype).view(-1, 1).clamp_min(1.0) + mean = (x * mask).sum(dim=-1) / denom + + centered = (x - mean.unsqueeze(-1)) * mask + var_denom = (lengths - 1).clamp_min(1).to(dtype=x.dtype).view(-1, 1) if unbiased else denom + var = centered.pow(2).sum(dim=-1) / var_denom + std = torch.sqrt(var.clamp_min(eps)) + return torch.cat([mean, std], dim=1) + + def forward(self, x, lengths=None): + x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T) + x = self.head(x) + if lengths is not None: + lengths = lengths.to(device=x.device, dtype=torch.long).clamp(min=1) + + for name, module in self.xvector.named_children(): + if name == "stats": + x = self._masked_stats_pooling(x, lengths) if lengths is not None else module(x) + continue + + x = module(x) + if name == "tdnn" and lengths is not None: + lengths = self._conv_output_lengths( + lengths, + kernel_size=self._TDNN_KERNEL_SIZE, + stride=self._TDNN_STRIDE, + padding=self._TDNN_PADDING, + ) + + return x + + +# ============================================================================ +# SpeakerXVectorFeatures wrapper (from modules/speaker/encoder.py) +# ============================================================================ + + +class SpeakerXVectorFeatures(nn.Module): + """Speaker embedding extractor based on 3D-Speaker CAM++.""" + + def __init__( + self, + sample_rate=_SPEAKER_FBANK_SAMPLE_RATE, + campplus_embedding_size=512, + max_audio_seconds=10.0, + ): + super().__init__() + + self.sample_rate = sample_rate + self.max_audio_seconds = float(max_audio_seconds) + self.model = CAMPPlus( + feat_dim=_SPEAKER_FBANK_N_MELS, + embedding_size=campplus_embedding_size, + ) + self.resample = None + if self.sample_rate != _SPEAKER_FBANK_SAMPLE_RATE: + self.resample = torchaudio.transforms.Resample( + orig_freq=sample_rate, + new_freq=_SPEAKER_FBANK_SAMPLE_RATE, + ) + + for param in self.model.parameters(): + param.requires_grad = False + + @staticmethod + def _normalize_lengths(lengths, batch_size, max_length, device, *, min_length): + if lengths is None: + return torch.full( + (batch_size,), + max_length, + device=device, + dtype=torch.long, + ) + return lengths.to(device=device, dtype=torch.long).clamp( + min=min_length, + max=max_length, + ) + + def _crop_audio(self, audio, audio_lengths=None): + original_lengths = self._normalize_lengths( + audio_lengths, + audio.size(0), + audio.size(-1), + audio.device, + min_length=0, + ) + if self.max_audio_seconds <= 0: + return audio, original_lengths, original_lengths, torch.zeros_like(original_lengths) + + max_input_length = round(self.sample_rate * self.max_audio_seconds) + cropped_audio = [] + cropped_lengths = [] + starts = [] + + for index, total_length_tensor in enumerate(original_lengths): + total_length = int(total_length_tensor.item()) + cropped_length = min(total_length, max_input_length) + start = random.randint(0, total_length - cropped_length) if total_length > cropped_length else 0 + cropped_audio.append(audio[index, start : start + cropped_length]) + cropped_lengths.append(cropped_length) + starts.append(start) + + return ( + pad_sequence( + cropped_audio, + batch_first=True, + padding_value=0.0, + ), + original_lengths, + torch.tensor( + cropped_lengths, + device=audio.device, + dtype=torch.long, + ), + torch.tensor(starts, device=audio.device, dtype=torch.long), + ) + + def _crop_fbank( + self, + fbank, + fbank_lengths, + original_audio_lengths, + cropped_audio_lengths, + starts, + ): + original_fbank_lengths = self._normalize_lengths( + fbank_lengths, + fbank.size(0), + fbank.size(1), + fbank.device, + min_length=1, + ) + cropped_fbank = [] + cropped_fbank_lengths = [] + + for index, total_feat_length_tensor in enumerate(original_fbank_lengths): + total_audio_length = int(original_audio_lengths[index].item()) + total_feat_length = int(total_feat_length_tensor.item()) + start_audio = int(starts[index].item()) + end_audio = start_audio + int(cropped_audio_lengths[index].item()) + + if total_audio_length > 0: + start_feat = math.floor(start_audio * total_feat_length / total_audio_length) + end_feat = math.ceil(end_audio * total_feat_length / total_audio_length) + else: + start_feat = 0 + end_feat = 1 + + start_feat = min(start_feat, total_feat_length - 1) + end_feat = min(max(end_feat, start_feat + 1), total_feat_length) + cropped_fbank.append(fbank[index, start_feat:end_feat]) + cropped_fbank_lengths.append(end_feat - start_feat) + + return pad_sequence( + cropped_fbank, + batch_first=True, + padding_value=0.0, + ), torch.tensor( + cropped_fbank_lengths, + device=fbank.device, + dtype=torch.long, + ) + + def _extract_fbank_batch(self, audio, audio_lengths): + if self.resample is not None: + audio = self.resample(audio) + audio_lengths = torch.ceil(audio_lengths.float() * (_SPEAKER_FBANK_SAMPLE_RATE / self.sample_rate)).long() + + audio_cpu = audio.detach().cpu() + features = [] + + for index, valid_length_tensor in enumerate(audio_lengths): + valid_length = int(valid_length_tensor.item()) + waveform = audio_cpu[index, :valid_length] + if waveform.numel() == 0: + waveform = audio_cpu.new_zeros(1) + features.append( + extract_speaker_fbank( + waveform, + sample_rate=_SPEAKER_FBANK_SAMPLE_RATE, + ) + ) + + fbank_lengths = torch.tensor( + [feature.size(0) for feature in features], + device=audio.device, + dtype=torch.long, + ) + fbank = pad_sequence( + features, + batch_first=True, + padding_value=0.0, + ).to(device=audio.device, dtype=audio.dtype) + return fbank, fbank_lengths + + @torch.no_grad() + @torch.autocast(enabled=False, device_type="cuda") + def forward(self, audio, audio_lengths=None, fbank=None, fbank_lengths=None, **_kwargs): + self.model.eval() + audio = audio.float() + if audio.dim() == 3: + if audio.size(1) != 1: + raise ValueError(f"Speaker encoder expects mono audio, got shape {tuple(audio.shape)}.") + audio = audio[:, 0] + elif audio.dim() != 2: + raise ValueError(f"Speaker encoder expects a 2D or 3D audio tensor, got shape {tuple(audio.shape)}.") + + audio, original_audio_lengths, cropped_audio_lengths, starts = self._crop_audio( + audio, + audio_lengths=audio_lengths, + ) + + if fbank is None: + fbank, fbank_lengths = self._extract_fbank_batch( + audio, + cropped_audio_lengths, + ) + else: + if not isinstance(fbank, torch.Tensor): + raise TypeError("Speaker encoder expects `fbank` to be a torch.Tensor.") + if fbank.dim() != 3 or fbank.size(0) != audio.size(0): + raise ValueError( + f"Speaker encoder expects `fbank` with shape (B, T, F) and matching batch size, got {tuple(fbank.shape)}." # noqa: E501 + ) + fbank, fbank_lengths = self._crop_fbank( + fbank.to(device=audio.device, dtype=torch.float32), + fbank_lengths, + original_audio_lengths, + cropped_audio_lengths, + starts, + ) + + return self.model(fbank, lengths=fbank_lengths) diff --git a/vllm_omni/model_executor/models/dots_tts/dots_tts_talker.py b/vllm_omni/model_executor/models/dots_tts/dots_tts_talker.py new file mode 100644 index 000000000..368a3fd1e --- /dev/null +++ b/vllm_omni/model_executor/models/dots_tts/dots_tts_talker.py @@ -0,0 +1,1426 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""dots.tts talker — vLLM-native AR base LM + audio side path. + +Mirrors upstream rednote-hilab/dots.tts (pinned @ a393d2e): + + * Qwen2.5-1.5B base LM runs under vLLM PagedAttention (stock + ``Qwen2Model``); weights come from the checkpoint's ``llm.model.*`` + namespace. ``DotsTTSConfig`` hoists the LM fields to top-level + attributes. + * Everything else is side computation, invisible to the vLLM + scheduler and driven once per step by ``_finish_decode``: + DiT flow-matching head (N-step Euler, fp32 integration state, + deterministic per-request noise) → io_helper denormalize → + patch_encoder AR loopback (produces the next step's LLM input + embedding) + streaming AudioVAE decode (per-request + ``BigVGANStreamState`` sliding window, so every latent patch is + decoded with real left conv context) → 48 kHz wav chunks pushed to + ``_audio_queue`` → eos_proj stop signal via ``_results_queue`` + (threshold 0.8, upstream default; the stop step also drains the + vocoder's 2-frame lookahead tail via ``stream_flush``). + * Cross-step state is isolated per request in ``_RequestState`` + (keyed by request_id); eviction is deferred to the end of + ``forward()`` so a finishing request's audio still drains. + * The CAM++ speaker encoder is constructed and its weights load, but + voice cloning is not exposed yet — generation is zero-shot and the + DiT falls back to its null conditioning (``fm_null_g_cond``). + +Debugging: set ``DOTS_TTS_BETA_TRACE=1`` for an env-gated per-patch +rms/max trace across the full side path (zero overhead when unset). + +Reference implementations: ``vllm_omni/model_executor/models/voxcpm2/`` +(architectural template: vLLM-native base LM + side path + per-request +state) and ``ming_flash_omni`` (AudioVAE weight-loading pattern). +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import os +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Any + +import torch +import torch.nn as nn +from vllm.config import VllmConfig +from vllm.logger import init_logger +from vllm.model_executor.models.qwen2 import Qwen2Model +from vllm.model_executor.models.utils import AutoWeightsLoader, maybe_prefix +from vllm.sequence import IntermediateTensors + +from vllm_omni.model_executor.models.dots_tts.dots_tts_dit import DiT +from vllm_omni.model_executor.models.dots_tts.dots_tts_patch_encoder import ( + VAESemanticEncoder, +) +from vllm_omni.model_executor.models.dots_tts.dots_tts_speaker_encoder import ( + SpeakerXVectorFeatures, +) +from vllm_omni.model_executor.models.dots_tts.dots_tts_vocoder import ( + AudioVAE, + AudioVAEConfig, +) +from vllm_omni.model_executor.models.output_templates import OmniOutput + +logger = init_logger(__name__) + + +def _require_config_block(hf_config: Any, name: str) -> dict[str, Any]: + """Fetch a sub-config dict (``vocoder`` / ``DiT`` / ``PatchEncoder``) + from the checkpoint's config.json, failing loudly when absent — + silently falling back to another checkpoint's values would produce + garbage audio, not an error.""" + block = getattr(hf_config, name, None) + if not isinstance(block, dict) or not block: + raise ValueError( + f"dots.tts checkpoint config.json has no '{name}' block — cannot " + "build the audio side path. Expected the upstream dots.tts config " + "layout (e.g. rednote-hilab/dots.tts-soar)." + ) + return block + + +def _build_audio_vae_config(hf_config: Any) -> AudioVAEConfig: + # AudioVAEConfig declares exactly the checkpoint's 17 ``vocoder`` keys, + # so an unknown / missing key raises TypeError instead of guessing. + return AudioVAEConfig(**_require_config_block(hf_config, "vocoder")) + + +# Architecture constants used by the per-request FM workspace math. All +# released dots.tts checkpoints (soar / base / mf) share these values; +# _validate_architecture_constants refuses a checkpoint that disagrees +# instead of silently generating garbage. +_FM_HIDDEN = 1024 # DiT.hidden_size (also LLM↔DiT projection target) +_LATENT_DIM = 128 # AudioVAE.latent_dim +_LATENT_PATCH_SIZE = 4 # config.patch_size — DiT samples 4 latent frames / audio patch +_HIDDEN_PATCH_SIZE = 1 # upstream core.py hardcodes self.hidden_patch_size = 1 +_MAX_AUDIO_PATCHES = 1024 # ~164 s @ 4×1920/48k; bounds per-request FM static buffer +_DIT_NUM_STEPS = int( + os.environ.get("DOTS_TTS_DIT_NUM_STEPS", "10") +) # env-gated; upstream default 10 for fixed-step Euler +_DIT_GUIDANCE_SCALE = 1.2 # upstream default guidance_scale for soar +_DIT_NOISE_SEED = 20260601 # base seed for per-request FM noise (voxcpm2 parity) +_PATCH_ENCODER_OUT_DS_RATE = 2 # patch_size / in_ds_rate = 4 / 2 (VAESemanticEncoder hardcodes in_ds_rate=2) + + +class _IOHelper: + """Latent stats wrapper. Holds (mean, var) over the 128-dim VAE latent + space and normalizes / denormalizes between DiT-internal space (which + operates on normalized latents) and AudioVAE / patch_encoder input space + (which expects raw latents). + + Mirrors upstream IOHelper (rednote-hilab/dots.tts @ a393d2e + models/dots_tts/core.py:684). Plain class (not nn.Module) so it stays + out of state_dict() and load_weights() doesn't have to mute its buffers. + Stats are loaded from latent_stats.pt — independent of safetensors. + """ + + def __init__(self, latent_stats_path: str | None = None) -> None: + if latent_stats_path is None: + self.global_mean: torch.Tensor | None = None + self.global_var: torch.Tensor | None = None + return + stats = torch.load(latent_stats_path, weights_only=False) + self.global_mean = torch.as_tensor(stats["mean"]) + self.global_var = torch.as_tensor(stats["var"]) + + def normalize(self, x: torch.Tensor) -> torch.Tensor: + if self.global_mean is None or self.global_var is None: + return x + mean = self.global_mean.to(device=x.device, dtype=x.dtype) + var = self.global_var.to(device=x.device, dtype=x.dtype) + return (x - mean) / torch.sqrt(var) + + def denormalize(self, x: torch.Tensor) -> torch.Tensor: + if self.global_mean is None or self.global_var is None: + return x + mean = self.global_mean.to(device=x.device, dtype=x.dtype) + var = self.global_var.to(device=x.device, dtype=x.dtype) + return x * torch.sqrt(var) + mean + + +# Per-request state container for AR-loop continuity. All cross-step +# side-path state lives here, keyed by request_id, so concurrent requests +# stay fully isolated (the voxcpm2 _RequestState pattern). +@dataclass +class _RequestState: + request_id: str + # AR loopback: patch_encoder's previous decode output, fed to the LM + # as next-step inputs_embeds. Shape: [1, llm_hidden] = [1, 1536]. + # (patch_encoder collapses its internal _PATCH_ENCODER_OUT_DS_RATE + # positions into one LLM token via out_proj; see + # VAESemanticEncoder._project_embeddings.) _finish_decode writes a + # fresh tensor every call. + curr_embed_for_next: torch.Tensor | None = None + # Stop-signal cache: _finish_decode populates precomputed_stop_logits; + # compute_logits drains it. + precomputed_stop_logits: torch.Tensor | None = None + is_stopping: bool = False + prefill_completed: bool = False + # Per-request FM noise counter: draw #n of this request hashes to a + # deterministic Generator seed (see _run_dit_n_step_euler), so outputs + # are reproducible run-to-run and concurrent requests cannot perturb + # each other's noise streams (voxcpm2 _fill_deterministic_cfm_noise). + noise_step: int = 0 + # Per-request FM static workspace (lazy-allocated by + # _initialize_request_fm_state on first _finish_decode call). Sized for + # _MAX_AUDIO_PATCHES × (_HIDDEN_PATCH_SIZE + _LATENT_PATCH_SIZE) = 1024 × 5 + # = 5120 positions. + fm_sequence: torch.Tensor | None = None # [1, fm_capacity, _FM_HIDDEN] + fm_cfg_sequence: torch.Tensor | None = None # [1, fm_capacity, _FM_HIDDEN] + fm_null_g_cond: torch.Tensor | None = None # [1, _FM_HIDDEN] + fm_seq_len: int = 0 + fm_capacity: int = 0 + # Speaker x-vector after _xvec_proj. Zero-shot generation leaves this + # None → DiT falls back to fm_null_g_cond; voice-clone wiring is a + # follow-up. + g_cond: torch.Tensor | None = None + # AR-loop bookkeeping. + # patch_encoder_state holds conv_tail + per-layer KV caches for the + # patch_encoder's streaming decode (mirrors upstream state.patch_encoder_state). + # Lazy-allocated by _run_patch_encoder_loopback on first audio step. + patch_encoder_state: Any | None = None + # Review M2: streaming vocoder state (BigVGANStreamState — LSTM hidden + # + decoder sliding latent window). Gives each patch real left conv + # context; output lags input by decoder.stream_lookahead (2 frames = + # 3840 samples). Lazy-allocated by _run_vocoder_stream_step. + vocoder_stream_state: Any | None = None + + +# DiT transformer hyperparameters from rednote-hilab/dots.tts-soar config.json +# (the ``DiT`` block). Mirrors the shape of an upstream pydantic config: DiT's +# constructor calls ``transformer_config.to_dict()`` and reads ``.hidden_size`` +# / ``.num_layers``, so this @dataclass plus the ``to_dict`` helper satisfies +# that minimal interface without dragging in upstream's config system. +# +# Note: ``attn_dropout`` in the JSON spells differently from MultiHeadAttention's +# ``attn_drop`` kwarg — the value gets absorbed by ``**_kwargs`` and the MHA +# default 0.0 applies. Same numeric outcome for the soar checkpoint (both 0.0), +# so this is benign; flagging it because it would mask non-default values if +# upstream ever raised this field. +@dataclass +class _DiTConfig: + num_layers: int = 18 + num_heads: int = 16 + hidden_size: int = 1024 + ffn_hidden_size: int = 4096 + modulation: bool = True + qkv_bias: bool = False + qk_norm: bool = True + attn_dropout: float = 0.0 + dropout: float = 0.0 + norm_layer: str = "RMSNorm" + alibi_bias: bool = False + rotary_bias: bool = True + rotary_theta: float = 10000.0 + + def to_dict(self) -> dict[str, Any]: + return dataclasses.asdict(self) + + +def _build_dit_config(hf_config: Any) -> _DiTConfig: + # _DiTConfig declares exactly the checkpoint's 13 ``DiT`` keys, so an + # unknown key raises TypeError instead of being silently dropped. + return _DiTConfig(**_require_config_block(hf_config, "DiT")) + + +# Patch encoder hyperparameters from rednote-hilab/dots.tts-soar config.json +# (the top-level ``patch_size`` and the ``PatchEncoder`` sub-block). +# +# Two layers of config: +# * Outer (``_PatchEncoderConfig``): VAESemanticEncoder reads ``.patch_size`` +# and forwards ``.PatchEncoder`` to its inner SuperviseEncoder. +# * Inner (``_PatchEncoderInner``): the constructor uses BOTH attribute +# access (``config.PatchEncoder.hidden_size``) and dict-style +# ``config.get("hidden_size", 1024)``. Upstream's pydantic ConfigBase +# supports both natively; we add a tiny ``.get()`` helper to a dataclass. +# +# Upstream's ``_EncoderConfig`` also defines qk_norm / rotary_bias / +# rotary_theta / input_dim, but ``SuperviseEncoder.__init__`` only reads six +# keys (num_layers / num_heads / hidden_size / ffn_hidden_size / norm_layer / +# causal). The rest are swallowed and MultiHeadAttention falls back to its +# own defaults — matching upstream behaviour means *not* plumbing them through. +@dataclass +class _PatchEncoderInner: + num_layers: int = 24 + num_heads: int = 16 + hidden_size: int = 1024 + ffn_hidden_size: int = 4096 + norm_layer: str = "RMSNorm" + causal: bool = True + + def get(self, key: str, default: Any = None) -> Any: + return getattr(self, key, default) + + +@dataclass +class _PatchEncoderConfig: + patch_size: int = 4 + PatchEncoder: _PatchEncoderInner = dataclasses.field(default_factory=_PatchEncoderInner) + + +def _build_patch_encoder_config(hf_config: Any) -> _PatchEncoderConfig: + block = _require_config_block(hf_config, "PatchEncoder") + # SuperviseEncoder reads only the six declared fields; upstream's + # pydantic config swallows the other checkpoint keys, so filtering + # here matches upstream behaviour exactly. + inner_fields = {f.name for f in dataclasses.fields(_PatchEncoderInner)} + inner = _PatchEncoderInner(**{k: v for k, v in block.items() if k in inner_fields}) + return _PatchEncoderConfig( + patch_size=int(hf_config.patch_size), + PatchEncoder=inner, + ) + + +def _validate_architecture_constants(hf_config: Any, dit_config: _DiTConfig) -> None: + """The FM workspace helpers size buffers off module-level constants; + a checkpoint that disagrees would corrupt the DiT conditioning, so + refuse it up front (making these fully config-driven is deferred + until a released checkpoint actually differs).""" + checks = [ + ("latent_dim", getattr(hf_config, "latent_dim", None), _LATENT_DIM), + ("patch_size", getattr(hf_config, "patch_size", None), _LATENT_PATCH_SIZE), + ("DiT.hidden_size", dit_config.hidden_size, _FM_HIDDEN), + ] + mismatches = [f"{name}={got!r} (expected {want})" for name, got, want in checks if got != want] + if mismatches: + raise ValueError( + "dots.tts checkpoint config disagrees with this integration's " + "architecture constants: " + ", ".join(mismatches) + ) + + +class DotsTTSForConditionalGeneration(nn.Module): + """dots.tts AR talker: Qwen2.5 base LM + DiT / AudioVAE / CAM++ side path.""" + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + + self.vllm_config = vllm_config + self.config = vllm_config.model_config.hf_config + + # vllm-omni capability flags — gpu_model_runner checks these to decide + # whether to call preprocess() / postprocess() / accept multimodal + # outputs. Without these, the runner bypasses our preprocess hook + # and _pending_requests stays empty (forward never dispatches the + # side-path). Voxcpm2 sets the same three. + self.have_multimodal_outputs = True + self.has_preprocess = True + self.has_postprocess = True + + self.model = Qwen2Model( + vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "model"), + ) + self.make_empty_intermediate_tensors = self.model.make_empty_intermediate_tensors + + # Side-path module hyperparameters come off the checkpoint's + # config.json blocks (fail loudly if a block is absent); the FM + # workspace constants are cross-checked against the same config. + dit_config = _build_dit_config(self.config) + _validate_architecture_constants(self.config, dit_config) + llm_hidden = self.config.hidden_size + fm_hidden = dit_config.hidden_size + latent_dim = int(self.config.latent_dim) + xvec_dim = int(getattr(self.config, "campplus_embedding_size", 512)) + + self._audio_vae = AudioVAE(_build_audio_vae_config(self.config)) + # Upstream serializes vocoder.safetensors with weight_norm folded into + # plain `weight` on the decoder (encoder kept weight_norm — it's not in + # the synthesis hot path). Match that layout before load_weights runs, + # so checkpoint keys align with our state_dict 1:1. + self._audio_vae.remove_weight_norm() + # AudioVAE / speaker encoder stay fp32 — upstream only casts + # `model.core` to bf16 (runtime.py:81), leaving vocoder + xvector + # extractor in fp32 because their kernels (conv1d / fbank) need + # higher precision. Without this _audio_vae would inherit talker's + # bf16 dtype via vLLM's auto-cast and dtype-mismatch under + # stream_step (input .float() vs bf16 weights). + + # DiT flow-matching head. Dimensions per upstream core.py:101-104: + # in_dim = config.DiT.hidden_size (DiT internal space) + # out_dim = config.latent_dim (AudioVAE input space) + # mode is "flow_matching" for soar/base; "meanflow" only for the mf + # checkpoint (handled by a separate factory in a later step). + self._head = DiT( + in_dim=fm_hidden, + out_dim=latent_dim, + transformer_config=dit_config, + mode="flow_matching", + ) + + # Patch encoder: closes the AR loop by mapping each DiT-emitted audio + # latent patch back into the LLM hidden space, so Qwen2 "hears" what it + # has already generated. Upstream core.py:75 calls this with + # in_dim=latent_dim and out_dim=llm.config.hidden_size. + self._patch_encoder = VAESemanticEncoder( + in_dim=latent_dim, + out_dim=llm_hidden, + config=_build_patch_encoder_config(self.config), + ) + + # Five thin projectors (upstream core.py:82-113). Soar dimensions: + # llm_hidden=1536, fm_hidden=1024, latent_dim=128, xvec_dim=512. + # These are kept as direct attributes (no vendor file) — they are flat + # nn.Linear / nn.Sequential, and live at the top of the checkpoint + # namespace under their own short prefixes (see load_weights). + self._hidden_proj = nn.Linear(llm_hidden, fm_hidden) + self._latent_proj = nn.Linear(latent_dim, fm_hidden) + self._coordinate_proj = nn.Linear(latent_dim, fm_hidden) + self._xvec_proj = nn.Sequential( + nn.Linear(xvec_dim, fm_hidden), + nn.LayerNorm(fm_hidden), + ) + # eos_proj is the stop predictor — feeds LLM hidden through a 2-layer + # MLP and produces a 2-way logit (continue / stop). + self._eos_proj = nn.Sequential( + nn.Linear(llm_hidden, llm_hidden), + nn.SiLU(), + nn.Linear(llm_hidden, 2), + ) + + # CAM++ speaker encoder (3D-Speaker), produces the x-vector from + # reference audio for voice cloning. sample_rate matches AudioVAE's + # input space (48 kHz); the wrapper resamples to 16k internally for + # CAM++. Weights are frozen (upstream does `requires_grad = False`). + self._speaker_encoder = SpeakerXVectorFeatures( + sample_rate=self._audio_vae.sample_rate, + campplus_embedding_size=xvec_dim, + max_audio_seconds=float(getattr(self.config, "xvec_max_audio_seconds", 10.0)), + ) + + # Pin AudioVAE + speaker encoder to fp32 (mirror upstream + # runtime.py:81 where only `model.core` is cast to bf16). vLLM's + # auto-cast would otherwise lower these to bf16, breaking + # stream_step (input .float() vs bf16 weights) and the fbank + # pipeline. + self._audio_vae.float() + self._speaker_encoder.float() + + # Latent stats — independent of safetensors (torch.load on + # latent_stats.pt sitting next to the checkpoint files). Used by + # _finish_decode to bridge between DiT's normalized latent space and + # the raw latent space expected by patch_encoder + AudioVAE. + latent_stats_path = self._resolve_latent_stats_path(vllm_config.model_config.model) + if latent_stats_path is not None: + self._io_helper = _IOHelper(latent_stats_path) + else: + logger.warning( + "latent_stats.pt not resolvable for %s; running with " + "identity normalize/denormalize. Wav will be ~10x quiet " + "and distorted (no sqrt(var) denormalize).", + vllm_config.model_config.model, + ) + self._io_helper = _IOHelper() + + # (β) diagnostic: env-gated per-patch rms/max trace across the full + # side path (LLM hidden → DiT → denormalize → patch_encoder → wav → + # eos). Zero overhead when DOTS_TTS_BETA_TRACE is unset (single + # bool check per step); kept as the model's debugging instrument + # (cf. voxcpm2's enable_profiling runtime knob). + self._beta_trace = bool(os.environ.get("DOTS_TTS_BETA_TRACE")) + + # Per-request state plumbing. + # _active_states: per-request state keyed by request_id (created in + # preprocess prefill, evicted by _flush_deferred_cleanup). + # _pending_requests: per-step list of (req_id, is_prefill, embeds, + # span_len) populated by preprocess(), consumed by forward(). + # _deferred_cleanup_ids: requests vLLM has marked finished but whose + # side-path audio still needs draining in the current forward step; + # evicted from _active_states at the end of forward(). + self._active_states: dict[str, _RequestState] = {} + self._pending_requests: list[tuple[str, bool, torch.Tensor, int]] = [] + self._deferred_cleanup_ids: set[str] = set() + + # Audio side-channel: + # _audio_queue: _finish_decode pushes (req_id, audio_tensor) tuples + # here; make_omni_output drains them into multimodal_outputs each + # step. + # _sample_rate: cached from AudioVAE config (= 48000 for soar). + self._audio_queue: list[tuple[str, torch.Tensor]] = [] + self._sample_rate = self._audio_vae.sample_rate + + # Stop-signal side-channel: + # _results_queue: _finish_decode pushes (req_id, stop_logits) tuples + # here; compute_logits drains them into the [bsz, vocab_size] + # logits tensor each step. + self._results_queue: list[tuple[str, torch.Tensor | None]] = [] + + logger.info( + "DotsTTS talker built (base_lm=Qwen2[28L,12H,1536d], " + "audio_vae=AudioVAE, dit=DiT[18L,16H,1024d], " + "patch_encoder=VAESemanticEncoder[24L,16H,1024d], " + "5 projectors, speaker_encoder=CAM++[7.2M], " + "io_helper=%s, model=%s, dit_steps=%d); " + "_finish_decode: DiT → denormalize → patch_encoder AR + " + "streaming AudioVAE decode → wav + eos_proj → stop signal.", + "loaded" if self._io_helper.global_mean is not None else "identity", + vllm_config.model_config.model, + _DIT_NUM_STEPS, + ) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **kwargs: Any, + ) -> torch.Tensor | IntermediateTensors: + """Drive the base LM, then the per-request audio side path. + + One vLLM step: base LM forward under PagedAttention, then slice + the scaffold hidden states per request (in _pending_requests + order) and run _finish_decode on each — DiT sampling, streaming + AudioVAE decode, AR loopback, and stop signal all happen there. + vLLM only ever sees the returned hidden states; audio leaves + through the _audio_queue side channel. + """ + output = self.model(input_ids, positions, intermediate_tensors, inputs_embeds) + if isinstance(output, IntermediateTensors): + return output + if isinstance(output, tuple): + output = output[0] + + # Per-request side-path: slice scaffold_hidden by token span_len + # in the order preprocess() appended to _pending_requests. + token_offset = 0 + for req_id, is_prefill, _embeds, span_len in self._pending_requests: + req_hidden = output[token_offset : token_offset + span_len] + token_offset += span_len + self._finish_decode(req_id, req_hidden, is_prefill) + + # End-of-step cleanup + self._pending_requests.clear() + self._flush_deferred_cleanup() + + return output + + # ── vllm-omni protocol methods ── + + @staticmethod + def _resolve_latent_stats_path(model_arg: str) -> str | None: + """Resolve latent_stats.pt to a local filesystem path. + + ``vllm_config.model_config.model`` is typically an HF repo ID like + ``"rednote-hilab/dots.tts-soar"`` — joining it with ``"latent_ + stats.pt"`` gives a non-existent path. Try the local-dir form + first (covers ``--model /local/path``), then fall back to HF + cache lookup with ``local_files_only=True`` so we never trigger + a download from this side (the file was fetched alongside the + safetensors at load time). + """ + local = os.path.join(model_arg, "latent_stats.pt") + if os.path.exists(local): + return local + try: + from huggingface_hub import hf_hub_download + + return hf_hub_download( + repo_id=model_arg, + filename="latent_stats.pt", + local_files_only=True, + ) + except Exception: + return None + + def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor: + """Map token ids to embeddings via the base LM's embed_tokens. + + Required by vLLM's ``is_text_generation_model`` Protocol check + (interfaces_base._check_vllm_model_embed_input_ids). Without this + the model fails ``--runner generate`` validation at engine startup. + """ + return self.model.embed_tokens(input_ids) + + def preprocess( + self, + input_ids: torch.Tensor, + input_embeds: torch.Tensor | None = None, + **info_dict: Any, + ) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]: + """Per-step embedding + per-request state registration. + + Prefill: embed the wrapped prompt + ``[文本][文本对应语音]`` via the LM's + embed_tokens. Caller (serving layer or test harness) is + responsible for the wrap — talker just embeds whatever it gets. + + Decode (span_len == 1): use ``state.curr_embed_for_next`` from + the previous step's AR loopback (DiT → patch_encoder output); + zero fallback only if the loopback hasn't produced one yet. + + Voice clone is intentionally not wired here — zero-shot only. + + Returns ``(input_ids unchanged, embeds [span_len, hidden_size], + {})``. ``forward()`` consumes ``self._pending_requests`` to + slice scaffold_hidden per request. + """ + # voxcpm2 protocol compatibility: flatten additional_information. + additional = info_dict.get("additional_information") + if isinstance(additional, dict): + merged = {k: v for k, v in info_dict.items() if k != "additional_information"} + for k, v in additional.items(): + merged.setdefault(k, v) + info_dict = merged + + span_len = int(input_ids.shape[0]) + dev = input_ids.device + req_id = info_dict.get("request_id", "default") + state = self._active_states.get(req_id) + # Prefill is detected by state lifecycle (not span_len), so decode + # can use span_len=1 with a multi-position curr_embed_for_next. + is_prefill = state is None or not state.prefill_completed + + if is_prefill: + if state is None: + state = _RequestState(request_id=req_id) + self._active_states[req_id] = state + # Reset per-step state on every prefill — previous run is stale. + state.curr_embed_for_next = None + state.precomputed_stop_logits = None + state.is_stopping = False + state.prefill_completed = False + state.noise_step = 0 + # Reset AR-loop state on every prefill. + state.patch_encoder_state = None + state.vocoder_stream_state = None + embeds = self.model.embed_tokens(input_ids) + else: + curr = state.curr_embed_for_next + if curr is not None: + embeds = curr.to(dev) # [1, llm_hidden] + else: + # First decode step before AR loop closed — zero fallback. + embed_dtype = self.model.embed_tokens.weight.dtype + embeds = torch.zeros( + 1, + self.config.hidden_size, + device=dev, + dtype=embed_dtype, + ) + + self._pending_requests.append((req_id, is_prefill, embeds, span_len)) + return input_ids, embeds, {} + + def postprocess(self, *args: Any, **kwargs: Any) -> dict: + return {} + + def on_requests_finished(self, finished_req_ids: Iterable[str]) -> None: + """Mark finished requests for deferred eviction. + + vLLM scheduler calls this BEFORE forward() to notify that certain + requests have completed. We can't drop their _RequestState yet — + the current forward step still needs to drain side-path audio + chunks. Real eviction happens at the end of forward() via + _flush_deferred_cleanup. + """ + for req_id in finished_req_ids: + if req_id in self._active_states: + self._deferred_cleanup_ids.add(req_id) + + def _flush_deferred_cleanup(self) -> None: + """Evict states marked by on_requests_finished. Called at the + tail of forward().""" + for req_id in self._deferred_cleanup_ids: + self._active_states.pop(req_id, None) + self._deferred_cleanup_ids.clear() + + def _finish_decode( + self, + req_id: str, + req_hidden: torch.Tensor, + is_prefill: bool, + ) -> None: + """One request's per-step side path: DiT → AR loopback → wav → stop. + + One vLLM step = one audio patch. patch_encoder.decode_patch + collapses its internal _PATCH_ENCODER_OUT_DS_RATE positions into + a single LLM input embedding (see VAESemanticEncoder._project_ + embeddings), so vLLM's span_len=1 decode matches naturally — no + slot bookkeeping needed. + + Per-call flow (prefill or decode): + 1. _hidden_proj(last_hidden) → fm_sequence (+1 position) + 2. DiT N-step Euler → audio_patch [1, 4, 128] (normalized) + 3. _latent_proj(audio_patch) → fm_sequence (+4 positions, normalized) + 4. io_helper.denormalize(audio_patch) → audio_patch_raw + 5. patch_encoder(audio_patch_raw) → next-step embed [1, 1536] + 6. AudioVAE stream_step(audio_patch_raw) → wav chunk pushed to + _audio_queue (prefill's patch #0 included; output lags 2 + frames behind input) + 7. (decode only) eos_proj(last_hidden) → stop_logits pushed to + _results_queue; threshold > 0.8 flips state.is_stopping and + stream_flush drains the decoder's lookahead tail. + """ + state = self._active_states.get(req_id) + if state is None: + return + + if state.fm_sequence is None: + self._initialize_request_fm_state( + state, + device=req_hidden.device, + dtype=req_hidden.dtype, + ) + + # This step's append would overflow the per-request FM buffer. + # Stop gracefully (same path as a model-decided stop below) instead + # of raising past _append_hidden_chunk/_append_history_chunk — + # those raises are engine-fatal, not request-fatal, and would take + # every other in-flight and future request down with this one. + if state.fm_seq_len + _HIDDEN_PATCH_SIZE + _LATENT_PATCH_SIZE > state.fm_capacity: + state.is_stopping = True + stop_logits = torch.tensor([[0.0, 1.0]], device=req_hidden.device, dtype=req_hidden.dtype) + state.precomputed_stop_logits = stop_logits + tail = self._audio_vae.stream_flush(state.vocoder_stream_state) + if tail.size(-1) > 0: + self._audio_queue.append((req_id, tail.reshape(-1))) + self._results_queue.append((req_id, stop_logits)) + return + + # 1. Append last LLM hidden to fm_sequence (+1 position). + last_hidden = req_hidden[-_HIDDEN_PATCH_SIZE:].unsqueeze(0) + + self._append_hidden_chunk(state, last_hidden) + + # 2. DiT N-step Euler → audio latent patch [1, 4, 128] (normalized). + audio_patch = self._run_dit_n_step_euler(state) + + # 3. Append latent to fm_sequence (+4 positions). Stays in + # normalized space — DiT's KV cache lives in that space. + self._append_history_chunk(state, audio_patch) + + # 4. Denormalize for downstream consumers (patch_encoder, AudioVAE). + audio_patch_raw = self._io_helper.denormalize(audio_patch) + + # 5. patch_encoder AR loopback → next-step inputs_embeds [1, 1536]. + next_embeds = self._run_patch_encoder_loopback(state, audio_patch_raw) + state.curr_embed_for_next = next_embeds.squeeze(0).detach() + + # 6. Streaming AudioVAE decode → wav chunk. Runs on prefill too: + # upstream zero-shot emits every DiT patch including the first + # (_generate_latents_stream drops one patch only under prompt + # prefill / voice clone), and our prefill patch #0 corresponds + # to upstream's first decode-loop patch. + wav = self._run_vocoder_stream_step(state, audio_patch_raw) + if wav.size(-1) > 0: + self._audio_queue.append((req_id, wav.reshape(-1))) + + if is_prefill: + if self._beta_trace: + self._beta_trace_log( + state, + is_prefill=True, + last_hidden=last_hidden, + audio_patch=audio_patch, + audio_patch_raw=audio_patch_raw, + next_embeds=next_embeds, + wav=wav if wav.size(-1) > 0 else None, + prob_stop=None, + ) + state.prefill_completed = True + # Keep len(_results_queue) == bsz: compute_logits pairs queue + # entries with batch rows positionally, so every request in the + # batch must push exactly one entry per step (voxcpm2_talker.py + # prefill placeholder pattern). None → forced continue via the + # stop_logits-is-None branch in compute_logits. + self._results_queue.append((req_id, None)) + return + + # 7. eos_proj → stop probability. Mirrors upstream + # model.py:1681 — softmax([continue, stop]) on detached LLM + # hidden, threshold the stop slot at 0.8 (upstream default + # eos_threshold). When stop fires for this step, compute_logits + # drains the high-stop logit on the next dispatch and vLLM + # terminates the request. + eos_logits = self._eos_proj(last_hidden.detach()).softmax(dim=-1) + stop_logits = eos_logits.squeeze(0) + state.precomputed_stop_logits = stop_logits + if eos_logits[0, -1, 1].item() > 0.8: + state.is_stopping = True + # Upstream flushes after its decode loop (model.py:1898); our + # request ends when vLLM sees this step's stop token, so drain + # the decoder's 2-frame lookahead tail in the same step. + # Requests cut by max_tokens/abort never reach this path and + # lose the 40 ms tail — no engine hook fires late enough to + # flush after the final step. + tail = self._audio_vae.stream_flush(state.vocoder_stream_state) + if tail.size(-1) > 0: + self._audio_queue.append((req_id, tail.reshape(-1))) + self._results_queue.append((req_id, stop_logits)) + + if self._beta_trace: + self._beta_trace_log( + state, + is_prefill=False, + last_hidden=last_hidden, + audio_patch=audio_patch, + audio_patch_raw=audio_patch_raw, + next_embeds=next_embeds, + wav=wav if wav.size(-1) > 0 else None, + prob_stop=eos_logits[0, -1, 1].item(), + ) + + def _beta_trace_log( + self, + state: _RequestState, + *, + is_prefill: bool, + last_hidden: torch.Tensor, + audio_patch: torch.Tensor, + audio_patch_raw: torch.Tensor, + next_embeds: torch.Tensor, + wav: torch.Tensor | None, + prob_stop: float | None, + ) -> None: + """(β) diagnostic — patch label matches output.wav indexing in + decode rows; prefill row labels itself ``pre``.""" + + def rmsmax(t: torch.Tensor) -> tuple[float, float]: + f = t.detach().float() + return f.pow(2).mean().sqrt().item(), f.abs().max().item() + + patch_label = "pre" if is_prefill else f"{state.fm_seq_len // 5 - 2:2d}" + req_tail = state.request_id[-8:] + hid_rms, hid_max = rmsmax(last_hidden) + dit_rms, dit_max = rmsmax(audio_patch) + raw_rms, raw_max = rmsmax(audio_patch_raw) + nxt_rms, nxt_max = rmsmax(next_embeds) + if wav is not None: + w_rms, w_max = rmsmax(wav) + wav_part = f"wav:rms={w_rms:.3f} max={w_max:.3f}" + else: + wav_part = "wav:rms= - max= - " + stop_part = f"stop={prob_stop:.3f}" if prob_stop is not None else "stop= - " + logger.info( + f"[β req={req_tail} patch={patch_label}] " + f"hid:rms={hid_rms:.3f} max={hid_max:.3f} | " + f"dit:rms={dit_rms:.3f} max={dit_max:.3f} | " + f"raw:rms={raw_rms:.3f} max={raw_max:.3f} | " + f"next:rms={nxt_rms:.3f} max={nxt_max:.3f} | " + f"{wav_part} | {stop_part}" + ) + + # ── FM helpers (per-request workspace + DiT N-step Euler) ── + + def _initialize_request_fm_state( + self, + state: _RequestState, + *, + device: torch.device, + dtype: torch.dtype, + ) -> None: + """Lazy-allocate per-request FM workspace. + + Mirrors upstream _allocate_generate_state (model.py:413) but per- + request (no workspace sharing) so concurrent requests stay + isolated. + """ + fm_capacity = _MAX_AUDIO_PATCHES * (_HIDDEN_PATCH_SIZE + _LATENT_PATCH_SIZE) + state.fm_capacity = fm_capacity + state.fm_sequence = torch.zeros( + (1, fm_capacity, _FM_HIDDEN), + device=device, + dtype=dtype, + ) + state.fm_cfg_sequence = torch.zeros( + (1, fm_capacity, _FM_HIDDEN), + device=device, + dtype=dtype, + ) + state.fm_null_g_cond = torch.zeros( + (1, _FM_HIDDEN), + device=device, + dtype=dtype, + ) + state.fm_seq_len = 0 + + def _append_hidden_chunk( + self, + state: _RequestState, + hidden_chunk: torch.Tensor, + ) -> None: + """Project last _HIDDEN_PATCH_SIZE LLM hiddens via _hidden_proj into fm_sequence. + + Mirrors upstream _append_hidden_chunk (model.py:1308): cfg buffer + gets the null-projected counterpart (``_hidden_proj`` of zero) so + CFG dropout sees no past hidden, only past audio history. + """ + last_hidden = hidden_chunk[:, -_HIDDEN_PATCH_SIZE:, :] + projected = self._hidden_proj(last_hidden) + null_projected = self._hidden_proj(torch.zeros_like(last_hidden)) + end = state.fm_seq_len + projected.size(1) + if end > state.fm_capacity: + raise RuntimeError(f"FM buffer overflow on hidden append: end={end} cap={state.fm_capacity}") + state.fm_sequence[:, state.fm_seq_len : end].copy_(projected.to(state.fm_sequence.dtype)) + state.fm_cfg_sequence[:, state.fm_seq_len : end].copy_(null_projected.to(state.fm_cfg_sequence.dtype)) + state.fm_seq_len = end + + def _append_history_chunk( + self, + state: _RequestState, + latent_chunk: torch.Tensor, + ) -> None: + """Project audio latent via _latent_proj into fm_sequence (history append). + + Mirrors upstream _append_history_chunk (model.py:1325): cfg + buffer gets the same value (latent history is shared between + conditional and unconditional CFG paths — only the hidden chunk + gets nulled). + """ + history_latent = self._latent_proj(latent_chunk) + end = state.fm_seq_len + history_latent.size(1) + if end > state.fm_capacity: + raise RuntimeError(f"FM buffer overflow on latent append: end={end} cap={state.fm_capacity}") + state.fm_sequence[:, state.fm_seq_len : end].copy_(history_latent.to(state.fm_sequence.dtype)) + state.fm_cfg_sequence[:, state.fm_seq_len : end].copy_(history_latent.to(state.fm_cfg_sequence.dtype)) + state.fm_seq_len = end + + def _build_fm_attn_mask( + self, + state: _RequestState, + total_len: int, + device: torch.device, + ) -> torch.Tensor: + """Build DiT attention mask for one decode step. + + Mirrors upstream _build_fm_attn_mask (model.py:1203). Layout: + * [0, fm_seq_len) — committed history. Causal over the past + up to a non-causal tail block of size _HIDDEN_PATCH_SIZE that + sees itself and the new noise patch (the just-appended hidden). + * [latent_start, total_len) — new noise patch, bidirectional + with the committed history and itself. + """ + attn_mask = torch.zeros((1, total_len, total_len), device=device, dtype=torch.bool) + latent_start = total_len - _LATENT_PATCH_SIZE + block_start = state.fm_seq_len - _HIDDEN_PATCH_SIZE + if block_start > 0: + causal_mask = torch.ones((block_start, block_start), device=device, dtype=torch.bool).triu(1).logical_not() + attn_mask[:, :block_start, :block_start] = causal_mask + attn_mask[:, block_start : state.fm_seq_len, : state.fm_seq_len] = True + attn_mask[:, block_start : state.fm_seq_len, latent_start:] = True + attn_mask[:, latent_start:, : state.fm_seq_len] = True + attn_mask[:, latent_start:, latent_start:] = True + if latent_start > state.fm_seq_len: + padding_indices = torch.arange(state.fm_seq_len, latent_start, device=device) + attn_mask[:, padding_indices, padding_indices] = True + return attn_mask + + def _build_fm_pos_ids( + self, + state: _RequestState, + total_len: int, + device: torch.device, + ) -> torch.Tensor: + """Build DiT position ids for one decode step. + + Mirrors upstream _build_fm_pos_ids (model.py:1236). Position ids + stay fp32 — DiT's RotaryEmbedding does its own dtype handling. + """ + pos_ids = torch.zeros((1, total_len), device=device, dtype=torch.float32) + latent_start = total_len - _LATENT_PATCH_SIZE + if state.fm_seq_len > 0: + pos_ids[:, : state.fm_seq_len] = torch.arange( + state.fm_seq_len, + device=device, + dtype=pos_ids.dtype, + ) + pos_ids[:, latent_start:] = torch.arange( + state.fm_seq_len, + state.fm_seq_len + _LATENT_PATCH_SIZE, + device=device, + dtype=pos_ids.dtype, + ) + return pos_ids + + def _run_dit_n_step_euler( + self, + state: _RequestState, + *, + num_steps: int = _DIT_NUM_STEPS, + guidance_scale: float = _DIT_GUIDANCE_SCALE, + ) -> torch.Tensor: + """DiT N-step Euler integration → audio latent patch [1, 4, 128]. + + Manual Euler loop (upstream uses torchdyn.odeint; we manualize to + avoid the dep and keep control flow explicit). Each step: + 1. z_proj = _coordinate_proj(z), placed into the last + _LATENT_PATCH_SIZE positions of input_sequence / cfg_sequence. + 2. Batch [conditional, unconditional] → DiT velocity field. + 3. CFG blend: v_c + guidance_scale * (v_c - v_u). + 4. Euler: z += v * dt. + + Mirrors upstream fm_solver_step (core.py:295) + _flow_matching_step_fm + (core.py:463), reduced to a fixed-step Euler integrator. + """ + assert state.fm_sequence is not None + assert state.fm_cfg_sequence is not None + assert state.fm_null_g_cond is not None + device = state.fm_sequence.device + dtype = state.fm_sequence.dtype + total_len = state.fm_seq_len + _LATENT_PATCH_SIZE + + # Workspace: committed history + 4 noise slots (overwritten per step). + input_sequence = torch.zeros((1, total_len, _FM_HIDDEN), device=device, dtype=dtype) + input_sequence[:, : state.fm_seq_len] = state.fm_sequence[:, : state.fm_seq_len] + cfg_sequence = torch.zeros((1, total_len, _FM_HIDDEN), device=device, dtype=dtype) + cfg_sequence[:, : state.fm_seq_len] = state.fm_cfg_sequence[:, : state.fm_seq_len] + + attn_mask = self._build_fm_attn_mask(state, total_len, device) + pos_ids = self._build_fm_pos_ids(state, total_len, device) + + g_cond = state.g_cond if state.g_cond is not None else state.fm_null_g_cond + g_cond = g_cond.to(device=device, dtype=dtype) + g_cond_batched = torch.cat([g_cond, torch.zeros_like(g_cond)], dim=0) + + latent_start = total_len - _LATENT_PATCH_SIZE + # fp32 ODE integration (same fix as Ming's CFM sampler, PR #4341): + # bf16's 7-bit mantissa loses the small per-step increments in the + # z += v*dt accumulation (hidden-state cos drift vs upstream past + # step ~7) and quantizes the linspace timesteps. Integration + # state stays fp32; DiT matmuls still run bf16 under the autocast + # block below, and the result is cast back to the sequence dtype + # on return. + # Deterministic per-request noise (voxcpm2 _fill_deterministic_cfm_ + # noise pattern): hash seed:request_key:draw# into a private + # Generator instead of the global CUDA RNG, so outputs reproduce + # run-to-run and concurrent requests cannot perturb each other's + # noise streams. request_key strips the engine's per-run "_" + # uuid suffix so replay across runs keys on the stable batch index. + request_key = state.request_id.split("_", 1)[0] + if not request_key.isdigit(): + request_key = state.request_id + noise_key = f"{_DIT_NOISE_SEED}:{request_key}:{state.noise_step}".encode() + digest = hashlib.blake2b(noise_key, digest_size=8).digest() + gen = torch.Generator(device=device) + gen.manual_seed(int.from_bytes(digest, "little") & 0x7FFF_FFFF_FFFF_FFFF) + z = torch.empty((1, _LATENT_PATCH_SIZE, _LATENT_DIM), device=device, dtype=torch.float32) + z.normal_(generator=gen) + state.noise_step += 1 + dt = 1.0 / num_steps + times = torch.linspace(0.0, 1.0, num_steps + 1, device=device, dtype=torch.float32) + + # Match upstream's autocast wrapper (model.py:280). DiT's + # TimestepEmbedder internally forces fp32 (`.float()` on freqs); + # without autocast the fp32 output hits a bf16 Linear → dtype + # mismatch. autocast bridges this for matmuls inside DiT. + use_amp = device.type == "cuda" and dtype in (torch.float16, torch.bfloat16) + with torch.autocast( + device_type=device.type if device.type == "cuda" else "cuda", + dtype=dtype if use_amp else torch.float32, + enabled=use_amp, + ): + for step in range(num_steps): + t = times[step].reshape(1) + z_proj = self._coordinate_proj(z) + z_c = input_sequence.clone() + z_c[:, latent_start:] = z_proj + z_u = cfg_sequence.clone() + z_u[:, latent_start:] = z_proj + z_batched = torch.cat([z_c, z_u], dim=0) + t_batched = t.repeat(2) + vt = self._head( + x=z_batched, + timesteps=t_batched, + attn_mask=attn_mask, + pos_ids=pos_ids, + g_cond=g_cond_batched, + ) + # Upcast DiT output before CFG blend + Euler update so the + # accumulation arithmetic stays fp32 (autocast only affects + # matmuls; these pointwise ops keep their input dtype). + vt = vt[:, latent_start:].float() + vt_c, vt_u = vt[0:1], vt[1:2] + velocity = vt_c + guidance_scale * (vt_c - vt_u) + z = z + velocity * dt + return z.to(dtype) + + def _run_patch_encoder_loopback( + self, + state: _RequestState, + audio_patch_raw: torch.Tensor, + ) -> torch.Tensor: + """Decode one audio latent patch → 1 LLM input embed. + + Mirrors upstream _consume_audio_patch (model.py:1557) — patch_encoder + side only. Lazy-allocates state.patch_encoder_state on first call. + Returns ``llm_embedding`` of shape ``[1, 1, llm_hidden]`` — the + _PATCH_ENCODER_OUT_DS_RATE encoder positions fold into the feature + axis before ``out_proj``, not into separate output positions. + + ``audio_patch_raw`` must already be denormalized (raw VAE latent + space). Caller (_finish_decode) runs io_helper.denormalize on + DiT's normalized output before passing in. + """ + if state.patch_encoder_state is None: + state.patch_encoder_state = self._patch_encoder.init_decode_state( + max_audio_patch_count=_MAX_AUDIO_PATCHES, + batch_size=1, + device=audio_patch_raw.device, + dtype=audio_patch_raw.dtype, + ) + pe_state = state.patch_encoder_state + positions = ( + torch.arange( + _PATCH_ENCODER_OUT_DS_RATE, + device=audio_patch_raw.device, + dtype=torch.long, + ) + + pe_state.seq_len + ) + llm_embedding, conv_tail = self._patch_encoder.decode_patch( + audio_patch_raw, + pe_state.conv_tail, + pe_state.layer_caches, + positions, + ) + pe_state.conv_tail.copy_(conv_tail) + pe_state.seq_len += _PATCH_ENCODER_OUT_DS_RATE + return llm_embedding + + def _run_vocoder_stream_step( + self, + state: _RequestState, + audio_patch_raw: torch.Tensor, + ) -> torch.Tensor: + """Streaming AudioVAE decode of one latent patch (review M2). + + Mirrors upstream generate_audio_stream (model.py:1881-1900): a + per-request BigVGANStreamState carries the decoder's LSTM hidden + and sliding latent window across steps, so each patch is decoded + with real left context instead of inference_from_latents' + zero-padded isolated boundary. Output lags input by + decoder.stream_lookahead (2 frames): the first call emits 2 frames + (3840 samples), steady state 4 (7680); the stop path in + _finish_decode drains the tail via stream_flush. + + audio_patch_raw: [1, T=4, D=128] denormalized latents. + Returns wav chunk [1, 1, n_samples] (n_samples may be 0). + """ + if state.vocoder_stream_state is None: + state.vocoder_stream_state = self._audio_vae.init_stream_state( + batch_size=1, + chunk_size=_LATENT_PATCH_SIZE, + ) + # stream_step expects [B, latent_dim, T]; the vocoder is pinned + # fp32 (__init__), so cast the (possibly bf16) latents up. + return self._audio_vae.stream_step( + audio_patch_raw.transpose(1, 2).float(), + state.vocoder_stream_state, + ) + + def make_omni_output( + self, + model_outputs: torch.Tensor | OmniOutput, + **kwargs: Any, + ) -> OmniOutput: + """Drain self._audio_queue into the OmniOutput's multimodal_outputs. + + Output contract is **delta** (SKILL I1): each call emits only + audio produced since the previous step; the engine's + _consolidate_multimodal_tensors concatenates across steps for + offline consumers. + + Same-request multi-chunk: chunks for the same req_id within one + step are cat'd together. Multi-request: one entry per req_id + in multimodal_outputs["model_outputs"]. + + Note: ``sr`` is not a declared OmniPayload key but the engine / + serving consumer reads it (voxcpm2 convention). + """ + if isinstance(model_outputs, OmniOutput): + return model_outputs + + mm: dict[str, Any] = {} + if self._audio_queue: + audio_by_req: dict[str, torch.Tensor] = {} + for req_id, audio in self._audio_queue: + if audio is None: + continue + if req_id in audio_by_req: + audio_by_req[req_id] = torch.cat( + [audio_by_req[req_id].reshape(-1), audio.reshape(-1)], + dim=0, + ) + else: + audio_by_req[req_id] = audio + if audio_by_req: + sr = torch.tensor(self._sample_rate, dtype=torch.int32) + ready_req_ids = list(audio_by_req) + chunks = [audio_by_req[req_id].reshape(-1) for req_id in ready_req_ids] + mm["model_outputs"] = chunks + mm["sr"] = [sr for _ in ready_req_ids] + # sparse_audio: ["1"] flags GPUARModelRunner to skip the default + # payload["hidden"] = scaffold_hidden injection (gpu_ar_model_ + # runner.py:1114). Otherwise prefill scaffold (24 hidden × 1536 + # = 29184 numbers observed) leaks into mm["audio"] via the + # output_processor "hidden" → target_key="audio" rename + # (output_processor.py:84), prepending ~0.61 s of noise. + mm["meta"] = {"req_id": ready_req_ids, "sparse_audio": ["1"]} + self._audio_queue.clear() + else: + # Empty-audio step (prefill) still needs the sparse marker so the + # engine doesn't bleed scaffold_hidden into mm["audio"]. + mm["model_outputs"] = [] + mm["sr"] = [] + mm["meta"] = {"req_id": [], "sparse_audio": ["1"]} + + if self._beta_trace: + payload = mm.get("model_outputs") + if payload: + for i, t in enumerate(payload): + logger.info( + f"[β-emit] mm[model_outputs][{i}]: shape={tuple(t.shape)} " + f"dtype={t.dtype} samples={t.numel()} " + f"rms={t.float().pow(2).mean().sqrt().item():.4f} " + f"max={t.abs().max().item():.4f}" + ) + else: + logger.info("[β-emit] mm empty / no model_outputs") + mo_kind = type(model_outputs).__name__ + mo_info = f"shape={tuple(model_outputs.shape)}" if hasattr(model_outputs, "shape") else "no_shape" + logger.info(f"[β-emit] text_hidden_states arg: {mo_kind} {mo_info}") + + return OmniOutput(text_hidden_states=model_outputs, multimodal_outputs=mm) + + def compute_logits( + self, + hidden_states: torch.Tensor | OmniOutput, + sampling_metadata: Any = None, + ) -> torch.Tensor | None: + """Per-step stop signal logits (vLLM sampling protocol). + + Encodes continue/stop into ``logits[i, 0]`` / ``logits[i, 1]`` + (voxcpm2 convention — rest of vocab stays at -inf so the sampler + picks between the two slots). Source of stop logits is + ``self._results_queue``, populated by forward()'s _finish_decode. + When the queue is empty (preprocess-only steps), defaults to + all-continue. + """ + if isinstance(hidden_states, OmniOutput): + hidden_states = hidden_states.text_hidden_states + if hidden_states is None: + return None + + bsz = hidden_states.shape[0] + logits = torch.full( + (bsz, self.config.vocab_size), + float("-inf"), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + if self._results_queue: + for i, (req_id, stop_logits) in enumerate(self._results_queue): + if i >= bsz: + break + state = self._active_states.get(req_id) + if stop_logits is not None: + if state is not None and state.is_stopping: + logits[i, 0] = 0.0 + logits[i, 1] = 1.0 + else: + # is_stopping=False means prob_stop <= 0.8 — force + # continue. Feeding the raw softmax (continue, stop) + # pair as logits would let the greedy sampler compare + # them directly, silently lowering the effective stop + # threshold to 0.5 (early stop, swallowed endings). + logits[i, 0] = 1.0 + if state is not None: + state.precomputed_stop_logits = None + else: + # No stop signal pushed this step (prefill placeholder). + # Default to continue — stop is driven by eos_proj. + logits[i, 0] = 1.0 # continue + self._results_queue.clear() + else: + logits[:, 0] = 1.0 # all continue + + return logits + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Load all dots.tts checkpoint weights — step 6b. + + Routing branches in a single pass over the input iterator (it can only + be consumed once): + + * **AudioVAE**: keys from ``vocoder.safetensors`` (no parent prefix). + Matched via set membership against ``self._audio_vae.state_dict()``; + acts as the final catch-all so it must run last. + * **DiT**: keys from ``model.safetensors`` under ``velocity_field_predictor.``. + * **patch_encoder**: keys from ``model.safetensors`` under ``patch_encoder.``. + * **Five projectors** (``hidden_proj`` / ``latent_proj`` / + ``coordinate_proj`` / ``xvec_proj`` / ``eos_proj``): each lives at + its own top-level prefix; dispatched through a shared table. + * **Qwen2 base LM**: keys under ``llm.model.``, handed off to vLLM's + Qwen2Model fuser (q/k/v_proj -> qkv_proj, gate/up_proj -> + gate_up_proj). ``llm.lm_head.*`` is silently dropped (soar uses + ``tie_word_embeddings=True`` and our ``self.model`` is a Qwen2Model + with no lm_head module). + * **CAM++ speaker encoder**: keys from ``speaker_encoder.safetensors`` + (no parent prefix, same convention as AudioVAE). Routed by set + membership against ``self._speaker_encoder.state_dict()`` (keys are + ``model.*`` / ``resample.kernel``, no distinguishing top-level + prefix). **Placed before the AudioVAE fallback** so its keys don't + fall through to the catch-all. + """ + vae_state_keys = set(self._audio_vae.state_dict().keys()) + dit_state_keys = set(self._head.state_dict().keys()) + patch_state_keys = set(self._patch_encoder.state_dict().keys()) + speaker_state_keys = set(self._speaker_encoder.state_dict().keys()) + + # (top-level prefix in checkpoint, local attr name, local nn.Module) + projector_specs: list[tuple[str, str, nn.Module]] = [ + ("hidden_proj.", "_hidden_proj", self._hidden_proj), + ("latent_proj.", "_latent_proj", self._latent_proj), + ("coordinate_proj.", "_coordinate_proj", self._coordinate_proj), + ("xvec_proj.", "_xvec_proj", self._xvec_proj), + ("eos_proj.", "_eos_proj", self._eos_proj), + ] + projector_state_keys = {prefix: set(mod.state_dict().keys()) for prefix, _, mod in projector_specs} + projector_matched: dict[str, list[tuple[str, torch.Tensor]]] = {prefix: [] for prefix, _, _ in projector_specs} + + matched_vae: list[tuple[str, torch.Tensor]] = [] + matched_dit: list[tuple[str, torch.Tensor]] = [] + matched_patch: list[tuple[str, torch.Tensor]] = [] + matched_llm: list[tuple[str, torch.Tensor]] = [] + matched_speaker: list[tuple[str, torch.Tensor]] = [] + skipped_lm_head = 0 + + DIT_PREFIX = "velocity_field_predictor." + PATCH_PREFIX = "patch_encoder." + LLM_MODEL_PREFIX = "llm.model." + LLM_LM_HEAD_PREFIX = "llm.lm_head." + for name, tensor in weights: + if name.startswith(DIT_PREFIX): + candidate = name[len(DIT_PREFIX) :] + if candidate in dit_state_keys: + matched_dit.append((candidate, tensor)) + continue + if name.startswith(PATCH_PREFIX): + candidate = name[len(PATCH_PREFIX) :] + if candidate in patch_state_keys: + matched_patch.append((candidate, tensor)) + continue + if name.startswith(LLM_LM_HEAD_PREFIX): + # Qwen2.5-1.5B-Base has tie_word_embeddings=True, and our + # ``self.model`` is a Qwen2Model (no lm_head submodule). Drop. + skipped_lm_head += 1 + continue + if name.startswith(LLM_MODEL_PREFIX): + candidate = name[len(LLM_MODEL_PREFIX) :] + # Hand off untouched — Qwen2Model.load_weights does its own + # fusion (q/k/v_proj -> qkv_proj, gate/up_proj -> gate_up_proj) + # and tolerates unknown keys, so no local key set to gate. + matched_llm.append((candidate, tensor)) + continue + # Projectors: 5 candidate top-level prefixes, first hit wins. + hit_projector = False + for prefix, _attr, _mod in projector_specs: + if name.startswith(prefix): + candidate = name[len(prefix) :] + if candidate in projector_state_keys[prefix]: + projector_matched[prefix].append((candidate, tensor)) + hit_projector = True + break + if hit_projector: + continue + # Speaker encoder: set-membership match (no distinguishing prefix + # in checkpoint; placed before the AudioVAE fallback so its keys + # don't get silently dropped by the catch-all). + if name in speaker_state_keys: + matched_speaker.append((name, tensor)) + continue + candidate = name[len("vocoder.") :] if name.startswith("vocoder.") else name + if candidate in vae_state_keys: + matched_vae.append((candidate, tensor)) + + loaded: set[str] = set() + + if matched_vae: + vae_loader = AutoWeightsLoader(self._audio_vae) + loaded_vae = vae_loader.load_weights(iter(matched_vae)) + loaded.update(f"_audio_vae.{name}" for name in loaded_vae) + logger.info( + "DotsTTS load_weights: loaded %d/%d AudioVAE tensors.", + len(loaded_vae), + len(vae_state_keys), + ) + + if matched_dit: + dit_loader = AutoWeightsLoader(self._head) + loaded_dit = dit_loader.load_weights(iter(matched_dit)) + loaded.update(f"_head.{name}" for name in loaded_dit) + logger.info( + "DotsTTS load_weights: loaded %d/%d DiT tensors.", + len(loaded_dit), + len(dit_state_keys), + ) + + if matched_patch: + patch_loader = AutoWeightsLoader(self._patch_encoder) + loaded_patch = patch_loader.load_weights(iter(matched_patch)) + loaded.update(f"_patch_encoder.{name}" for name in loaded_patch) + logger.info( + "DotsTTS load_weights: loaded %d/%d patch_encoder tensors.", + len(loaded_patch), + len(patch_state_keys), + ) + + any_projector_matched = False + for prefix, attr_name, mod in projector_specs: + matched = projector_matched[prefix] + if not matched: + continue + any_projector_matched = True + proj_loader = AutoWeightsLoader(mod) + loaded_proj = proj_loader.load_weights(iter(matched)) + loaded.update(f"{attr_name}.{name}" for name in loaded_proj) + logger.info( + "DotsTTS load_weights: loaded %d/%d %s tensors.", + len(loaded_proj), + len(projector_state_keys[prefix]), + prefix[:-1], + ) + + if matched_llm: + # vLLM's Qwen2Model.load_weights does its own fusion + tolerates + # unknown keys. It returns the set of param names it actually + # accepted; re-prefix with ``model.`` for our return value (our + # local attr is ``self.model``). + loaded_llm = self.model.load_weights(iter(matched_llm)) + loaded.update(f"model.{name}" for name in loaded_llm) + logger.info( + "DotsTTS load_weights: loaded %d Qwen2 tensors (fed %d; %d llm.lm_head.* skipped — tied embeddings).", + len(loaded_llm), + len(matched_llm), + skipped_lm_head, + ) + + if matched_speaker: + speaker_loader = AutoWeightsLoader(self._speaker_encoder) + loaded_speaker = speaker_loader.load_weights(iter(matched_speaker)) + loaded.update(f"_speaker_encoder.{name}" for name in loaded_speaker) + logger.info( + "DotsTTS load_weights: loaded %d/%d CAM++ speaker_encoder tensors.", + len(loaded_speaker), + len(speaker_state_keys), + ) + + if ( + not matched_vae + and not matched_dit + and not matched_patch + and not any_projector_matched + and not matched_llm + and not matched_speaker + ): + logger.warning( + "DotsTTS load_weights: no AudioVAE / DiT / " + "patch_encoder / projector / Qwen2 / speaker_encoder keys matched." + ) + + return loaded diff --git a/vllm_omni/model_executor/models/dots_tts/dots_tts_vocoder.py b/vllm_omni/model_executor/models/dots_tts/dots_tts_vocoder.py new file mode 100644 index 000000000..fe17b5c0b --- /dev/null +++ b/vllm_omni/model_executor/models/dots_tts/dots_tts_vocoder.py @@ -0,0 +1,1434 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 The vLLM-Omni team. +# Copyright (c) rednote-hilab. All rights reserved. +# Adapted from: +# https://github.com/rednote-hilab/dots.tts/tree/a393d2e/src/dots_tts/modules +# +# Merged from six upstream files (preserved as-is, inference behaviour +# unchanged; only internal imports were folded inline): +# - modules/vocoder/config.py -> AudioVAEConfig +# - modules/backbone/layers.py -> Conv1d, ConvTranspose1d +# (Mlp/MultiHeadAttention/RotaryEmbedding +# are not vendored here; they belong +# to the DiT side-path module added later) +# - modules/vocoder/alias_free_filter.py -> sinc, kaiser_sinc_filter1d, +# LowPassFilter1d +# - modules/vocoder/alias_free_resample.py-> UpSample1d, DownSample1d +# - modules/vocoder/alias_free_act.py -> Snake, SnakeBeta, Activation1d +# - modules/vocoder/bigvgan.py -> AudioVAE and all its sub-modules +# +# Per-section upstream attributions are preserved at each section header. +# AudioVAEConfig is the only piece that diverges from upstream: pydantic +# ConfigBase replaced with a stdlib @dataclass (extra fields not exercised by +# any shipped checkpoint). + +from __future__ import annotations + +import itertools +import math +from dataclasses import dataclass, field +from fractions import Fraction + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import pow, sin +from torch.nn import Parameter +from torch.nn.utils import remove_weight_norm, weight_norm + +# ============================================================================ +# Configuration (adapted from modules/vocoder/config.py) +# ---------------------------------------------------------------------------- +# Upstream subclasses a pydantic ``ConfigBase`` with ``extra="allow"``; both the +# dots.tts-soar and dots.tts-mf checkpoint ``config.json`` pass only the 17 +# declared fields below, so the escape hatch is never exercised. ``.get()`` +# helper preserves bigvgan.py's ``h.get(...)`` call sites without modification. +# ``num_decoder_lookahead`` / ``num_encoder_lookahead`` are looked up via +# ``h.get(...)`` but not present in any shipped config; they fall back to 2. +# ============================================================================ + + +@dataclass +class AudioVAEConfig: + sample_rate: int = 24000 + upsample_rates: list[int] = field(default_factory=list) + upsample_kernel_sizes: list[int] = field(default_factory=list) + upsample_initial_channel: int = 1536 + resblock: str = "1" + resblock_kernel_sizes: list[int] = field(default_factory=list) + resblock_dilation_sizes: list[list[int]] = field(default_factory=list) + downsample_rates: list[int] = field(default_factory=list) + downsample_channels: list[int] = field(default_factory=list) + activation: str = "snakebeta" + snake_logscale: bool = True + latent_dim: int = 128 + causal: bool = False + mi_num_layers: int = 4 + causal_encoder: bool = False + use_bias_at_final: bool = True + use_tanh_at_final: bool = True + + num_decoder_lookahead: int = 2 + num_encoder_lookahead: int = 2 + + def get(self, key: str, default=None): + return getattr(self, key, default) + + +# ============================================================================ +# Conv layers (vendored from modules/backbone/layers.py) +# ---------------------------------------------------------------------------- +# Only the Conv1d / ConvTranspose1d helpers are needed by the AudioVAE. +# Mlp / MultiHeadAttention / RotaryEmbedding from the same upstream file are +# reserved for the DiT flow-matching head, which gets its own vendored module +# in a later step. +# ============================================================================ + + +class Conv1d(nn.Conv1d): + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int = 1, + stride: int = 1, + dilation: int = 1, + groups: int = 1, + padding_mode: str = "zeros", + bias: bool = True, + padding=None, + causal: bool = False, + **_kwargs, + ): + self.causal = causal + if padding is None: + if causal: + padding = 0 + self.left_padding = dilation * (kernel_size - 1) + else: + padding = int((kernel_size * dilation - dilation) / 2) + + super().__init__( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + padding_mode=padding_mode, + bias=bias, + ) + + self.in_channels = in_channels + + def forward(self, x): + if self.causal: + x = F.pad(x.unsqueeze(2), (self.left_padding, 0, 0, 0)).squeeze(2) + return super().forward(x) + + +class ConvTranspose1d(nn.ConvTranspose1d): + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int = 1, + output_padding: int = 0, + groups: int = 1, + bias: bool = True, + dilation: int = 1, + padding=None, + padding_mode: str = "zeros", + causal: bool = False, + **_kwargs, + ): + if padding is None: + padding = 0 if causal else (kernel_size - stride) // 2 + if causal: + assert padding == 0, "padding is not allowed in causal ConvTranspose1d." + assert kernel_size == 2 * stride, "kernel_size must be equal to 2*stride in Causal ConvTranspose1d." + + super().__init__( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + output_padding=output_padding, + groups=groups, + bias=bias, + dilation=dilation, + padding_mode=padding_mode, + ) + + self.causal = causal + self.stride = stride + + def forward(self, x): + x = super().forward(x) + if self.causal: + x = x[:, :, : -self.stride] + return x + + +# ============================================================================ +# Alias-free low-pass filter (vendored from modules/vocoder/alias_free_filter.py) +# ---------------------------------------------------------------------------- +# Adapted from https://github.com/junjun3518/alias-free-torch under +# the Apache License 2.0. The sinc fallback is adopted from adefossez's +# julius.core.sinc (MIT). +# ============================================================================ + + +if "sinc" in dir(torch): + sinc = torch.sinc +else: + + def sinc(x: torch.Tensor): + """ + Implementation of sinc, i.e. sin(pi * x) / (pi * x) + __Warning__: Different to julius.sinc, the input is multiplied by `pi`! + """ + return torch.where( + x == 0, + torch.tensor(1.0, device=x.device, dtype=x.dtype), + torch.sin(math.pi * x) / math.pi / x, + ) + + +def kaiser_sinc_filter1d(cutoff, half_width, kernel_size): # return filter [1,1,kernel_size] + even = kernel_size % 2 == 0 + half_size = kernel_size // 2 + + # For kaiser window + delta_f = 4 * half_width + A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95 + if A > 50.0: + beta = 0.1102 * (A - 8.7) + elif A >= 21.0: + beta = 0.5842 * (A - 21) ** 0.4 + 0.07886 * (A - 21.0) + else: + beta = 0.0 + window = torch.kaiser_window(kernel_size, beta=beta, periodic=False) + + # ratio = 0.5/cutoff -> 2 * cutoff = 1 / ratio + if even: + time = torch.arange(-half_size, half_size) + 0.5 + else: + time = torch.arange(kernel_size) - half_size + if cutoff == 0: + filter_ = torch.zeros_like(time) + else: + filter_ = 2 * cutoff * window * sinc(2 * cutoff * time) + # Normalize filter to have sum = 1, otherwise we will have a small leakage + # of the constant component in the input signal. + filter_ /= filter_.sum() + filter = filter_.view(1, 1, kernel_size) + + return filter + + +class LowPassFilter1d(nn.Module): + def __init__( + self, + cutoff=0.5, + half_width=0.6, + stride: int = 1, + padding: bool = True, + padding_mode: str = "replicate", + kernel_size: int = 12, + channels: int = 1, + causal: bool = True, + fixed_filter: bool = False, + ): + # kernel_size should be even number for stylegan3 setup, + # in this implementation, odd number is also possible. + super().__init__() + if cutoff < -0.0: + raise ValueError("Minimum cutoff must be larger than zero.") + if cutoff > 0.5: + raise ValueError("A cutoff above 0.5 does not make sense.") + self.kernel_size = kernel_size + if causal: + self.pad_left = kernel_size - 1 + self.pad_right = 0 + else: + self.even = kernel_size % 2 == 0 + self.pad_left = kernel_size // 2 - int(self.even) + self.pad_right = kernel_size // 2 + self.stride = stride + self.padding = padding + self.padding_mode = padding_mode + self.fixed_filter = fixed_filter + filter = kaiser_sinc_filter1d(cutoff, half_width, kernel_size) + if fixed_filter: + self.register_buffer("filter", filter) + else: + self.filter = nn.Parameter(filter.expand(channels, -1, -1).clone()) + + # input [B, C, T] + def forward(self, x): + _, C, _ = x.shape + if self.padding: + x = F.pad(x, (self.pad_left, self.pad_right), mode=self.padding_mode) + if self.fixed_filter: + out = F.conv1d(x, self.filter.expand(C, -1, -1), stride=self.stride, groups=C) + else: + out = F.conv1d(x, self.filter, stride=self.stride, groups=C) + return out + + +# ============================================================================ +# Alias-free up/down sampling (vendored from modules/vocoder/alias_free_resample.py) +# ---------------------------------------------------------------------------- +# Adapted from https://github.com/junjun3518/alias-free-torch (Apache 2.0). +# ============================================================================ + + +class UpSample1d(nn.Module): + def __init__(self, ratio=2, kernel_size=None, channels=None, causal=True, fixed_filter=False): + super().__init__() + self.ratio = ratio + self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size + self.stride = ratio + self.channels = channels + self.causal = causal + self.fixed_filter = fixed_filter + if causal: + self.pad = 0 + else: + self.pad = self.kernel_size // ratio - 1 + self.pad_left = self.pad * self.stride + (self.kernel_size - self.stride) // 2 + self.pad_right = self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2 + filter = kaiser_sinc_filter1d(cutoff=0.5 / ratio, half_width=0.6 / ratio, kernel_size=self.kernel_size) + if self.fixed_filter: + self.register_buffer("filter", filter) + else: + self.filter = nn.Parameter(filter.expand(channels, -1, -1).clone()) + + # x: [B, C, T] + def forward(self, x): + _, C, _ = x.shape + x = F.pad(x, (self.pad, self.pad), mode="replicate") + if self.fixed_filter: + x = self.ratio * F.conv_transpose1d(x, self.filter.expand(C, -1, -1), stride=self.stride, groups=C) + else: + x = self.ratio * F.conv_transpose1d(x, self.filter, stride=self.stride, groups=C) + if self.causal: + x = x[..., : -(self.kernel_size - self.stride)] + else: + x = x[..., self.pad_left : -self.pad_right] + + return x + + +class DownSample1d(nn.Module): + def __init__(self, ratio=2, kernel_size=None, channels=None, causal=True, fixed_filter=False): + super().__init__() + self.ratio = ratio + self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size + self.lowpass = LowPassFilter1d( + cutoff=0.5 / ratio, + half_width=0.6 / ratio, + stride=ratio, + kernel_size=self.kernel_size, + channels=channels, + causal=causal, + fixed_filter=fixed_filter, + ) + + def forward(self, x): + return self.lowpass(x) + + +# ============================================================================ +# Snake activations + alias-free Activation1d (vendored from +# modules/vocoder/alias_free_act.py) +# ---------------------------------------------------------------------------- +# Adapted from https://github.com/junjun3518/alias-free-torch (Apache 2.0). +# Snake / SnakeBeta paper: https://arxiv.org/abs/2006.08195 +# ============================================================================ + + +class Activation1d(nn.Module): + def __init__( + self, + activation, + up_ratio: int = 2, + down_ratio: int = 2, + up_kernel_size: int = 12, + down_kernel_size: int = 12, + causal=True, + fixed_filter=False, + ): + super().__init__() + # causal=False + self.up_ratio = up_ratio + self.down_ratio = down_ratio + self.act = activation + self.upsample = UpSample1d( + up_ratio, + up_kernel_size, + activation.in_features, + causal=causal, + fixed_filter=fixed_filter, + ) + self.downsample = DownSample1d( + down_ratio, + down_kernel_size, + activation.in_features, + causal=causal, + fixed_filter=fixed_filter, + ) + + # x: [B,C,T] + def forward(self, x): + x = self.upsample(x) + x = self.act(x) + return self.downsample(x) + + +class Snake(nn.Module): + """ + Implementation of a sine-based periodic activation function + Shape: + - Input: (B, C, T) + - Output: (B, C, T), same shape as the input + Parameters: + - alpha - trainable parameter + """ + + def __init__(self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False): + super().__init__() + self.in_features = in_features + + # initialize alpha + self.alpha_logscale = alpha_logscale + if self.alpha_logscale: # log scale alphas initialized to zeros + self.alpha = Parameter(torch.zeros(in_features) * alpha) + else: # linear scale alphas initialized to ones + self.alpha = Parameter(torch.ones(in_features) * alpha) + + self.alpha.requires_grad = alpha_trainable + + self.no_div_by_zero = 0.000000001 + + def forward(self, x): + # Snake := x + 1/a * sin^2 (xa) + alpha = self.alpha.unsqueeze(0).unsqueeze(-1) # line up with x to [B, C, T] + if self.alpha_logscale: + alpha = torch.exp(alpha) + return x + (1.0 / (alpha + self.no_div_by_zero)) * pow(sin(x * alpha), 2) + + +class SnakeBeta(nn.Module): + """ + A modified Snake function which uses separate parameters for the magnitude + of the periodic components. + Shape: + - Input: (B, C, T) + - Output: (B, C, T), same shape as the input + Parameters: + - alpha - trainable parameter that controls frequency + - beta - trainable parameter that controls magnitude + """ + + def __init__(self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False): + super().__init__() + self.in_features = in_features + + # initialize alpha + self.alpha_logscale = alpha_logscale + if self.alpha_logscale: # log scale alphas initialized to zeros + self.alpha = Parameter(torch.zeros(in_features) * alpha) + self.beta = Parameter(torch.zeros(in_features) * alpha) + else: # linear scale alphas initialized to ones + self.alpha = Parameter(torch.ones(in_features) * alpha) + self.beta = Parameter(torch.ones(in_features) * alpha) + + self.alpha.requires_grad = alpha_trainable + self.beta.requires_grad = alpha_trainable + + self.no_div_by_zero = 0.000000001 + + def forward(self, x): + # SnakeBeta := x + 1/b * sin^2 (xa) + alpha = self.alpha.unsqueeze(0).unsqueeze(-1) # line up with x to [B, C, T] + beta = self.beta.unsqueeze(0).unsqueeze(-1) + if self.alpha_logscale: + alpha = torch.exp(alpha) + beta = torch.exp(beta) + return x + (1.0 / (beta + self.no_div_by_zero)) * pow(sin(x * alpha), 2) + + +# ============================================================================ +# BigVGAN AudioVAE (vendored from modules/vocoder/bigvgan.py) +# ---------------------------------------------------------------------------- +# Copyright (c) 2022 NVIDIA CORPORATION. +# Licensed under the MIT license. +# Upstream layout flattened: the only modification is that the three +# original ``from dots_tts.modules.{backbone,vocoder}.* import ...`` lines +# are dropped, since those names are already defined above in this file. +# ============================================================================ + + +@dataclass(slots=True) +class BigVGANStreamState: + lstm_hidden: tuple[torch.Tensor, torch.Tensor] + decoder: DecoderStreamState + + +@dataclass(slots=True) +class DecoderStreamState: + window: torch.Tensor + chunk_size: int + total_frames: int = 0 + emitted_frames: int = 0 + + +def _empty_chunk( + ref: torch.Tensor, + *, + channels: int | None = None, + dtype: torch.dtype | None = None, +) -> torch.Tensor: + return ref.new_zeros( + (ref.size(0), channels or ref.size(1), 0), + dtype=dtype or ref.dtype, + ) + + +def _module_state_device_dtype(module: nn.Module) -> tuple[torch.device, torch.dtype]: + for name in ("weight", "bias", "filter"): + tensor = getattr(module, name, None) + if tensor is not None: + return tensor.device, tensor.dtype + for tensor in itertools.chain(module.parameters(), module.buffers()): + return tensor.device, tensor.dtype + raise RuntimeError(f"Unable to infer state dtype/device for {type(module).__name__}.") + + +def _stream_state_zeros( + batch_size: int, + channels: int, + length: int, + *, + device: torch.device, + dtype: torch.dtype, +) -> torch.Tensor: + return torch.zeros( + (batch_size, channels, max(0, int(length))), + device=device, + dtype=dtype, + ) + + +def init_weights(m, mean=0.0, std=0.01): + classname = m.__class__.__name__ + if classname.find("Conv") != -1: + m.weight.data.normal_(mean, std) + + +class Conv1d_S(nn.Module): + "Conv1d for spectral normalisation and orthogonal initialisation" + + def __init__( + self, + in_channels, + out_channels, + kernel_size=1, + stride=1, + dilation=1, + groups=1, + causal=False, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.stride = stride + self.dilation = dilation + self.groups = groups + self.causal = causal + pad = 0 if causal else dilation * (kernel_size - 1) // 2 + self.causal_pad = dilation * (kernel_size - 1) if causal else 0 + + self.layer = weight_norm( + nn.Conv1d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=pad, + dilation=dilation, + groups=groups, + ) + ) + + def forward(self, inputs): + if self.causal and self.causal_pad > 0: + inputs = F.pad(inputs, (self.causal_pad, 0)) + return self.layer(inputs) + + +class SLSTM(nn.Module): + """ + LSTM without worrying about the hidden state, nor the layout of the data. + Expects input as convolutional layout. + """ + + def __init__( + self, + dimension: int, + num_layers: int = 2, + skip: bool = True, + bidirectional: bool = False, + ): + super().__init__() + self.skip = skip + self.bidirectional = bidirectional + self.lstm = nn.LSTM( + input_size=dimension, + hidden_size=dimension, + num_layers=num_layers, + bidirectional=bidirectional, + batch_first=True, + ) + self._stream_num_layers = num_layers + self._stream_weight_ih = tuple(getattr(self.lstm, f"weight_ih_l{layer_idx}") for layer_idx in range(num_layers)) + self._stream_weight_hh = tuple(getattr(self.lstm, f"weight_hh_l{layer_idx}") for layer_idx in range(num_layers)) + self._stream_bias_ih = tuple(getattr(self.lstm, f"bias_ih_l{layer_idx}") for layer_idx in range(num_layers)) + self._stream_bias_hh = tuple(getattr(self.lstm, f"bias_hh_l{layer_idx}") for layer_idx in range(num_layers)) + if self.bidirectional: + self.proj_out = nn.Linear(dimension * 2, dimension) + + def forward(self, x): + y, _ = self.lstm(x) + if self.bidirectional: + y = self.proj_out(y) + if self.skip: + y = y + x + return y + + def stream_step( + self, + x: torch.Tensor, + hidden: tuple[torch.Tensor, torch.Tensor] | None, + ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]: + if self.bidirectional: + raise RuntimeError("Streaming only supports unidirectional SLSTM.") + + residual = x + if hidden is None: + hidden = self.init_stream_state(x.size(0)) + + hidden_h, hidden_c = hidden + next_hidden_h = [] + next_hidden_c = [] + for layer_idx in range(self._stream_num_layers): + layer_input = x + hx = hidden_h[layer_idx] + cx = hidden_c[layer_idx] + outputs = [] + weight_ih = self._stream_weight_ih[layer_idx] + weight_hh = self._stream_weight_hh[layer_idx] + bias_ih = self._stream_bias_ih[layer_idx] + bias_hh = self._stream_bias_hh[layer_idx] + + for frame_idx in range(x.size(1)): + gates = F.linear(layer_input[:, frame_idx, :], weight_ih, bias_ih) + gates = gates + F.linear(hx, weight_hh, bias_hh) + input_gate, forget_gate, cell_gate, output_gate = gates.chunk(4, dim=-1) + input_gate = torch.sigmoid(input_gate) + forget_gate = torch.sigmoid(forget_gate) + cell_gate = torch.tanh(cell_gate) + output_gate = torch.sigmoid(output_gate) + cx = forget_gate * cx + input_gate * cell_gate + hx = output_gate * torch.tanh(cx) + outputs.append(hx) + + x = torch.stack(outputs, dim=1) + next_hidden_h.append(hx) + next_hidden_c.append(cx) + + y = x + if self.skip: + y = y + residual + return y, (torch.stack(next_hidden_h, dim=0), torch.stack(next_hidden_c, dim=0)) + + def init_stream_state( + self, + batch_size: int, + ) -> tuple[torch.Tensor, torch.Tensor]: + num_directions = 2 if self.bidirectional else 1 + state_shape = ( + self.lstm.num_layers * num_directions, + batch_size, + self.lstm.hidden_size, + ) + weight = self.lstm.weight_hh_l0 + return ( + weight.new_zeros(state_shape), + weight.new_zeros(state_shape), + ) + + +class ResStack(nn.Module): + def __init__(self, channel, kernel_size=3, base=3, nums=4, causal=False): + super().__init__() + + self.layers = nn.ModuleList([]) + for i in range(nums): + dil = base**i + pad1 = dil * (kernel_size - 1) if causal else dil + pad2 = (kernel_size - 1) if causal else 1 + block = [ + nn.LeakyReLU(), + ] + if causal and pad1 > 0: + block.append(nn.ConstantPad1d((pad1, 0), 0.0)) + block.append( + nn.utils.weight_norm( + nn.Conv1d( + channel, + channel, + kernel_size=kernel_size, + dilation=dil, + padding=0 if causal else pad1, + ) + ) + ) + block.append(nn.LeakyReLU()) + if causal and pad2 > 0: + block.append(nn.ConstantPad1d((pad2, 0), 0.0)) + block.append( + nn.utils.weight_norm( + nn.Conv1d( + channel, + channel, + kernel_size=kernel_size, + dilation=1, + padding=0 if causal else pad2, + ) + ) + ) + self.layers.append(nn.Sequential(*block)) + + def forward(self, x): + for layer in self.layers: + x = x + layer(x) + return x + + +class Encoder(nn.Module): + def __init__( + self, + in_channels=1, + out_channels=100, + base_channels=12, + proj_kernel_size=3, + stack_kernel_size=3, + stack_dilation_base=2, + stacks=6, + channels=(12, 24, 48, 96, 192, 384, 768), + down_sample_factors=(2, 2, 2, 2, 4, 4), + causal=False, + lookahead=0, + ): + super().__init__() + + act_slope = 0.2 + layers = [] + # pre proj_layer + layers += [ + Conv1d_S( + in_channels, + base_channels, + kernel_size=proj_kernel_size, + stride=1, + causal=causal, + ), + nn.LeakyReLU(act_slope, True), + ] + + # channels: [512, 256, 128, 64], upsample_factors: [5, 2, 2] + for (in_c, out_c), down_f in zip(itertools.pairwise(channels), down_sample_factors, strict=True): + layers += [ + Conv1d_S(in_c, out_c, kernel_size=down_f * 2, stride=down_f, causal=causal), + ResStack(out_c, stack_kernel_size, stack_dilation_base, stacks, causal=causal), + nn.LeakyReLU(act_slope, True), + ] + + # post layers + if lookahead > 0: + layers += [ + Conv1d_S( + channels[-1], + out_channels, + kernel_size=lookahead * 2 + 1, + stride=1, + causal=False, + ), + ] + else: + layers += [ + Conv1d_S( + channels[-1], + out_channels, + kernel_size=proj_kernel_size, + stride=1, + causal=causal, + ), + ] + self.generator = nn.Sequential(*layers) + + def forward(self, conditions, _z_inputs=None): + return self.generator(conditions) + + +class AMPBlock1(torch.nn.Module): + def __init__( + self, + h, + channels, + kernel_size=3, + dilation=(1, 3, 5), + activation=None, + causal=True, + ): + super().__init__() + self.h = h + + self.convs1 = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[0], + causal=causal, + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[1], + causal=causal, + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[2], + causal=causal, + ) + ), + ] + ) + self.convs1.apply(init_weights) + + self.convs2 = nn.ModuleList( + [ + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, causal=causal)), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, causal=causal)), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, causal=causal)), + ] + ) + self.convs2.apply(init_weights) + + self.num_layers = len(self.convs1) + len(self.convs2) # total number of conv layers + + if activation == "snake": # periodic nonlinearity with snake function and anti-aliasing + self.activations = nn.ModuleList( + [ + Activation1d( + activation=Snake(channels, alpha_logscale=h.snake_logscale), + causal=causal, + fixed_filter=True, + ) + for _ in range(self.num_layers) + ] + ) + elif activation == "snakebeta": # periodic nonlinearity with snakebeta function and anti-aliasing + self.activations = nn.ModuleList( + [ + Activation1d( + activation=SnakeBeta(channels, alpha_logscale=h.snake_logscale), + causal=causal, + fixed_filter=True, + ) + for _ in range(self.num_layers) + ] + ) + else: + raise NotImplementedError( + "activation incorrectly specified. check the config file and look for 'activation'." + ) + + def forward(self, x): + acts1, acts2 = self.activations[::2], self.activations[1::2] + for c1, c2, a1, a2 in zip(self.convs1, self.convs2, acts1, acts2, strict=True): + xt = a1(x) + xt = c1(xt) + xt = a2(xt) + xt = c2(xt) + x = xt + x + + return x + + def remove_weight_norm(self): + for layer in self.convs1: + remove_weight_norm(layer) + for layer in self.convs2: + remove_weight_norm(layer) + + +class AMPBlock2(torch.nn.Module): + def __init__(self, h, channels, kernel_size=3, dilation=(1, 3), activation=None, causal=True): + super().__init__() + self.h = h + + self.convs = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[0], + causal=causal, + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[1], + causal=causal, + ) + ), + ] + ) + self.convs.apply(init_weights) + + self.num_layers = len(self.convs) # total number of conv layers + + if activation == "snake": # periodic nonlinearity with snake function and anti-aliasing + self.activations = nn.ModuleList( + [ + Activation1d( + activation=Snake(channels, alpha_logscale=h.snake_logscale), + causal=causal, + fixed_filter=True, + ) + for _ in range(self.num_layers) + ] + ) + elif activation == "snakebeta": # periodic nonlinearity with snakebeta function and anti-aliasing + self.activations = nn.ModuleList( + [ + Activation1d( + activation=SnakeBeta(channels, alpha_logscale=h.snake_logscale), + causal=causal, + fixed_filter=True, + ) + for _ in range(self.num_layers) + ] + ) + else: + raise NotImplementedError( + "activation incorrectly specified. check the config file and look for 'activation'." + ) + + def forward(self, x): + for c, a in zip(self.convs, self.activations, strict=True): + xt = a(x) + xt = c(xt) + x = xt + x + + return x + + def remove_weight_norm(self): + for layer in self.convs: + remove_weight_norm(layer) + + +class Decoder(nn.Module): + def __init__(self, h): + super().__init__() + self.h = h + causal = h.causal + self._stream_window_sizes: dict[int, int] = {} + + self.num_kernels = len(h.resblock_kernel_sizes) + self.num_upsamples = len(h.upsample_rates) + + num_decoder_lookahead = h.get("num_decoder_lookahead", 2) + # pre conv + self.conv_pre = weight_norm( + Conv1d( + h.latent_dim, + h.upsample_initial_channel, + kernel_size=2 * num_decoder_lookahead + 1, + stride=1, + causal=False, + ) + ) + + # define which AMPBlock to use. BigVGAN uses AMPBlock1 as default + resblock = AMPBlock1 if h.resblock == "1" else AMPBlock2 + + # transposed conv-based upsamplers. does not apply anti-aliasing + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(h.upsample_rates, h.upsample_kernel_sizes, strict=True)): + self.ups.append( + nn.ModuleList( + [ + weight_norm( + ConvTranspose1d( + h.upsample_initial_channel // (2**i), + h.upsample_initial_channel // (2 ** (i + 1)), + k, + u, + causal=causal, + ) + ) + ] + ) + ) + + # residual blocks using anti-aliased multi-periodicity composition modules (AMP) + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = h.upsample_initial_channel // (2 ** (i + 1)) + for k, d in zip(h.resblock_kernel_sizes, h.resblock_dilation_sizes, strict=True): + self.resblocks.append(resblock(h, ch, k, d, activation=h.activation, causal=causal)) + + # post conv + if h.activation == "snake": # periodic nonlinearity with snake function and anti-aliasing + activation_post = Snake(ch, alpha_logscale=h.snake_logscale) + self.activation_post = Activation1d(activation=activation_post, causal=causal, fixed_filter=False) + elif h.activation == "snakebeta": # periodic nonlinearity with snakebeta function and anti-aliasing + activation_post = SnakeBeta(ch, alpha_logscale=h.snake_logscale) + self.activation_post = Activation1d(activation=activation_post, causal=causal, fixed_filter=False) + else: + raise NotImplementedError( + "activation incorrectly specified. check the config file and look for 'activation'." + ) + + self.conv_post = weight_norm(Conv1d(ch, 1, 7, 1, causal=causal, bias=h.get("use_bias_at_final", True))) + + # weight initialization + for i in range(len(self.ups)): + self.ups[i].apply(init_weights) + self.conv_post.apply(init_weights) + + def forward(self, z): + # pre conv + x = self.conv_pre(z) + + for i in range(self.num_upsamples): + # upsampling + for i_up in range(len(self.ups[i])): + x = self.ups[i][i_up](x) + # AMP blocks + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i * self.num_kernels + j](x) + else: + xs += self.resblocks[i * self.num_kernels + j](x) + x = xs / self.num_kernels + + # post conv + x = self.activation_post(x) + x = self.conv_post(x) + if self.h.get("use_tanh_at_final", True): + x = torch.tanh(x) + else: + x = torch.clamp(x, min=-1.0, max=1.0) # Bound the output to [-1, 1] + + return x + + @property + def stream_lookahead(self) -> int: + return (self.conv_pre.kernel_size[0] - 1) // 2 + + @staticmethod + def _conv1d_left_context(layer) -> int: + dilation = layer.dilation[0] if isinstance(layer.dilation, tuple) else layer.dilation + kernel_size = layer.kernel_size[0] if isinstance(layer.kernel_size, tuple) else layer.kernel_size + if getattr(layer, "causal", False): + return dilation * (kernel_size - 1) + return layer.padding[0] if isinstance(layer.padding, tuple) else layer.padding + + @staticmethod + def _convtranspose1d_left_context(layer) -> int: + stride = layer.stride[0] if isinstance(layer.stride, tuple) else layer.stride + kernel_size = layer.kernel_size[0] if isinstance(layer.kernel_size, tuple) else layer.kernel_size + if not getattr(layer, "causal", False): + raise NotImplementedError("Streaming only supports causal ConvTranspose1d.") + if kernel_size != 2 * stride: + raise ValueError( + "Streaming ConvTranspose1d expects kernel_size == 2 * stride, got " + f"kernel_size={kernel_size} stride={stride}." + ) + return 1 + + @classmethod + def _activation_left_context(cls, activation: Activation1d) -> int: + upsample = activation.upsample + downsample = activation.downsample.lowpass + if not upsample.causal or not downsample.padding or downsample.pad_right != 0: + raise NotImplementedError("Streaming only supports causal alias-free activations.") + ratio = int(upsample.ratio) + if ratio != int(downsample.stride): + raise ValueError( + "Alias-free activation expects matched up/down ratios, got " + f"up_ratio={ratio} down_ratio={downsample.stride}." + ) + total_left = (upsample.kernel_size - 1) + (downsample.kernel_size - 1) + return (total_left + ratio - 1) // ratio + + @classmethod + def _ampblock_left_context(cls, block) -> int: + if isinstance(block, AMPBlock1): + left_context = 0 + acts1 = block.activations[::2] + acts2 = block.activations[1::2] + for conv1, conv2, act1, act2 in zip( + block.convs1, + block.convs2, + acts1, + acts2, + strict=True, + ): + left_context += ( + cls._activation_left_context(act1) + + cls._conv1d_left_context(conv1) + + cls._activation_left_context(act2) + + cls._conv1d_left_context(conv2) + ) + return left_context + if isinstance(block, AMPBlock2): + left_context = 0 + for conv, activation in zip(block.convs, block.activations, strict=True): + left_context += cls._activation_left_context(activation) + cls._conv1d_left_context(conv) + return left_context + raise TypeError(f"Unsupported resblock type: {type(block).__name__}.") + + def _stream_left_context(self) -> int: + left_context = Fraction(self._conv1d_left_context(self.conv_pre), 1) + current_scale = Fraction(1, 1) + for stage_idx, upsample_layers in enumerate(self.ups): + for upsample in upsample_layers: + left_context += current_scale * self._convtranspose1d_left_context(upsample) + stride = upsample.stride[0] if isinstance(upsample.stride, tuple) else upsample.stride + current_scale /= int(stride) + + stage_start = stage_idx * self.num_kernels + stage_end = stage_start + self.num_kernels + stage_context = max(self._ampblock_left_context(block) for block in self.resblocks[stage_start:stage_end]) + left_context += current_scale * stage_context + + left_context += current_scale * self._activation_left_context(self.activation_post) + left_context += current_scale * self._conv1d_left_context(self.conv_post) + return int(left_context.__ceil__()) + + def stream_window_size(self, chunk_size: int) -> int: + if chunk_size < 1: + raise ValueError(f"chunk_size must be >= 1, got {chunk_size}.") + cached = self._stream_window_sizes.get(chunk_size) + if cached is not None: + return cached + + window_size = chunk_size + self.stream_lookahead + self._stream_left_context() + self._stream_window_sizes[chunk_size] = window_size + return window_size + + def remove_weight_norm(self): + for upsample_layers in self.ups: + for upsample_layer in upsample_layers: + remove_weight_norm(upsample_layer) + for resblock in self.resblocks: + resblock.remove_weight_norm() + remove_weight_norm(self.conv_pre) + remove_weight_norm(self.conv_post) + + +class AudioVAE(nn.Module): + def __init__(self, h: AudioVAEConfig): + super().__init__() + self.config = h + + self.h = h + self.hop_size = int(np.prod(h.downsample_rates)) + self.sample_rate = h.sample_rate + self.decoder_lookahead = int(h.get("num_decoder_lookahead", 2)) + + self.audio_encoder = Encoder( + out_channels=h.latent_dim, + down_sample_factors=h.downsample_rates, + channels=h.downsample_channels, + causal=h.causal_encoder, + lookahead=h.get("num_encoder_lookahead", 2), + ) + + intermediate_size = h.latent_dim * 4 + self.enc_mi_layer = nn.Sequential( + nn.Linear(h.latent_dim, intermediate_size), + SLSTM(intermediate_size, num_layers=h.mi_num_layers), + nn.Linear(intermediate_size, h.latent_dim), + ) + self.dec_mi_layer = nn.Sequential( + nn.Linear(h.latent_dim, intermediate_size), + SLSTM(intermediate_size, num_layers=h.mi_num_layers), + nn.Linear(intermediate_size, h.latent_dim), + ) + self.pre_proj = Conv1d( + in_channels=h.latent_dim, + out_channels=h.latent_dim * 2, + kernel_size=1, + stride=1, + ) + self.post_proj = Conv1d(in_channels=h.latent_dim, out_channels=h.latent_dim, kernel_size=1, stride=1) + + self.decoder = Decoder(h) + + def inference(self, data): + latents = self.extract_latents(data["sample"]) + return {"sample": self.inference_from_latents(latents)} + + @torch.autocast(enabled=False, device_type="cuda") + def extract_latents(self, x, do_sample=False): + x = x.float() + x = self.audio_encoder(x) + x = x.permute(0, 2, 1) + x = self.enc_mi_layer(x) + x = x.permute(0, 2, 1) + x = self.pre_proj(x) + if do_sample: + m_q, logs_q = torch.split(x, self.h.latent_dim, dim=1) + x = m_q + torch.randn_like(m_q) * torch.exp(logs_q) + return x + + def inference_from_latents(self, x, do_sample=True, noise_scale=1.0): + if do_sample: + assert x.size(1) == self.h.latent_dim * 2, f"Input must be like [B, D, H], got {x.shape}" + m_q, logs_q = torch.split(x, self.h.latent_dim, dim=1) + x = m_q + torch.randn_like(m_q) * torch.exp(logs_q) * noise_scale + else: + assert x.size(1) == self.h.latent_dim, f"Input must be like [B, D, H], got {x.shape}" + x = self.post_proj(x) + x = x.permute(0, 2, 1) + x = self.dec_mi_layer(x) + x = x.permute(0, 2, 1) + return self.decoder(x) + + def _validate_stream_latents(self, latents: torch.Tensor) -> None: + if latents.ndim != 3: + raise ValueError( + f"Streaming latents must have shape [batch, latent_dim, frames], got {tuple(latents.shape)}." + ) + if latents.size(1) != self.h.latent_dim: + raise ValueError(f"Streaming latent_dim must be {self.h.latent_dim}, got {latents.size(1)}.") + + def init_stream_state( + self, + batch_size: int = 1, + chunk_size: int = 8, + ) -> BigVGANStreamState: + if not self.h.causal: + raise RuntimeError("Strict streaming requires a causal vocoder.") + if batch_size < 1: + raise ValueError(f"batch_size must be >= 1, got {batch_size}.") + window_size = self.decoder.stream_window_size(chunk_size) + device, dtype = _module_state_device_dtype(self.decoder.conv_pre) + return BigVGANStreamState( + lstm_hidden=self.dec_mi_layer[1].init_stream_state(batch_size), + decoder=DecoderStreamState( + window=_stream_state_zeros( + batch_size, + self.h.latent_dim, + window_size, + device=device, + dtype=dtype, + ), + chunk_size=int(chunk_size), + ), + ) + + def _decode_stream_latents( + self, + latents: torch.Tensor, + lstm_hidden: tuple[torch.Tensor, torch.Tensor], + ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]: + self._validate_stream_latents(latents) + latents = latents.float() + x = self.post_proj(latents) + x = x.permute(0, 2, 1) + x = self.dec_mi_layer[0](x) + x, lstm_hidden = self.dec_mi_layer[1].stream_step(x, lstm_hidden) + x = self.dec_mi_layer[2](x) + decoder_dtype = self.decoder.conv_pre.weight.dtype + return x.permute(0, 2, 1).to(dtype=decoder_dtype), lstm_hidden + + def _prepare_stream_decoder_input( + self, + latents: torch.Tensor, + state: BigVGANStreamState, + ) -> torch.Tensor: + decoder_input, state.lstm_hidden = self._decode_stream_latents( + latents, + state.lstm_hidden, + ) + return decoder_input + + def _append_stream_decoder_input_tensor( + self, + decoder_input: torch.Tensor, + window: torch.Tensor, + valid_frames: torch.Tensor, + ) -> torch.Tensor: + if window.dtype != decoder_input.dtype: + window = window.to(dtype=decoder_input.dtype) + chunk_size = int(decoder_input.size(-1)) + if chunk_size >= window.size(-1): + raise ValueError(f"decoder window size {window.size(-1)} must be larger than chunk_size {chunk_size}.") + positions = torch.arange( + window.size(-1), + device=window.device, + dtype=valid_frames.dtype, + ) + clipped_valid = valid_frames.clamp(min=0, max=window.size(-1)) + combined = torch.cat( + [window, decoder_input.new_zeros(window.size(0), window.size(1), chunk_size)], + dim=-1, + ) + insert_index = clipped_valid + torch.arange( + chunk_size, + device=window.device, + dtype=valid_frames.dtype, + ) + combined.scatter_( + -1, + insert_index.view(1, 1, -1).expand_as(decoder_input), + decoder_input, + ) + new_valid = (clipped_valid + chunk_size).clamp(max=window.size(-1)) + start = (clipped_valid + chunk_size - window.size(-1)).clamp(min=0) + gather_index = (start + positions).clamp(max=combined.size(-1) - 1) + gathered = combined.gather( + -1, + gather_index.view(1, 1, -1).expand_as(window), + ) + mask = (positions < new_valid).to(dtype=window.dtype).view(1, 1, -1) + return gathered * mask + + def _append_stream_decoder_input( + self, + decoder_input: torch.Tensor, + state: BigVGANStreamState, + ) -> torch.Tensor: + decoder_state = state.decoder + chunk_size = int(decoder_input.size(-1)) + if chunk_size != decoder_state.chunk_size: + raise ValueError(f"Streaming chunk_size must stay fixed at {decoder_state.chunk_size}, got {chunk_size}.") + window = decoder_state.window + valid_frames = min(decoder_state.total_frames, window.size(-1)) + valid_frames_tensor = window.new_tensor(valid_frames, dtype=torch.int64) + new_window = self._append_stream_decoder_input_tensor( + decoder_input, + window, + valid_frames_tensor, + ) + decoder_state.window = new_window + decoder_state.total_frames += chunk_size + return new_window + + def compiled_stream_step( + self, + latents: torch.Tensor, + hidden_h: torch.Tensor, + hidden_c: torch.Tensor, + window: torch.Tensor, + valid_frames: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + decoder_input, (hidden_h, hidden_c) = self._decode_stream_latents( + latents, + (hidden_h, hidden_c), + ) + new_window = self._append_stream_decoder_input_tensor( + decoder_input, + window, + valid_frames, + ) + audio_window = self.decode_stream_window(new_window) + return audio_window, hidden_h, hidden_c, new_window + + def decode_stream_window(self, window: torch.Tensor) -> torch.Tensor: + decoder_dtype = self.decoder.conv_pre.weight.dtype + if window.dtype != decoder_dtype: + window = window.to(dtype=decoder_dtype) + return self.decoder(window) + + def _slice_stream_audio_window( + self, + audio_window: torch.Tensor, + state: BigVGANStreamState, + *, + final: bool, + ) -> torch.Tensor: + decoder_state = state.decoder + stable_end = ( + decoder_state.total_frames if final else max(0, decoder_state.total_frames - self.decoder.stream_lookahead) + ) + if stable_end <= decoder_state.emitted_frames: + return _empty_chunk(audio_window, channels=1) + + window_size = decoder_state.window.size(-1) + valid_frames = min(decoder_state.total_frames, window_size) + window_start = decoder_state.total_frames - valid_frames + if decoder_state.emitted_frames < window_start: + raise RuntimeError("Decoder stream window is too short for fixed-graph decoding.") + + local_start = decoder_state.emitted_frames - window_start + local_end = stable_end - window_start + sample_start = local_start * self.hop_size + sample_end = local_end * self.hop_size + decoder_state.emitted_frames = stable_end + return audio_window[..., sample_start:sample_end] + + def stream_step( + self, + latents: torch.Tensor, + state: BigVGANStreamState, + ) -> torch.Tensor: + decoder_input = self._prepare_stream_decoder_input(latents, state) + window = self._append_stream_decoder_input(decoder_input, state) + audio_window = self.decode_stream_window(window) + return self._slice_stream_audio_window(audio_window, state, final=False) + + def stream_flush(self, state: BigVGANStreamState) -> torch.Tensor: + audio_window = self.decode_stream_window(state.decoder.window) + return self._slice_stream_audio_window(audio_window, state, final=True) + + def remove_weight_norm(self): + self.decoder.remove_weight_norm() + + +__all__ = ["AudioVAEConfig", "AudioVAE"] diff --git a/vllm_omni/model_executor/models/dots_tts/pipeline.py b/vllm_omni/model_executor/models/dots_tts/pipeline.py new file mode 100644 index 000000000..22b50c629 --- /dev/null +++ b/vllm_omni/model_executor/models/dots_tts/pipeline.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""dots.tts pipeline topology (frozen). + +Single-stage AR TTS: text → 48 kHz waveform in one pass. +Architecture: Qwen2.5-1.5B base LM (vLLM-native) → DiTAR / MeanFlow head +→ AudioVAE.decode → BigVGAN → 48 kHz mono. + +Speaker conditioning via CAM++ x-vector injected in preprocess(). + +Modeled after voxcpm2's single-stage pipeline (same "vLLM-native base LM ++ side-path computation" pattern), but simplified: no residual LM, no FSQ. +""" + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +DOTS_TTS_PIPELINE = PipelineConfig( + model_type="dots_tts", + model_arch="DotsTTSForConditionalGeneration", + default_deploy_config_name="dots_tts.yaml", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="latent_generator", + execution_type=StageExecutionType.LLM_AR, + input_sources=(), + final_output=True, + final_output_type="audio", + owns_tokenizer=True, + engine_output_type="audio", + sampling_constraints={ + "detokenize": False, + # Stop signal slot in compute_logits (voxcpm2 convention — + # talker emits logits[i, 1] = 1.0 when prob_stop > 0.8, vLLM + # matches against this list to terminate the request). Not + # Qwen2.5 EOS (151643) — that ID would never come out of the + # 2-slot continue/stop softmax our compute_logits builds. + "stop_token_ids": [1], + }, + ), + ), +) diff --git a/vllm_omni/model_executor/models/registry.py b/vllm_omni/model_executor/models/registry.py index 498f9eb85..530fb1712 100644 --- a/vllm_omni/model_executor/models/registry.py +++ b/vllm_omni/model_executor/models/registry.py @@ -301,6 +301,12 @@ _OMNI_MODELS = { "voxcpm2_talker", "VoxCPM2TalkerForConditionalGeneration", ), + ## dots.tts + "DotsTTSForConditionalGeneration": ( + "dots_tts", + "dots_tts_talker", + "DotsTTSForConditionalGeneration", + ), ## Voxtral TTS "VoxtralTTSForConditionalGeneration": ( "voxtral_tts", diff --git a/vllm_omni/transformers_utils/configs/__init__.py b/vllm_omni/transformers_utils/configs/__init__.py index 88a23dc0c..cdf105d69 100644 --- a/vllm_omni/transformers_utils/configs/__init__.py +++ b/vllm_omni/transformers_utils/configs/__init__.py @@ -20,6 +20,7 @@ _CLASS_TO_MODULE: dict[str, str] = { "FishSpeechFastARConfig": "vllm_omni.transformers_utils.configs.fish_speech", "GLMTTSConfig": "vllm_omni.transformers_utils.configs.glm_tts", "VoxCPM2Config": "vllm_omni.transformers_utils.configs.voxcpm2", + "DotsTTSConfig": "vllm_omni.transformers_utils.configs.dots_tts", "VoxtralTTSConfig": "vllm_omni.transformers_utils.configs.voxtral_tts", "CosyVoice3Config": "vllm_omni.transformers_utils.configs.cosyvoice3", "OmniVoiceConfig": "vllm_omni.transformers_utils.configs.omnivoice", @@ -42,6 +43,7 @@ __all__ = [ "FishSpeechFastARConfig", "GLMTTSConfig", "VoxCPM2Config", + "DotsTTSConfig", "VoxtralTTSConfig", "CosyVoice3Config", "OmniVoiceConfig", @@ -70,6 +72,7 @@ def __dir__(): # Eagerly import all config modules so their AutoConfig.register() side-effects # run as soon as `vllm_omni.transformers_utils.configs` is imported. from vllm_omni.transformers_utils.configs import cosyvoice3 as _cosyvoice3 # noqa: F401, E402 +from vllm_omni.transformers_utils.configs import dots_tts as _dots_tts # noqa: F401, E402 from vllm_omni.transformers_utils.configs import fish_speech as _fish_speech # noqa: F401, E402 from vllm_omni.transformers_utils.configs import glm_tts as _glm_tts # noqa: F401, E402 from vllm_omni.transformers_utils.configs import higgs_audio_v3 as _higgs_audio_v3 # noqa: F401, E402 diff --git a/vllm_omni/transformers_utils/configs/dots_tts.py b/vllm_omni/transformers_utils/configs/dots_tts.py new file mode 100644 index 000000000..1950a2b3a --- /dev/null +++ b/vllm_omni/transformers_utils/configs/dots_tts.py @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""HF config for dots.tts. + +dots.tts splits its config across two files in the HF checkpoint: + * ``config.json`` — top-level dots.tts fields (latent_dim / patch_size + / PatchEncoder / DiT / vocoder / etc.). + * ``llm_config.json`` — full Qwen2 LM config (vocab_size / hidden_size / + num_hidden_layers / rope_theta / etc.). + +``AutoConfig.from_pretrained`` only consumes ``config.json``: its dots.tts +sub-blocks (``vocoder`` / ``DiT`` / ``PatchEncoder``) arrive through +``**kwargs`` and land as attributes, which the talker's config factories +consume directly. The Qwen2 LM fields live in ``llm_config.json``, which +AutoConfig never reads — this class hoists them to top-level attributes +with defaults matching the released checkpoints (soar / base / mf all ship +the same Qwen2.5-1.5B backbone). vLLM's ``Qwen2Model.__init__`` then reads +``config.hidden_size`` / ``config.vocab_size`` / etc. through +``get_text_config()``, the same pattern voxcpm2 follows. + +Future work: read ``llm_config.json`` during ``from_pretrained`` so a +checkpoint with a different LM shape works without editing this file. + +Upstream reference: + https://huggingface.co/rednote-hilab/dots.tts-soar/blob/main/llm_config.json +""" + +from __future__ import annotations + +from transformers import AutoConfig +from transformers.configuration_utils import PretrainedConfig + + +class DotsTTSConfig(PretrainedConfig): + """Configuration for dots.tts integration (Qwen2.5-1.5B-Base AR backbone).""" + + model_type = "dots_tts" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + # -- top-level dots.tts params -- + lm_config: dict | None = None, + # -- LM defaults (overridden by lm_config dict if present) -- + # All values match dots.tts-soar's llm_config.json. + vocab_size: int = 151672, + hidden_size: int = 1536, + intermediate_size: int = 8960, + num_hidden_layers: int = 28, + num_attention_heads: int = 12, + num_key_value_heads: int = 2, + max_position_embeddings: int = 131072, + rms_norm_eps: float = 1e-6, + rope_theta: float = 1_000_000.0, + tie_word_embeddings: bool = True, + hidden_act: str = "silu", + bos_token_id: int = 151643, + eos_token_id: int = 151643, + **kwargs, + ): + super().__init__( + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + # The dots.tts sub-blocks (vocoder / DiT / PatchEncoder / latent_dim + # / patch_size / ...) arrive via **kwargs and are stored as + # attributes by PretrainedConfig; the talker's config factories + # read them from there. + self.lm_config = lm_config or {} + + # -- Hoist LM parameters to top-level for vLLM's Qwen2Model -- + lm = self.lm_config + self.vocab_size = lm.get("vocab_size", vocab_size) + self.hidden_size = lm.get("hidden_size", hidden_size) + self.intermediate_size = lm.get("intermediate_size", intermediate_size) + self.num_hidden_layers = lm.get("num_hidden_layers", num_hidden_layers) + self.num_attention_heads = lm.get("num_attention_heads", num_attention_heads) + self.num_key_value_heads = lm.get("num_key_value_heads", num_key_value_heads) + self.max_position_embeddings = lm.get("max_position_embeddings", max_position_embeddings) + self.rms_norm_eps = lm.get("rms_norm_eps", rms_norm_eps) + self.rope_theta = lm.get("rope_theta", rope_theta) + self.hidden_act = lm.get("hidden_act", hidden_act) + self.head_dim = self.hidden_size // self.num_attention_heads + + # vLLM's Qwen2Model reads ``max_window_layers`` and checks against + # ``num_hidden_layers`` only when ``use_sliding_window`` is true. + # soar disables sliding window — set both to the same value so the + # assertion (if executed) stays vacuous. + self.use_sliding_window = lm.get("use_sliding_window", False) + self.sliding_window = lm.get("sliding_window", None) + self.max_window_layers = lm.get("max_window_layers", self.num_hidden_layers) + + def get_text_config(self, **kwargs): + """Return self — LM fields are hoisted to top-level attributes.""" + return self + + +AutoConfig.register("dots_tts", DotsTTSConfig) + +__all__ = ["DotsTTSConfig"] From 596c16a550aa134faf7f3dcfa0f8adf513ccd9ce Mon Sep 17 00:00:00 2001 From: Bo Li <22713281+bobboli@users.noreply.github.com> Date: Fri, 14 Aug 2026 08:46:59 +0800 Subject: [PATCH 005/267] [Kernel] Fuse Q/K RMSNorm and RoPE (#5990) Signed-off-by: Bo Li <22713281+bobboli@users.noreply.github.com> Signed-off-by: lishunyang12 Signed-off-by: Mengjie Zhao Co-authored-by: lishunyang12 Co-authored-by: Claude Opus 5 --- .../layers/test_fused_qk_norm_rope.py | 82 +++++ tests/e2e/accuracy/ltx/run_ltx_reference.py | 12 +- .../diffusion/cache/teacache/extractors.py | 7 +- .../diffusion/layers/fused_qk_norm_rope.py | 299 ++++++++++++++++++ .../minimax_h3/minimax_h3_transformer.py | 47 ++- 5 files changed, 420 insertions(+), 27 deletions(-) create mode 100644 tests/diffusion/layers/test_fused_qk_norm_rope.py create mode 100644 vllm_omni/diffusion/layers/fused_qk_norm_rope.py diff --git a/tests/diffusion/layers/test_fused_qk_norm_rope.py b/tests/diffusion/layers/test_fused_qk_norm_rope.py new file mode 100644 index 000000000..40888556a --- /dev/null +++ b/tests/diffusion/layers/test_fused_qk_norm_rope.py @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch +import torch.nn.functional as F +from vllm.triton_utils import HAS_TRITON + +pytestmark = [pytest.mark.core_model, pytest.mark.cuda, pytest.mark.diffusion] + +_HEAD_DIM = 128 +_ROTARY_DIM = 96 +_EPS = 1e-5 + + +def _reference(q, k, q_weight, k_weight, rope_table): + q = F.rms_norm(q, (_HEAD_DIM,), q_weight, _EPS) + k = F.rms_norm(k, (_HEAD_DIM,), k_weight, _EPS) + half = _ROTARY_DIM // 2 + cos = rope_table[..., :half].unsqueeze(1) + sin = rope_table[..., half:].unsqueeze(1) + + def apply(x): + first = x[..., :half] + second = x[..., half:_ROTARY_DIM] + return torch.cat( + ( + first * cos - second * sin, + second * cos + first * sin, + x[..., _ROTARY_DIM:], + ), + dim=-1, + ) + + return apply(q), apply(k) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +@pytest.mark.skipif(not HAS_TRITON, reason="Triton required") +@pytest.mark.parametrize("seq_len", [1, 257, 1024]) +def test_fused_qk_norm_rope_matches_bf16_reference(seq_len): + from vllm_omni.diffusion.layers.fused_qk_norm_rope import ( + fused_qk_norm_rope, + ) + + torch.manual_seed(17) + heads = 14 + qkv = torch.randn( + seq_len, + heads * _HEAD_DIM * 3, + device="cuda", + dtype=torch.bfloat16, + ) + q = qkv[:, : heads * _HEAD_DIM].view(seq_len, heads, _HEAD_DIM) + k = qkv[:, heads * _HEAD_DIM : 2 * heads * _HEAD_DIM].view( + seq_len, + heads, + _HEAD_DIM, + ) + q_weight = torch.randn(_HEAD_DIM, device="cuda", dtype=torch.bfloat16) + k_weight = torch.randn(_HEAD_DIM, device="cuda", dtype=torch.bfloat16) + freqs = torch.randn(seq_len, _ROTARY_DIM // 2, device="cuda") + rope_table = torch.cat((torch.cos(freqs), torch.sin(freqs)), dim=-1).to(torch.bfloat16) + + expected_q, expected_k = _reference( + q, + k, + q_weight, + k_weight, + rope_table, + ) + actual_q, actual_k = fused_qk_norm_rope( + q, + k, + q_weight, + k_weight, + rope_table, + _EPS, + ) + + torch.testing.assert_close(actual_q, expected_q, atol=0.0625, rtol=0.02) + torch.testing.assert_close(actual_k, expected_k, atol=0.0625, rtol=0.02) diff --git a/tests/e2e/accuracy/ltx/run_ltx_reference.py b/tests/e2e/accuracy/ltx/run_ltx_reference.py index 3ab794662..ccabd4596 100644 --- a/tests/e2e/accuracy/ltx/run_ltx_reference.py +++ b/tests/e2e/accuracy/ltx/run_ltx_reference.py @@ -260,15 +260,9 @@ def _unwrap_omni_output(output: Any) -> tuple[Any, Any, int]: multimodal_output = frames.multimodal_output or {} audio = multimodal_output.get("audio") audio_sample_rate = multimodal_output.get("audio_sample_rate") - if frames.is_pipeline_output and isinstance(frames.request_output, OmniRequestOutput): - frames = frames.request_output - multimodal_output = frames.multimodal_output or {} - audio = multimodal_output.get("audio", audio) - audio_sample_rate = multimodal_output.get("audio_sample_rate", audio_sample_rate) - if isinstance(frames, OmniRequestOutput): - if not frames.images: - raise ValueError("No video frames found in OmniRequestOutput") - frames = frames.images + if not frames.images: + raise ValueError("No video frames found in OmniRequestOutput") + frames = frames.images if isinstance(frames, list) and len(frames) == 1: frames = frames[0] diff --git a/vllm_omni/diffusion/cache/teacache/extractors.py b/vllm_omni/diffusion/cache/teacache/extractors.py index 5f1ca1d22..8689c2c54 100644 --- a/vllm_omni/diffusion/cache/teacache/extractors.py +++ b/vllm_omni/diffusion/cache/teacache/extractors.py @@ -1263,6 +1263,7 @@ def extract_minimax_h3_context( _BF16_DTYPE, _FORWARD_SUPPORTED_KWARGS, MINIMAX_H3_ADALN_MODALITY_NUM, + _build_rope_table, _modulate_scale_shift, _required_kwarg, ) @@ -1318,7 +1319,7 @@ def extract_minimax_h3_context( raise ValueError(f"inverse_indices must be [{seq_len}], got {list(inverse_indices.shape)}") device = x.device - rope_freqs = module.rope(img_position_ids).to(device) + rope_table = _build_rope_table(module.rope(img_position_ids).to(device)) decoder_input, t_emb = module._embed( x=x, @@ -1351,7 +1352,7 @@ def extract_minimax_h3_context( def run_transformer_blocks() -> tuple[torch.Tensor, ...]: hidden, block_rope, block_combined = module.sp_prepare( decoder_input, - rope_freqs, + rope_table, combined_indices, ) for block in module.blocks: @@ -1359,7 +1360,7 @@ def extract_minimax_h3_context( hidden, t_emb=t_emb, combined_indices=block_combined, - rope_freqs=block_rope, + rope_table=block_rope, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen, packed_total=seq_len, diff --git a/vllm_omni/diffusion/layers/fused_qk_norm_rope.py b/vllm_omni/diffusion/layers/fused_qk_norm_rope.py new file mode 100644 index 000000000..64462d6c2 --- /dev/null +++ b/vllm_omni/diffusion/layers/fused_qk_norm_rope.py @@ -0,0 +1,299 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Q/K RMSNorm followed by packed non-interleaved RoPE. + +The public contract is shared by diffusion attention implementations: + +* ``q`` and ``k`` are ``[tokens, heads, head_dim]``; +* norm weights are one-dimensional ``[head_dim]`` tensors; +* ``rope_table`` is ``[tokens, rotary_dim]`` and stores + ``[cos(theta), sin(theta)]`` with ``theta`` of width ``rotary_dim // 2``. + +The CUDA fast path fuses RMSNorm and RoPE without materializing normalized Q/K +or rotary-product intermediates. Unsupported inputs use the eager reference. +""" + +from __future__ import annotations + +import torch +import torch.nn.functional as F +from torch.library import Library +from vllm.platforms import current_platform +from vllm.triton_utils import HAS_TRITON, tl, triton +from vllm.utils.torch_utils import direct_register_custom_op + +_FUSED_HEAD_DIM = 128 +_FUSED_ROTARY_DIM = 96 +_HEADS_PER_PROGRAM = 8 + + +def _apply_rope_table( + x: torch.Tensor, + rope_table: torch.Tensor, + rotary_dim: int, +) -> torch.Tensor: + half = rotary_dim // 2 + cos = rope_table[..., :half].to(x.dtype).unsqueeze(1) + sin = rope_table[..., half:].to(x.dtype).unsqueeze(1) + first = x[..., :half] + second = x[..., half:rotary_dim] + return torch.cat( + ( + first * cos - second * sin, + second * cos + first * sin, + x[..., rotary_dim:], + ), + dim=-1, + ) + + +if HAS_TRITON: + + @triton.jit + def _rms_norm_rope_kernel( + x_ptr, + weight_ptr, + rope_table_ptr, + out_ptr, + x_stride_t, + x_stride_h, + x_stride_d, + rope_stride_t, + out_stride_t, + out_stride_h, + out_stride_d, + num_heads: tl.constexpr, + head_dim: tl.constexpr, + rotary_half: tl.constexpr, + eps: tl.constexpr, + heads_per_program: tl.constexpr, + ): + token = tl.program_id(0) + head_group = tl.program_id(1) + heads = head_group * heads_per_program + tl.arange(0, heads_per_program) + dims = tl.arange(0, head_dim) + mask = heads[:, None] < num_heads + offsets = token * x_stride_t + heads[:, None] * x_stride_h + dims[None, :] * x_stride_d + + x = tl.load(x_ptr + offsets, mask=mask, other=0.0).to(tl.float32) + weight = tl.load(weight_ptr + dims).to(tl.float32) + inv_rms = tl.rsqrt(tl.sum(x * x, axis=1) / head_dim + eps) + normalized = (x * inv_rms[:, None] * weight[None, :]).to(tl.bfloat16) + + rotary_dim = rotary_half * 2 + pair_dims = tl.where( + dims < rotary_half, + dims + rotary_half, + tl.where(dims < rotary_dim, dims - rotary_half, dims), + ) + pair_offsets = token * x_stride_t + heads[:, None] * x_stride_h + pair_dims[None, :] * x_stride_d + pair_x = tl.load(x_ptr + pair_offsets, mask=mask, other=0.0).to(tl.float32) + pair_weight = tl.load(weight_ptr + pair_dims).to(tl.float32) + pair_normalized = (pair_x * inv_rms[:, None] * pair_weight[None, :]).to(tl.bfloat16) + + freq_dims = tl.where( + dims < rotary_half, + dims, + tl.where(dims < rotary_dim, dims - rotary_half, 0), + ) + table_offsets = token * rope_stride_t + freq_dims + cos = tl.load(rope_table_ptr + table_offsets).to(tl.float32) + sin = tl.load(rope_table_ptr + table_offsets + rotary_half).to(tl.float32) + first = normalized.to(tl.float32) * cos - pair_normalized.to(tl.float32) * sin + second = normalized.to(tl.float32) * cos + pair_normalized.to(tl.float32) * sin + output = tl.where( + dims < rotary_dim, + tl.where(dims < rotary_half, first, second), + normalized.to(tl.float32), + ) + + out_offsets = token * out_stride_t + heads[:, None] * out_stride_h + dims[None, :] * out_stride_d + tl.store(out_ptr + out_offsets, output, mask=mask) + + +def _eager_qk_norm_rope( + q: torch.Tensor, + k: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rope_table: torch.Tensor, + eps: float, + head_dim: int, + rotary_dim: int, +) -> tuple[torch.Tensor, torch.Tensor]: + q_norm = F.rms_norm(q, (head_dim,), q_weight, eps) + k_norm = F.rms_norm(k, (head_dim,), k_weight, eps) + return ( + _apply_rope_table(q_norm, rope_table, rotary_dim), + _apply_rope_table(k_norm, rope_table, rotary_dim), + ) + + +def _fused_cuda_supported( + q: torch.Tensor, + k: torch.Tensor, + head_dim: int, + rotary_dim: int, +) -> bool: + return ( + HAS_TRITON + and current_platform.is_cuda() + and q.is_cuda + and k.is_cuda + and q.dtype == torch.bfloat16 + and k.dtype == torch.bfloat16 + and head_dim == _FUSED_HEAD_DIM + and rotary_dim == _FUSED_ROTARY_DIM + ) + + +def _launch_fused_rms_norm_rope( + x: torch.Tensor, + weight: torch.Tensor, + rope_table: torch.Tensor, + eps: float, +) -> torch.Tensor: + tokens, heads, head_dim = x.shape + rotary_half = rope_table.shape[-1] // 2 + out = torch.empty(x.shape, dtype=x.dtype, device=x.device) + if tokens == 0: + return out + grid = (tokens, triton.cdiv(heads, _HEADS_PER_PROGRAM)) + _rms_norm_rope_kernel[grid]( + x, + weight, + rope_table, + out, + x.stride(0), + x.stride(1), + x.stride(2), + rope_table.stride(0), + out.stride(0), + out.stride(1), + out.stride(2), + num_heads=heads, + head_dim=head_dim, + rotary_half=rotary_half, + eps=eps, + heads_per_program=_HEADS_PER_PROGRAM, + num_warps=8, + ) + return out + + +def _fused_qk_norm_rope_impl( + q: torch.Tensor, + k: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rope_table: torch.Tensor, + eps: float, + head_dim: int, + rotary_dim: int, +) -> tuple[torch.Tensor, torch.Tensor]: + if not _fused_cuda_supported(q, k, head_dim, rotary_dim): + return _eager_qk_norm_rope( + q, + k, + q_weight, + k_weight, + rope_table, + eps, + head_dim, + rotary_dim, + ) + return ( + _launch_fused_rms_norm_rope(q, q_weight, rope_table, eps), + _launch_fused_rms_norm_rope(k, k_weight, rope_table, eps), + ) + + +def _fused_qk_norm_rope_fake( + q: torch.Tensor, + k: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rope_table: torch.Tensor, + eps: float, + head_dim: int, + rotary_dim: int, +) -> tuple[torch.Tensor, torch.Tensor]: + del q_weight, k_weight, rope_table, eps, head_dim, rotary_dim + return torch.empty_like(q), torch.empty_like(k) + + +_OMNI_OP_LIB = Library("vllm_omni", "FRAGMENT") +if not hasattr(torch.ops.vllm_omni, "fused_qk_norm_rope"): + direct_register_custom_op( + op_name="fused_qk_norm_rope", + op_func=_fused_qk_norm_rope_impl, + fake_impl=_fused_qk_norm_rope_fake, + mutates_args=[], + target_lib=_OMNI_OP_LIB, + ) + + +def fused_qk_norm_rope( + q: torch.Tensor, + k: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rope_table: torch.Tensor, + eps: float, + *, + head_dim: int | None = None, + rotary_dim: int | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Apply Q/K RMSNorm and packed non-interleaved RoPE.""" + if q.ndim != 3 or k.ndim != 3: + raise ValueError(f"q and k must be [tokens, heads, head_dim], got {q.shape} and {k.shape}") + if q.shape[0] != k.shape[0] or q.shape[2] != k.shape[2]: + raise ValueError(f"q and k shapes are incompatible: {q.shape} and {k.shape}") + if q.dtype != k.dtype or q.device != k.device: + raise ValueError("q and k must have the same dtype and device") + if q.dtype not in (torch.bfloat16, torch.float16, torch.float32): + raise TypeError(f"Fused QK RMSNorm/RoPE requires floating inputs, got {q.dtype}") + + head_dim = q.shape[-1] if head_dim is None else head_dim + rotary_dim = rope_table.shape[-1] if rotary_dim is None else rotary_dim + if q.shape[-1] != head_dim: + raise ValueError(f"Expected q/k head_dim={head_dim}, got {q.shape[-1]}") + if rotary_dim <= 0 or rotary_dim > head_dim or rotary_dim % 2: + raise ValueError(f"rotary_dim must be even and in [2, {head_dim}], got {rotary_dim}") + if q_weight.shape != (head_dim,) or k_weight.shape != (head_dim,): + raise ValueError(f"Expected norm weights [{head_dim}], got {tuple(q_weight.shape)} and {tuple(k_weight.shape)}") + if q_weight.device != q.device or k_weight.device != q.device: + raise ValueError("Q/K norm weights must be on the activation device") + if rope_table.device != q.device or rope_table.dtype != q.dtype: + raise ValueError("rope_table must have the same dtype and device as q/k") + if rope_table.shape != (q.shape[0], rotary_dim): + raise ValueError(f"Expected rope_table [{q.shape[0]}, {rotary_dim}], got {tuple(rope_table.shape)}") + + q_weight = q_weight.contiguous() + k_weight = k_weight.contiguous() + rope_table = rope_table.contiguous() + if not _fused_cuda_supported(q, k, head_dim, rotary_dim): + return _fused_qk_norm_rope_impl( + q, + k, + q_weight, + k_weight, + rope_table, + eps, + head_dim, + rotary_dim, + ) + return torch.ops.vllm_omni.fused_qk_norm_rope( + q, + k, + q_weight, + k_weight, + rope_table, + eps, + head_dim, + rotary_dim, + ) + + +__all__ = ["fused_qk_norm_rope"] diff --git a/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py b/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py index 5e180aa23..659b64bc4 100644 --- a/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py +++ b/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py @@ -33,6 +33,7 @@ from vllm_omni.diffusion.distributed.sp_plan import ( SequenceParallelInput, SequenceParallelOutput, ) +from vllm_omni.diffusion.layers.fused_qk_norm_rope import fused_qk_norm_rope from vllm_omni.diffusion.layers.norm import RMSNorm from vllm_omni.diffusion.layers.rope import RotaryEmbedding @@ -237,6 +238,15 @@ class MiniMaxH3Rope(nn.Module): return torch.cat((half, half), dim=-1) # [S, 96] +def _build_rope_table(freqs: torch.Tensor) -> torch.Tensor: + """Materialize H3's packed ``[cos(freqs[:48]), sin(freqs[:48])]`` table.""" + half = freqs.shape[-1] // 2 + return torch.cat( + (torch.cos(freqs[..., :half]), torch.sin(freqs[..., :half])), + dim=-1, + ).to(_BF16_DTYPE) + + class MiniMaxH3TimeEmbedder(nn.Module): def __init__( self, @@ -453,7 +463,7 @@ class MiniMaxH3Attention(nn.Module): self, x: torch.Tensor, *, - rope_freqs: torch.Tensor | None, + rope_table: torch.Tensor | None, cu_seqlens: torch.Tensor, max_seqlen: int, packed_total: int | None = None, @@ -479,11 +489,18 @@ class MiniMaxH3Attention(nn.Module): q = q.view(total, self.num_heads, self.head_dim) k = k.view(total, self.num_kv_heads, self.head_dim) v = v.view(total, self.num_kv_heads, self.head_dim) - q = self.q_norm(q) - k = self.k_norm(k) - if rope_freqs is not None: - q = self._apply_rope(q, rope_freqs) - k = self._apply_rope(k, rope_freqs) + if rope_table is None: + q = self.q_norm(q) + k = self.k_norm(k) + else: + q, k = fused_qk_norm_rope( + q, + k, + self.q_norm.weight, + self.k_norm.weight, + rope_table, + self.q_norm.variance_epsilon, + ) # The packed layout uses a second document for alignment padding. # Local/Ulysses backends unpad it, while Ring keeps aligned rows for @@ -628,7 +645,7 @@ class MiniMaxH3TokenRefinerBlock(nn.Module): ) -> torch.Tensor: x = x + self.attn( self.norm1(x), - rope_freqs=None, + rope_table=None, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen, ) @@ -707,7 +724,7 @@ class MiniMaxH3DiTBlock(nn.Module): *, t_emb: torch.Tensor, combined_indices: torch.Tensor, - rope_freqs: torch.Tensor, + rope_table: torch.Tensor, cu_seqlens: torch.Tensor, max_seqlen: int, packed_total: int, @@ -735,7 +752,7 @@ class MiniMaxH3DiTBlock(nn.Module): h = _modulate_scale_shift(h, shift_msa, scale_msa, combined_indices, dtype=_BF16_DTYPE) h = self.attn( h, - rope_freqs=rope_freqs, + rope_table=rope_table, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen, packed_total=packed_total, @@ -817,10 +834,10 @@ class MiniMaxH3SPPrepare(nn.Module): def forward( self, hidden_states: torch.Tensor, - rope_freqs: torch.Tensor, + rope_table: torch.Tensor, combined_indices: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - return hidden_states, rope_freqs, combined_indices + return hidden_states, rope_table, combined_indices class MiniMaxH3SPGather(nn.Module): @@ -1153,8 +1170,8 @@ class MiniMaxH3DiTModel(nn.Module): if inverse_indices.shape[0] != seq_len: raise ValueError(f"inverse_indices must be [{seq_len}], got {list(inverse_indices.shape)}") device = x.device - # Compute RoPE frequencies over the full packed sequence. - rope_freqs = self.rope(img_position_ids).to(device) + # Compute the packed RoPE table once for all transformer blocks. + rope_table = _build_rope_table(self.rope(img_position_ids).to(device)) decoder_input, t_emb = self._embed( x=x, @@ -1175,7 +1192,7 @@ class MiniMaxH3DiTModel(nn.Module): hidden = decoder_input cu_seqlens = cu_seqlens.to(device) - block_rope = rope_freqs + block_rope = rope_table block_combined = combined_indices hidden, block_rope, block_combined = self.sp_prepare( @@ -1188,7 +1205,7 @@ class MiniMaxH3DiTModel(nn.Module): hidden, t_emb=t_emb, combined_indices=block_combined, - rope_freqs=block_rope, + rope_table=block_rope, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen, packed_total=seq_len, From 8a5bf2fb3f1ab0f7242c72d25f694652fe957256 Mon Sep 17 00:00:00 2001 From: Samit <285365963@qq.com> Date: Fri, 14 Aug 2026 12:04:15 +0800 Subject: [PATCH 006/267] [doc]: Update latest news section for verl-omni release (#6187) Signed-off-by: Samit <285365963@qq.com> --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 418d0b38d..e6abc70d7 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Easy, fast, and cheap omni-modality model serving for everyone --- *Latest News* 🔥 +- [2026/08] [VeRL-Omni](https://github.com/verl-project/verl-omni) `v0.2.0` is released: faster diffusion RL powered by vLLM-Omni (request-level/step-wise batching with FA3), rebuilt Qwen3-Omni multimodal training (DPO & GSPO), plus LTX-2.3, Qwen-Image-Edit support and more. See the [release notes](https://github.com/verl-project/verl-omni/releases/tag/v0.2.0). - [2026/08] We released [0.26.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.26.0) - aligned with the vLLM 0.26 release line, featuring [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) joint video/audio generation, an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md), distributed layerwise diffusion offload, and broader model, hardware, streaming, TTS, and quantization support. - [2026/07] We released [0.24.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.24.0) - aligned with the vLLM 0.24 release line, expanding production-ready coverage across TTS, speech, diffusion, image/video generation, and robot-policy serving, with major Omni stage runtime refactoring, diffusion request-level batching, async output materialization, quantization/cache/memory improvements, and broad CUDA/ROCm/XPU/NPU support. - [2026/06] Starting with [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0), vLLM-Omni publishes a stable release aligned with every even-numbered upstream vLLM minor version. [0.16.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.16.0), [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0), [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0), and [0.22.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.22.0) continued this cadence, expanding omni and world-model support with [NVIDIA Cosmos3](recipes/cosmos3/Cosmos3-Nano.md) and DreamZero, adding models such as MiniCPM-o 4.5, MOSS-TTS, and Lance, and advancing TTS, diffusion, distributed execution, quantization, RL integration through [VeRL-Omni](https://github.com/verl-project/verl-omni), and CUDA/ROCm/MUSA/NPU/XPU coverage. From 5d50bdc2ab7691b98fa0371ab2d9c534ca801991 Mon Sep 17 00:00:00 2001 From: zhengjia Date: Fri, 14 Aug 2026 12:14:55 +0800 Subject: [PATCH 007/267] [Perf] Cap diffusion worker thread count before spawn (#6165) Signed-off-by: zjli2013 Co-authored-by: Cursor --- vllm_omni/diffusion/executor/multiproc_executor.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vllm_omni/diffusion/executor/multiproc_executor.py b/vllm_omni/diffusion/executor/multiproc_executor.py index 7de46b5cf..7cc6488c1 100644 --- a/vllm_omni/diffusion/executor/multiproc_executor.py +++ b/vllm_omni/diffusion/executor/multiproc_executor.py @@ -18,6 +18,7 @@ import zmq from vllm.distributed.device_communicators.shm_broadcast import Handle, MessageQueue from vllm.logger import init_logger from vllm.v1.engine.exceptions import EngineDeadError +from vllm.v1.executor.multiproc_executor import set_multiprocessing_worker_envs from vllm_omni.diffusion.data import SHUTDOWN_MESSAGE, AsyncDiffusionOutput, AsyncOutputKind, DiffusionOutput from vllm_omni.diffusion.executor.abstract import DiffusionExecutor @@ -294,6 +295,10 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): logger.info("Starting server...") num_gpus = cast(int, od_config.num_gpus) + # Without this, every worker inherits one Torch thread per core, so an + # N-GPU run oversubscribes the host by N x core_count. Honours a + # user-provided OMP_NUM_THREADS. + set_multiprocessing_worker_envs() mp.set_start_method("spawn", force=True) processes = [] From 2b08419d8be88f3f78852e42bb70a0bd46b289ae Mon Sep 17 00:00:00 2001 From: Shenglei Fu <117230642+ShengleiFu@users.noreply.github.com> Date: Fri, 14 Aug 2026 00:36:56 -0700 Subject: [PATCH 008/267] [CI] Per-model, per-entry-mode code coverage collection (#5593) Signed-off-by: Shenglei Fu Signed-off-by: Shenglei Fu <117230642+ShengleiFu@users.noreply.github.com> Co-authored-by: Claude Sonnet 5 Co-authored-by: Alicia <115451386+congw729@users.noreply.github.com> Co-authored-by: wangyu <53896905+yenuo26@users.noreply.github.com> Co-authored-by: Hongsheng Liu --- .buildkite/common/ci_mirror_hardwares.yml | 7 ++ .buildkite/common/scripts/run_cov_split.sh | 82 ++++++++++++++++++++ .buildkite/cuda/test-merge.yml | 28 +++++-- .gitignore | 2 + docs/contributing/ci/ci_settings.md | 90 ++++++++++++++++++++++ pyproject.toml | 15 ++++ tests/buildkite/test_upload_pipeline.py | 46 +++++++++++ 7 files changed, 263 insertions(+), 7 deletions(-) create mode 100755 .buildkite/common/scripts/run_cov_split.sh diff --git a/.buildkite/common/ci_mirror_hardwares.yml b/.buildkite/common/ci_mirror_hardwares.yml index 00c6e4f82..e68cbd382 100644 --- a/.buildkite/common/ci_mirror_hardwares.yml +++ b/.buildkite/common/ci_mirror_hardwares.yml @@ -9,6 +9,11 @@ # a2b3_npu_1 — ascend-a2b3, resource_class npu-1 (A2/B3 CI image) # a2b3_npu_4 — ascend-a2b3, resource_class npu-4 # a3_npu_2 — ascend-a3, resource_class npu-2 +# +# The ``l4_*`` presets run under the ``docker`` plugin, which does not put +# ``buildkite-agent`` on PATH inside the container the way the ``kubernetes`` +# presets do, so they set ``mount-buildkite-agent: true``; without it any step +# calling ``buildkite-agent`` (artifact upload, annotate, meta-data) fails there. x-npu-k8s-plugins: &npu_k8s_plugins - kubernetes: @@ -26,6 +31,7 @@ mirror_hardwares: image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT always-pull: true propagate-environment: true + mount-buildkite-agent: true shm-size: "8gb" environment: - HF_HOME=/fsx/hf_cache @@ -41,6 +47,7 @@ mirror_hardwares: image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT always-pull: true propagate-environment: true + mount-buildkite-agent: true shm-size: "8gb" environment: - HF_HOME=/fsx/hf_cache diff --git a/.buildkite/common/scripts/run_cov_split.sh b/.buildkite/common/scripts/run_cov_split.sh new file mode 100755 index 000000000..fe068519c --- /dev/null +++ b/.buildkite/common/scripts/run_cov_split.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Run a model's E2E tests once per entry mode as separate coverage runs and upload +# the reports, so per-mode attribution survives (see vllm-project/vllm-omni#5332). +# +# Usage: run_cov_split.sh --model-id [--offline ] [--online ] \ +# -- +# +# --model-id Names the artifacts: coverage--{offline,online}-.xml.gz. +# Use the model's directory name under vllm_omni/*/models/. +# --offline Pytest path(s) for the offline half. Repeatable. +# --online Pytest path(s) for the online half. Repeatable. +# -- Everything after this is forwarded verbatim to every pytest run, +# quoting preserved: the job's own -m expression, --run-level, and +# anything else it needs (e.g. --test-config-file). +# +# At least one of --offline/--online is required, so a job with only one mode runs +# just that half. Total runtime is bounded by the step's timeout_in_minutes. +# +# Exits non-zero if any half or the upload failed. +set -uo pipefail + +MODEL_ID="" +OFFLINE=() +ONLINE=() +PYTEST_ARGS=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --model-id) MODEL_ID="$2"; shift 2 ;; + --offline) OFFLINE+=("$2"); shift 2 ;; + --online) ONLINE+=("$2"); shift 2 ;; + --) shift; PYTEST_ARGS=("$@"); break ;; + *) echo "run_cov_split.sh: unknown argument: $1" >&2; exit 2 ;; + esac +done + +if [[ -z "${MODEL_ID}" ]]; then + echo "run_cov_split.sh: missing --model-id" >&2 + exit 2 +fi + +if ((${#OFFLINE[@]} == 0 && ${#ONLINE[@]} == 0)); then + echo "run_cov_split.sh: need at least one of --offline / --online" >&2 + exit 2 +fi + +# Two steps can test the same model, and artifacts are scoped to the build (a +# nightly one carries the ready, merge and nightly tiers at once), so unsuffixed +# reports would share a path -- a glob download then races to rename onto one file. +# Not BUILDKITE_JOB_ID: the step id survives retries, so attempts collapse under +# Buildkite's latest-attempt lookup instead of accumulating. A parallel step would +# need BUILDKITE_PARALLEL_JOB as well; no pilot is one. +REPORT_SUFFIX="${BUILDKITE_STEP_ID:-local}" +OFFLINE_REPORT="coverage-${MODEL_ID}-offline-${REPORT_SUFFIX}.xml" +ONLINE_REPORT="coverage-${MODEL_ID}-online-${REPORT_SUFFIX}.xml" + +run_half() { + local mode="$1" + shift + local report="coverage-${MODEL_ID}-${mode}-${REPORT_SUFFIX}.xml" + echo "--- coverage: ${MODEL_ID} ${mode}" + pytest -s -v "$@" "${PYTEST_ARGS[@]}" --cov=vllm_omni --cov-report="xml:${report}" + local status=$? + # Cobertura lists every statement in vllm_omni whether or not it ran, so the + # report is ~7MB no matter how little the job covers. The repetition is what + # dominates it, and gzip takes that to ~400KB. + gzip -9 -f "${report}" || status=1 + return "${status}" +} + +# Clear both modes, not just the ones being run: the upload glob matches both, so +# a report left by an earlier run would otherwise be published as if the mode it +# belongs to had run this time. +rm -f "${OFFLINE_REPORT}" "${ONLINE_REPORT}" "${OFFLINE_REPORT}.gz" "${ONLINE_REPORT}.gz" + +EXIT=0 +((${#OFFLINE[@]})) && { run_half offline "${OFFLINE[@]}" || EXIT=1; } +((${#ONLINE[@]})) && { run_half online "${ONLINE[@]}" || EXIT=1; } + +buildkite-agent artifact upload "coverage-${MODEL_ID}-*-${REPORT_SUFFIX}.xml.gz" || EXIT=1 + +exit "${EXIT}" diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index 363e8fbdc..970ab5a5a 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -94,13 +94,19 @@ steps: - vllm_omni/model_executor/models/qwen3_tts/ - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - vllm_omni/deploy/qwen3_tts.yaml + - .buildkite/common/scripts/run_cov_split.sh + - pyproject.toml + # 40 -> 60: the coverage split runs the model twice. + timeout_in_minutes: 60 commands: - | - timeout 40m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model' - " + export VLLM_LOGGING_LEVEL=DEBUG + export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 + .buildkite/common/scripts/run_cov_split.sh \ + --model-id qwen3_tts \ + --offline tests/e2e/offline_inference/test_qwen3_tts_base.py \ + --online tests/e2e/online_serving/test_qwen3_tts_base.py \ + -- -m 'advanced_model and cuda' --run-level 'advanced_model' mirror_hardwares: l4_1 - label: "TTS · Higgs-Audio-V3 Test" @@ -203,9 +209,17 @@ steps: - vllm_omni/diffusion/models/bagel/ - vllm_omni/model_executor/stage_input_processors/bagel.py - vllm_omni/deploy/bagel.yaml - timeout_in_minutes: 20 + - .buildkite/common/scripts/run_cov_split.sh + - pyproject.toml + # 20 -> 40: the coverage split runs the model twice. + timeout_in_minutes: 40 commands: - - pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model' + - | + .buildkite/common/scripts/run_cov_split.sh \ + --model-id bagel \ + --offline tests/e2e/offline_inference/test_bagel.py \ + --online tests/e2e/online_serving/test_bagel.py \ + -- -m 'advanced_model and cuda' --run-level 'advanced_model' mirror_hardwares: h100_1 - label: "Diffusion · Qwen Image Test" diff --git a/.gitignore b/.gitignore index 3c24e40ae..f11b57388 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,8 @@ htmlcov/ .cache nosetests.xml coverage.xml +coverage-*.xml +coverage-*.xml.gz *.cover *.py,cover .hypothesis/ diff --git a/docs/contributing/ci/ci_settings.md b/docs/contributing/ci/ci_settings.md index ff4b8e107..62717ddc9 100644 --- a/docs/contributing/ci/ci_settings.md +++ b/docs/contributing/ci/ci_settings.md @@ -327,6 +327,96 @@ On PR builds, `upload_pipeline.py` logs `skip '…' (no changes under …)` for - Use **groups** for dashboard readability; keep **E2E Test** as the group name CUDA diff filtering expects for `--e2e` nightly runs on main. - Prefix labels by model domain: **Omni ·**, **TTS ·**, **Diffusion ·**, **Simple ·**, etc., matching existing steps. +### Per-model coverage + +Pilot (v1): two Merge-tier (L3) jobs in `.buildkite/cuda/test-merge.yml` upload +per-model, per-entry-mode coverage as Buildkite artifacts — "Diffusion · Bagel +Test" and "TTS · Qwen3-TTS Base Test". Both run on a single GPU so the pilot is +cheap to reproduce. This is a pilot, not a rollout: no dashboard/visualization +lives in this repo, and nothing is gated on coverage. + +**Naming convention**: `coverage---.xml.gz`, where +`` is the model's directory name under +`vllm_omni/diffusion/models//` or +`vllm_omni/model_executor/models//` (e.g. `z_image`, `qwen3_omni`), +`` is `online` or `offline`, and `` is `BUILDKITE_STEP_ID` +(`local` outside Buildkite). `gunzip` before feeding a report to a Cobertura +consumer. + +Reports are gzipped because `--cov=vllm_omni` sets coverage's `source`, which +makes it walk the package and emit one `` element per statement for every +file — including the ones a single model's tests never import. That inventory, +not the coverage, is the size: a Bagel report is 7.1 MB, of which 96.5% is +`` elements and 77% belongs to the 840 of 1169 files that run never +imported. A run covering zero lines produces the same 7 MB. gzip takes it to +roughly 410 KB and changes nothing about the data. + +`` is what keeps two steps that cover the same model apart. Artifacts are +scoped to the build, and a nightly build carries the ready, merge and nightly tiers +at once, so several same-model steps land in one namespace — `qwen3_tts` already has +both a Base and a CustomVoice job, and `minicpmo_4_5` a base and a duplex one. Two +uploads on one path do not fail loudly: an exact-name download can report the path +as ambiguous, and a glob download fetches every match concurrently and renames each +onto the same destination, so whichever finishes last wins. The step id remains the +logical step identifier across retries, while Buildkite's default artifact lookup +selects only the latest attempt — `BUILDKITE_JOB_ID` would instead give every +attempt its own filename, leaving a failed attempt's partial report to be picked up +alongside the real one. A step using `parallelism`/matrix would need +`BUILDKITE_PARALLEL_JOB` added, since its jobs share a step id; neither pilot is +one. + +**Opting in a new model**: replace the job's combined `pytest` command with +[`run_cov_split.sh`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/run_cov_split.sh), +which runs each mode, names the reports, uploads them, and fails the step if +either half or the upload failed: + +```yaml +commands: + - | + .buildkite/common/scripts/run_cov_split.sh \ + --model-id \ + --offline \ + --online \ + -- +``` + +Everything after `--` is forwarded verbatim to both runs with quoting preserved, +so an expression like `-m 'advanced_model and cuda'` survives intact and a job can +pass anything else it needs. `--offline`/`--online` are repeatable and only one is +required, so a job with tests in a single mode runs just that half. Runtime is +bounded solely by the step's `timeout_in_minutes` — raise it, since the split +loads the model once per mode. + +Before splitting, confirm each half actually collects at least one test under the job's forwarded filters — pytest exits with code 5 ("no tests collected") if a half is empty, which the script reports as a failure, red-ing a job that used to pass as one combined invocation. + +Also check the tests' `num_cards` against the job's `mirror_hardwares` preset. +`cuda_marks` turns `num_cards > 1` into `skipif(device_count() < num_cards)`, so a +test asking for more GPUs than the preset provides is silently skipped and its +coverage artifact comes back empty. Splitting such a job produces a file that +looks fine and measures nothing — "Diffusion · Z Image Test" is the current +example (online tests want 4 cards, the job runs on `l4_1`), which is why it is +not a coverage pilot. + +Online-mode coverage depends on the `[tool.coverage.run]` `patch`/`sigterm` +settings in `pyproject.toml`: online tests launch the server with +`subprocess.Popen` and stop it with SIGTERM, and without those settings the XML +reflects only the pytest parent process, not the server's code paths. + +The upload depends on `buildkite-agent` being callable inside the container. The +`kubernetes` presets (`h100_*`, `*_npu_*`) provide it; the `docker` ones (`l4_*`) +only do because they set `mount-buildkite-agent: true`. A new docker preset that +runs a coverage job needs the same. + +List both `run_cov_split.sh` and `pyproject.toml` in every opted-in job's +`source_file_dependencies` — both change what the job measures, so without them a +change there is filtered out of normal PR builds and only surfaces in a later +nightly. `tests/buildkite/test_upload_pipeline.py` asserts this for the pilots. +Editing only the surrounding CI YAML still does not schedule them, so a PR that +touches just the wiring needs a full E2E run (or the commands run on a GPU host) +to produce artifacts. When checking a new model's +artifacts, compare `lines-covered` between the online and offline XML rather than +just confirming both files exist. + ### Validation checklist | Check | Command / location | diff --git a/pyproject.toml b/pyproject.toml index 0e91bfae6..48273fd20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -301,6 +301,21 @@ filterwarnings = [ "ignore:.*does not have '__bases__' attribute.*:UserWarning", ] +[tool.coverage.run] +core = "sysmon" +source = ["vllm_omni"] +# Online-serving tests launch the server via subprocess.Popen and stop it with +# SIGTERM. pytest-cov no longer instruments subprocesses implicitly, and a +# SIGTERM-ed process discards its data, so without patch/sigterm a coverage run +# over an online test measures only the pytest parent. +patch = ["subprocess"] +concurrency = ["multiprocessing", "thread"] +sigterm = true + +[tool.coverage.report] +show_missing = true +skip_covered = true + [tool.typos.default] extend-ignore-identifiers-re = [ ".*_thw", diff --git a/tests/buildkite/test_upload_pipeline.py b/tests/buildkite/test_upload_pipeline.py index 24a2a731e..ee45a64d3 100644 --- a/tests/buildkite/test_upload_pipeline.py +++ b/tests/buildkite/test_upload_pipeline.py @@ -5,6 +5,7 @@ import sys from pathlib import Path import pytest +import yaml sys.path.insert(0, str(Path(__file__).resolve().parents[2] / ".buildkite" / "common" / "scripts")) @@ -139,3 +140,48 @@ def test_mirror_hardwares_a2b3_npu_4_expands_agents_image_and_plugins() -> None: assert step["plugins"][0]["kubernetes"]["podSpecPatch"]["imagePullSecrets"] == [ {"name": "swr-secret"}, ] + + +COVERAGE_PILOT_LABELS = ("Diffusion · Bagel Test", "TTS · Qwen3-TTS Base Test") +MERGE_PIPELINE = Path(".buildkite/cuda/test-merge.yml") + + +def _surviving_labels(changed_files: list[str]) -> set[str]: + doc = yaml.safe_load(MERGE_PIPELINE.read_text(encoding="utf-8")) + rendered = _render_test_pipeline(doc, changed_files=changed_files) + labels: set[str] = set() + + def walk(steps: list | None) -> None: + for step in steps or []: + if not isinstance(step, dict): + continue + if "label" in step: + labels.add(step["label"]) + walk(step.get("steps")) + + walk(rendered.get("steps")) + return labels + + +@pytest.mark.parametrize( + "changed_file", + [ + ".buildkite/common/scripts/run_cov_split.sh", + "pyproject.toml", + ], +) +def test_coverage_pilots_selected_by_their_shared_inputs(changed_file: str) -> None: + """The split helper and the coverage config both change what these jobs measure. + + Without them in source_file_dependencies the jobs are filtered out of normal + PR builds, so a regression would only surface in a later nightly. + """ + labels = _surviving_labels([changed_file]) + for pilot in COVERAGE_PILOT_LABELS: + assert pilot in labels, f"{pilot} dropped when {changed_file} changed" + + +def test_coverage_pilots_not_selected_by_unrelated_change() -> None: + labels = _surviving_labels(["vllm_omni/entrypoints/openai/serving_chat.py"]) + for pilot in COVERAGE_PILOT_LABELS: + assert pilot not in labels From bc73546c065844136b16e3a8e799cc0bd36bc2ca Mon Sep 17 00:00:00 2001 From: TJian Date: Fri, 14 Aug 2026 15:38:47 +0800 Subject: [PATCH 009/267] [ROCm] [CI] Migrate CI to mi300x for v0.27.x (#5886) Signed-off-by: tjtanaa Signed-off-by: TJian Co-authored-by: Hongsheng Liu --- .buildkite/amd/scripts/run-amd-test.sh | 232 +++++++----------- .buildkite/amd/test-amd-merge.yml | 79 +++--- .buildkite/amd/test-amd-ready.yml | 81 +++--- .buildkite/amd/test-template-amd-omni.j2 | 112 ++++++++- docker/Dockerfile.rocm | 11 +- pyproject.toml | 1 + .../ar_diffusion/test_paged_attention.py | 14 +- tests/diffusion/attention/test_flash_attn.py | 13 +- .../distributed/test_sequence_parallel.py | 12 +- .../lora/test_images_generations_lora.py | 16 ++ tests/engine/test_duplex_import_boundary.py | 18 ++ .../openai_api/test_decode_audio_url.py | 3 +- .../ar_diffusion/kv_cache/paged_attention.py | 44 ++++ 13 files changed, 403 insertions(+), 233 deletions(-) diff --git a/.buildkite/amd/scripts/run-amd-test.sh b/.buildkite/amd/scripts/run-amd-test.sh index c49614e70..c1c80c495 100755 --- a/.buildkite/amd/scripts/run-amd-test.sh +++ b/.buildkite/amd/scripts/run-amd-test.sh @@ -1,153 +1,103 @@ #!/bin/bash -# vllm-omni customized version -# Based on: vllm/.buildkite/scripts/hardware_ci/run-amd-test.sh -# Last synced: 2025-12-15 -# Modifications: docker image name for vllm-omni -# This script runs test inside the corresponding ROCm docker container. -set -o pipefail +# Run vLLM-Omni ROCm tests directly in the MI300 Kubernetes pod. The pod's +# container image is selected by test-template-amd-omni.j2; MI300 has no DinD. +set -euo pipefail -# Export Python path -export PYTHONPATH=".." +: "${PYTHONFAULTHANDLER:=1}" +: "${HF_HOME:=/home/buildkite-agent/huggingface}" +: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}" +: "${HF_HUB_ETAG_TIMEOUT:=60}" +: "${MIOPEN_DEBUG_CONV_DIRECT:=0}" +: "${MIOPEN_DEBUG_CONV_GEMM:=0}" +: "${VLLM_ROCM_USE_AITER:=0}" +export PYTHONFAULTHANDLER HF_HOME HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT +export MIOPEN_DEBUG_CONV_DIRECT MIOPEN_DEBUG_CONV_GEMM VLLM_ROCM_USE_AITER +export PYTORCH_ROCM_ARCH="" +export PYTHONPATH="${PYTHONPATH:-..}" + +if [[ "${VLLM_CI_DOCKER_DISABLED:-0}" != "1" ]]; then + echo "Error: MI300 CI must run natively with Docker disabled." >&2 + exit 1 +fi + +if [[ -n "${TEST_COMMANDS:-}" ]]; then + commands="${TEST_COMMANDS}" +else + commands="$*" +fi +if [[ -z "${commands}" ]]; then + echo "Error: No test commands provided." >&2 + exit 1 +fi + +echo "--- Native in-pod ROCm CI" + +job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-local}}" +job_id="${job_id//[^A-Za-z0-9_.-]/_}" +native_cache_root="/tmp/vllm-omni-native-${job_id}" +TMPDIR="${native_cache_root}/tmp" +TORCHINDUCTOR_CACHE_DIR="${native_cache_root}/torchinductor" +TRITON_CACHE_DIR="${native_cache_root}/triton" +VLLM_CACHE_ROOT="${native_cache_root}/vllm" +XDG_CACHE_HOME="${native_cache_root}/xdg" +HF_DATASETS_CACHE="${native_cache_root}/huggingface/datasets" +export TMPDIR TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT +export XDG_CACHE_HOME HF_DATASETS_CACHE +mkdir -p "${TMPDIR}" "${TORCHINDUCTOR_CACHE_DIR}" "${TRITON_CACHE_DIR}" \ + "${VLLM_CACHE_ROOT}" "${XDG_CACHE_HOME}" "${HF_HOME}" \ + "${HF_DATASETS_CACHE}" + +if [[ "${VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE:-0}" == "1" ]]; then + if ! command -v findmnt >/dev/null 2>&1; then + echo "Error: findmnt is required to verify the Hugging Face cache mount." >&2 + exit 1 + fi + hf_mount=$(findmnt -n -T "${HF_HOME}" -o TARGET 2>/dev/null || true) + if [[ -z "${hf_mount}" || "${hf_mount}" == "/" ]]; then + echo "Error: MI300 CI requires a persistent volume at or above ${HF_HOME}." >&2 + exit 1 + fi +fi -# Print ROCm version -echo "--- ROCm info" rocminfo -# cleanup older docker images -cleanup_docker() { - # Get Docker's root directory - docker_root=$(docker info -f '{{.DockerRootDir}}') - if [ -z "$docker_root" ]; then - echo "Failed to determine Docker root directory." - exit 1 - fi - echo "Docker root directory: $docker_root" - # Check disk usage of the filesystem where Docker's root directory is located - disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//') - # Define the threshold - threshold=70 - if [ "$disk_usage" -gt "$threshold" ]; then - echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..." - # Remove dangling images (those that are not tagged and not used by any container) - docker image prune -f - # Remove unused volumes / force the system prune for old images as well. - docker volume prune -f && docker system prune --force --filter "until=72h" --all - echo "Docker images and volumes cleanup completed." - else - echo "Disk usage is below $threshold%. No cleanup needed." - fi -} +expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}" +python3 - <<'PY' +import os -# Call the cleanup docker function -cleanup_docker +import torch -echo "--- Pulling container" -## Temporary change to use AMD Docker Hub to store the vllm-omni image -# to bypass the rate limit issue with ECR Public Gallery. -# Images are now stored in a separate repository for vllm-omni, instead of vllm-ci. -# TODO: @tjtanaa point back to ECR Public Gallery -# once the amd agents are configured to use ECR Public Gallery. -# image_name="public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:${BUILDKITE_COMMIT}-rocm-omni" -image_name="rocm/vllm-omni:${BUILDKITE_COMMIT}" -container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)" +expected = int(os.environ.get("VLLM_CI_EXPECTED_GPU_COUNT", "1")) +assert torch.version.hip, "PyTorch is not a ROCm build" +assert torch.cuda.is_available(), "ROCm GPU is not available to PyTorch" +actual = torch.cuda.device_count() +assert actual == expected, f"Expected {expected} ROCm GPU(s), found {actual}" +PY -# TODO: @tjtanaa uncomment this once the amd agents are configured to use ECR Public Gallery. -# # Install AWS CLI to authenticate to ECR Public Gallery to get higher rate limit for pulling images -# sudo apt-get update && sudo apt-get install -y awscli -## Use safe docker login helper to prevent race conditions -# source "$(dirname "${BASH_SOURCE[0]}")/../../common/scripts/docker_login_ecr_public.sh" -# safe_docker_login_ecr_public - -## Pull the container from AMD Docker Hub - -docker pull "${image_name}" - -remove_docker_container() { - docker ps -aq --filter "name=${container_name}" | xargs -r docker rm -f || true - docker image rm -f "${image_name}" || true -} -trap remove_docker_container EXIT - -echo "--- Running container" - -HF_CACHE="$(realpath ~)/huggingface" -mkdir -p "${HF_CACHE}" -HF_MOUNT="/root/.cache/huggingface" - -if [[ -n "${TEST_COMMAND:-}" ]]; then - commands="$TEST_COMMAND" -else - commands="$@" -fi -echo "Commands:$commands" - -PARALLEL_JOB_COUNT=8 -MYPYTHONPATH=".." - -# Test that we're launching on the machine that has -# proper access to GPUs -render_gid=$(getent group render | cut -d: -f3) -if [[ -z "$render_gid" ]]; then - echo "Error: 'render' group not found. This is required for GPU access." >&2 - exit 1 -fi - -# check if the command contains shard flag, we will run all shards in parallel because the host have 8 GPUs. -# TODO: @tjtanaa reenable to run VLLM_ROCM_USE_AITER=1 when AITER is shipped with prebuilt kernels. -if [[ $commands == *"--shard-id="* ]]; then - # assign job count as the number of shards used - commands=$(echo "$commands" | sed -E "s/--num-shards[[:blank:]]*=[[:blank:]]*[0-9]*/--num-shards=${PARALLEL_JOB_COUNT} /g" | sed 's/ \\ / /g') - for GPU in $(seq 0 $(($PARALLEL_JOB_COUNT-1))); do - # assign shard-id for each shard - commands_gpu=$(echo "$commands" | sed -E "s/--shard-id[[:blank:]]*=[[:blank:]]*[0-9]*/--shard-id=${GPU} /g" | sed 's/ \\ / /g') - echo "Shard ${GPU} commands:$commands_gpu" - echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES" - docker run \ - --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \ - --network=host \ - --shm-size=16gb \ - --group-add "$render_gid" \ - -e MIOPEN_DEBUG_CONV_DIRECT=0 \ - -e MIOPEN_DEBUG_CONV_GEMM=0 \ - -e VLLM_ROCM_USE_AITER=0 \ - -e HIP_VISIBLE_DEVICES="${GPU}" \ - -e HF_TOKEN \ - -v "${HF_CACHE}:${HF_MOUNT}" \ - -e "HF_HOME=${HF_MOUNT}" \ - -e "PYTHONPATH=${MYPYTHONPATH}" \ - --name "${container_name}_${GPU}" \ - "${image_name}" \ - /bin/bash -c "${commands_gpu}" \ - |& while read -r line; do echo ">>Shard $GPU: $line"; done & - PIDS+=($!) - done - #wait for all processes to finish and collect exit codes - for pid in "${PIDS[@]}"; do - wait "${pid}" - STATUS+=($?) - done - for st in "${STATUS[@]}"; do - if [[ ${st} -ne 0 ]]; then - echo "One of the processes failed with $st" - exit "${st}" +echo "Commands:${commands}" +if /bin/bash -o pipefail -c ' +set -E +test_status=0 +trap '\'' + command_status=$? + # Keep running subsequent commands, but retain a failure for the final exit. + # Prefer a test failure over pytest "no tests collected" when both occur. + if (( test_status == 0 || (test_status == 5 && command_status != 5) )); then + test_status=${command_status} fi - done +'\'' ERR +eval "$1" +exit "${test_status}" +' _ "${commands}"; then + exit 0 else - echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES" - docker run \ - --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \ - --network=host \ - --shm-size=16gb \ - --group-add "$render_gid" \ - -e MIOPEN_DEBUG_CONV_DIRECT=0 \ - -e MIOPEN_DEBUG_CONV_GEMM=0 \ - -e VLLM_ROCM_USE_AITER=0 \ - -e HF_TOKEN \ - -v "${HF_CACHE}:${HF_MOUNT}" \ - -e "HF_HOME=${HF_MOUNT}" \ - -e "PYTHONPATH=${MYPYTHONPATH}" \ - --name "${container_name}" \ - "${image_name}" \ - /bin/bash -c "${commands}" + exit_code=$? fi + +if [[ ${exit_code} -eq 5 && "${VLLM_CI_ALLOW_NO_TESTS:-0}" == "1" ]]; then + echo "Pytest collected no tests; VLLM_CI_ALLOW_NO_TESTS=1, treating exit code 5 as success." + exit 0 +fi + +exit "${exit_code}" diff --git a/.buildkite/amd/test-amd-merge.yml b/.buildkite/amd/test-amd-merge.yml index 4e80d1968..fbe3b0c0f 100644 --- a/.buildkite/amd/test-amd-merge.yml +++ b/.buildkite/amd/test-amd-merge.yml @@ -7,54 +7,64 @@ steps: - group: ":card_index_dividers: Simple Test" steps: - - label: "Simple · Diffusion & Model Executor Test (1/2)" - agent_pool: mi325_1 + - label: "Simple · Model Executor Test · Shard %N/%t" + agent_pool: mi300_1 + parallelism: 2 depends_on: amd-build - timeout_in_minutes: 70 mirror_hardwares: [amdproduction] grade: Blocking commands: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - # CPU tests on ROCm gpu machines are very slow, so we give them more time. - - "timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:coverage-model-executor-$$BUILDKITE_PARALLEL_JOB.xml" - - label: "Simple · Diffusion & Model Executor Test (2/2)" - agent_pool: mi325_1 + - label: "Simple · Diffusion Test · Shard %N/%t" + agent_pool: mi300_1 + parallelism: 2 depends_on: amd-build - timeout_in_minutes: 70 mirror_hardwares: [amdproduction] grade: Blocking commands: - export VLLM_ROCM_USE_AITER=0 - # CPU tests on ROCm gpu machines are very slow, so we give them more time. - - "timeout 60m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM + # that is not supported on CPU + - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:coverage-diffusion-$$BUILDKITE_PARALLEL_JOB.xml" + + - label: "Simple · Engine&Entrypoints Test" + agent_pool: mi300_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export VLLM_ROCM_USE_AITER=0 + - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" - label: "Simple · Other Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build - timeout_in_minutes: 50 mirror_hardwares: [amdproduction] grade: Blocking commands: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" - group: ":card_index_dividers: Diffusion Test" steps: - label: "Diffusion · LoRA Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking + artifact_paths: + - lora-server.log commands: - - timeout 20m pytest -s -v tests/diffusion/lora/test_images_generations_lora.py + - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model" 2>&1 | tee lora-server.log - label: "Diffusion Model CPU offloading Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -62,7 +72,7 @@ steps: - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - | - timeout 20m bash -c ' + bash -c ' set +e pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py EXIT1=\$? @@ -72,7 +82,7 @@ steps: ' - label: "Diffusion Sequence Parallelism Test (Need 4 GPUs)" - agent_pool: mi325_4 + agent_pool: mi300_4 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -84,7 +94,7 @@ steps: # merge-only tests - label: "Diffusion Tensor Parallelism Test" - agent_pool: mi325_2 + agent_pool: mi300_2 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -92,7 +102,7 @@ steps: - pytest -s -v tests/diffusion/distributed/test_tensor_parallel.py - label: "Diffusion Model Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -100,7 +110,7 @@ steps: - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py - label: "Diffusion GPU Worker Test" - agent_pool: mi325_2 + agent_pool: mi300_2 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -108,7 +118,7 @@ steps: - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py - label: "Diffusion · Wan22 Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -121,7 +131,7 @@ steps: ## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274 # - label: "Audio Generation Model Test" -# agent_pool: mi325_1 +# agent_pool: mi300_1 # depends_on: amd-build # mirror_hardwares: [amdproduction] # grade: Blocking @@ -132,7 +142,7 @@ steps: # - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model - label: "Engine Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -147,7 +157,7 @@ steps: - group: ":card_index_dividers: E2E Test" steps: - label: "Omni Model Test Qwen2-5-Omni" - agent_pool: mi325_4 + agent_pool: mi300_4 depends_on: amd-build mirror_hardwares: [amdproduction] commands: @@ -155,7 +165,7 @@ steps: - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model" - label: "Omni Model Test Qwen3-Omni" - agent_pool: mi325_2 + agent_pool: mi300_2 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -163,22 +173,23 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model' - - label: "Qwen3-TTS CustomVoice E2E Test" - agent_pool: mi325_1 + - label: "Qwen3-TTS CustomVoice E2E Test · Shard %N/%t" + agent_pool: mi300_1 + parallelism: 2 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking commands: - | - timeout 50m bash -c ' + bash -c ' export VLLM_LOGGING_LEVEL=DEBUG export VLLM_WORKER_MULTIPROC_METHOD=spawn export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model" + pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model" --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB ' - label: "Qwen3-TTS Base E2E Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -196,7 +207,7 @@ steps: # Need to debug before reneable numerical changes across large PRs # # split Bagel Model Test with H100 (Real Weights) into three tests # - label: "Bagel Text2Img Model Test (1/3)" -# agent_pool: mi325_1 +# agent_pool: mi300_1 # depends_on: amd-build # mirror_hardwares: [amdproduction] # grade: Blocking @@ -208,7 +219,7 @@ steps: # - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory" -k "rocm" # - label: "Bagel Img2Img Model Test (2/3)" -# agent_pool: mi325_1 +# agent_pool: mi300_1 # depends_on: amd-build # mirror_hardwares: [amdproduction] # grade: Blocking @@ -220,7 +231,7 @@ steps: # - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm" # - label: "Bagel Online Serving Test (3/3)" -# agent_pool: mi325_1 +# agent_pool: mi300_1 # depends_on: amd-build # mirror_hardwares: [amdproduction] # grade: Blocking diff --git a/.buildkite/amd/test-amd-ready.yml b/.buildkite/amd/test-amd-ready.yml index 8f99ce909..b8fca2f88 100644 --- a/.buildkite/amd/test-amd-ready.yml +++ b/.buildkite/amd/test-amd-ready.yml @@ -7,8 +7,9 @@ steps: - group: ":card_index_dividers: Simple Test" steps: - - label: "Simple · Diffusion & Model Executor Test" - agent_pool: mi325_1 + - label: "Simple · Model Executor Test · Shard %N/%t" + agent_pool: mi300_1 + parallelism: 2 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -16,10 +17,30 @@ steps: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:coverage-model-executor-$$BUILDKITE_PARALLEL_JOB.xml" + + - label: "Simple · Diffusion Test" + agent_pool: mi300_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export VLLM_ROCM_USE_AITER=0 + # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM + # that is not supported on CPU + - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + + - label: "Simple · Engine&Entrypoints Test" + agent_pool: mi300_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export VLLM_ROCM_USE_AITER=0 + - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" - label: "Simple · Other Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -27,19 +48,19 @@ steps: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" - label: "Custom Pipeline Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking commands: - - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model" + - pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model" - label: "Voxtral TTS CUDA Unit Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -49,33 +70,25 @@ steps: - group: ":card_index_dividers: Diffusion Test" steps: - label: "Diffusion · Batch Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking commands: - - timeout 20m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model" + - pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model" - label: "Diffusion · Offloader Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking commands: - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - | - timeout 20m bash -c ' - set +e - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py - EXIT1=\$? - pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py - EXIT2=\$? - exit \$((EXIT1 | EXIT2)) - ' + - pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model" - label: "Diffusion · Cache Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -83,10 +96,10 @@ steps: - export GPU_ARCHS=gfx942 - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - timeout 15m pytest -s -v -m "core_model and cache and diffusion and not distributed_cuda and L4" + - pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model" - label: "Diffusion Sequence Parallelism Test" - agent_pool: mi325_2 + agent_pool: mi300_2 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -96,7 +109,7 @@ steps: - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model - label: "Diffusion GPU Worker Test" - agent_pool: mi325_2 + agent_pool: mi300_2 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -104,7 +117,7 @@ steps: - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py - label: "Diffusion · Model Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -112,7 +125,7 @@ steps: - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model" - label: "Diffusion · Wan22 Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -126,7 +139,7 @@ steps: ## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274 # - label: "Audio Generation Model Test" -# agent_pool: mi325_1 +# agent_pool: mi300_1 # depends_on: amd-build # mirror_hardwares: [amdproduction] # grade: Blocking @@ -137,7 +150,7 @@ steps: # - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model - label: "Engine Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -149,7 +162,7 @@ steps: ' # - label: "Omni Model Test Qwen3-Omni" -# agent_pool: mi325_2 +# agent_pool: mi300_2 # depends_on: amd-build # mirror_hardwares: [amdproduction] # grade: Blocking @@ -162,7 +175,7 @@ steps: - group: ":card_index_dividers: E2E Test" steps: - label: "Qwen3-TTS E2E Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -173,7 +186,7 @@ steps: - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model" - label: "CosyVoice3-TTS E2E Test" - agent_pool: mi325_1 + agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking @@ -186,7 +199,7 @@ steps: # TODO: Bagel test on ROCm is very unstable. @tjtanaa # Need to debug before reneable numerical changes across large PRs # - label: "Bagel Text2Img Model Test" -# agent_pool: mi325_1 +# agent_pool: mi300_1 # depends_on: amd-build # mirror_hardwares: [amdproduction] # grade: Blocking @@ -198,7 +211,7 @@ steps: # - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm" # - label: "Bagel Img2Img Model Test" -# agent_pool: mi325_1 +# agent_pool: mi300_1 # depends_on: amd-build # mirror_hardwares: [amdproduction] # grade: Blocking @@ -210,7 +223,7 @@ steps: # - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm" # - label: "Bagel Online Serving Test" -# agent_pool: mi325_1 +# agent_pool: mi300_1 # depends_on: amd-build # mirror_hardwares: [amdproduction] # grade: Blocking diff --git a/.buildkite/amd/test-template-amd-omni.j2 b/.buildkite/amd/test-template-amd-omni.j2 index 2f1a9aed3..e2042b96a 100644 --- a/.buildkite/amd/test-template-amd-omni.j2 +++ b/.buildkite/amd/test-template-amd-omni.j2 @@ -1,11 +1,65 @@ {# vllm-omni customized version Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/test-template-amd.j2 - Last synced: 2025-12-15 - Modifications: Removed unused CUDA/NVIDIA logic, keeping only AMD tests + Last synced: 2026-08-04 + Modifications: Use the shared MI300 queues and Omni command-passing contract #} {% set docker_image_amd = "rocm/vllm-omni:$BUILDKITE_COMMIT" %} {% set default_working_dir = "/app/vllm-omni" %} +{# MI300 runs tests natively in Kubernetes; Docker-in-Docker is unavailable. #} +{% macro amd_native_k8s_plugin(indent, step) -%} +{{ indent }}plugins: +{{ indent }} - kubernetes: +{{ indent }} podSpecPatch: +{{ indent }} automountServiceAccountToken: false +{{ indent }} securityContext: +{{ indent }} seccompProfile: +{{ indent }} type: RuntimeDefault +{{ indent }} imagePullSecrets: +{{ indent }} - name: docker-config +{{ indent }} containers: +{{ indent }} - name: container-0 +{{ indent }} image: {{ docker_image_amd }} +{{ indent }} imagePullPolicy: Always +{{ indent }} securityContext: +{{ indent }} allowPrivilegeEscalation: false +{{ indent }} capabilities: +{{ indent }} add: +{{ indent }} - IPC_LOCK +{{ indent }} resources: +{{ indent }} limits: +{{ indent }} amd.com/gpu: "{{ step.agent_pool[-1:] }}" +{{ indent }} requests: +{{ indent }} amd.com/gpu: "{{ step.agent_pool[-1:] }}" +{{ indent }} volumeMounts: +{{ indent }} - name: devshm +{{ indent }} mountPath: /dev/shm +{{ indent }} env: +{{ indent }} - name: AMD_CI_RUNTIME +{{ indent }} value: native +{{ indent }} - name: NATIVE_CI +{{ indent }} value: "true" +{{ indent }} - name: VLLM_CI_DOCKER_DISABLED +{{ indent }} value: "1" +{{ indent }} - name: VLLM_CI_EXPECTED_GPU_COUNT +{{ indent }} value: "{{ step.agent_pool[-1:] }}" +{{ indent }} - name: HF_HOME +{{ indent }} value: /home/buildkite-agent/huggingface +{{ indent }} - name: MIOPEN_DEBUG_CONV_DIRECT +{{ indent }} value: "0" +{{ indent }} - name: MIOPEN_DEBUG_CONV_GEMM +{{ indent }} value: "0" +{{ indent }} - name: VLLM_ROCM_USE_AITER +{{ indent }} value: "0" +{{ indent }} - name: PYTORCH_ROCM_ARCH +{{ indent }} value: "" +{{ indent }} volumes: +{{ indent }} - name: devshm +{{ indent }} emptyDir: +{{ indent }} medium: Memory +{{ indent }} sizeLimit: 16Gi +{%- endmacro %} + {% if env %} env: {% for name in env %} @@ -53,19 +107,31 @@ steps: {% if step.agent_pool %} queue: amd_{{ step.agent_pool }} {% else %} - queue: amd_mi325_1 + queue: amd_mi300_1 {% endif %} +{{ amd_native_k8s_plugin(" ", step) }} {% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %} {% set indented_cmd = cmd_body | replace("\n", "\n ") %} command: bash .buildkite/amd/scripts/run-amd-test.sh env: - DOCKER_BUILDKIT: "1" + AMD_CI_RUNTIME: "native" + NATIVE_CI: "true" + VLLM_CI_DOCKER_DISABLED: "1" + VLLM_CI_EXPECTED_GPU_COUNT: "{{ step.agent_pool[-1:] }}" + VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE: "1" + HF_HOME: "/home/buildkite-agent/huggingface" + HF_HUB_DOWNLOAD_TIMEOUT: "300" + HF_HUB_ETAG_TIMEOUT: "60" + MIOPEN_DEBUG_CONV_DIRECT: "0" + MIOPEN_DEBUG_CONV_GEMM: "0" + VLLM_ROCM_USE_AITER: "0" + PYTORCH_ROCM_ARCH: "" {% if step.env %} {% for name in step.env %} {{ name }}: "{{ step.env[name] }}" {% endfor %} {% endif %} - TEST_COMMAND: |- + TEST_COMMANDS: |- (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }} {% if env %} {% for name in env %} @@ -82,6 +148,15 @@ steps: {% endif %} {{ indented_cmd | safe }} priority: 100 +{% if step.parallelism %} + parallelism: {{ step.parallelism }} +{% endif %} +{% if step.artifact_paths %} + artifact_paths: +{% for path in step.artifact_paths %} + - "{{ path }}" +{% endfor %} +{% endif %} {% if step.timeout_in_minutes %} timeout_in_minutes: {{ step.timeout_in_minutes }} {% endif %} @@ -100,19 +175,31 @@ steps: {% if step.agent_pool %} queue: amd_{{ step.agent_pool }} {% else %} - queue: amd_mi325_1 + queue: amd_mi300_1 {% endif %} +{{ amd_native_k8s_plugin(" ", step) }} {% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %} {% set indented_cmd = cmd_body | replace("\n", "\n ") %} command: bash .buildkite/amd/scripts/run-amd-test.sh env: - DOCKER_BUILDKIT: "1" + AMD_CI_RUNTIME: "native" + NATIVE_CI: "true" + VLLM_CI_DOCKER_DISABLED: "1" + VLLM_CI_EXPECTED_GPU_COUNT: "{{ step.agent_pool[-1:] }}" + VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE: "1" + HF_HOME: "/home/buildkite-agent/huggingface" + HF_HUB_DOWNLOAD_TIMEOUT: "300" + HF_HUB_ETAG_TIMEOUT: "60" + MIOPEN_DEBUG_CONV_DIRECT: "0" + MIOPEN_DEBUG_CONV_GEMM: "0" + VLLM_ROCM_USE_AITER: "0" + PYTORCH_ROCM_ARCH: "" {% if step.env %} {% for name in step.env %} {{ name }}: "{{ step.env[name] }}" {% endfor %} {% endif %} - TEST_COMMAND: |- + TEST_COMMANDS: |- (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }} {% if env %} {% for name in env %} @@ -129,6 +216,15 @@ steps: {% endif %} {{ indented_cmd | safe }} priority: 100 +{% if step.parallelism %} + parallelism: {{ step.parallelism }} +{% endif %} +{% if step.artifact_paths %} + artifact_paths: +{% for path in step.artifact_paths %} + - "{{ path }}" +{% endfor %} +{% endif %} {% if step.timeout_in_minutes %} timeout_in_minutes: {{ step.timeout_in_minutes }} {% endif %} diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index 6b5a12309..4283efc77 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.26.0 +ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.27.0 FROM ${BASE_IMAGE} AS base # Declare a variable to know if we want to use the nightly build or the stable build. @@ -10,7 +10,7 @@ FROM ${BASE_IMAGE} AS base # we should swap over to use stable release ASAP. # We should avoid relying on custom commits. ARG USE_NIGHTLY_BUILD=0 -ARG VLLM_VERSION_OR_COMMIT_HASH=v0.26.0 +ARG VLLM_VERSION_OR_COMMIT_HASH=v0.27.0 ARG ARG_PYTORCH_ROCM_ARCH ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}} @@ -79,9 +79,16 @@ RUN printf '%s\n' \ ' os._exit(_exit_code)' \ > ${COMMON_WORKDIR}/vllm-omni/conftest.py +# Some optional dependencies pull in the CPU ONNX Runtime distribution. The +# CPU and ROCm wheels own the same import path and cannot coexist, so remove +# the CPU wheel and reinstall the ROCm wheel after dependency resolution. RUN cd ${COMMON_WORKDIR}/vllm-omni && \ uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \ --no-cache-dir ".[dev]" --no-build-isolation && \ + python3 -m pip uninstall -y onnxruntime && \ + python3 -m pip install --no-cache-dir --force-reinstall --no-deps \ + "onnxruntime-rocm>=1.22.2" && \ + python3 -c "import importlib.metadata as m; import onnxruntime as ort; assert 'ROCMExecutionProvider' in ort.get_available_providers(), ort.get_available_providers(); assert m.version('onnxruntime-rocm'); assert not any(d.metadata['Name'] == 'onnxruntime' for d in m.distributions())" && \ python3 -c "from torchcodec.decoders import VideoDecoder" RUN ln -sf /usr/bin/python3 /usr/bin/python diff --git a/pyproject.toml b/pyproject.toml index 48273fd20..160c6bc2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dev = [ "pytest-asyncio==1.4.0", "pytest-cov==7.1.0", "pytest-mock==3.15.1", + "pytest-shard==0.1.2", "websockets==16.1", "pytest-subtests==0.15.0", "datasets==3.6.0", diff --git a/tests/diffusion/ar_diffusion/test_paged_attention.py b/tests/diffusion/ar_diffusion/test_paged_attention.py index 0b66c01a0..27135874f 100644 --- a/tests/diffusion/ar_diffusion/test_paged_attention.py +++ b/tests/diffusion/ar_diffusion/test_paged_attention.py @@ -60,9 +60,18 @@ def _dense_attention(query: torch.Tensor, key: torch.Tensor, value: torch.Tensor return torch.einsum("bhqk,bkhd->bqhd", probs, value) -def _cuda_flash_attn_usable() -> bool: +def _gpu_flash_attn_usable() -> bool: if not torch.cuda.is_available(): return False + if torch.version.hip is not None: + for module in ("aiter", "flash_attn"): + try: + imported = __import__(module, fromlist=["flash_attn_varlen_func"]) + if getattr(imported, "flash_attn_varlen_func", None) is not None: + return True + except ImportError: + pass + return False try: spec = find_spec("vllm.vllm_flash_attn") if spec is None or spec.origin is None: @@ -227,12 +236,11 @@ def test_paged_attention_matches_dense_reference_cpu(history_chunks, action_len, assert st.adapter(POS).completed_chunks == before + (1 if commit_current else 0) -@pytest.mark.skipif(not _cuda_flash_attn_usable(), reason="usable CUDA FlashAttention is required") +@pytest.mark.skipif(not _gpu_flash_attn_usable(), reason="usable GPU FlashAttention is required") @pytest.mark.parametrize("history_chunks", [1, 3]) @pytest.mark.parametrize("action_len", [0, 3]) @pytest.mark.parametrize("commit_current", [False, True]) def test_paged_attention_matches_dense_reference_gpu(history_chunks, action_len, commit_current): - pytest.importorskip("vllm.vllm_flash_attn") torch.manual_seed(0) device = torch.device("cuda") dtype = torch.float16 diff --git a/tests/diffusion/attention/test_flash_attn.py b/tests/diffusion/attention/test_flash_attn.py index e3f5355bf..5924beabf 100644 --- a/tests/diffusion/attention/test_flash_attn.py +++ b/tests/diffusion/attention/test_flash_attn.py @@ -265,10 +265,15 @@ def test_fa_vs_sdpa(): print(f"Max absolute difference (valid region): {max_diff:.6f}") print(f"Mean absolute difference (valid region): {mean_diff:.6f}") - # Assert that outputs are close - # Using higher tolerance for bfloat16 and different implementations - assert max_diff < 0.01, f"Max difference {max_diff} exceeds threshold 0.01" - assert mean_diff < 0.001, f"Mean difference {mean_diff} exceeds threshold 0.001" + # AITER's ROCm BF16 kernel is deterministic, but differs from ROCm SDPA by + # about 1.3% relative error. An absolute-only bound misclassifies that + # expected rounding difference for larger output values (for example, + # -2.65625 vs -2.6875). Keep the tighter historical bounds elsewhere. + if current_omni_platform.is_rocm(): + torch.testing.assert_close(output_fa_valid, output_sdpa_valid, rtol=1e-2, atol=1.5e-2) + else: + assert max_diff < 0.01, f"Max difference {max_diff} exceeds threshold 0.01" + assert mean_diff < 0.001, f"Mean difference {mean_diff} exceeds threshold 0.001" print("✓ Case 2 PASSED: FA and SDPA outputs are very close!") diff --git a/tests/diffusion/distributed/test_sequence_parallel.py b/tests/diffusion/distributed/test_sequence_parallel.py index d43d18c49..f083e1d26 100644 --- a/tests/diffusion/distributed/test_sequence_parallel.py +++ b/tests/diffusion/distributed/test_sequence_parallel.py @@ -416,8 +416,8 @@ def test_sp_correctness_advanced(model_name: str): @pytest.mark.skipif( - not (current_omni_platform.is_cuda() or current_omni_platform.is_xpu()), - reason="Only tested on CUDA and XPU", + not (current_omni_platform.is_cuda() or current_omni_platform.is_rocm() or current_omni_platform.is_xpu()), + reason="Only tested on CUDA, ROCM and XPU", ) @pytest.mark.diffusion @pytest.mark.parallel @@ -447,8 +447,8 @@ def test_local_sp_padding_mask(monkeypatch: pytest.MonkeyPatch) -> None: @pytest.mark.skipif( - not (current_omni_platform.is_cuda() or current_omni_platform.is_xpu()), - reason="Only tested on CUDA and XPU", + not (current_omni_platform.is_cuda() or current_omni_platform.is_rocm() or current_omni_platform.is_xpu()), + reason="Only tested on CUDA, ROCM and XPU", ) @pytest.mark.diffusion @pytest.mark.parallel @@ -468,8 +468,8 @@ def test_local_sp_padding_mask_no_padding(monkeypatch: pytest.MonkeyPatch) -> No @pytest.mark.skipif( - not (current_omni_platform.is_cuda() or current_omni_platform.is_xpu()), - reason="Only tested on CUDA and XPU", + not (current_omni_platform.is_cuda() or current_omni_platform.is_rocm() or current_omni_platform.is_xpu()), + reason="Only tested on CUDA, ROCM and XPU", ) @pytest.mark.diffusion @pytest.mark.parallel diff --git a/tests/diffusion/lora/test_images_generations_lora.py b/tests/diffusion/lora/test_images_generations_lora.py index dbdf59647..a01bcdae3 100644 --- a/tests/diffusion/lora/test_images_generations_lora.py +++ b/tests/diffusion/lora/test_images_generations_lora.py @@ -24,11 +24,20 @@ from safetensors.torch import save_file from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServer, OmniServerParams +from vllm_omni.platforms import current_omni_platform os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" MODEL = "Tongyi-MAI/Z-Image-Turbo" +# ROCm's default FLASH_ATTN diffusion backend resolves to AITER. On a cold CI +# worker, JIT-building its MHA kernel can take longer than the server readiness +# timeout. This test validates LoRA switching rather than attention-backend or +# compiled-execution performance, so use the eager SDPA path on ROCm only. +ROCM_SERVER_ARGS = ( + ["--enforce-eager", "--diffusion-attention-backend", "TORCH_SDPA"] if current_omni_platform.is_rocm() else [] +) + PROMPT = "a photo of a cat sitting on a laptop keyboard" SIZE = "256x256" @@ -42,7 +51,14 @@ server_params = [ server_args=[ "--num-gpus", "1", + *ROCM_SERVER_ARGS, ], + # A cold-cache Z-Image load can take slightly more than the + # orchestrator's 900s default on CI (weights alone have been + # observed at ~690s and full orchestrator readiness at ~906s). + # Match the large-model offline runner's startup allowance without + # weakening timeout detection for every online-serving test. + init_timeout=1800, ), id="zimage_turbo_lora", ), diff --git a/tests/engine/test_duplex_import_boundary.py b/tests/engine/test_duplex_import_boundary.py index 6eae2e698..d04ed2e70 100644 --- a/tests/engine/test_duplex_import_boundary.py +++ b/tests/engine/test_duplex_import_boundary.py @@ -3,6 +3,7 @@ from __future__ import annotations +import os import subprocess import sys from pathlib import Path @@ -14,9 +15,18 @@ REPO_ROOT = Path(__file__).resolve().parents[2] def _assert_isolated_import_succeeds(script: str) -> None: + # These tests validate Python import/package boundaries and do not use a + # GPU. Other tests in the combined entrypoints/engine suite may temporarily + # configure only one of the CUDA/ROCm visibility variables. Do not let an + # unrelated, inconsistent device-selection environment prevent the child + # interpreter from reaching the boundary assertion. + env = os.environ.copy() + env.pop("CUDA_VISIBLE_DEVICES", None) + env.pop("HIP_VISIBLE_DEVICES", None) result = subprocess.run( [sys.executable, "-c", script], cwd=REPO_ROOT, + env=env, capture_output=True, text=True, timeout=60, @@ -25,6 +35,14 @@ def _assert_isolated_import_succeeds(script: str) -> None: assert result.returncode == 0, result.stderr +def test_isolated_import_ignores_conflicting_gpu_visibility(monkeypatch: pytest.MonkeyPatch) -> None: + """Import-boundary checks must not depend on GPU state leaked by prior tests.""" + monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0") + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1") + + _assert_isolated_import_succeeds("import vllm_omni.outputs") + + def test_stable_engine_imports_do_not_load_experimental_duplex() -> None: _assert_isolated_import_succeeds(""" import sys diff --git a/tests/entrypoints/openai_api/test_decode_audio_url.py b/tests/entrypoints/openai_api/test_decode_audio_url.py index 2a5c5be47..5dff3b052 100644 --- a/tests/entrypoints/openai_api/test_decode_audio_url.py +++ b/tests/entrypoints/openai_api/test_decode_audio_url.py @@ -5,6 +5,7 @@ import asyncio import base64 import os +import tempfile import httpx import pytest @@ -101,7 +102,7 @@ class TestDecodeAudioUrlSuffixSanitization: try: # ".." contains non-alnum dots, so falls back to .wav assert path.endswith(".wav") - assert os.path.dirname(path) == "/tmp" + assert os.path.dirname(path) == tempfile.gettempdir() finally: _cleanup(path) diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py b/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py index bcd06b63a..f16ecc9e9 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py @@ -365,6 +365,22 @@ def _resolve_fa_version(head_size: int) -> int: return version +def _rocm_flash_attn_varlen_func(): + """Resolve a ROCm varlen kernel, preferring AITER when available. + + The caller gathers paged KV into packed tensors before invoking this + function, avoiding AITER releases whose ``block_table`` kernel is broken. + """ + try: + from aiter import flash_attn_varlen_func + + return flash_attn_varlen_func + except ImportError: + from flash_attn import flash_attn_varlen_func + + return flash_attn_varlen_func + + def ar_diffusion_paged_attention( query: torch.Tensor, key_cache: torch.Tensor, @@ -401,6 +417,34 @@ def ar_diffusion_paged_attention( softmax_scale, causal=causal, ) + elif torch.version.hip is not None: + # vllm.vllm_flash_attn contains CUDA-only extensions. ROCm's AITER and + # upstream flash-attn expose the standard cu_seqlens_k API instead of + # vLLM's seqused_k/fa_version API. The ROCm flash-attn paged kernel also + # requires 128-token blocks, while AR-Diffusion uses frame-aligned + # 16-token blocks, so gather the visible blocks on-device first. + flash_attn_varlen_func = _rocm_flash_attn_varlen_func() + cu_seqlens_k = torch.cat([seq_lens.new_zeros(1), torch.cumsum(seq_lens, dim=0, dtype=torch.int32)]) + positions = torch.arange(int(max_seq_len), device=query_flat.device) + logical_blocks = torch.div(positions, key_cache.shape[1], rounding_mode="floor") + offsets = positions % key_cache.shape[1] + physical_blocks = block_table[:, logical_blocks].long() + gathered_k = key_cache[physical_blocks, offsets] + gathered_v = value_cache[physical_blocks, offsets] + valid = positions.unsqueeze(0) < seq_lens.unsqueeze(1) + packed_k = gathered_k[valid] + packed_v = gathered_v[valid] + out = flash_attn_varlen_func( + q=query_flat, + k=packed_k, + v=packed_v, + cu_seqlens_q=query_start_loc, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=int(max_query_len), + max_seqlen_k=int(max_seq_len), + softmax_scale=float(softmax_scale), + causal=causal, + ) else: from vllm.vllm_flash_attn import flash_attn_varlen_func From b300f666568e6b83a1e8893a3e13fa71afa88aa5 Mon Sep 17 00:00:00 2001 From: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com> Date: Fri, 14 Aug 2026 03:45:55 -0400 Subject: [PATCH 010/267] [Bugfix] Fix HSDP compatibility with the new online FP8 linear method (#5677) Signed-off-by: roG0d --- docs/user_guide/diffusion/parallelism/hsdp.md | 36 +++++++++++++++++++ tests/diffusion/quantization/test_hsdp_fp8.py | 34 ++++++++++++++++-- vllm_omni/diffusion/quantization/hsdp_fp8.py | 15 +++++--- 3 files changed, 79 insertions(+), 6 deletions(-) diff --git a/docs/user_guide/diffusion/parallelism/hsdp.md b/docs/user_guide/diffusion/parallelism/hsdp.md index 96a357c86..8a8aa5ef9 100644 --- a/docs/user_guide/diffusion/parallelism/hsdp.md +++ b/docs/user_guide/diffusion/parallelism/hsdp.md @@ -64,6 +64,41 @@ omni = Omni( ) ``` +### Combined with Online FP8 Quantization + +HSDP can be combined with online FP8 quantization for diffusion models that +support both features. HSDP shards the quantized weights across GPUs to reduce +the per-GPU weight memory footprint. + +```python +from vllm_omni import Omni +from vllm_omni.diffusion.data import DiffusionParallelConfig + +omni = Omni( + model="", + quantization="fp8", + parallel_config=DiffusionParallelConfig( + use_hsdp=True, + hsdp_shard_size=2, + ), +) +``` + +For online serving: + +```bash +vllm serve \ + --omni \ + --quantization fp8 \ + --use-hsdp \ + --hsdp-shard-size 2 +``` + +The model must support both HSDP and online FP8 quantization. HSDP remains +incompatible with Tensor Parallelism. See the +[FP8 Quantization Guide](../../quantization/fp8.md) for supported models and +hardware. + --- ## Example Script @@ -147,3 +182,4 @@ For detailed instructions on adding HSDP support to new models, see the [HSDP Co 1. ✅ **Enable HSDP** - Set `use_hsdp=True` and `hsdp_shard_size` to reduce per-GPU memory for large models 2. ✅ **Combine with SP** - Use together with `ulysses_degree` for video models requiring both memory reduction and sequence parallelism 3. ⚠️ **Incompatible with TP** - `tensor_parallel_size` must be 1 when HSDP is enabled +4. ✅ **Combine with online FP8** - Use `quantization="fp8"` or `--quantization fp8` to reduce the sharded weight memory footprint diff --git a/tests/diffusion/quantization/test_hsdp_fp8.py b/tests/diffusion/quantization/test_hsdp_fp8.py index 482235ff2..ce939dbf3 100644 --- a/tests/diffusion/quantization/test_hsdp_fp8.py +++ b/tests/diffusion/quantization/test_hsdp_fp8.py @@ -36,15 +36,23 @@ class _ToyKernel: # --- helper to build a toy module simulating online-FP8 post-load state --- -def _make_fp8_toy_module(out_features: int = 16, in_features: int = 32): +def _make_fp8_toy_module( + out_features: int = 16, + in_features: int = 32, + *, + quant_method_cls=None, +): """Return a module whose weight is ``qweight.t()`` (non-contiguous) and whose ``quant_method`` is an ``Fp8LinearMethod`` with a ``_ToyKernel``. """ from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod + if quant_method_cls is None: + quant_method_cls = Fp8LinearMethod + # Create the quant-method instance without calling __init__ (avoids # heavyweight upstream config dependencies). isinstance() still works. - qm = object.__new__(Fp8LinearMethod) + qm = object.__new__(quant_method_cls) qm.fp8_linear = _ToyKernel() # randn does not support float8 on CPU; zeros does and the test only @@ -104,6 +112,28 @@ def test_rewrites_non_contiguous_weight(): assert module.weight.shape == (16, 32) +def test_rewrites_per_tensor_online_fp8_weight(): + """Regression test for vLLM's new online FP8 frontend (#45463).""" + from vllm.model_executor.layers.quantization.online.fp8 import ( + Fp8PerTensorOnlineLinearMethod, + ) + + module = _make_fp8_toy_module( + 16, + 32, + quant_method_cls=Fp8PerTensorOnlineLinearMethod, + ) + + n_rewritten_layers = prepare_fp8_layers_for_fsdp(module) + + assert n_rewritten_layers == 1 + assert module.weight.is_contiguous() + assert module.weight.shape == (16, 32) + w, *_ = module.quant_method.fp8_linear._get_layer_params(module) + assert w.shape == (32, 16) + assert w.stride() == (1, 32) + + def test_patched_get_layer_params_returns_transposed_view(): module = _make_fp8_toy_module(16, 32) prepare_fp8_layers_for_fsdp(module) diff --git a/vllm_omni/diffusion/quantization/hsdp_fp8.py b/vllm_omni/diffusion/quantization/hsdp_fp8.py index 7c4666e29..6ffa8f2b4 100644 --- a/vllm_omni/diffusion/quantization/hsdp_fp8.py +++ b/vllm_omni/diffusion/quantization/hsdp_fp8.py @@ -33,9 +33,17 @@ import types from torch import nn from vllm.logger import init_logger from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod +from vllm.model_executor.layers.quantization.online.fp8 import ( + Fp8PerTensorOnlineLinearMethod, +) logger = init_logger(__name__) +_FP8_TRANSPOSED_WEIGHT_METHODS = ( + Fp8LinearMethod, + Fp8PerTensorOnlineLinearMethod, +) + def _build_transposed_get_layer_params(original_bound_method): """Return a ``_get_layer_params`` replacement that transposes ``w``. @@ -55,9 +63,8 @@ def _build_transposed_get_layer_params(original_bound_method): def prepare_fp8_layers_for_fsdp(model: nn.Module) -> int: """Make online-FP8 linear layers in ``model`` FSDP2-compatible. - For every layer whose ``quant_method`` is an :class:`Fp8LinearMethod` - and whose weight is currently a non-contiguous transpose view, this - function: + For every layer whose quantization method stores FP8 weights as a + non-contiguous transpose view, this function: 1. Replaces ``layer.weight`` with the underlying ``(out, in)`` row-major contiguous storage so FSDP2 ``fully_shard`` accepts it. @@ -76,7 +83,7 @@ def prepare_fp8_layers_for_fsdp(model: nn.Module) -> int: patched_kernel_ids: set[int] = set() for module in model.modules(): qm = getattr(module, "quant_method", None) - if not isinstance(qm, Fp8LinearMethod): + if not isinstance(qm, _FP8_TRANSPOSED_WEIGHT_METHODS): continue weight = getattr(module, "weight", None) From 022d7a7695ffc3ce7329c38656e388cc11aecb8d Mon Sep 17 00:00:00 2001 From: R0CKSTAR Date: Fri, 14 Aug 2026 15:49:26 +0800 Subject: [PATCH 011/267] [Perf][Diffusion] Avoid redundant MiniMax-H3 reference video scans (#6064) Signed-off-by: Xiaodong Ye --- .../minimax_h3/test_minimax_h3_contract.py | 210 +++++++++++++++++- .../models/minimax_h3/reference_video.py | 133 ++++++++--- 2 files changed, 300 insertions(+), 43 deletions(-) diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py index 60ed7090f..f872e381f 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import json +import sys from multiprocessing.reduction import ForkingPickler from types import SimpleNamespace from unittest.mock import Mock @@ -1428,34 +1429,219 @@ def test_r7_r8_ref2va_video_segment_matrix(monkeypatch, tmp_path, case, start_ti assert transcode_calls[0][1]["target_frame_count"] == 209 -def test_ref2va_qwen_sampling_decodes_prepared_video_once(monkeypatch): +def test_ref2va_qwen_sampling_uses_one_selective_decode(monkeypatch): from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module - decoded = np.arange(25 * 2 * 2 * 3, dtype=np.uint8).reshape(25, 2, 2, 3) - load_calls = [] + decoded = np.arange(3 * 2 * 2 * 3, dtype=np.uint8).reshape(3, 2, 2, 3) + decode_calls = [] monkeypatch.setattr( reference_video_module, "_probe_video", - lambda _path: {"frame_count": len(decoded)}, + lambda _path: { + "frame_count": 25, + "width": 2, + "height": 2, + }, ) monkeypatch.setattr( reference_video_module, - "load_video_frames", - lambda path: load_calls.append(path) or decoded, + "_decode_video_frames_ffmpeg", + lambda path, **kwargs: decode_calls.append((path, kwargs)) or decoded, ) sampled = reference_video_module.sample_reference_video_frames("prepared.mp4") - assert load_calls == ["prepared.mp4"] - for frame, expected in zip( - sampled["frames"], - (decoded[0], decoded[12], decoded[24]), - strict=True, - ): + assert decode_calls == [ + ( + "prepared.mp4", + { + "frame_count": 25, + "indices": [0, 12, 24], + "width": 2, + "height": 2, + }, + ) + ] + for frame, expected in zip(sampled["frames"], decoded, strict=True): np.testing.assert_array_equal(frame, expected) assert sampled["block_timestamps"] == [0.25, 1.0] +@pytest.mark.parametrize( + ("indices", "frame_count"), + [([0, 12], 25), (None, 2)], +) +def test_ref2va_ffmpeg_decode_reads_exact_rgb_frames(monkeypatch, indices, frame_count): + from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module + + expected = np.arange(2 * 2 * 3 * 3, dtype=np.uint8).reshape(2, 2, 3, 3) + observed = [] + + def fake_run(command, **kwargs): + observed.append((command, kwargs)) + return SimpleNamespace(stdout=expected.tobytes()) + + monkeypatch.setattr(reference_video_module.subprocess, "run", fake_run) + + actual = reference_video_module._decode_video_frames_ffmpeg( + "prepared.mp4", + frame_count=frame_count, + indices=indices, + width=3, + height=2, + ) + + np.testing.assert_array_equal(actual, expected) + command, kwargs = observed[0] + assert command[0] == "ffmpeg" + assert command[command.index("-frames:v") + 1] == "2" + if indices is None: + assert "-vf" not in command + else: + assert command[command.index("-vf") + 1] == "select=eq(n\\,0)+eq(n\\,12)" + assert command[-1] == "pipe:1" + assert kwargs == {"check": True, "capture_output": True} + + +def test_ref2va_load_video_frames_uses_ffmpeg_without_decord(monkeypatch): + from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module + + expected = np.zeros((2, 2, 3, 3), dtype=np.uint8) + decode_calls = [] + monkeypatch.setitem(sys.modules, "decord", None) + monkeypatch.setattr( + reference_video_module, + "_probe_video", + lambda _path: { + "frame_count": 2, + "width": 3, + "height": 2, + }, + ) + monkeypatch.setattr( + reference_video_module, + "_decode_video_frames_ffmpeg", + lambda path, **kwargs: decode_calls.append((path, kwargs)) or expected, + ) + + actual = reference_video_module.load_video_frames("prepared.mp4") + + assert actual is expected + assert decode_calls == [ + ( + "prepared.mp4", + { + "frame_count": 2, + "width": 3, + "height": 2, + }, + ) + ] + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"frame_count": 0, "width": 3, "height": 2}, "video has no frames"), + ({"frame_count": 1, "width": 0, "height": 2}, "invalid dimensions"), + ( + {"frame_count": 1, "indices": [], "width": 3, "height": 2}, + "invalid frame indices", + ), + ], +) +def test_ref2va_ffmpeg_decode_rejects_invalid_metadata(kwargs, message): + from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module + from vllm_omni.errors import OmniClientError + + with pytest.raises(OmniClientError, match=message): + reference_video_module._decode_video_frames_ffmpeg("prepared.mp4", **kwargs) + + +def test_ref2va_probe_uses_container_frame_count_without_scan(monkeypatch): + from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module + + calls = [] + probe = { + "streams": [ + { + "codec_type": "video", + "width": 4, + "height": 2, + "r_frame_rate": "24/1", + "nb_frames": "3", + "duration": "0.125", + "sample_aspect_ratio": "1:1", + } + ], + "format": {"duration": "0.125", "size": "123"}, + } + monkeypatch.setattr( + reference_video_module.subprocess, + "run", + lambda command, **kwargs: calls.append((command, kwargs)) or SimpleNamespace(stdout=json.dumps(probe)), + ) + + metadata = reference_video_module._probe_video("prepared.mp4") + + assert metadata["frame_count"] == 3 + assert len(calls) == 1 + assert "-count_frames" not in calls[0][0] + + +def test_ref2va_probe_counts_frames_only_when_metadata_is_missing(monkeypatch): + from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module + + calls = [] + stream = { + "codec_type": "video", + "width": 4, + "height": 2, + "r_frame_rate": "24/1", + "duration": "0.125", + "sample_aspect_ratio": "1:1", + } + first_probe = { + "streams": [stream], + "format": {"duration": "0.125", "size": "123"}, + } + second_probe = { + "streams": [{**stream, "nb_read_frames": "3"}], + "format": {"duration": "0.125", "size": "123"}, + } + monkeypatch.setattr( + reference_video_module.subprocess, + "run", + lambda command, **kwargs: calls.append((command, kwargs)) + or SimpleNamespace(stdout=json.dumps(first_probe if len(calls) == 1 else second_probe)), + ) + + metadata = reference_video_module._probe_video("prepared.mp4") + + assert metadata["frame_count"] == 3 + assert len(calls) == 2 + assert "-count_frames" in calls[1][0] + + +def test_ref2va_ffmpeg_decode_rejects_incomplete_output(monkeypatch): + from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module + from vllm_omni.errors import OmniClientError + + monkeypatch.setattr( + reference_video_module.subprocess, + "run", + lambda *_args, **_kwargs: SimpleNamespace(stdout=b"short"), + ) + + with pytest.raises(OmniClientError, match="decoded 5 bytes.*expected 18"): + reference_video_module._decode_video_frames_ffmpeg( + "prepared.mp4", + frame_count=1, + width=3, + height=2, + ) + + def test_ref2va_two_video_recipe_tolerates_container_rounding(monkeypatch, tmp_path): from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module diff --git a/vllm_omni/diffusion/models/minimax_h3/reference_video.py b/vllm_omni/diffusion/models/minimax_h3/reference_video.py index 0ecd69f31..18c8de5b4 100644 --- a/vllm_omni/diffusion/models/minimax_h3/reference_video.py +++ b/vllm_omni/diffusion/models/minimax_h3/reference_video.py @@ -44,31 +44,33 @@ def _nearest_multiple(value: float, multiple: int) -> int: def _probe_video(path: str) -> dict[str, Any]: - result = subprocess.run( - [ - "ffprobe", - "-v", - "error", - "-count_frames", - "-show_entries", - ( - "stream=codec_type,codec_name,width,height,r_frame_rate,nb_read_frames,nb_frames,duration," - "sample_aspect_ratio:format=format_name,size,duration" - ), - "-of", - "json", - path, - ], - check=True, - capture_output=True, - text=True, + show_entries = ( + "stream=codec_type,codec_name,width,height,r_frame_rate,nb_read_frames,nb_frames,duration," + "sample_aspect_ratio:format=format_name,size,duration" ) + command = ["ffprobe", "-v", "error", "-show_entries", show_entries, "-of", "json", path] + result = subprocess.run(command, check=True, capture_output=True, text=True) probe = json.loads(result.stdout) streams = probe.get("streams") or [] video_streams = [stream for stream in streams if stream.get("codec_type") == "video"] if not video_streams: raise OmniClientError(f"media has no video stream: {path}") stream = video_streams[0] + raw_count = stream.get("nb_read_frames") or stream.get("nb_frames") + if raw_count in (None, "", "N/A"): + result = subprocess.run( + ["ffprobe", "-v", "error", "-count_frames", *command[3:]], + check=True, + capture_output=True, + text=True, + ) + probe = json.loads(result.stdout) + streams = probe.get("streams") or [] + video_streams = [stream for stream in streams if stream.get("codec_type") == "video"] + if not video_streams: + raise OmniClientError(f"media has no video stream: {path}") + stream = video_streams[0] + raw_count = stream.get("nb_read_frames") or stream.get("nb_frames") try: numerator, denominator = str(stream["r_frame_rate"]).split("/", 1) fps = float(numerator) / float(denominator) @@ -76,7 +78,6 @@ def _probe_video(path: str) -> dict[str, Any]: raise OmniClientError(f"cannot determine video FPS: {path}") from exc if not math.isfinite(fps) or fps <= 0: raise OmniClientError(f"cannot determine video FPS: {path}") - raw_count = stream.get("nb_read_frames") or stream.get("nb_frames") if raw_count in (None, "", "N/A"): raise OmniClientError(f"cannot determine video frame count: {path}") raw_duration = stream.get("duration") @@ -404,13 +405,13 @@ def load_video_frames(path: str) -> np.ndarray: try: import decord except ImportError: - import av - - with av.open(path) as container: - frames = [frame.to_ndarray(format="rgb24") for frame in container.decode(video=0)] - if not frames: - raise OmniClientError(f"video has no frames: {path}") - return np.stack(frames) + meta = _probe_video(path) + return _decode_video_frames_ffmpeg( + path, + frame_count=int(meta["frame_count"]), + width=int(meta["width"]), + height=int(meta["height"]), + ) reader = decord.VideoReader(path) if len(reader) <= 0: @@ -419,6 +420,70 @@ def load_video_frames(path: str) -> np.ndarray: return frames.asnumpy() if hasattr(frames, "asnumpy") else np.asarray(frames) +def _decode_video_frames_ffmpeg( + path: str, + *, + frame_count: int, + width: int, + height: int, + indices: list[int] | None = None, +) -> np.ndarray: + frame_count = int(frame_count) + width = int(width) + height = int(height) + if frame_count <= 0: + raise OmniClientError(f"video has no frames: {path}") + if width <= 0 or height <= 0: + raise OmniClientError(f"video has invalid dimensions {width}x{height}: {path}") + if indices is not None and (not indices or any(index < 0 or index >= frame_count for index in indices)): + raise OmniClientError(f"invalid frame indices for {path}: {indices}") + + output_frame_count = frame_count if indices is None else len(indices) + command = [ + "ffmpeg", + "-loglevel", + "error", + "-threads", + "0", + "-i", + path, + "-map", + "0:v:0", + "-an", + ] + if indices is not None: + select = "+".join(f"eq(n\\,{index})" for index in indices) + command.extend(["-vf", f"select={select}"]) + command.extend( + [ + "-vsync", + "0", + "-frames:v", + str(output_frame_count), + "-f", + "rawvideo", + "-pix_fmt", + "rgb24", + "pipe:1", + ] + ) + result = subprocess.run( + command, + check=True, + capture_output=True, + ) + frame_size = width * height * 3 + expected_size = output_frame_count * frame_size + if len(result.stdout) != expected_size: + raise OmniClientError(f"decoded {len(result.stdout)} bytes from {path}, expected {expected_size}") + return np.frombuffer(result.stdout, dtype=np.uint8).reshape( + output_frame_count, + height, + width, + 3, + ) + + def sample_reference_video_frames(prepared_path: str) -> dict[str, Any]: meta = _probe_video(prepared_path) ratio = MINIMAX_H3_FPS / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS @@ -434,11 +499,17 @@ def sample_reference_video_frames(prepared_path: str) -> dict[str, Any]: if not indices: raise OmniClientError(f"no frames sampled from {prepared_path}") - # The preparation step emits a lossless RGB stream. Decode it once and - # sample from that array so Qwen3VL sees exactly the same pixels as the - # video VAE, without starting one full-video ffmpeg decode per sample. - decoded_frames = load_video_frames(prepared_path) - frames = [np.asarray(decoded_frames[source_index]) for source_index in indices] + # The preparation step emits a lossless RGB stream. Decode only the sampled + # frames in one ffmpeg process so Qwen3VL sees the exact prepared pixels + # without decoding the entire high-bitrate stream into host memory. + decoded_frames = _decode_video_frames_ffmpeg( + prepared_path, + frame_count=int(meta["frame_count"]), + indices=indices, + width=meta["width"], + height=meta["height"], + ) + frames = [np.asarray(frame) for frame in decoded_frames] timestamps = [index / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS for index in range(len(indices))] timestamps += [timestamps[-1]] * ((-len(timestamps)) % MINIMAX_H3_QWEN_TEMPORAL_PATCH) From 728640f4d9bb7c1b68d646e2a4c59ce1ce45de9c Mon Sep 17 00:00:00 2001 From: R0CKSTAR Date: Fri, 14 Aug 2026 15:51:23 +0800 Subject: [PATCH 012/267] [Bugfix][MUSA][Qwen Image] Avoid complex RoPE alias guards (#6110) Signed-off-by: Xiaodong Ye --- .../diffusion/models/qwen_image/qwen_image_transformer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py b/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py index 67e2d0db7..c82ace9c4 100644 --- a/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py +++ b/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py @@ -642,10 +642,10 @@ class QwenImageCrossAttention(nn.Module): txt_query = self.norm_added_q(txt_query) txt_key = self.norm_added_k(txt_key) - img_cos = vid_freqs.real.to(img_query.dtype) - img_sin = vid_freqs.imag.to(img_query.dtype) - txt_cos = txt_freqs.real.to(txt_query.dtype) - txt_sin = txt_freqs.imag.to(txt_query.dtype) + img_cos = torch.real(vid_freqs).to(img_query.dtype) + img_sin = torch.imag(vid_freqs).to(img_query.dtype) + txt_cos = torch.real(txt_freqs).to(txt_query.dtype) + txt_sin = torch.imag(txt_freqs).to(txt_query.dtype) img_query = self.rope(img_query, img_cos, img_sin) img_key = self.rope(img_key, img_cos, img_sin) From ffd11c181e5fa4e558ba2570f326fb41371c0421 Mon Sep 17 00:00:00 2001 From: Wenjia Li Date: Fri, 14 Aug 2026 17:49:26 +0800 Subject: [PATCH 013/267] [Perf][NPU] Fuse MiniMax H3 Qwen3-VL RoPE (#6061) Signed-off-by: Wenjia Li --- .../test_minimax_h3_qwen3vl_rope.py | 104 ++++++++++++++++++ vllm_omni/platforms/npu/models/minimax_h3.py | 43 ++++++++ vllm_omni/platforms/npu/platform.py | 2 + 3 files changed, 149 insertions(+) create mode 100644 tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py create mode 100644 vllm_omni/platforms/npu/models/minimax_h3.py diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py new file mode 100644 index 000000000..9ac3243bb --- /dev/null +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +from vllm_omni.platforms import current_omni_platform + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion] + + +@pytest.mark.skipif(not current_omni_platform.is_npu(), reason="requires Ascend NPU") +@pytest.mark.parametrize("num_q_heads,num_kv_heads,seq_len", [(8, 2, 16), (64, 8, 1)]) +def test_qwen3_vl_npu_rope_matches_bnsd_reference( + num_q_heads: int, + num_kv_heads: int, + seq_len: int, +) -> None: + from vllm_omni.platforms.npu.models.minimax_h3 import _apply_rotary_pos_emb_npu + + device = torch.device("npu") + dtype = torch.bfloat16 + q = torch.randn(1, num_q_heads, seq_len, 128, device=device, dtype=dtype) + k = torch.randn(1, num_kv_heads, seq_len, 128, device=device, dtype=dtype) + cos = torch.randn(1, seq_len, 128, device=device, dtype=dtype) + sin = torch.randn(1, seq_len, 128, device=device, dtype=dtype) + + q_out, k_out = _apply_rotary_pos_emb_npu(q, k, cos, sin) + cos_bnsd = cos.unsqueeze(1) + sin_bnsd = sin.unsqueeze(1) + + def rotate_half(x: torch.Tensor) -> torch.Tensor: + x1, x2 = x.chunk(2, dim=-1) + return torch.cat((-x2, x1), dim=-1) + + torch.testing.assert_close( + q_out, + q * cos_bnsd + rotate_half(q) * sin_bnsd, + atol=2e-2, + rtol=2e-2, + ) + torch.testing.assert_close( + k_out, + k * cos_bnsd + rotate_half(k) * sin_bnsd, + atol=2e-2, + rtol=2e-2, + ) + + +@pytest.mark.skipif(not current_omni_platform.is_npu(), reason="requires Ascend NPU") +def test_qwen3_vl_npu_rope_preserves_batched_mrope_positions() -> None: + """Keep per-sample Qwen3-VL M-RoPE frequencies on the BNSD NPU path.""" + from types import SimpleNamespace + + from vllm_omni.diffusion.models.minimax_h3.encoder import ( + MiniMaxH3Qwen3VLTextRotaryEmbedding, + ) + from vllm_omni.platforms.npu.models.minimax_h3 import _apply_rotary_pos_emb_npu + + device = torch.device("npu") + dtype = torch.bfloat16 + batch_size, seq_len, head_dim = 2, 16, 128 + config = SimpleNamespace( + hidden_size=head_dim, + num_attention_heads=1, + rope_parameters={ + "rope_theta": 1_000_000.0, + "mrope_section": [24, 20, 20], + }, + ) + rope = MiniMaxH3Qwen3VLTextRotaryEmbedding(config).to(device) + positions = torch.arange(seq_len, device=device) + position_ids = torch.stack( + ( + torch.stack((positions, positions + 7)), + torch.stack((positions * 2, positions * 2 + 3)), + torch.stack((positions * 3, positions * 3 + 5)), + ) + ) + q = torch.randn(batch_size, 8, seq_len, head_dim, device=device, dtype=dtype) + k = torch.randn(batch_size, 2, seq_len, head_dim, device=device, dtype=dtype) + cos, sin = rope(q, position_ids) + + q_out, k_out = _apply_rotary_pos_emb_npu(q, k, cos, sin) + cos_bnsd = cos.unsqueeze(1) + sin_bnsd = sin.unsqueeze(1) + + def rotate_half(x: torch.Tensor) -> torch.Tensor: + x1, x2 = x.chunk(2, dim=-1) + return torch.cat((-x2, x1), dim=-1) + + assert not torch.equal(cos[0], cos[1]) + torch.testing.assert_close( + q_out, + q * cos_bnsd + rotate_half(q) * sin_bnsd, + atol=2e-2, + rtol=2e-2, + ) + torch.testing.assert_close( + k_out, + k * cos_bnsd + rotate_half(k) * sin_bnsd, + atol=2e-2, + rtol=2e-2, + ) diff --git a/vllm_omni/platforms/npu/models/minimax_h3.py b/vllm_omni/platforms/npu/models/minimax_h3.py new file mode 100644 index 000000000..a05abd654 --- /dev/null +++ b/vllm_omni/platforms/npu/models/minimax_h3.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""NPU patches for the MiniMax H3 Qwen3-VL text encoder.""" + +from __future__ import annotations + +import torch +from vllm.logger import init_logger + +from vllm_omni.platforms.npu.layers.rotary_embedding import ( + npu_rotary_mul_with_bsnd_fallback, +) + +logger = init_logger(__name__) + +_PATCHED = False + + +def _apply_rotary_pos_emb_npu( + q: torch.Tensor, + k: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Apply Qwen3-VL text RoPE with fused BNSD rotary multiplication.""" + return ( + npu_rotary_mul_with_bsnd_fallback(q, cos, sin, unsqueeze_dim=1), + npu_rotary_mul_with_bsnd_fallback(k, cos, sin, unsqueeze_dim=1), + ) + + +def apply_minimax_h3_qwen3vl_patch() -> None: + """Route MiniMax H3 Qwen3-VL text RoPE to the Ascend fused operator.""" + global _PATCHED + if _PATCHED: + return + + from vllm_omni.diffusion.models.minimax_h3 import encoder + + encoder._apply_rotary_pos_emb = _apply_rotary_pos_emb_npu + _PATCHED = True + logger.debug("Applied NPU fused RoPE patch for MiniMax H3 Qwen3-VL text encoder") diff --git a/vllm_omni/platforms/npu/platform.py b/vllm_omni/platforms/npu/platform.py index 525725fcf..16f87b21d 100644 --- a/vllm_omni/platforms/npu/platform.py +++ b/vllm_omni/platforms/npu/platform.py @@ -36,6 +36,7 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform): from vllm_ascend.utils import adapt_patch from vllm_omni.platforms.npu._310p import apply_patches as apply_310p_patches + from vllm_omni.platforms.npu.models.minimax_h3 import apply_minimax_h3_qwen3vl_patch from vllm_omni.platforms.npu.models.qwen3_tts_code2wav import ( apply_qwen3_tts_code2wav_patch, ) @@ -44,6 +45,7 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform): ) adapt_patch(is_global_patch=True) + apply_minimax_h3_qwen3vl_patch() apply_qwen3_tts_code2wav_patch() apply_qwen3_tts_tokenizer_v2_patch() apply_310p_patches() From 8ecd1f6d5cc91aab8a475a861213720b336e2f65 Mon Sep 17 00:00:00 2001 From: Wenjia Li Date: Fri, 14 Aug 2026 19:47:17 +0800 Subject: [PATCH 014/267] [Perf][NPU] Fuse MiniMax H3 Qwen3-VL SwiGLU (#6167) Signed-off-by: Wenjia Li --- .../test_minimax_h3_qwen3vl_swiglu.py | 46 +++++++++++++++++++ vllm_omni/platforms/npu/models/minimax_h3.py | 36 +++++++++++++-- vllm_omni/platforms/npu/platform.py | 8 ++++ 3 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_swiglu.py diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_swiglu.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_swiglu.py new file mode 100644 index 000000000..2664990b4 --- /dev/null +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_swiglu.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch +import torch.nn.functional as F + +from vllm_omni.platforms import current_omni_platform + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion] + + +@pytest.mark.skipif(not current_omni_platform.is_npu(), reason="requires Ascend NPU") +@pytest.mark.parametrize("shape", [(1, 16, 256), (2, 1, 256)]) +def test_qwen3_vl_npu_swiglu_matches_split_reference( + shape: tuple[int, int, int], +) -> None: + from vllm_omni.platforms.npu.models.minimax_h3 import npu_swiglu_from_packed + + packed = torch.randn(*shape, device="npu", dtype=torch.bfloat16) + gate, up = packed.chunk(2, dim=-1) + + torch.testing.assert_close( + npu_swiglu_from_packed(packed), + F.silu(gate) * up, + atol=2e-2, + rtol=2e-2, + ) + + +@pytest.mark.skipif(not current_omni_platform.is_npu(), reason="requires Ascend NPU") +def test_qwen3_vl_npu_swiglu_matches_two_projection_reference() -> None: + from vllm_omni.platforms.npu.models.minimax_h3 import npu_swiglu_from_packed + + x = torch.randn(1, 16, 128, device="npu", dtype=torch.bfloat16) + weight = torch.randn(256, 128, device="npu", dtype=torch.bfloat16) + packed = F.linear(x, weight) + gate = F.linear(x, weight[:128]) + up = F.linear(x, weight[128:]) + + torch.testing.assert_close( + npu_swiglu_from_packed(packed), + F.silu(gate) * up, + atol=2e-2, + rtol=2e-2, + ) diff --git a/vllm_omni/platforms/npu/models/minimax_h3.py b/vllm_omni/platforms/npu/models/minimax_h3.py index a05abd654..78e0b3f14 100644 --- a/vllm_omni/platforms/npu/models/minimax_h3.py +++ b/vllm_omni/platforms/npu/models/minimax_h3.py @@ -5,7 +5,11 @@ from __future__ import annotations +from typing import Any + import torch +import torch.nn.functional as F +import torch_npu from vllm.logger import init_logger from vllm_omni.platforms.npu.layers.rotary_embedding import ( @@ -14,7 +18,8 @@ from vllm_omni.platforms.npu.layers.rotary_embedding import ( logger = init_logger(__name__) -_PATCHED = False +_ROPE_PATCHED = False +_SWIGLU_PATCHED = False def _apply_rotary_pos_emb_npu( @@ -32,12 +37,35 @@ def _apply_rotary_pos_emb_npu( def apply_minimax_h3_qwen3vl_patch() -> None: """Route MiniMax H3 Qwen3-VL text RoPE to the Ascend fused operator.""" - global _PATCHED - if _PATCHED: + global _ROPE_PATCHED + if _ROPE_PATCHED: return from vllm_omni.diffusion.models.minimax_h3 import encoder encoder._apply_rotary_pos_emb = _apply_rotary_pos_emb_npu - _PATCHED = True + _ROPE_PATCHED = True logger.debug("Applied NPU fused RoPE patch for MiniMax H3 Qwen3-VL text encoder") + + +def npu_swiglu_from_packed(gate_up: torch.Tensor) -> torch.Tensor: + """Apply fused SwiGLU to a packed gate/up projection tensor.""" + return torch_npu.npu_swiglu(gate_up, dim=-1) + + +def _forward_minimax_h3_qwen3vl_text_mlp_npu(self: Any, x: torch.Tensor) -> torch.Tensor: + """Run the Qwen3-VL MLP with one packed GEMM and fused SwiGLU.""" + gate_up = F.linear(x, self.gate_up_proj.weight) + return self.down_proj(npu_swiglu_from_packed(gate_up)) + + +def apply_minimax_h3_qwen3vl_swiglu_patch() -> None: + """Route MiniMax H3 Qwen3-VL text MLP to the Ascend fused SwiGLU path.""" + global _SWIGLU_PATCHED + if _SWIGLU_PATCHED: + return + + from vllm_omni.diffusion.models.minimax_h3 import encoder + + encoder.MiniMaxH3Qwen3VLTextMLP.forward = _forward_minimax_h3_qwen3vl_text_mlp_npu + _SWIGLU_PATCHED = True diff --git a/vllm_omni/platforms/npu/platform.py b/vllm_omni/platforms/npu/platform.py index 16f87b21d..b17f447c4 100644 --- a/vllm_omni/platforms/npu/platform.py +++ b/vllm_omni/platforms/npu/platform.py @@ -82,6 +82,14 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform): def init_diffusion_model_runner_runtime(cls, vllm_config: Any, od_config: Any, device: torch.device) -> None: from vllm_ascend.ascend_forward_context import set_mc2_mask, set_mc2_tokens_capacity + from vllm_omni.platforms.npu.models.minimax_h3 import ( + apply_minimax_h3_qwen3vl_swiglu_patch, + ) + + # The patch imports the MiniMax encoder, which depends on + # current_omni_platform. Run it only after platform construction has + # completed, but before the diffusion pipeline is loaded. + apply_minimax_h3_qwen3vl_swiglu_patch() set_mc2_tokens_capacity(vllm_config, od_config.max_num_seqs, 1) set_mc2_mask(vllm_config, device) From f7a5146093bc4d16b5607901a8a2509d785777db Mon Sep 17 00:00:00 2001 From: Alex Brooks Date: Fri, 14 Aug 2026 13:13:00 -0600 Subject: [PATCH 015/267] [Feat/Bugfix] Fix API Server model tag <-> model sync & flag normalization (#3805) Signed-off-by: Alex Brooks --- vllm_omni/entrypoints/openai/api_server.py | 29 ++++++++-------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 16114bcb6..1ba26096f 100755 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -43,6 +43,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ChatCompletionResponse, ) +from vllm.entrypoints.openai.cli_args import make_arg_parser # yapf conflicts with isort for this block # yapf: disable @@ -3826,26 +3827,16 @@ async def omni_wakeup(request: OmniWakeupRequest, raw_request: Request): if __name__ == "__main__": - import asyncio - - from vllm.entrypoints.openai.cli_args import make_arg_parser - parser = TrackingArgumentParser(description="vLLM-Omni OpenAI-Compatible REST API server") parser = make_arg_parser(parser) - registered_flags = set() - for action in parser._actions: - registered_flags.update(action.option_strings) - - # FIXME - this is broken on `main`. `make_arg_parser` does not handle omni engine args properly. - if "--omni" not in registered_flags: - parser.add_argument("--omni", action="store_true", default=False, help="Enable vLLM-Omni mode.") - if "--enable-sleep-mode" not in registered_flags: - parser.add_argument( - "--enable-sleep-mode", action="store_true", default=False, help="Enable GPU memory pool for sleep mode." - ) + # Ensure that passing --omni won't crash the server. + # NOTE: the value here does not matter since we are always running the Omni server + # when __main__ is called, i.e., --omni is only used when called through the entrypoints. + parser.add_argument("--omni", action="store_true", default=False) args = parser.parse_args() - if not hasattr(args, "model_tag"): - setattr(args, "model_tag", args.model) - if hasattr(args, "model_tag") and args.model_tag is None: - args.model_tag = args.model + # sync args.model to model_tag, because if we pass the model positionally, + # args.model will be the default from vLLM's ModelConfig (currently + # Qwen/Qwen3-0.6B) and crash cryptically. + if args.model_tag is not None: + args.model = args.model_tag asyncio.run(omni_run_server(args)) From 4b6d6d5cf79506d2a6ea697da11f1b2bdceccda8 Mon Sep 17 00:00:00 2001 From: Joshna-Medisetty Date: Fri, 14 Aug 2026 13:23:41 -0700 Subject: [PATCH 016/267] [Bugfix][XPU][Tests]Scope XPU pytest to explicit paths to fix collection crash (#6175) Signed-off-by: Joshna Medisetty Co-authored-by: Cursor Co-authored-by: Chendi.Xue --- .buildkite/intel/scripts/run-xpu-test.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.buildkite/intel/scripts/run-xpu-test.sh b/.buildkite/intel/scripts/run-xpu-test.sh index 742e39b78..9fb7267c7 100755 --- a/.buildkite/intel/scripts/run-xpu-test.sh +++ b/.buildkite/intel/scripts/run-xpu-test.sh @@ -87,8 +87,9 @@ time timeout -k 30 30m docker run \ echo $ZE_AFFINITY_MASK pip install tblib==3.1.0 cd /workspace/vllm-omni - pytest -v -s -m "core_model and xpu and B60" + XPU_TEST_PATHS="tests/diffusion tests/dfx tests/e2e" + pytest -v -s $XPU_TEST_PATHS -m "core_model and xpu and B60" pytest -v -s tests/diffusion/quantization/test_mxfp8_config.py - pytest -v -s -m "advanced_model and xpu and B60" - pytest -v -s -m "omni and xpu and B60" + pytest -v -s $XPU_TEST_PATHS -m "advanced_model and xpu and B60" + pytest -v -s $XPU_TEST_PATHS -m "omni and xpu and B60" ' From fe54ecc38845bd2a14419c57bf7c0a7a761e7161 Mon Sep 17 00:00:00 2001 From: Alex Brooks Date: Fri, 14 Aug 2026 17:05:11 -0600 Subject: [PATCH 017/267] [Frontend] Loosen Validation for TTS Models with No Uploaded Speakers (#5878) Signed-off-by: Alex Brooks --- .../test_omnivoice_expansion.py | 121 +++++++++++++++++- vllm_omni/entrypoints/openai/api_server.py | 3 +- .../entrypoints/openai/protocol/audio.py | 10 ++ .../entrypoints/openai/serving_speech.py | 51 ++++++-- 4 files changed, 172 insertions(+), 13 deletions(-) diff --git a/tests/e2e/online_serving/test_omnivoice_expansion.py b/tests/e2e/online_serving/test_omnivoice_expansion.py index ab35b74ea..78326f74c 100644 --- a/tests/e2e/online_serving/test_omnivoice_expansion.py +++ b/tests/e2e/online_serving/test_omnivoice_expansion.py @@ -14,9 +14,10 @@ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" import pytest from tests.helpers.mark import hardware_test -from tests.helpers.media import load_test_audio_data_url +from tests.helpers.media import get_asset_path, load_test_audio_data_url from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path +from vllm_omni.entrypoints.openai.serving_speech import _DEFAULT_VOICE_NAME try: from transformers import HiggsAudioV2TokenizerModel # noqa: F401 @@ -140,3 +141,121 @@ class TestOmniVoiceVoiceCloning: "min_audio_bytes": _DEFAULT_MIN_AUDIO_BYTES, } openai_client.send_audio_speech_request(request_config) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True) +@pytest.mark.parametrize("voice", [_DEFAULT_VOICE_NAME, {"id": _DEFAULT_VOICE_NAME}]) +def test_speech_with_voice_param_accepted(omni_server, openai_client, voice) -> None: + """The default voice should be accepted as both a plain string and a + VoiceID dict, even when the model has no uploaded speakers.""" + request_config = { + "model": omni_server.model, + "input": get_prompt("text"), + "voice": voice, + "response_format": "wav", + "timeout": 180.0, + "min_audio_bytes": _DEFAULT_MIN_AUDIO_BYTES, + } + openai_client.send_audio_speech_request(request_config) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True) +def test_unknown_voice_rejected(omni_server, openai_client) -> None: + """An unrecognized voice name should be rejected with a 400.""" + request_config = { + "model": omni_server.model, + "input": get_prompt("text"), + "voice": "nonexistent_voice", + "status_code": 400, + "err_message": "Invalid voice", + } + openai_client.send_audio_speech_request(request_config) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True) +def test_voice_request_before_and_after_clone_registration(omni_server, openai_client) -> None: + """The default voice should always be listed and usable, even after + a different cloned voice is registered.""" + speech_config = { + "model": omni_server.model, + "input": get_prompt("text"), + "voice": _DEFAULT_VOICE_NAME, + } + + # 1) "default" should appear in the voices list before any uploads + voices = openai_client.send_audio_voices_list_http_request()[0] + assert _DEFAULT_VOICE_NAME in voices.json_body["voices"] + + # 2) Speech with the default voice should succeed + openai_client.send_audio_speech_request(speech_config) + + # 3) Register a cloned voice under a different name + register_config = { + "data": {"name": "bar", "consent": "test-consent"}, + "files": { + "audio_sample": ( + "clone_2.wav", + get_asset_path("qwen3_tts/clone_2.wav").read_bytes(), + "audio/wav", + ), + }, + } + openai_client.send_audio_voices_create_http_request(register_config) + + # 4) "default" should still be listed alongside the new voice + voices = openai_client.send_audio_voices_list_http_request()[0] + assert _DEFAULT_VOICE_NAME in voices.json_body["voices"] + assert "bar" in voices.json_body["voices"] + + # 5) Speech with the default voice should still succeed + openai_client.send_audio_speech_request(speech_config) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True) +def test_registered_default_voice_overrides_placeholder(omni_server, openai_client) -> None: + """When a real voice is uploaded under the default name, it should be + used as a cloned voice; after deletion, the placeholder behavior resumes.""" + speech_config = { + "model": omni_server.model, + "input": get_prompt("text"), + "voice": _DEFAULT_VOICE_NAME, + } + + # 1) Placeholder default works before any registration + openai_client.send_audio_speech_request(speech_config) + + # 2) Register a real voice under the default name + register_config = { + "data": {"name": _DEFAULT_VOICE_NAME, "consent": "test-consent"}, + "files": { + "audio_sample": ( + "clone_2.wav", + get_asset_path("qwen3_tts/clone_2.wav").read_bytes(), + "audio/wav", + ), + }, + } + openai_client.send_audio_voices_create_http_request(register_config) + + # 3) After creating the voice with the default name, it's visible as an uploaded_voice + voices = openai_client.send_audio_voices_list_http_request()[0] + assert _DEFAULT_VOICE_NAME in voices.json_body["voices"] + uploaded_names = [v["name"] for v in voices.json_body["uploaded_voices"]] + assert _DEFAULT_VOICE_NAME in uploaded_names + openai_client.send_audio_speech_request(speech_config) + + # 4) Delete the uploaded default voice + openai_client.send_audio_voices_delete_http_request( + {"name": _DEFAULT_VOICE_NAME}, + ) + + # 5) Ensure the placeholder behavior is consistent, but it's not an uploaded voice now + voices = openai_client.send_audio_voices_list_http_request()[0] + assert _DEFAULT_VOICE_NAME in voices.json_body["voices"] + uploaded_names = [v["name"] for v in voices.json_body["uploaded_voices"]] + assert _DEFAULT_VOICE_NAME not in uploaded_names + openai_client.send_audio_speech_request(speech_config) diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 1ba26096f..71020bd84 100755 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -1482,8 +1482,7 @@ async def list_voices(raw_request: Request): status_code=HTTPStatus.NOT_FOUND, ) - # Get all speakers (both model built-in and uploaded) - speakers = sorted(handler.supported_speakers) if handler.supported_speakers else [] + speakers = sorted(handler._get_available_voices()) # Get uploaded speakers details uploaded_speakers = [] diff --git a/vllm_omni/entrypoints/openai/protocol/audio.py b/vllm_omni/entrypoints/openai/protocol/audio.py index b2b55bc52..e309f1cb7 100644 --- a/vllm_omni/entrypoints/openai/protocol/audio.py +++ b/vllm_omni/entrypoints/openai/protocol/audio.py @@ -59,6 +59,16 @@ class OpenAICreateSpeechRequest(BaseModel): validation_alias=AliasChoices("voice", "speaker"), description="Speaker/voice to use. For Qwen3-TTS: vivian, ryan, aiden, etc.", ) + + @field_validator("voice", mode="before") + @classmethod + def _normalize_voice(cls, voice: str | dict[str, str]) -> str: + if isinstance(voice, dict): + if "id" not in voice: + raise ValueError("voice dict must contain 'id'") + return voice["id"] + return voice + instructions: str | None = Field( default=None, description="Instructions for voice style/emotion (maps to 'instruct' for Qwen3-TTS)", diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 109777282..3612a2ff1 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -134,6 +134,13 @@ _QWEN3_TTS_REF_AUDIO_CACHE_KEY = "_qwen3_tts_ref_audio_cache_key" _TTS_MAX_INSTRUCTIONS_LENGTH = 500 _TTS_MAX_NEW_TOKENS_MAX = 4096 _MING_DEFAULT_PROMPT = MING_DEFAULT_PROMPT +_DEFAULT_VOICE_NAME = "default" + + +def _is_default_voice(voice, supported_speakers): + """Check if a lowercased voice name is the placeholder default and not + an actual registered/built-in speaker.""" + return voice == _DEFAULT_VOICE_NAME and voice not in supported_speakers def _create_wav_header(sample_rate: int, num_channels: int = 1, bits_per_sample: int = 16) -> bytes: @@ -470,7 +477,14 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self._audex_tta_rvq = None self._voxcpm2_split_map: dict[int, list[int]] = {} - logger.info("Loaded %d supported speakers: %s", len(self.supported_speakers), sorted(self.supported_speakers)) + if self.supported_speakers: + logger.info( + "Loaded %d supported speakers: %s", len(self.supported_speakers), sorted(self.supported_speakers) + ) + else: + logger.info( + "No built-in speakers configured; only '%s' and uploaded voices are available", _DEFAULT_VOICE_NAME + ) # Batch configuration self._batch_max_items: int = getattr(self.engine_client, "tts_batch_max_items", 32) @@ -759,6 +773,10 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # 3. Default fallback return _TTS_MAX_INSTRUCTIONS_LENGTH + def _get_available_voices(self) -> set[str]: + """Get all voice names accepted by the API, including the placeholder default.""" + return self.supported_speakers | self.uploaded_speakers.keys() | {_DEFAULT_VOICE_NAME} + def _load_supported_speakers(self) -> set[str]: """Load supported speakers (case-insensitive) from the model configuration.""" if self._tts_model_type == "ming_flash_omni_tts": @@ -3557,6 +3575,20 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if not artifact_ready: self._discard_ref_audio_artifact_warmup(request_id) + def _get_normalized_voice(self, voice: str | None) -> str | None: + """Get the normalized voice to be used; currently this means that + the voice is a: + - lowercase str if it's a valid supported/uploaded speaker + - None if the voice is the placeholder default or not provided + """ + if voice is not None: + voice = voice.lower() + if voice not in self.uploaded_speakers and voice not in self.supported_speakers: + raise ValueError( + f"Invalid voice '{voice}'. Supported: {', '.join(sorted(self._get_available_voices()))}" + ) + return voice + async def _create_diffusion_speech( self, request: OpenAICreateSpeechRequest, @@ -3573,11 +3605,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if fmt_err: return self._diffusion_error_response(fmt_err, status_code=400) - if request.voice: - voice_lower = request.voice.lower() - if voice_lower not in self.uploaded_speakers and voice_lower not in self.supported_speakers: - all_voices = sorted(self.uploaded_speakers.keys() | self.supported_speakers) - raise ValueError(f"Invalid voice '{request.voice}'. Supported: {', '.join(all_voices) or 'none'}") + request.voice = self._get_normalized_voice(request.voice) has_inline_ref_audio = request.ref_audio is not None err = self._apply_uploaded_speaker(request) @@ -3592,10 +3620,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if request.ref_text: prompt["ref_text"] = request.ref_text if request.voice: - voice_lower = request.voice.lower() - if voice_lower in self.uploaded_speakers and not has_inline_ref_audio: - prompt["voice_name"] = voice_lower - prompt["voice_created_at"] = self._voice_created_at(voice_lower) + if request.voice in self.uploaded_speakers and not has_inline_ref_audio: + prompt["voice_name"] = request.voice + prompt["voice_created_at"] = self._voice_created_at(request.voice) if request.language: prompt["lang"] = request.language if request.instructions: @@ -3734,6 +3761,10 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): Raw audio streaming yields each Code2Wav chunk as raw bytes as soon as it is decoded. Raw WAV streaming emits a header with placeholder size values first. """ + if request.voice is not None: + if _is_default_voice(request.voice.lower(), self.supported_speakers): + request.voice = None + if self._diffusion_mode: return await self._create_diffusion_speech(request) From b95c73742d3a8a2485b464bb3925cdcf1e18588a Mon Sep 17 00:00:00 2001 From: MrlixiangWE <102979255+MrlixiangWE@users.noreply.github.com> Date: Sat, 15 Aug 2026 07:07:28 +0800 Subject: [PATCH 018/267] [Bugfix] Stop reading the removed OmniRequestOutput.request_output accessor (#6172) Signed-off-by: MrlixiangWE --- docs/features/custom_pipeline.md | 2 +- docs/getting_started/quickstart.md | 5 +- .../examples/offline_inference/glm_image.md | 4 +- .../offline_inference/text_to_image.md | 6 +- .../image_to_video/README.md | 2 +- .../offline_inference/minicpmo/end2end.py | 2 +- .../offline_inference/text_to_image/README.md | 6 +- .../test_flux2_vae_patch_parallel.py | 10 +--- .../test_vae_decode_parallelism.py | 3 +- .../test_minimax_h3_quantization_quality.py | 6 -- tests/docs/test_example_output_accessors.py | 55 +++++++++++++++++++ tools/baselines/omni_hash_baseline.py | 5 +- 12 files changed, 74 insertions(+), 32 deletions(-) create mode 100644 tests/docs/test_example_output_accessors.py diff --git a/docs/features/custom_pipeline.md b/docs/features/custom_pipeline.md index 78d680f26..887dfa769 100644 --- a/docs/features/custom_pipeline.md +++ b/docs/features/custom_pipeline.md @@ -101,7 +101,7 @@ outputs = omni.generate( ) # Access custom trajectory data -output = outputs[0].request_output +output = outputs[0] print(f"Trajectory timesteps shape: {output.metrics['trajectory_timesteps'].shape}") print(f"Trajectory latents shape: {output.latents.shape}") ``` diff --git a/docs/getting_started/quickstart.md b/docs/getting_started/quickstart.md index f9673eac7..45ff311cb 100644 --- a/docs/getting_started/quickstart.md +++ b/docs/getting_started/quickstart.md @@ -47,7 +47,7 @@ if __name__ == "__main__": omni = Omni(model="Tongyi-MAI/Z-Image-Turbo") prompt = "a cup of coffee on the table" outputs = omni.generate(prompt) - images = outputs[0].request_output.images + images = outputs[0].images images[0].save("coffee.png") ``` @@ -75,8 +75,7 @@ if __name__ == "__main__": ] omni_outputs = omni.generate(prompts) for i_prompt, prompt_output in enumerate(omni_outputs): - this_request_output = prompt_output.request_output - this_images = this_request_output.images + this_images = prompt_output.images for i_image, image in enumerate(this_images): image.save(f"p{i_prompt}-img{i_image}.jpg") print("saved to", f"p{i_prompt}-img{i_image}.jpg") diff --git a/docs/user_guide/examples/offline_inference/glm_image.md b/docs/user_guide/examples/offline_inference/glm_image.md index 09d927ff7..267819cc5 100644 --- a/docs/user_guide/examples/offline_inference/glm_image.md +++ b/docs/user_guide/examples/offline_inference/glm_image.md @@ -43,7 +43,7 @@ if __name__ == "__main__": ), ], ) - outputs[0].request_output.images[0].save("output.png") + outputs[0].images[0].save("output.png") ``` ## Image-to-Image (Image Editing) @@ -72,7 +72,7 @@ if __name__ == "__main__": ), ], ) - outputs[0].request_output.images[0].save("output.png") + outputs[0].images[0].save("output.png") ``` ## Generation Parameters diff --git a/docs/user_guide/examples/offline_inference/text_to_image.md b/docs/user_guide/examples/offline_inference/text_to_image.md index f631cd962..24cab3a3c 100644 --- a/docs/user_guide/examples/offline_inference/text_to_image.md +++ b/docs/user_guide/examples/offline_inference/text_to_image.md @@ -55,7 +55,7 @@ if __name__ == "__main__": omni = Omni(model="Qwen/Qwen-Image") prompt = "a cup of coffee on the table" outputs = omni.generate(prompt) - images = outputs[0].request_output.images + images = outputs[0].images images[0].save("coffee.png") ``` @@ -179,7 +179,7 @@ if __name__ == "__main__": ] outputs = omni.generate(prompts) for i, output in enumerate(outputs): - output.request_output.images[0].save(f"{i}.jpg") + output.images[0].save(f"{i}.jpg") ``` !!! info @@ -207,7 +207,7 @@ if __name__ == "__main__": } ]) for i, output in enumerate(outputs): - output.request_output.images[0].save(f"{i}.jpg") + output.images[0].save(f"{i}.jpg") ``` You can also pass a negative prompt via the CLI argument `--negative-prompt`: diff --git a/examples/offline_inference/image_to_video/README.md b/examples/offline_inference/image_to_video/README.md index c8f7fe617..050448571 100644 --- a/examples/offline_inference/image_to_video/README.md +++ b/examples/offline_inference/image_to_video/README.md @@ -96,7 +96,7 @@ if __name__ == "__main__": ), ) from diffusers.utils import export_to_video - frames = outputs[0].request_output.images + frames = outputs[0].images export_to_video(frames, "quick_test_i2v.mp4", fps=16) ``` diff --git a/examples/offline_inference/minicpmo/end2end.py b/examples/offline_inference/minicpmo/end2end.py index 857f1f00f..8e536735d 100644 --- a/examples/offline_inference/minicpmo/end2end.py +++ b/examples/offline_inference/minicpmo/end2end.py @@ -335,7 +335,7 @@ def main(args): print(f"use_tts (tts_bos): {use_tts}") for stage_outputs in omni_generator: - output = stage_outputs.request_output + output = stage_outputs if stage_outputs.final_output_type == "text": request_id = output.request_id text_output = output.outputs[0].text diff --git a/examples/offline_inference/text_to_image/README.md b/examples/offline_inference/text_to_image/README.md index b71c998b3..cfe05255d 100644 --- a/examples/offline_inference/text_to_image/README.md +++ b/examples/offline_inference/text_to_image/README.md @@ -74,7 +74,7 @@ if __name__ == "__main__": omni = Omni(model="Qwen/Qwen-Image") prompt = "a cup of coffee on the table" outputs = omni.generate(prompt) - images = outputs[0].request_output.images + images = outputs[0].images images[0].save("coffee.png") ``` @@ -265,7 +265,7 @@ if __name__ == "__main__": ] outputs = omni.generate(prompts) for i, output in enumerate(outputs): - output.request_output.images[0].save(f"{i}.jpg") + output.images[0].save(f"{i}.jpg") ``` !!! info @@ -301,7 +301,7 @@ if __name__ == "__main__": } ]) for i, output in enumerate(outputs): - output.request_output.images[0].save(f"{i}.jpg") + output.images[0].save(f"{i}.jpg") ``` You can also pass a negative prompt via the CLI argument `--negative-prompt`: diff --git a/tests/diffusion/distributed/test_flux2_vae_patch_parallel.py b/tests/diffusion/distributed/test_flux2_vae_patch_parallel.py index e861bd61a..8de3a42f8 100644 --- a/tests/diffusion/distributed/test_flux2_vae_patch_parallel.py +++ b/tests/diffusion/distributed/test_flux2_vae_patch_parallel.py @@ -53,14 +53,10 @@ def _diff_metrics(a: Image.Image, b: Image.Image) -> tuple[float, float]: def _extract_single_image(outputs) -> Image.Image: first_output = outputs[0] assert first_output.final_output_type == "image" - if not hasattr(first_output, "request_output") or not first_output.request_output: - raise ValueError("No request_output found in OmniRequestOutput") + if not isinstance(first_output, OmniRequestOutput): + raise ValueError(f"Expected an OmniRequestOutput, got {type(first_output).__name__}") - req_out = first_output.request_output - if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"): - raise ValueError("Invalid request_output structure or missing 'images' key") - - images = req_out.images + images = first_output.images if images is None or len(images) != 1: raise ValueError(f"Expected 1 image, got {0 if images is None else len(images)}") return images[0] diff --git a/tests/diffusion/distributed/test_vae_decode_parallelism.py b/tests/diffusion/distributed/test_vae_decode_parallelism.py index c3996fc23..28ded4639 100644 --- a/tests/diffusion/distributed/test_vae_decode_parallelism.py +++ b/tests/diffusion/distributed/test_vae_decode_parallelism.py @@ -72,8 +72,7 @@ def _to_float_array(data: Any) -> np.ndarray: def _extract_output_array(output: Any) -> np.ndarray: - req_out = output.request_output - payload = req_out.images[0] + payload = output.images[0] return _to_float_array(payload) diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py index 80d99895f..1ba382703 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py @@ -142,12 +142,6 @@ def _generate_joint_output(omni, config: QualityTestConfig): first = outputs[0] worker_peak_memory_mb = float(getattr(first, "peak_memory_mb", 0.0) or 0.0) - if hasattr(first, "request_output") and isinstance(first.request_output, list): - first = first.request_output[0] - worker_peak_memory_mb = max( - worker_peak_memory_mb, - float(getattr(first, "peak_memory_mb", 0.0) or 0.0), - ) if not hasattr(first, "images") or not first.images: raise ValueError("Could not extract video frames from H3 output") frames = first.images[0] diff --git a/tests/docs/test_example_output_accessors.py b/tests/docs/test_example_output_accessors.py new file mode 100644 index 000000000..e7a618890 --- /dev/null +++ b/tests/docs/test_example_output_accessors.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Guard the public accessor that examples and docs use to read omni outputs. + +``OmniRequestOutput`` used to hold a nested ``RequestOutput`` in a +``request_output`` field. Since #5146 it *inherits* ``RequestOutput``, so the +generation content lives on the object itself and the nested field is gone. + +Call sites that kept the old accessor only fail once a real model runs, which +means a GPU nightly job is the first thing to notice. These checks pin both +halves of the contract on CPU instead: the field is really gone, and no doc, +example, test or tool still reaches for it. +""" + +import re +from pathlib import Path + +import pytest + +from vllm_omni.outputs import OmniRequestOutput + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +ROOT_DIR = Path(__file__).parents[2] +SCANNED_DIRS = ("docs", "examples", "tests", "tools") +SCANNED_SUFFIXES = (".py", ".md") + +# ``.request_outputs`` (plural, an OutputProcessor batch) is a different and +# still-valid attribute, so require a non-identifier character after the name. +NESTED_ACCESSOR = re.compile(r"\.request_output(?![\w])") + + +def _iter_scanned_files(): + for directory in SCANNED_DIRS: + for path in sorted((ROOT_DIR / directory).rglob("*")): + if path.suffix in SCANNED_SUFFIXES and path.resolve() != Path(__file__).resolve(): + yield path + + +def test_omni_request_output_has_no_nested_request_output(): + assert not hasattr(OmniRequestOutput(request_id="req-accessor"), "request_output") + + +def test_no_source_reads_the_nested_request_output(): + offenders = [] + for path in _iter_scanned_files(): + for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if NESTED_ACCESSOR.search(line): + offenders.append(f"{path.relative_to(ROOT_DIR)}:{lineno}: {line.strip()}") + + assert not offenders, ( + "OmniRequestOutput no longer nests a RequestOutput; read the content off the " + "output itself (e.g. `outputs[0].images`) instead of `.request_output`:\n" + "\n".join(offenders) + ) diff --git a/tools/baselines/omni_hash_baseline.py b/tools/baselines/omni_hash_baseline.py index 8d8f5ff13..a1644aeb3 100644 --- a/tools/baselines/omni_hash_baseline.py +++ b/tools/baselines/omni_hash_baseline.py @@ -159,11 +159,10 @@ def _run_once(model: str, yaml_path: str, *, async_mode: bool) -> list[dict]: wav: bytes = b"" # Batch size 1: submit a single prompt so scheduling is deterministic. for step in omni.generate([_text_prompt(question)], _sampling_params(), py_generator=False): - out = step.request_output if step.final_output_type == "text": - text = out.outputs[0].text # exact text, no strip -> truly bit-identical + text = step.outputs[0].text # exact text, no strip -> truly bit-identical elif step.final_output_type == "audio": - wav = _audio_bytes(out.outputs[0].multimodal_output["audio"]) + wav = _audio_bytes(step.outputs[0].multimodal_output["audio"]) if text is None: raise RuntimeError(f"prompt_id={prompt_id} async={async_mode}: produced no text output") if not wav: From d671e6b0db1fe93d59efa23b435d51f0215f779e Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:01:20 +0800 Subject: [PATCH 019/267] [CI] Expand E2E source_file_dependencies for shared model code (#5994) Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com> --- .buildkite/cuda/test-merge.yml | 117 ++++-------------- .buildkite/cuda/test-ready.yml | 103 +++++---------- docs/contributing/ci/ci_settings.md | 15 ++- tests/buildkite/test_upload_pipeline.py | 57 +++++---- tests/e2e/offline_inference/test_bagel.py | 2 +- .../e2e/offline_inference/test_flux2_klein.py | 4 +- .../e2e/offline_inference/test_voxcpm2_tts.py | 9 +- tests/e2e/offline_inference/test_z_image.py | 2 +- tests/e2e/online_serving/test_bagel.py | 6 +- .../e2e/online_serving/test_higgs_audio_v3.py | 27 ++-- .../online_serving/test_qwen_image_edit.py | 9 +- .../online_serving/test_qwen_image_layered.py | 3 +- tests/e2e/online_serving/test_voxcpm2_tts.py | 5 +- tests/e2e/online_serving/test_z_image.py | 4 +- 14 files changed, 135 insertions(+), 228 deletions(-) diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index 970ab5a5a..69913bf55 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -76,6 +76,9 @@ steps: - tests/e2e/online_serving/test_qwen3_tts_customvoice.py - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py - vllm_omni/model_executor/models/qwen3_tts/ + - vllm_omni/model_executor/models/common/qwen3_code_predictor.py + - vllm_omni/model_executor/models/common/snake_activation.py + - vllm_omni/model_executor/models/common/alias_free_activation.py - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - vllm_omni/deploy/qwen3_tts.yaml commands: @@ -92,6 +95,9 @@ steps: - tests/e2e/online_serving/test_qwen3_tts_base.py - tests/e2e/offline_inference/test_qwen3_tts_base.py - vllm_omni/model_executor/models/qwen3_tts/ + - vllm_omni/model_executor/models/common/qwen3_code_predictor.py + - vllm_omni/model_executor/models/common/snake_activation.py + - vllm_omni/model_executor/models/common/alias_free_activation.py - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - vllm_omni/deploy/qwen3_tts.yaml - .buildkite/common/scripts/run_cov_split.sh @@ -109,27 +115,15 @@ steps: -- -m 'advanced_model and cuda' --run-level 'advanced_model' mirror_hardwares: l4_1 - - label: "TTS · Higgs-Audio-V3 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_higgs_audio_v3.py - - vllm_omni/model_executor/models/higgs_audio_v3/ - - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py - - vllm_omni/deploy/higgs_multimodal_qwen3.yaml - commands: - - | - timeout 20m bash -c " - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_USE_DEEP_GEMM=0 - export VLLM_MOE_USE_DEEP_GEMM=0 - pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'advanced_model' --run-level 'advanced_model' - " - mirror_hardwares: l4_1 - - label: "Omni · Qwen3-Omni Test" source_file_dependencies: - tests/e2e/offline_inference/test_qwen3_omni.py - tests/e2e/online_serving/test_qwen3_omni.py - vllm_omni/model_executor/models/qwen3_omni/ + - vllm_omni/model_executor/models/common/qwen3_code_predictor.py + - vllm_omni/model_executor/models/common/snake_activation.py + - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py + - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py - vllm_omni/deploy/qwen3_omni_moe.yaml timeout_in_minutes: 50 @@ -141,6 +135,11 @@ steps: source_file_dependencies: - tests/e2e/offline_inference/test_minicpmo_4_5.py - tests/e2e/online_serving/test_minicpmo_4_5.py + - vllm_omni/model_executor/models/minicpmo_4_5/ + - vllm_omni/model_executor/models/step_audio2/ + - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/ + - vllm_omni/model_executor/models/cosyvoice3/utils.py + - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py - vllm_omni/deploy/minicpmo_4_5.yaml timeout_in_minutes: 50 commands: @@ -162,70 +161,35 @@ steps: - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py + - vllm_omni/deploy/minicpmo_4_5.yaml - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py - vllm_omni/deploy/minicpmo_4_5_duplex.yaml - vllm_omni/experimental/fullduplex/ - vllm_omni/model_executor/models/minicpmo_4_5/ + - vllm_omni/model_executor/models/step_audio2/ + - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/ + - vllm_omni/model_executor/models/cosyvoice3/utils.py - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py - vllm_omni/core/sched/ - vllm_omni/outputs/multimodal_accumulation.py - vllm_omni/engine/stage_init_utils.py - vllm_omni/engine/async_omni_engine.py + - vllm_omni/engine/orchestrator.py - vllm_omni/entrypoints/cli/serve.py + - vllm_omni/entrypoints/openai/api_server.py + - vllm_omni/entrypoints/openai/duplex_capability.py + - vllm_omni/config/stage_config.py timeout_in_minutes: 50 commands: - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model' mirror_hardwares: h100_1 - - label: "TTS · VoxCPM2 Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_voxcpm2_tts.py - - tests/e2e/online_serving/test_voxcpm2_tts.py - - vllm_omni/model_executor/models/voxcpm2/ - - vllm_omni/deploy/voxcpm2.yaml - timeout_in_minutes: 30 - commands: - - | - timeout 30m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/offline_inference/test_voxcpm2_tts.py tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model' - " - mirror_hardwares: l4_1 - - - label: "Diffusion · Qwen Image Edit Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen_image_edit.py - - vllm_omni/diffusion/models/qwen_image/ - timeout_in_minutes: 20 - commands: - - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model' - mirror_hardwares: h100_1 - - label: "Diffusion · Bagel Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_bagel.py - - tests/e2e/online_serving/test_bagel.py - - vllm_omni/model_executor/models/bagel/ - - vllm_omni/diffusion/models/bagel/ - - vllm_omni/model_executor/stage_input_processors/bagel.py - - vllm_omni/deploy/bagel.yaml - - .buildkite/common/scripts/run_cov_split.sh - - pyproject.toml - # 20 -> 40: the coverage split runs the model twice. - timeout_in_minutes: 40 - commands: - - | - .buildkite/common/scripts/run_cov_split.sh \ - --model-id bagel \ - --offline tests/e2e/offline_inference/test_bagel.py \ - --online tests/e2e/online_serving/test_bagel.py \ - -- -m 'advanced_model and cuda' --run-level 'advanced_model' - mirror_hardwares: h100_1 - - label: "Diffusion · Qwen Image Test" source_file_dependencies: - tests/e2e/online_serving/test_qwen_image.py - vllm_omni/diffusion/models/qwen_image/ + - vllm_omni/diffusion/models/dmd2/ + - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py commands: - | timeout 40m bash -c ' @@ -234,23 +198,12 @@ steps: ' mirror_hardwares: h100_1 - - label: "Diffusion · Qwen Image Layered Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen_image_layered.py - - vllm_omni/diffusion/models/qwen_image/ - commands: - - | - timeout 40m bash -c " - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'advanced_model and cuda' --run-level 'advanced_model' - " - mirror_hardwares: h100_1 - - label: "Diffusion · Wan22 Test" source_file_dependencies: - tests/e2e/offline_inference/test_wan22_t2v.py - tests/e2e/online_serving/test_wan22_t2v.py - vllm_omni/diffusion/models/wan2_2/ + - vllm_omni/diffusion/models/dmd2/ commands: - | timeout 40m bash -c " @@ -258,21 +211,3 @@ steps: pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' " mirror_hardwares: h100_1 - - - label: "Diffusion · Z Image Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_z_image.py - - tests/e2e/online_serving/test_z_image.py - - vllm_omni/diffusion/models/z_image/ - timeout_in_minutes: 30 - commands: - - pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' - mirror_hardwares: l4_1 - - label: "Diffusion · FLUX.2 Klein Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_flux2_klein.py - - vllm_omni/diffusion/models/flux2_klein/ - timeout_in_minutes: 30 - commands: - - pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' - mirror_hardwares: l4_1 diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index f416631ff..8dbe7bd71 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -169,6 +169,10 @@ steps: source_file_dependencies: - tests/e2e/online_serving/test_qwen3_omni.py - vllm_omni/model_executor/models/qwen3_omni/ + - vllm_omni/model_executor/models/common/qwen3_code_predictor.py + - vllm_omni/model_executor/models/common/snake_activation.py + - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py + - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py - vllm_omni/deploy/qwen3_omni_moe.yaml commands: @@ -181,6 +185,11 @@ steps: - label: "Omni · MiniCPM-o 4.5 Offline Test" source_file_dependencies: - tests/e2e/offline_inference/test_minicpmo_4_5.py + - vllm_omni/model_executor/models/minicpmo_4_5/ + - vllm_omni/model_executor/models/step_audio2/ + - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/ + - vllm_omni/model_executor/models/cosyvoice3/utils.py + - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py - vllm_omni/deploy/minicpmo_4_5.yaml commands: - | @@ -192,6 +201,11 @@ steps: - label: "Omni · MiniCPM-o 4.5 Online Test" source_file_dependencies: - tests/e2e/online_serving/test_minicpmo_4_5.py + - vllm_omni/model_executor/models/minicpmo_4_5/ + - vllm_omni/model_executor/models/step_audio2/ + - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/ + - vllm_omni/model_executor/models/cosyvoice3/utils.py + - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py - vllm_omni/deploy/minicpmo_4_5.yaml commands: - | @@ -210,16 +224,24 @@ steps: - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py + - vllm_omni/deploy/minicpmo_4_5.yaml - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py - vllm_omni/deploy/minicpmo_4_5_duplex.yaml - vllm_omni/experimental/fullduplex/ - vllm_omni/model_executor/models/minicpmo_4_5/ + - vllm_omni/model_executor/models/step_audio2/ + - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/ + - vllm_omni/model_executor/models/cosyvoice3/utils.py - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py - vllm_omni/core/sched/ - vllm_omni/outputs/multimodal_accumulation.py - vllm_omni/engine/stage_init_utils.py - vllm_omni/engine/async_omni_engine.py + - vllm_omni/engine/orchestrator.py - vllm_omni/entrypoints/cli/serve.py + - vllm_omni/entrypoints/openai/api_server.py + - vllm_omni/entrypoints/openai/duplex_capability.py + - vllm_omni/config/stage_config.py commands: - | timeout 20m bash -c " @@ -239,25 +261,13 @@ steps: - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda' mirror_hardwares: h100_1 - - label: "TTS · VoxCPM2 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_voxcpm2_tts.py - - vllm_omni/model_executor/models/voxcpm2/ - - vllm_omni/deploy/voxcpm2.yaml - timeout_in_minutes: 30 - commands: - - | - timeout 30m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: l4_1 - - label: "TTS · Qwen3-TTS CustomVoice Test" source_file_dependencies: - tests/e2e/online_serving/test_qwen3_tts_customvoice.py - vllm_omni/model_executor/models/qwen3_tts/ + - vllm_omni/model_executor/models/common/qwen3_code_predictor.py + - vllm_omni/model_executor/models/common/snake_activation.py + - vllm_omni/model_executor/models/common/alias_free_activation.py - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - vllm_omni/deploy/qwen3_tts.yaml commands: @@ -272,6 +282,9 @@ steps: source_file_dependencies: - tests/e2e/online_serving/test_qwen3_tts_base.py - vllm_omni/model_executor/models/qwen3_tts/ + - vllm_omni/model_executor/models/common/qwen3_code_predictor.py + - vllm_omni/model_executor/models/common/snake_activation.py + - vllm_omni/model_executor/models/common/alias_free_activation.py - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - vllm_omni/deploy/qwen3_tts.yaml commands: @@ -284,52 +297,11 @@ steps: " mirror_hardwares: l4_1 - - label: "TTS · Higgs-Audio-V3 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_higgs_audio_v3.py - - vllm_omni/model_executor/models/higgs_audio_v3/ - - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py - - vllm_omni/deploy/higgs_multimodal_qwen3.yaml - commands: - - | - timeout 20m bash -c " - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_USE_DEEP_GEMM=0 - export VLLM_MOE_USE_DEEP_GEMM=0 - pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: l4_1 - - - label: "Diffusion · Qwen Image Edit Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen_image_edit.py - - vllm_omni/diffusion/models/qwen_image/ - commands: - - | - timeout 20m bash -c " - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: h100_1 - - - label: "Diffusion · Bagel Test" - source_file_dependencies: - - tests/e2e/online_serving/test_bagel.py - - vllm_omni/model_executor/models/bagel/ - - vllm_omni/diffusion/models/bagel/ - - vllm_omni/model_executor/stage_input_processors/bagel.py - - vllm_omni/deploy/bagel.yaml - commands: - - | - timeout 40m bash -c " - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: h100_1 - - label: "Diffusion · Wan22 Test" source_file_dependencies: - tests/e2e/online_serving/test_wan22_t2v.py - vllm_omni/diffusion/models/wan2_2/ + - vllm_omni/diffusion/models/dmd2/ commands: - | timeout 40m bash -c " @@ -342,6 +314,9 @@ steps: source_file_dependencies: - tests/e2e/online_serving/test_cosmos3.py - vllm_omni/diffusion/models/cosmos3/ + - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py + - vllm_omni/experimental/world_models/ + - vllm_omni/model_extras/cosmos3.py commands: - | timeout 40m bash -c " @@ -354,6 +329,8 @@ steps: source_file_dependencies: - tests/e2e/online_serving/test_qwen_image.py - vllm_omni/diffusion/models/qwen_image/ + - vllm_omni/diffusion/models/dmd2/ + - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py commands: - | timeout 40m bash -c " @@ -361,15 +338,3 @@ steps: pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model' " mirror_hardwares: h100_1 - - - label: "Diffusion · Qwen Image Layered Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen_image_layered.py - - vllm_omni/diffusion/models/qwen_image/ - commands: - - | - timeout 40m bash -c " - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: h100_1 diff --git a/docs/contributing/ci/ci_settings.md b/docs/contributing/ci/ci_settings.md index 62717ddc9..035e8c650 100644 --- a/docs/contributing/ci/ci_settings.md +++ b/docs/contributing/ci/ci_settings.md @@ -329,11 +329,13 @@ On PR builds, `upload_pipeline.py` logs `skip '…' (no changes under …)` for ### Per-model coverage -Pilot (v1): two Merge-tier (L3) jobs in `.buildkite/cuda/test-merge.yml` upload -per-model, per-entry-mode coverage as Buildkite artifacts — "Diffusion · Bagel -Test" and "TTS · Qwen3-TTS Base Test". Both run on a single GPU so the pilot is -cheap to reproduce. This is a pilot, not a rollout: no dashboard/visualization -lives in this repo, and nothing is gated on coverage. +Pilot (v1): one Merge-tier (L3) job in `.buildkite/cuda/test-merge.yml` uploads +per-model, per-entry-mode coverage as Buildkite artifacts — "TTS · Qwen3-TTS Base +Test". It runs on a single GPU so the pilot is cheap to reproduce. (A second +diffusion pilot, "Diffusion · Bagel Test", was retired from merge; reintroduce +another diffusion coverage job if a second pilot is needed again.) This is a +pilot, not a rollout: no dashboard/visualization lives in this repo, and nothing +is gated on coverage. **Naming convention**: `coverage---.xml.gz`, where `` is the model's directory name under @@ -410,7 +412,8 @@ runs a coverage job needs the same. List both `run_cov_split.sh` and `pyproject.toml` in every opted-in job's `source_file_dependencies` — both change what the job measures, so without them a change there is filtered out of normal PR builds and only surfaces in a later -nightly. `tests/buildkite/test_upload_pipeline.py` asserts this for the pilots. +nightly. `tests/buildkite/test_upload_pipeline.py` covers the filter behavior with +a synthetic job (it does not pin real merge labels). Editing only the surrounding CI YAML still does not schedule them, so a PR that touches just the wiring needs a full E2E run (or the commands run on a GPU host) to produce artifacts. When checking a new model's diff --git a/tests/buildkite/test_upload_pipeline.py b/tests/buildkite/test_upload_pipeline.py index ee45a64d3..bd0f50f5c 100644 --- a/tests/buildkite/test_upload_pipeline.py +++ b/tests/buildkite/test_upload_pipeline.py @@ -39,8 +39,6 @@ def _render(changed_files: list[str]) -> str: def test_bootstrap_if_injected_by_step_key() -> None: - import yaml - rendered = _render_bootstrap_pipeline( BOOTSTRAP_STEPS_TEMPLATE, decision=resolve_ci_decision([]), @@ -142,12 +140,7 @@ def test_mirror_hardwares_a2b3_npu_4_expands_agents_image_and_plugins() -> None: ] -COVERAGE_PILOT_LABELS = ("Diffusion · Bagel Test", "TTS · Qwen3-TTS Base Test") -MERGE_PIPELINE = Path(".buildkite/cuda/test-merge.yml") - - -def _surviving_labels(changed_files: list[str]) -> set[str]: - doc = yaml.safe_load(MERGE_PIPELINE.read_text(encoding="utf-8")) +def _surviving_labels(doc: dict, changed_files: list[str]) -> set[str]: rendered = _render_test_pipeline(doc, changed_files=changed_files) labels: set[str] = set() @@ -163,6 +156,29 @@ def _surviving_labels(changed_files: list[str]) -> set[str]: return labels +# Synthetic coverage-style job: shared inputs that change what the split measures. +_COVERAGE_SHARED_INPUTS_DOC = { + "steps": [ + { + "label": "Coverage Pilot", + "source_file_dependencies": [ + "tests/e2e/online_serving/test_example.py", + ".buildkite/common/scripts/run_cov_split.sh", + "pyproject.toml", + ], + "commands": [".buildkite/common/scripts/run_cov_split.sh --model-id example"], + }, + { + "label": "Unrelated Model Test", + "source_file_dependencies": [ + "tests/e2e/online_serving/test_other.py", + ], + "commands": ["pytest -sv tests/e2e/online_serving/test_other.py"], + }, + ], +} + + @pytest.mark.parametrize( "changed_file", [ @@ -170,18 +186,17 @@ def _surviving_labels(changed_files: list[str]) -> set[str]: "pyproject.toml", ], ) -def test_coverage_pilots_selected_by_their_shared_inputs(changed_file: str) -> None: - """The split helper and the coverage config both change what these jobs measure. - - Without them in source_file_dependencies the jobs are filtered out of normal - PR builds, so a regression would only surface in a later nightly. - """ - labels = _surviving_labels([changed_file]) - for pilot in COVERAGE_PILOT_LABELS: - assert pilot in labels, f"{pilot} dropped when {changed_file} changed" +def test_coverage_shared_inputs_select_dependent_job(changed_file: str) -> None: + """Jobs that list coverage shared inputs must stay selected when those files change.""" + labels = _surviving_labels(_COVERAGE_SHARED_INPUTS_DOC, [changed_file]) + assert "Coverage Pilot" in labels + assert "Unrelated Model Test" not in labels -def test_coverage_pilots_not_selected_by_unrelated_change() -> None: - labels = _surviving_labels(["vllm_omni/entrypoints/openai/serving_chat.py"]) - for pilot in COVERAGE_PILOT_LABELS: - assert pilot not in labels +def test_coverage_shared_inputs_ignored_for_unrelated_change() -> None: + labels = _surviving_labels( + _COVERAGE_SHARED_INPUTS_DOC, + ["vllm_omni/entrypoints/openai/serving_chat.py"], + ) + assert "Coverage Pilot" not in labels + assert "Unrelated Model Test" not in labels diff --git a/tests/e2e/offline_inference/test_bagel.py b/tests/e2e/offline_inference/test_bagel.py index 9ba7bdac4..580a46742 100644 --- a/tests/e2e/offline_inference/test_bagel.py +++ b/tests/e2e/offline_inference/test_bagel.py @@ -46,7 +46,7 @@ REFERENCE_TEXT_IMG2TEXT = ( _OMNI_RUNNER_PARAM = (MODEL_NAME, STAGE_CONFIG) pytestmark = [ - pytest.mark.advanced_model, + pytest.mark.slow, pytest.mark.diffusion, pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), ] diff --git a/tests/e2e/offline_inference/test_flux2_klein.py b/tests/e2e/offline_inference/test_flux2_klein.py index 9656772c9..ea3943454 100644 --- a/tests/e2e/offline_inference/test_flux2_klein.py +++ b/tests/e2e/offline_inference/test_flux2_klein.py @@ -72,7 +72,7 @@ def _send_inpaint_with_generator( ) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_flux2_klein_inpaint_deterministic(omni_runner_handler: OmniRunnerHandler): @@ -93,7 +93,7 @@ def test_flux2_klein_inpaint_deterministic(omni_runner_handler: OmniRunnerHandle ) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_flux2_klein_inpaint_different_seeds_different_output(omni_runner_handler: OmniRunnerHandler): diff --git a/tests/e2e/offline_inference/test_voxcpm2_tts.py b/tests/e2e/offline_inference/test_voxcpm2_tts.py index c10694a40..af55c51aa 100644 --- a/tests/e2e/offline_inference/test_voxcpm2_tts.py +++ b/tests/e2e/offline_inference/test_voxcpm2_tts.py @@ -39,8 +39,7 @@ def _extract_audio(multimodal_output: dict) -> torch.Tensor: return audio -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voxcpm2_zero_shot_001(omni_runner: OmniRunner) -> None: @@ -53,8 +52,7 @@ def test_voxcpm2_zero_shot_001(omni_runner: OmniRunner) -> None: assert 0.5 < duration_s < 30.0, f"Audio duration out of range: {duration_s:.2f}s" -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voxcpm2_voice_clone_002(omni_runner: OmniRunner) -> None: @@ -95,8 +93,7 @@ def test_voxcpm2_voice_clone_002(omni_runner: OmniRunner) -> None: assert 0.5 < duration_s < 30.0, f"Audio duration out of range: {duration_s:.2f}s" -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voxcpm2_prefill_decode_mixed_batch_003(omni_runner: OmniRunner) -> None: diff --git a/tests/e2e/offline_inference/test_z_image.py b/tests/e2e/offline_inference/test_z_image.py index 1770ccd85..edf158ffd 100644 --- a/tests/e2e/offline_inference/test_z_image.py +++ b/tests/e2e/offline_inference/test_z_image.py @@ -11,7 +11,7 @@ MODEL = "Tongyi-MAI/Z-Image-Turbo" PROMPT = "A high-detail studio photo of an orange tabby cat sitting on a laptop keyboard." -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}, num_cards={"cuda": 1, "rocm": 1, "xpu": 2}) @pytest.mark.parametrize("omni_runner", [(MODEL, None)], indirect=True) diff --git a/tests/e2e/online_serving/test_bagel.py b/tests/e2e/online_serving/test_bagel.py index b22762acd..dca846108 100644 --- a/tests/e2e/online_serving/test_bagel.py +++ b/tests/e2e/online_serving/test_bagel.py @@ -75,8 +75,7 @@ def _build_img2img_messages(prompt: str, image_b64: str) -> list[dict]: ] -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) @@ -98,8 +97,7 @@ def test_bagel_text2img_online(omni_server, openai_client) -> None: openai_client.send_diffusion_request(request_config) -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "H100", "rocm": "MI325"}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) diff --git a/tests/e2e/online_serving/test_higgs_audio_v3.py b/tests/e2e/online_serving/test_higgs_audio_v3.py index d26e6bff4..d1d592d21 100644 --- a/tests/e2e/online_serving/test_higgs_audio_v3.py +++ b/tests/e2e/online_serving/test_higgs_audio_v3.py @@ -59,8 +59,7 @@ _REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know w class TestHiggsAudioV3OnlineHappyPath: """Plain-text -> audio happy paths against the live HTTP server.""" - @pytest.mark.core_model - @pytest.mark.advanced_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_plain_text_wav(self, omni_server, openai_client) -> None: @@ -79,7 +78,7 @@ class TestHiggsAudioV3OnlineHappyPath: } ) - @pytest.mark.core_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_plain_text_with_max_new_tokens(self, omni_server, openai_client) -> None: @@ -97,8 +96,7 @@ class TestHiggsAudioV3OnlineHappyPath: ) @pytest.mark.skip(reason="issue#4411") - @pytest.mark.core_model - @pytest.mark.advanced_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_concurrent_plain_text(self, omni_server, openai_client) -> None: @@ -115,7 +113,7 @@ class TestHiggsAudioV3OnlineHappyPath: ) @pytest.mark.skip(reason="issue#4411") - @pytest.mark.advanced_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_plain_text_pcm_streaming(self, omni_server, openai_client) -> None: @@ -147,7 +145,7 @@ class TestHiggsAudioV3OnlineHappyPath: ) @pytest.mark.skip(reason="issue#4411") - @pytest.mark.advanced_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_concurrent_pcm_streaming(self, omni_server, openai_client) -> None: @@ -189,7 +187,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: They do not assert audio quality (out of scope for CI). """ - @pytest.mark.core_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_inline_emotion_and_expressive(self, omni_server, openai_client) -> None: @@ -205,7 +203,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: } ) - @pytest.mark.core_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_inline_style_whispering(self, omni_server, openai_client) -> None: @@ -221,7 +219,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: } ) - @pytest.mark.core_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_inline_prosody_speed_and_pitch(self, omni_server, openai_client) -> None: @@ -239,7 +237,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: } ) - @pytest.mark.core_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_inline_pause_mid_text(self, omni_server, openai_client) -> None: @@ -255,7 +253,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: } ) - @pytest.mark.core_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_inline_sfx_with_onomatopoeia(self, omni_server, openai_client) -> None: @@ -279,8 +277,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: class TestHiggsAudioV3OnlineVoiceClone: """Voice clone via the two payload shapes the model serves.""" - @pytest.mark.core_model - @pytest.mark.advanced_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voice_clone_ref_audio_ref_text(self, omni_server, openai_client) -> None: @@ -297,7 +294,7 @@ class TestHiggsAudioV3OnlineVoiceClone: } ) - @pytest.mark.core_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voice_clone_references_alias(self, omni_server, openai_client) -> None: diff --git a/tests/e2e/online_serving/test_qwen_image_edit.py b/tests/e2e/online_serving/test_qwen_image_edit.py index 738a2e50c..a02bc1b57 100644 --- a/tests/e2e/online_serving/test_qwen_image_edit.py +++ b/tests/e2e/online_serving/test_qwen_image_edit.py @@ -6,7 +6,7 @@ Online serving tests for the Qwen-Image-Edit family (image-to-image via chat com - ``test_single_image_to_image_001``: ``Qwen/Qwen-Image-Edit`` — one reference image, fixed 512×512. - ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2511`` — two reference images, fixed 512×512. -- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2511`` only, ``advanced_model`` — mixed input +- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2511`` only, ``slow`` — mixed input resolutions; ``extra_body`` uses per-output ``width``/``height`` lists; the test client sends one scalar-size request per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``). @@ -52,7 +52,7 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", @@ -82,8 +82,7 @@ def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenA openai_client.send_diffusion_request(request_config) -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", @@ -114,7 +113,7 @@ def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenA openai_client.send_diffusion_request(request_config) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", diff --git a/tests/e2e/online_serving/test_qwen_image_layered.py b/tests/e2e/online_serving/test_qwen_image_layered.py index b29609351..289e35764 100644 --- a/tests/e2e/online_serving/test_qwen_image_layered.py +++ b/tests/e2e/online_serving/test_qwen_image_layered.py @@ -44,8 +44,7 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler): diff --git a/tests/e2e/online_serving/test_voxcpm2_tts.py b/tests/e2e/online_serving/test_voxcpm2_tts.py index 19ddc0401..5597bf367 100644 --- a/tests/e2e/online_serving/test_voxcpm2_tts.py +++ b/tests/e2e/online_serving/test_voxcpm2_tts.py @@ -46,8 +46,7 @@ tts_server_params = [ ] -@pytest.mark.advanced_model -@pytest.mark.core_model +@pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) @@ -72,7 +71,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None: openai_client.send_audio_speech_request(request_config, request_num=MAX_CONCURRENT) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) diff --git a/tests/e2e/online_serving/test_z_image.py b/tests/e2e/online_serving/test_z_image.py index 2ef78ce0f..b1180f514 100644 --- a/tests/e2e/online_serving/test_z_image.py +++ b/tests/e2e/online_serving/test_z_image.py @@ -32,7 +32,7 @@ def _get_diffusion_feature_cases(): ] -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", @@ -60,7 +60,7 @@ def test_basic_001( openai_client.send_diffusion_request(request_config) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", From 35cacf020c823da56e5ad437128bc4e93a5a3ed1 Mon Sep 17 00:00:00 2001 From: mjZhaoElaine <38148457+mjZhaoElaine@users.noreply.github.com> Date: Fri, 14 Aug 2026 22:06:05 -0700 Subject: [PATCH 020/267] [Model] Add Gepard-1.0 native-AR FSQ/NanoCodec TTS (offline inference) (#5666) Signed-off-by: Mengjie Zhao Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> --- .buildkite/cuda/test-weekly.yml | 16 + docs/models/supported_models.md | 1 + .../text_to_speech/README.md | 53 ++ .../text_to_speech/gepard/end2end.py | 139 ++++ .../e2e/offline_inference/test_gepard_tts.py | 345 +++++++++ .../models/test_gepard_window.py | 363 +++++++++ .../models/test_gepard_wiring.py | 731 ++++++++++++++++++ vllm_omni/config/pipeline_registry.py | 2 + vllm_omni/deploy/gepard.yaml | 40 + .../model_executor/models/gepard/__init__.py | 8 + .../models/gepard/configuration_gepard.py | 215 ++++++ .../models/gepard/gepard_talker.py | 526 +++++++++++++ .../model_executor/models/gepard/nanocodec.py | 172 +++++ .../model_executor/models/gepard/pipeline.py | 45 ++ .../model_executor/models/gepard/prompt.py | 55 ++ vllm_omni/model_executor/models/registry.py | 6 + vllm_omni/worker/gpu_model_runner.py | 10 + 17 files changed, 2727 insertions(+) create mode 100644 examples/offline_inference/text_to_speech/gepard/end2end.py create mode 100644 tests/e2e/offline_inference/test_gepard_tts.py create mode 100644 tests/model_executor/models/test_gepard_window.py create mode 100644 tests/model_executor/models/test_gepard_wiring.py create mode 100644 vllm_omni/deploy/gepard.yaml create mode 100644 vllm_omni/model_executor/models/gepard/__init__.py create mode 100644 vllm_omni/model_executor/models/gepard/configuration_gepard.py create mode 100644 vllm_omni/model_executor/models/gepard/gepard_talker.py create mode 100644 vllm_omni/model_executor/models/gepard/nanocodec.py create mode 100644 vllm_omni/model_executor/models/gepard/pipeline.py create mode 100644 vllm_omni/model_executor/models/gepard/prompt.py diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml index cc7f99647..0fd3a06ce 100644 --- a/.buildkite/cuda/test-weekly.yml +++ b/.buildkite/cuda/test-weekly.yml @@ -108,6 +108,22 @@ steps: --run-level "full_model" mirror_hardwares: h100_1 + # Own step: the image has no NanoCodec, so this file import-skips out of + # the mark-selected TTS steps above. The overrides keep nemo_toolkit[tts] + # from downgrading transformers; see + # examples/offline_inference/text_to_speech/README.md. + - label: "TTS · Gepard-1.0 · L4" + timeout_in_minutes: 60 + commands: + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - printf '%s\n' "transformers>=5.5.3" "huggingface-hub>=1.0" > /tmp/gepard-overrides.txt + - uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" "nemo_toolkit[tts]==2.7.3" --overrides /tmp/gepard-overrides.txt + - >- + pytest -s -v tests/e2e/offline_inference/test_gepard_tts.py + -m "tts and cuda" + --run-level "full_model" + mirror_hardwares: l4_1 + - label: "Diffusion · L4" timeout_in_minutes: 60 commands: diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 995afc28d..e6a38640f 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -98,6 +98,7 @@ th { |`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | | — | | `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | — | | `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | +| `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | | — | |`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — | The `Recipe` column links the corresponding deployment recipe. `Published` is diff --git a/examples/offline_inference/text_to_speech/README.md b/examples/offline_inference/text_to_speech/README.md index 64e9b4d42..12f1967ee 100644 --- a/examples/offline_inference/text_to_speech/README.md +++ b/examples/offline_inference/text_to_speech/README.md @@ -16,6 +16,7 @@ list of supported architectures across all modalities, see |---|---|---|---|---|---|---| | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 24 kHz | | Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ | — | 44.1 kHz | +| Gepard-1.0 | `nineninesix/gepard-1.0` | single (native AR + NanoCodec) | — (zero-shot; cloning WIP) | — (serving WIP) | zero-shot | 22.05 kHz | | GLM-TTS | `zai-org/GLM-TTS` | 2 (AR + DiT) | ✓ (required) | ✓ | — | 24 kHz | | Ming-omni-tts | `inclusionAI/Ming-omni-tts-0.5B` | 2 (AR + audio VAE) | ✓ | ✓ | style / IP / dialect / TTA / podcast | 44.1 kHz | | Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | single (talker only) | — (caption-controlled) | — | style / IP / basic captions | 44.1 kHz | @@ -395,6 +396,58 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \ --- +## Gepard-1.0 + +Single-stage native AR TTS at 22.05 kHz. Pipeline: `Qwen3.5 backbone → 32 FSQ codebook heads (one frame/step) → NeMo NanoCodec`. Backbone runs under vLLM paged attention; the 32-head sampling + learned embedding feedback ride the native-AR runner hooks. `enforce_eager` (CUDA graph is a perf follow-up). + +### Prerequisites +The NanoCodec decoder needs NeMo (NVIDIA Open Model License), installed separately: + +```bash +uv pip install "nemo_toolkit[tts]==2.7.3" \ + --overrides <(printf '%s\n' "transformers>=5.5.3" "huggingface-hub>=1.0") +``` + +The overrides are what make that command work. `nemo_toolkit[tts]` declares +`transformers~=4.57.0`, so a plain install downgrades the version +`requirements/common.txt` pins and takes `huggingface-hub` back to 0.x with it — after +which `import vllm` fails. Both packages are named here rather than left to resolve on +their own: transformers 5.x happens to require `huggingface-hub` 1.x, but relying on +that would make the second repair an accident of the first. `--constraint` cannot be +used instead, since it narrows a version range rather than overriding a dependency a +package declares, and the two sets never intersect. NeMo 2.7.3 decodes correctly +against transformers 5.x, so nothing is lost by holding it there. + +(The same clash is why this is documented here rather than declared in +`pyproject.toml`: an extra resolves together with the base dependencies, so +`vllm-omni[gepard]` would be unsatisfiable rather than opt-in. SoulX-Singer English +SVS documents `nemo_toolkit[asr]` the same way.) + +On a host whose CUDA toolkit cannot build kernels for the GPU — no `nvcc`/`ninja`, or a +consumer Blackwell (`sm_120`) card — also set `VLLM_USE_FLASHINFER_SAMPLER=0`. vLLM's +sampler JIT-compiles through FlashInfer on first use, and where that build fails the engine +dies inside `profile_run` long after every import has succeeded, which reads as a model +failure rather than an environment one. The PyTorch sampler it falls back to costs nothing +here. + +### Quick start (zero-shot, default voice) +```bash +python examples/offline_inference/text_to_speech/gepard/end2end.py \ + --text "Hello, this is a Gepard demo." +``` + +### Voice cloning +Not yet — PR1 is zero-shot only (the learned `null_prefix` default voice). Reference-audio cloning (`ref_compressor` speaker prefix) is a follow-up PR. + +### Notes +- Output: 22.05 kHz mono WAV. +- First run downloads two checkpoints, not one: the model itself, and the NanoCodec decoder it names in `codec_id` (`nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps` by default). Both are fetched automatically; expect the first startup to be correspondingly slower. +- Offline only for now; the `/v1/audio/speech` adapter is a follow-up PR. +- Deploy config: `vllm_omni/deploy/gepard.yaml` (the example's default; copy it and pass `--deploy-config` to change it). +- Generation length and reproducibility are stage settings, not script flags. One output token is one audio frame, so `max_tokens` in the YAML is the frame budget; `seed` makes the in-model 32-head sampling reproducible. The script deliberately passes no `SamplingParams`: one supplied by a caller replaces the stage defaults wholesale rather than merging, which would drop the pipeline's stop token and run every request to `max_tokens`. +- Text length: short texts are repeated internally to match the training layout (the checkpoint's `text_repetition` block); the upper bound is the stage's `max_model_len`, enforced by the engine. Empty text is rejected rather than voiced. +- `VLLM_GEPARD_GREEDY=1` swaps the 32-head Gumbel-max sampling for argmax, for reproducible comparisons that do not depend on a seed. + ## VoxCPM2 Single-stage native AR TTS at 48 kHz. Pipeline: `feat_encoder → MiniCPM4 → FSQ → residual_lm → LocDiT → AudioVAE`. diff --git a/examples/offline_inference/text_to_speech/gepard/end2end.py b/examples/offline_inference/text_to_speech/gepard/end2end.py new file mode 100644 index 000000000..2f6b61d89 --- /dev/null +++ b/examples/offline_inference/text_to_speech/gepard/end2end.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Offline inference example for Gepard-1.0 TTS via vLLM-Omni. + +Single-stage native-AR pipeline: a Qwen3.5 backbone samples one 32-code FSQ +frame per step; the NeMo NanoCodec decodes the frames to a 22.05 kHz mono +waveform. Zero-shot: the default learned voice, no reference audio. + +Generation length and seed are stage settings rather than flags here. One +output token is one audio frame, so ``max_tokens`` in the deploy YAML is the +frame budget; pass a copy of the YAML via ``--deploy-config`` to change it. + +Usage: + python end2end.py --text "Hello, this is Gepard speaking." +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import soundfile as sf +import torch + +import vllm_omni +from vllm_omni.utils.tracking_parser import TrackingArgumentParser + +os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + +from vllm_omni import Omni # noqa: E402 + +MODEL = "nineninesix/gepard-1.0" +SAMPLE_RATE = 22050 +# Absolute: the loader hands the path straight to OmegaConf, which resolves a +# bare name against the current working directory, not the package. +DEFAULT_DEPLOY_CONFIG = str(Path(vllm_omni.__file__).resolve().parent / "deploy" / "gepard.yaml") + + +def build_request(text: str) -> dict: + """Build an Omni request payload for Gepard (zero-shot, no ref audio). + + The model only consumes the [speaker slots, SOT, text, EOT, SOS] layout, so + a bare text prompt would have no speaker slots and no SOS. + """ + from transformers import AutoTokenizer + + from vllm_omni.model_executor.models.gepard.configuration_gepard import GepardConfig + from vllm_omni.model_executor.models.gepard.prompt import build_gepard_prompt_ids + + # From the checkpoint's own sidecar, which is what the worker builds its + # config from too. A layout that disagrees with the trained one does not + # fail, it just degrades the speech. + cfg = GepardConfig.from_checkpoint(MODEL) + tokenizer = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True) + prompt_token_ids = build_gepard_prompt_ids( + tokenizer(text, add_special_tokens=False)["input_ids"], + config=cfg, + ) + return { + "prompt_token_ids": prompt_token_ids, + "additional_information": {"text": [text]}, + } + + +def save_audio(waveform: torch.Tensor, path: str, sample_rate: int = SAMPLE_RATE) -> None: + sf.write(path, waveform.float().numpy(), sample_rate) + seconds = waveform.numel() / sample_rate + print( + f" Saved {path} ({tuple(waveform.shape)}, {sample_rate} Hz, " + f"{seconds:.2f}s, peak {float(waveform.abs().max()):.3f})" + ) + + +def main(args) -> None: + omni = Omni( + model=MODEL, + deploy_config=args.deploy_config, + stage_init_timeout=args.stage_init_timeout, + init_timeout=args.init_timeout, + ) + + # No explicit SamplingParams: one supplied here would replace the stage + # defaults rather than merge over them, dropping the pipeline's stop token. + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + print(f"Synthesizing: {args.text!r}") + inputs = build_request(text=args.text) + + for i, stage_outputs in enumerate(omni.generate(inputs)): + for j, out in enumerate(stage_outputs.outputs): + mm = out.multimodal_output + if mm is None: + print(f" [req {i}] No audio output.") + continue + # The consolidation path renames "model_outputs" to "audio". + audio = mm.get("audio") + if audio is None: + audio = mm.get("model_outputs") + if isinstance(audio, list): + audio = audio[0] if len(audio) else None + if audio is None: + print(f" [req {i}] No waveform in multimodal_output (keys: {list(mm)}).") + continue + sr_tensor = mm.get("sr") + sr = int(sr_tensor.item()) if hasattr(sr_tensor, "item") else SAMPLE_RATE + save_audio(audio.cpu(), str(output_dir / f"output_{i}_{j}.wav"), sr) + + print("Done.") + + +def parse_args(): + parser = TrackingArgumentParser(description="Gepard-1.0 offline TTS inference (zero-shot)") + parser.add_argument("--text", default="Hello, this is Gepard speaking.", help="Text to synthesize.") + parser.add_argument( + "--output-dir", + default=os.path.join( + os.environ.get("XDG_CACHE_HOME", os.path.join(os.path.expanduser("~"), ".cache")), + "gepard_output", + ), + help="Directory for WAV outputs (default: ~/.cache/gepard_output).", + ) + parser.add_argument( + "--deploy-config", + default=DEFAULT_DEPLOY_CONFIG, + help=( + "Path to the deploy YAML (default: the packaged vllm_omni/deploy/gepard.yaml). This must " + "stay set: without the YAML's `pipeline: gepard` pin the architectures fallback routes " + "Qwen3_5ForCausalLM to the diffusion registry. Copy it to change max_tokens or seed." + ), + ) + # The upstream defaults are too tight for a cold start. + parser.add_argument("--stage-init-timeout", type=int, default=900) + parser.add_argument("--init-timeout", type=int, default=1800) + return parser.parse_args() + + +if __name__ == "__main__": + main(parse_args()) diff --git a/tests/e2e/offline_inference/test_gepard_tts.py b/tests/e2e/offline_inference/test_gepard_tts.py new file mode 100644 index 000000000..b201842b4 --- /dev/null +++ b/tests/e2e/offline_inference/test_gepard_tts.py @@ -0,0 +1,345 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""E2E offline inference tests for the Gepard-1.0 single-stage native-AR TTS. + +Zero-shot only (the model's default learned voice). These tests need a GPU and +the NeMo NanoCodec, so the whole file runs in the weekly tier -- see the comment +above ``pytestmark``. The window arithmetic behind the streaming decode is +covered on CPU in ``tests/model_executor/models/test_gepard_window.py``. +""" + +from __future__ import annotations + +import functools +import tempfile +from pathlib import Path +from typing import NamedTuple + +import pytest +import soundfile as sf +import torch +from transformers import AutoTokenizer + +from tests.helpers.mark import hardware_test +from tests.helpers.media import convert_audio_file_to_text +from tests.helpers.runtime import OmniRunner # noqa: F401 (indirect fixture) +from tests.helpers.stage_config import get_deploy_config_path +from vllm_omni import Omni +from vllm_omni.model_executor.models.gepard.configuration_gepard import GepardConfig +from vllm_omni.model_executor.models.gepard.prompt import build_gepard_prompt_ids + +# The codec decoder is an optional dependency; skip rather than fail the engine +# inside the worker process when it is absent. The CI image does not carry it, +# so the weekly step installs it first -- see .buildkite/cuda/test-weekly.yml. +# Install recipe: examples/offline_inference/text_to_speech/README.md. +pytest.importorskip("nemo.collections.tts.models") + +MODEL_NAME = "nineninesix/gepard-1.0" +STAGE_CONFIG = get_deploy_config_path("gepard.yaml") +SAMPLE_RATE = 22050 +# NanoCodec upsamples one FSQ frame to exactly this many samples. +SAMPLES_PER_FRAME = 1024 + + +@functools.cache +def _config() -> GepardConfig: + """The checkpoint's own layout, not the class defaults. + + The worker builds its config from the ``gepard_config.json`` sidecar, so a + prompt assembled from the defaults would silently use a different layout + the day the sidecar moves — and a mismatched layout degrades the speech + rather than failing, which is the hardest way for this to be found. + """ + return GepardConfig.from_checkpoint(MODEL_NAME) + + +# Prompt -> a word that appears in no other prompt, so a clip can be traced +# back to the request that asked for it. +# +# These are not interchangeable. Batching perturbs every request's sampling +# trajectory, and on some trajectories this checkpoint's stop head never fires; +# this set was screened over 10 batched runs with no runaway. Re-screen the +# whole set when changing one, not just the row changed. +_DISTINGUISHABLE_PROMPTS = { + "He drinks coffee every morning.": "coffee", + "Machine learning is interesting.": "learning", + "Please close the window before leaving.": "window", + "My favorite color is purple.": "purple", +} + +# Above text_repetition.apply_below, so the layout is voiced once instead of +# repeated. Every other prompt here takes the repeated branch. +_LONG_TEXT = ( + "The morning train was late again, so I walked the last two miles along the " + "river and reached the office just before nine." +) + +_OMNI_RUNNER_PARAM = ( + MODEL_NAME, + STAGE_CONFIG, + {"trust_remote_code": True}, +) + +# Only a common word is checked, which needs less ASR than the shared helper's +# ``small`` default; a miss escalates to it. +_ASR_MODEL = "base" +_ASR_ESCALATION_MODEL = "small" + +# Transcription dominates this file's runtime -- Whisper runs on CPU on a +# single-GPU runner -- so only two of the four concurrent clips are read as +# speech. The pairwise waveform comparison covers all four. +_TRANSCRIBED_CLIPS = 2 + +# No level marker at module scope: each test carries the one matching how often +# its claim needs re-checking, though the weekly step collects the file whole. +# There is deliberately no core_model row -- under dummy weights the stop head is +# random, so every request would run to gepard.yaml's max_tokens. +pytestmark = [ + pytest.mark.tts, + pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), +] + + +def _build_request(text: str) -> dict: + """Assemble the [speaker slots, SOT, text, EOT, SOS] prompt.""" + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True) + prompt_token_ids = build_gepard_prompt_ids( + tokenizer(text, add_special_tokens=False)["input_ids"], + config=_config(), + ) + return { + "prompt_token_ids": prompt_token_ids, + "additional_information": {"text": [text]}, + } + + +def _extract_audio(mm) -> torch.Tensor | None: + # The consolidation path renames "model_outputs" to "audio". + audio = mm.get("audio") + if audio is None: + audio = mm.get("model_outputs") + if isinstance(audio, list): + audio = audio[0] if len(audio) else None + return audio + + +class _Clip(NamedTuple): + wav: torch.Tensor + # Counts the frames the codec was handed using only what the engine + # returned, never the model's internals. + committed_frames: int + + +def _synthesize_all(omni: Omni, texts: list[str]) -> list[_Clip]: + """Submit every text in one call; return one clip per request, in + submission order. + + ``Omni.generate`` yields in completion order and never sorts, so the pairing + is restored from the request id prefix it assigns (``f"{i}_{uuid4()}"``). + Without that, the only testable claim is that the batch as a whole looks + plausible, which a stable swap between two requests satisfies. + + Nothing here authors a SamplingParams: a fresh one would replace the stage + defaults rather than merge over them, dropping the pipeline's stop token. + """ + outputs = omni.generate([_build_request(t) for t in texts]) + assert len(outputs) == len(texts), f"expected {len(texts)} outputs, got {len(outputs)}" + + by_index: dict[int, _Clip] = {} + for stage_outputs in outputs: + index = int(str(stage_outputs.request_id).split("_", 1)[0]) + waveform = None + frames = 0 + for out in stage_outputs.outputs: + # Explicit None check: `x or []` would ask an array for a truth value. + token_ids = out.token_ids if out.token_ids is not None else [] + # STOP sits just past head0's range, so a token equal to it is + # never a committed frame. + frames += sum(1 for t in token_ids if int(t) != _config().stop_token) + mm = out.multimodal_output + if mm is None: + continue + audio = _extract_audio(mm) + if audio is None: + continue + sr_t = mm.get("sr") + if hasattr(sr_t, "item"): + assert int(sr_t.item()) == SAMPLE_RATE + waveform = audio.cpu().float() + assert waveform is not None, "no audio output produced" + by_index[index] = _Clip(wav=waveform, committed_frames=frames) + + assert sorted(by_index) == list(range(len(texts))), ( + f"request ids did not cover every submitted prompt exactly once: got {sorted(by_index)}" + ) + return [by_index[i] for i in range(len(texts))] + + +def _synthesize(omni: Omni, text: str) -> _Clip: + return _synthesize_all(omni, [text])[0] + + +def _assert_clip_is_sane(clip: _Clip) -> None: + """The checks every clip has to pass, whatever produced it.""" + wav = clip.wav + assert wav.dim() == 1, f"expected mono 1-D, got shape {tuple(wav.shape)}" + assert wav.numel() > 0, "empty waveform" + assert torch.isfinite(wav).all(), "non-finite samples" + + # Frame conservation against the engine's own token count, not just + # divisibility. This is what catches a payload carrying non-audio data or a + # dropped tail; both pass every other check here. + assert wav.numel() == clip.committed_frames * SAMPLES_PER_FRAME, ( + f"{wav.numel()} samples for {clip.committed_frames} committed frames — " + f"expected exactly {clip.committed_frames * SAMPLES_PER_FRAME}" + ) + + # Anything far outside the waveform range is not codec output. + assert float(wav.abs().max()) <= 1.5, f"peak {float(wav.abs().max()):.2f} is out of range" + + +def _assert_stopped_on_its_own(clip: _Clip) -> None: + """The request must stop on its own rather than run to ``max_tokens``. + + Separate from the structural checks because it reads the stop head's + judgement, not the payload's shape: under ``load_format: dummy`` that head + is random and never fires, so every clip runs to the token cap. + """ + seconds = clip.wav.numel() / SAMPLE_RATE + assert 0.5 < seconds < 30.0, f"implausible duration {seconds:.2f}s — did STOP fire?" + + +def _transcribe(wav: torch.Tensor, tmp_dir: str, name: str, model_size: str = _ASR_MODEL) -> str: + path = str(Path(tmp_dir) / f"{name}.wav") + sf.write(path, wav.numpy(), SAMPLE_RATE) + return convert_audio_file_to_text(path, model_size=model_size, language="en").lower() + + +def _transcribe_for(wav: torch.Tensor, tmp_dir: str, name: str, accepts) -> str: + """Transcribe with the fast model, retrying once with the stronger one. + + ``accepts`` decides whether the fast transcript is usable. Escalating only + on a miss keeps the average clip at ``_ASR_MODEL`` while a genuine model + artifact still fails the caller's assertion, since the stronger ASR + mistranscribes it too. + """ + transcript = _transcribe(wav, tmp_dir, name) + if accepts(transcript): + return transcript + return _transcribe(wav, tmp_dir, name, model_size=_ASR_ESCALATION_MODEL) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize( + ("text", "keyword"), + [ + # The canonical path: one request, one transcript, the cheapest check + # that reads the audio as speech. + pytest.param( + "Hello, this is Gepard speaking.", + "hello", + marks=pytest.mark.advanced_model, + id="canonical", + ), + # No keyword: this row carries the structural and stopping checks over a + # second text and leaves the transcript to the row above. + pytest.param( + "The quick brown fox jumps over the lazy dog.", + None, + marks=pytest.mark.full_model, + id="second_text", + ), + ], +) +def test_gepard_offline_zero_shot(omni_runner, run_level: str, text: str, keyword: str | None) -> None: + """Zero-shot synthesis produces a finite mono waveform at 22.05 kHz. + + The transcript check is the one assertion here that reads the audio as + speech rather than as a tensor. It stands in for a parity comparison, which + needs a reference implementation this PR cannot run — noise, a repeated + loop or the wrong clip all pass every numeric check above it. + """ + clip = _synthesize(omni_runner, text) + _assert_clip_is_sane(clip) + + if run_level not in {"advanced_model", "full_model"}: + return # dummy weights: the audio is structural only, so do not read it + + _assert_stopped_on_its_own(clip) + if keyword is None: + return + + with tempfile.TemporaryDirectory() as tmp_dir: + transcript = _transcribe_for(clip.wav, tmp_dir, "zero_shot", lambda t: keyword in t) + assert keyword in transcript, f"expected {keyword!r} in transcript, got {transcript!r}" + + +@pytest.mark.full_model +@hardware_test(res={"cuda": "L4"}, num_cards=1) +def test_gepard_offline_long_text_skips_repetition(omni_runner, run_level: str) -> None: + """The other branch of the prompt layout, end to end. + + A text at or above ``apply_below`` tokens is voiced once; shorter ones are + repeated to carry enough mass against the speaker prefix. The layout + assertion keeps this row honest — if a tokenizer change pushed this text + back under the threshold it would silently retest the repeated path. + """ + prompt_ids = _build_request(_LONG_TEXT)["prompt_token_ids"] + assert prompt_ids.count(_config().start_of_text) == 1, "_LONG_TEXT is no longer above apply_below" + + clip = _synthesize(omni_runner, _LONG_TEXT) + _assert_clip_is_sane(clip) + + if run_level not in {"advanced_model", "full_model"}: + return + + _assert_stopped_on_its_own(clip) + with tempfile.TemporaryDirectory() as tmp_dir: + transcript = _transcribe_for(clip.wav, tmp_dir, "long_text", lambda t: "river" in t) + assert "river" in transcript, f"expected 'river' in transcript, got {transcript!r}" + + +@pytest.mark.slow +@hardware_test(res={"cuda": "L4"}, num_cards=1) +def test_gepard_offline_concurrent_requests_stay_isolated(omni_runner, run_level: str) -> None: + """Four requests in one batch, each holding its own generation state. + + The prompts are deliberately unlike each other: cross-talk between the + per-request frame history, the codec window or the output routing shows up + as a transcript matching the wrong prompt, which similar prompts would + hide. ``max_num_seqs`` is 4, so these really do run together. + """ + texts = list(_DISTINGUISHABLE_PROMPTS) + clips = _synthesize_all(omni_runner, texts) + + for clip in clips: + _assert_clip_is_sane(clip) + + if run_level not in {"advanced_model", "full_model"}: + return # dummy weights: the audio is structural only, so do not read it + + for clip in clips: + _assert_stopped_on_its_own(clip) + + # All four clips, not just the transcribed ones: delivering one request's + # audio for another leaves two byte-identical waveforms. + for i in range(len(clips)): + for j in range(i + 1, len(clips)): + assert not torch.equal(clips[i].wav, clips[j].wav), ( + f"clips {i} and {j} are the same waveform — one request's audio was delivered for another" + ) + + # Each clip must say what its own prompt asked for: accepting any of the + # four would pass a stable permutation, which is what a mix-up produces. + keywords = set(_DISTINGUISHABLE_PROMPTS.values()) + with tempfile.TemporaryDirectory() as tmp_dir: + for i, clip in enumerate(clips[:_TRANSCRIBED_CLIPS]): + expected = _DISTINGUISHABLE_PROMPTS[texts[i]] + # Escalate on the expected word being absent, not on "no keyword + # at all" -- a neighbour's word is the failure under test. + transcript = _transcribe_for(clip.wav, tmp_dir, f"concurrent_{i}", lambda t: expected in t) + present = sorted(k for k in keywords if k in transcript) + assert present == [expected], ( + f"clip for {texts[i]!r} transcribed as {transcript!r}: expected {expected!r} and " + f"no other prompt's keyword, found {present} — requests bled into each other" + ) diff --git a/tests/model_executor/models/test_gepard_window.py b/tests/model_executor/models/test_gepard_window.py new file mode 100644 index 000000000..67c61c7ab --- /dev/null +++ b/tests/model_executor/models/test_gepard_window.py @@ -0,0 +1,363 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CPU coverage for Gepard's prompt layout, streaming-decode window and config. + +The e2e test needs a GPU, NeMo and the checkpoint, so it only runs nightly. +These pin the parts that are pure logic and break silently: which frames each +decode covers, which samples survive the lookback trim, the prompt layout, and +that the packaged deploy config sits where the example looks for it. + +The codec is replaced by a shift-invariant stand-in whose output depends only +on the codes, which makes "chunked emissions concatenate to one whole-history +decode" an exact assertion — the contract lookback exists to provide. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import torch +import torch.nn as nn + +import vllm_omni +from vllm_omni.config.stage_config import load_deploy_config +from vllm_omni.model_executor.models.gepard.configuration_gepard import GepardConfig +from vllm_omni.model_executor.models.gepard.gepard_talker import ( + CHUNK_FRAMES, + FIRST_CHUNK_FRAMES, + GepardTalkerForConditionalGeneration, + _GepardState, +) +from vllm_omni.model_executor.models.gepard.nanocodec import ( + LOOKBACK_FRAMES, + NanoCodec, + apply_end_of_speech_tail, +) +from vllm_omni.model_executor.models.gepard.prompt import build_gepard_prompt_ids + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +SAMPLES_PER_FRAME = 1024 +NUM_HEADS = 32 +SAMPLE_RATE = 22050 +# Text-token count at or above which the layout is left unrepeated, read from +# the checkpoint defaults so the boundary cases follow the config. +APPLY_BELOW = GepardConfig().text_repetition_apply_below + + +class _ShiftInvariantCodec: + """Frame tagged ``v`` -> samples ``v*SPF .. v*SPF+SPF-1``, wherever it sits.""" + + def __init__(self) -> None: + self.windows: list[int] = [] # frame count of every decode call + + def decode_from_codes(self, codes: torch.Tensor, codes_len: torch.Tensor): + assert codes.shape[0] == 1, "streaming decode is per-request" + assert codes.shape[1] == NUM_HEADS, f"expected {NUM_HEADS} code channels" + tags = codes[0, 0, :].to(torch.int64) # (T,) + self.windows.append(int(tags.shape[0])) + offsets = torch.arange(SAMPLES_PER_FRAME, dtype=torch.int64) + audio = (tags.unsqueeze(1) * SAMPLES_PER_FRAME + offsets).reshape(1, -1) + return audio.to(torch.float32), codes_len * SAMPLES_PER_FRAME + + +def _make_codec() -> tuple[NanoCodec, _ShiftInvariantCodec]: + codec = NanoCodec(codec_id="test/fake-codec", sample_rate=SAMPLE_RATE) + fake = _ShiftInvariantCodec() + codec._codec = fake # bypasses load(); NeMo is never imported + assert codec.is_loaded + return codec, fake + + +def _make_talker(codec: NanoCodec) -> GepardTalkerForConditionalGeneration: + """A real talker with only the audio-path attributes populated. + + ``__init__`` needs a VllmConfig and builds the backbone; the methods under + test touch only these four attributes. + """ + talker = GepardTalkerForConditionalGeneration.__new__(GepardTalkerForConditionalGeneration) + nn.Module.__init__(talker) # sets up _parameters/_buffers/_modules only + talker._codec = codec + talker._audio_queue = [] + talker._active_states = {} + talker._deferred_cleanup_ids = set() + return talker + + +def _frame(tag: int) -> torch.Tensor: + """One committed frame: 32 per-dimension codes, all carrying the tag.""" + return torch.full((NUM_HEADS,), tag, dtype=torch.long) + + +def _expected_samples(num_frames: int) -> torch.Tensor: + offsets = torch.arange(SAMPLES_PER_FRAME, dtype=torch.int64) + tags = torch.arange(num_frames, dtype=torch.int64).unsqueeze(1) + return (tags * SAMPLES_PER_FRAME + offsets).reshape(-1).to(torch.float32) + + +def _drain(talker) -> torch.Tensor: + """Concatenate and clear the queued deltas, as make_omni_output does.""" + if not talker._audio_queue: + return torch.empty(0) + out = torch.cat([a.reshape(-1) for _, a in talker._audio_queue]) + talker._audio_queue.clear() + return out + + +def _generate(talker, state, num_frames: int, *, stop_at_end: bool) -> None: + """Drive the commit loop the way forward() does, one frame per step.""" + for tag in range(num_frames): + state.frames.append(_frame(tag)) + state.frame_count += 1 + state.past_first_step = True + talker._emit_audio(state, is_final=stop_at_end and tag == num_frames - 1) + + +# Counts straddling both cadence thresholds, where an off-by-one hides. +@pytest.mark.parametrize( + "num_frames", + [ + 1, + 2, + FIRST_CHUNK_FRAMES - 1, + FIRST_CHUNK_FRAMES, + FIRST_CHUNK_FRAMES + 1, + CHUNK_FRAMES - 1, + CHUNK_FRAMES, + CHUNK_FRAMES + 1, + FIRST_CHUNK_FRAMES + CHUNK_FRAMES, + 79, + 200, + ], +) +def test_chunked_emissions_equal_a_whole_history_decode(num_frames: int) -> None: + codec, _fake = _make_codec() + talker = _make_talker(codec) + state = _GepardState(request_id="req-0") + + _generate(talker, state, num_frames, stop_at_end=True) + + torch.testing.assert_close(_drain(talker), _expected_samples(num_frames)) + assert state.emitted_frames == num_frames + + +def test_every_frame_is_emitted_exactly_once() -> None: + """The trim must not drop or duplicate samples at a chunk boundary.""" + codec, _fake = _make_codec() + talker = _make_talker(codec) + state = _GepardState(request_id="req-0") + num_frames = FIRST_CHUNK_FRAMES + 2 * CHUNK_FRAMES + 3 + + _generate(talker, state, num_frames, stop_at_end=True) + audio = _drain(talker) + + assert audio.numel() == num_frames * SAMPLES_PER_FRAME + # Tags are globally unique, so a duplicated or dropped frame shows up here. + assert len(torch.unique(audio)) == audio.numel() + + +def test_decode_windows_carry_lookback_context() -> None: + """Every decode after the first must re-decode LOOKBACK_FRAMES of context.""" + codec, fake = _make_codec() + talker = _make_talker(codec) + state = _GepardState(request_id="req-0") + num_frames = FIRST_CHUNK_FRAMES + 2 * CHUNK_FRAMES + + _generate(talker, state, num_frames, stop_at_end=True) + + assert len(fake.windows) >= 3, "expected a first chunk plus steady-state chunks" + assert fake.windows[0] == FIRST_CHUNK_FRAMES, "the first chunk has no left context" + for window in fake.windows[1:]: + assert window == CHUNK_FRAMES + LOOKBACK_FRAMES + + +def test_codec_runs_once_per_chunk_not_once_per_frame() -> None: + codec, fake = _make_codec() + talker = _make_talker(codec) + state = _GepardState(request_id="req-0") + num_frames = FIRST_CHUNK_FRAMES + 3 * CHUNK_FRAMES + + _generate(talker, state, num_frames, stop_at_end=True) + + assert len(fake.windows) == 4 + assert sum(fake.windows) < num_frames * 2 # far below one decode per frame + + +def test_final_emit_drains_a_partial_chunk() -> None: + """A request can end mid-chunk (max_tokens, STOP); the frames committed + since the last cadence boundary must still reach the queue. + + Arithmetic only: this drives ``_emit_audio`` directly, so it says nothing + about *whether* the engine reaches it on the request's last step. That + wiring lives in ``test_gepard_wiring.py`` — a bug there leaves this green. + """ + codec, _fake = _make_codec() + talker = _make_talker(codec) + state = _GepardState(request_id="req-0") + num_frames = FIRST_CHUNK_FRAMES + CHUNK_FRAMES + 5 + + _generate(talker, state, num_frames, stop_at_end=False) + assert state.emitted_frames < num_frames, "a partial chunk should still be pending" + + talker._emit_audio(state, is_final=True) + + torch.testing.assert_close(_drain(talker), _expected_samples(num_frames)) + assert state.emitted_frames == num_frames + + +def test_emitting_with_nothing_pending_is_a_no_op() -> None: + codec, fake = _make_codec() + talker = _make_talker(codec) + state = _GepardState(request_id="req-0") + + _generate(talker, state, 4, stop_at_end=True) + calls_after_flush = len(fake.windows) + talker._audio_queue.clear() + + talker._emit_audio(state, is_final=True) + + assert len(fake.windows) == calls_after_flush, "codec ran with no new frames" + assert talker._audio_queue == [] + + +def test_concurrent_requests_keep_separate_windows() -> None: + codec, _fake = _make_codec() + talker = _make_talker(codec) + states = [_GepardState(request_id=f"req-{i}") for i in range(2)] + + for state in states: + _generate(talker, state, FIRST_CHUNK_FRAMES, stop_at_end=True) + + by_req: dict[str, list[torch.Tensor]] = {} + for req_id, audio in talker._audio_queue: + by_req.setdefault(req_id, []).append(audio.reshape(-1)) + assert set(by_req) == {"req-0", "req-1"} + for chunks in by_req.values(): + torch.testing.assert_close(torch.cat(chunks), _expected_samples(FIRST_CHUNK_FRAMES)) + + +def test_end_of_speech_tail_fades_and_pads() -> None: + audio = torch.ones(SAMPLE_RATE) # 1 s of full-scale signal + out = apply_end_of_speech_tail(audio, SAMPLE_RATE, fade_ms=10.0, silence_ms=20.0) + + fade = int(SAMPLE_RATE * 10.0 / 1000.0) + pad = int(SAMPLE_RATE * 20.0 / 1000.0) + assert out.numel() == SAMPLE_RATE + pad + assert torch.all(out[-pad:] == 0.0), "trailing pad must be silent" + assert out[SAMPLE_RATE - fade - 1] == 1.0, "the fade must not reach back past fade_ms" + # Monotonically ramped to zero — the pad alone would leave the click. + ramp = out[SAMPLE_RATE - fade : SAMPLE_RATE] + assert torch.all(ramp[:-1] >= ramp[1:]) + assert ramp[-1] == 0.0 + + +def test_end_of_speech_tail_is_a_no_op_by_default() -> None: + audio = torch.ones(128) + out = apply_end_of_speech_tail(audio, SAMPLE_RATE, fade_ms=0.0, silence_ms=0.0) + assert out is audio + + +def _repeat_count(ids: list[int], cfg: GepardConfig) -> int: + """How many ``[SOT ... EOT]`` copies the layout carries.""" + return ids.count(cfg.start_of_text) + + +def test_gepard_prompt_layout() -> None: + """The prompt carries the speaker slots and exactly one SOS, at the end.""" + cfg = GepardConfig() + ids = build_gepard_prompt_ids([1, 2, 3], config=cfg) + + slots = [cfg.speaker_token_base + i for i in range(cfg.num_speaker_prefix)] + assert ids[: cfg.num_speaker_prefix] == slots + assert ids.count(cfg.start_of_speech) == 1, "only the canonical copy may carry SOS" + assert ids[-1] == cfg.start_of_speech, "the first frame is sampled from the SOS position" + + +def test_short_text_repeats_to_reach_the_target_token_mass() -> None: + """A short text is repeated until its region holds ~target_text_tokens.""" + cfg = GepardConfig() + text = [1, 2, 3, 4] # target 16 / 4 -> 4 copies + ids = build_gepard_prompt_ids(text, config=cfg) + + assert _repeat_count(ids, cfg) == 4 + assert ids.count(cfg.start_of_speech) == 1, "only the last copy opens the speech region" + assert ids[-1] == cfg.start_of_speech + + +@pytest.mark.parametrize( + ("num_text_tokens", "repeated"), + [ + (APPLY_BELOW - 1, True), + (APPLY_BELOW, False), + (APPLY_BELOW + 1, False), + ], +) +def test_apply_below_is_the_repetition_boundary(num_text_tokens: int, repeated: bool) -> None: + """Both sides of the threshold: at apply_below the text is left alone. + + The layout has to match training here or WER collapses, and the boundary + is where an off-by-one would be invisible in a single happy-path clip. + """ + cfg = GepardConfig() + ids = build_gepard_prompt_ids(list(range(1, num_text_tokens + 1)), config=cfg) + + assert (_repeat_count(ids, cfg) > 1) is repeated + + +def test_max_repeats_caps_a_one_token_text() -> None: + """target/1 would ask for 16 copies; the cap keeps the prefill bounded.""" + cfg = GepardConfig() + ids = build_gepard_prompt_ids([1], config=cfg) + + assert _repeat_count(ids, cfg) == cfg.text_repetition_max_repeats + + +def test_repetition_thresholds_are_read_from_the_config() -> None: + """The numbers come from the checkpoint sidecar, not from literals. + + A checkpoint whose text_repetition block differs must change the layout; + duplicating the values in the builder would silently ignore it. + """ + cfg = GepardConfig(text_repetition={"enabled": True, "target_text_tokens": 9, "apply_below": 8, "max_repeats": 2}) + assert _repeat_count(build_gepard_prompt_ids([1, 2, 3], config=cfg), cfg) == 2 # ceil(9/3)=3, capped at 2 + assert _repeat_count(build_gepard_prompt_ids(list(range(8)), config=cfg), cfg) == 1 # at apply_below + + +def test_repetition_can_be_disabled() -> None: + cfg = GepardConfig(text_repetition={"enabled": False}) + ids = build_gepard_prompt_ids([1], config=cfg) + + assert _repeat_count(ids, cfg) == 1 + assert ids[-1] == cfg.start_of_speech + + +def test_empty_text_is_rejected() -> None: + """An empty text would still assemble a valid layout and be voiced. + + ``[slots, SOT, EOT, SOS]`` parses, prefills and generates: the failure is + audible, not raised, which is the worst shape for a caller to debug. + """ + with pytest.raises(ValueError, match="at least one text token"): + build_gepard_prompt_ids([], config=GepardConfig()) + + +def test_packaged_deploy_config_is_where_the_example_looks_for_it() -> None: + """The example resolves its default off ``vllm_omni.__file__``; tests reach + the same YAML through ``tests.helpers.stage_config``, so nothing covered the + packaged location until this. + + ``pipeline: gepard`` is the load-bearing key: the checkpoint self-identifies + as qwen3_5_text, so without the pin the architectures fallback routes it to + the diffusion registry. + + This asserts the location, not the example's constant — importing the + example would pull ``Omni`` and NeMo into a CPU test. Running the quick + start verbatim stays a release check. + """ + path = Path(vllm_omni.__file__).resolve().parent / "deploy" / "gepard.yaml" + assert path.is_file(), f"packaged deploy config missing at {path}" + + deploy = load_deploy_config(path) + assert deploy.pipeline == "gepard" + assert deploy.stages, "deploy config declares no stages" diff --git a/tests/model_executor/models/test_gepard_wiring.py b/tests/model_executor/models/test_gepard_wiring.py new file mode 100644 index 000000000..b38b65639 --- /dev/null +++ b/tests/model_executor/models/test_gepard_wiring.py @@ -0,0 +1,731 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CPU coverage for the seam between Gepard's talker and the AR runner. + +``test_gepard_window.py`` pins the decode arithmetic *below* this layer, and +the e2e test exercises the layer *above* it but only along the path where the +stop head fires. Neither covers the wiring itself: whether anything calls the +final emit on a request's last step, and whether the payload it produces +survives the runner's sparse-audio routing. + +These drive the real ``preprocess`` -> ``forward`` -> ``make_omni_output`` +sequence through a stand-in for ``GPUARModelRunner.execute_model``, and use the +runner's own ``_resolve_sparse_mm_routing`` to decide what actually reaches the +caller. Only the LM backbone and the frame sampler are stubbed out; every +method under test is the real one, so a test here goes red when the engine +path breaks — which a test that calls the flush by hand cannot do. +""" + +from __future__ import annotations + +import logging + +import pytest +import torch +import torch.nn as nn + +from vllm_omni.model_executor.models.gepard.configuration_gepard import GepardConfig +from vllm_omni.model_executor.models.gepard.gepard_talker import ( + CHUNK_FRAMES, + FIRST_CHUNK_FRAMES, + GepardTalkerForConditionalGeneration, + _ReqInfo, +) +from vllm_omni.model_executor.models.gepard.nanocodec import NanoCodec +from vllm_omni.model_executor.models.gepard.prompt import build_gepard_prompt_ids +from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +SAMPLES_PER_FRAME = 1024 +NUM_HEADS = 32 +SAMPLE_RATE = 22050 +LOGGER_NAME = "vllm_omni.model_executor.models.gepard.gepard_talker" + +# Each request tags its frames from a distinct base, so a sample that reaches +# the wrong request is visible in the value, not just the length. +_TAG_STRIDE = 100_000 + + +class _TagCodec: + """Frame carrying tag ``v`` -> samples ``v*SPF .. v*SPF+SPF-1``, wherever it sits. + + Shift-invariant, so "what the caller received" can be compared against one + whole-history decode exactly. The tag rides channel 1: channel 0 is head0, + which the talker keeps inside its real (small) vocab. + """ + + def __init__(self) -> None: + self.decode_calls = 0 + + def decode_from_codes(self, codes: torch.Tensor, codes_len: torch.Tensor): + assert codes.shape[0] == 1, "streaming decode is per-request" + assert codes.shape[1] == NUM_HEADS, f"expected {NUM_HEADS} code channels" + self.decode_calls += 1 + tags = codes[0, 1, :].to(torch.int64) # (T,) + offsets = torch.arange(SAMPLES_PER_FRAME, dtype=torch.int64) + audio = (tags.unsqueeze(1) * SAMPLES_PER_FRAME + offsets).reshape(1, -1) + return audio.to(torch.float32), codes_len * SAMPLES_PER_FRAME + + +class _StubBackbone(nn.Module): + """Stands in for the Qwen3.5 backbone; only shapes matter to this seam.""" + + def __init__(self, hidden: int) -> None: + super().__init__() + self.hidden = hidden + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return torch.zeros(int(input_ids.shape[0]), self.hidden) + + def forward( + self, + input_ids: torch.Tensor | None = None, + positions: torch.Tensor | None = None, + intermediate_tensors=None, + inputs_embeds: torch.Tensor | None = None, + ) -> torch.Tensor: + src = inputs_embeds if inputs_embeds is not None else input_ids + return torch.zeros(int(src.shape[0]), self.hidden) + + +def _expected_samples(tag_base: int, num_frames: int) -> torch.Tensor: + offsets = torch.arange(SAMPLES_PER_FRAME, dtype=torch.int64) + tags = (torch.arange(num_frames, dtype=torch.int64) + tag_base).unsqueeze(1) + return (tags * SAMPLES_PER_FRAME + offsets).reshape(-1).to(torch.float32) + + +class _MiniEngine: + """Drives the talker the way ``GPUARModelRunner.execute_model`` does. + + Reproduces the three orderings the dropped-tail bug lived in: + + * ``on_requests_finished`` fires at the START of the step AFTER the one + that finished the request (it reads ``scheduler_output.finished_req_ids``); + * a finished request is not scheduled again, so it is absent from + ``req_ids_output_copy`` and from that step's routing; + * with nothing left to schedule there is no step at all, so no further + ``forward`` runs; + * a stop-token finish gets one more scheduled step, a budget finish does + not -- under async scheduling only the latter is guarded against it. + """ + + def __init__(self, talker: GepardTalkerForConditionalGeneration, *, max_model_len: int) -> None: + self.talker = talker + self.max_model_len = max_model_len + self.reqs: dict[str, dict] = {} + self.order: list[str] = [] + self._finished_last_step: list[str] = [] + self.delivered: dict[str, list[torch.Tensor]] = {} + self.committed_frames: dict[str, int] = {} + self.steps = 0 + self._tags_handed_out = 0 + + def add_request( + self, + req_id: str, + *, + max_tokens: int, + stop_after: int | None = None, + text_len: int = 3, + seed: int | None = None, + ) -> None: + """Submit a request. An id already seen is a resubmit of that id. + + Tags come from a counter, so a resubmitted id gets frames the previous + request could not have produced. + """ + prompt_ids = build_gepard_prompt_ids(list(range(10, 10 + text_len)), config=self.talker.config) + self._tags_handed_out += 1 + self.reqs[req_id] = { + "prompt_ids": torch.tensor(prompt_ids, dtype=torch.long), + "prompt_len": len(prompt_ids), + "max_tokens": max_tokens, + "stop_after": stop_after, + "stop_fired": False, + "seed": seed, + "computed": 0, + "generated": 0, + "finished": False, + "ghost_step_pending": False, + "next_token": None, + "tag_base": _TAG_STRIDE * self._tags_handed_out, + } + if req_id not in self.order: + self.order.append(req_id) + self.delivered[req_id] = [] + self.committed_frames[req_id] = 0 + + def abort(self, req_id: str) -> None: + """Kill a request the way an aborted client does: no STOP, no budget.""" + self.reqs[req_id]["finished"] = True + self._finished_last_step.append(req_id) + + def _live(self) -> list[str]: + return [r for r in self.order if not self.reqs[r]["finished"]] + + def step(self) -> bool: + """Run one engine step. False when there was nothing to schedule.""" + live = self._live() + if not live: + return False + talker = self.talker + self.steps += 1 + + # --- runner: notify the model of requests finished since last step --- + if self._finished_last_step: + talker.on_requests_finished(set(self._finished_last_step)) + self._finished_last_step = [] + + # --- runner: per-request preprocess, in input_batch.req_ids order --- + embeds_parts, ids_parts = [], [] + for req_id in live: + r = self.reqs[req_id] + is_prefill = r["computed"] < r["prompt_len"] + if is_prefill: + ids = r["prompt_ids"] + else: + ids = torch.tensor([r["next_token"] or 0], dtype=torch.long) + span = int(ids.shape[0]) + out_ids, out_embeds, _ = talker.preprocess( + input_ids=ids, + input_embeds=None, + request_id=req_id, + _omni_prompt_len=r["prompt_len"], + _omni_num_computed_tokens=r["computed"], + _omni_is_prefill=is_prefill, + _omni_max_tokens=r["max_tokens"], + _omni_seed=r["seed"], + ) + ids_parts.append(out_ids) + embeds_parts.append(out_embeds) + r["span"] = span + + inputs_embeds = torch.cat(embeds_parts, dim=0) + input_ids = torch.cat(ids_parts, dim=0) + positions = torch.arange(int(input_ids.shape[0]), dtype=torch.long) + + # --- runner: forward, then build the multimodal payload --- + hidden = talker.forward(input_ids, positions, inputs_embeds=inputs_embeds) + results = list(talker._results_queue) + omni_out = talker.make_omni_output(hidden) + mm = omni_out.multimodal_outputs + # compute_logits drains this; it does not touch the audio path. + talker._results_queue.clear() + + for req_id in live: + state = talker._active_states.get(req_id) + if state is not None: + self.committed_frames[req_id] = state.frame_count + + # --- runner: sparse-audio routing decides what reaches the caller --- + downstream, sparse_index, is_sparse = GPUARModelRunner._resolve_sparse_mm_routing( + engine_output_type="audio", + req_ids_output_copy=list(live), + downstream_req_ids=list(live), + multimodal_outputs=mm, + ) + assert is_sparse, "gepard always marks meta.sparse_audio" + for rid in downstream: + self.delivered[rid].append(mm["model_outputs"][sparse_index[rid]].reshape(-1)) + + # --- runner/scheduler bookkeeping: advance and detect finishes --- + stopped = {rid for rid, _head0, do_stop in results if do_stop} + for req_id in live: + r = self.reqs[req_id] + r["computed"] += r["span"] + r["generated"] = r["computed"] - r["prompt_len"] + 1 + r["next_token"] = 0 + length_cap = min(r["max_tokens"], self.max_model_len - r["prompt_len"]) + if r["ghost_step_pending"]: + # This step WAS the extra one. The engine finished the request + # on the stop token last step and discards whatever came back + # here, whether or not the model chose to stop again. + r["finished"] = True + self._finished_last_step.append(req_id) + elif req_id in stopped: + # Ends on the stop token: one more step is already scheduled. + r["ghost_step_pending"] = True + elif r["generated"] >= length_cap: + r["finished"] = True + self._finished_last_step.append(req_id) + return True + + def run(self, max_steps: int = 4000) -> None: + for _ in range(max_steps): + if not self.step(): + return + raise AssertionError("engine did not drain") + + def audio_for(self, req_id: str) -> torch.Tensor: + chunks = self.delivered[req_id] + return torch.cat(chunks) if chunks else torch.empty(0) + + +def _make_talker(*, max_model_len: int = 4096) -> tuple[GepardTalkerForConditionalGeneration, _TagCodec]: + """A real talker with only the audio-path collaborators populated. + + ``__init__`` needs a VllmConfig and builds the Qwen3.5 backbone; the seam + under test needs none of that, so the backbone and the frame sampler are + stubbed and everything else is the class's own code. + """ + cfg = GepardConfig() + # Fixed, not cfg.get_text_config().hidden_size: nothing in this seam reads + # the real width, and a small one keeps the test off the backbone config. + hidden = 64 + codec = NanoCodec(codec_id="test/fake-codec", sample_rate=SAMPLE_RATE) + fake = _TagCodec() + codec._codec = fake # bypasses load(); NeMo is never imported + assert codec.is_loaded + + talker = GepardTalkerForConditionalGeneration.__new__(GepardTalkerForConditionalGeneration) + nn.Module.__init__(talker) + talker.config = cfg + talker.model = _StubBackbone(hidden) + # requires_grad=False: preprocess writes the prefix into inputs_embeds + # in place, and production runs that under inference_mode. + talker.null_prefix = nn.Parameter(torch.zeros(cfg.num_speaker_prefix, hidden), requires_grad=False) + talker._codec = codec + talker._sample_rate = SAMPLE_RATE + talker._max_model_len = max_model_len + talker._active_states = {} + talker._pending_requests = [] + talker._results_queue = [] + talker._audio_queue = [] + talker._deferred_cleanup_ids = set() + return talker, fake + + +def _install_scripted_sampling(talker: GepardTalkerForConditionalGeneration, engine: _MiniEngine) -> None: + """Replace the 32-head sampler with a scripted one. + + Frames carry a per-request, per-step tag so the caller's audio is + self-describing. STOP fires once, on the request's configured frame: on the + extra step the engine then runs, re-firing is likely but not guaranteed, + and the model may not assume it. + """ + head0_vocab = talker.config.head0_vocab_size + + # Signature mirrors the real ``_sample_frame`` minus ``self``; the scripted + # sampler ignores ``generators`` because seeding is decided in + # ``_build_with_real_sampler``, which keeps the model's own sampler. + def _sample_frame(rows: torch.Tensor, generators: list[torch.Generator | None] | None = None): + n = int(rows.shape[0]) + head0 = torch.zeros(n, dtype=torch.long) + heads = torch.zeros(n, NUM_HEADS - 1, dtype=torch.long) + stop = torch.zeros(n, dtype=torch.bool) + for i, (req_id, _span, _samples_frame, *_rest) in enumerate(talker._pending_requests): + state = talker._active_states.get(req_id) + frame_index = state.frame_count if state is not None else 0 + tag = engine.reqs[req_id]["tag_base"] + frame_index + head0[i] = tag % head0_vocab + heads[i] = tag + r = engine.reqs[req_id] + if r["stop_after"] is not None and frame_index >= r["stop_after"] and not r["stop_fired"]: + r["stop_fired"] = True + stop[i] = True + return head0, heads, stop + + talker._sample_frame = _sample_frame + talker._audio_frame_embed = lambda h0, h: torch.zeros(int(h0.shape[0]), talker.model.hidden) + + +def _build(*, max_model_len: int = 4096) -> tuple[GepardTalkerForConditionalGeneration, _TagCodec, _MiniEngine]: + talker, codec = _make_talker(max_model_len=max_model_len) + engine = _MiniEngine(talker, max_model_len=max_model_len) + _install_scripted_sampling(talker, engine) + return talker, codec, engine + + +def _build_with_real_sampler(*, max_model_len: int = 4096) -> tuple[GepardTalkerForConditionalGeneration, _MiniEngine]: + """Same seam, but ``_sample_frame`` is the model's own 32-head sampler. + + The stub backbone hands every row the same hidden state, so the sampled + codes — and therefore the audio the codec tags them into — vary with + nothing but the Gumbel draws. That makes this the one place where "did the + caller's seed reach the audio" is a decidable question. + """ + talker, _codec = _make_talker(max_model_len=max_model_len) + engine = _MiniEngine(talker, max_model_len=max_model_len) + cfg = talker.config + talker.vocab_sizes = list(cfg.audio_head_levels) + talker.num_heads = cfg.num_audio_heads + talker.head0_vocab = cfg.head0_vocab_size + talker.stop_token = cfg.stop_token + talker.stop_threshold = cfg.stop_threshold + talker.temperature = cfg.temperature + talker._greedy = False + gather_idx, mask = GepardTalkerForConditionalGeneration._build_gather_mask(talker.vocab_sizes) + talker._cb_gather_idx = gather_idx + talker._cb_mask = mask + # Fixed weights: two engine runs must differ only by their RNG. + torch.manual_seed(0) + talker.fused_codebook_head = nn.Linear(talker.model.hidden, sum(talker.vocab_sizes)) + talker.stop_head = nn.Linear(talker.model.hidden, 1) + # Never stop: these requests end on their token budget, so the frame count + # is fixed and only the sampled codes can differ. + torch.nn.init.constant_(talker.stop_head.bias, -20.0) + talker._audio_frame_embed = lambda h0, h: torch.zeros(int(h0.shape[0]), talker.model.hidden) + return talker, engine + + +# -------------------------------------------------------------------------- +# The tail must reach the caller, whichever way the request ends. +# -------------------------------------------------------------------------- + + +def test_last_in_flight_request_delivers_every_committed_frame() -> None: + """The only request runs out of budget: there is no later step to flush in. + + This is the case a deferred flush cannot serve at all — nothing runs after + the final forward. + """ + _talker, _codec, engine = _build() + engine.add_request("req-0", max_tokens=FIRST_CHUNK_FRAMES + CHUNK_FRAMES + 5) + engine.run() + + frames = engine.committed_frames["req-0"] + assert frames > 0 + torch.testing.assert_close( + engine.audio_for("req-0"), + _expected_samples(engine.reqs["req-0"]["tag_base"], frames), + ) + + +@pytest.mark.parametrize( + "max_tokens", + [ + 1, + FIRST_CHUNK_FRAMES, + FIRST_CHUNK_FRAMES + 1, + 30, + FIRST_CHUNK_FRAMES + CHUNK_FRAMES, + 100, + 150, + ], +) +def test_budget_truncated_request_delivers_every_committed_frame(max_tokens: int) -> None: + """Lowering max_tokens must shorten the clip, never silently cut it back + to the last cadence boundary.""" + _talker, _codec, engine = _build() + engine.add_request("req-0", max_tokens=max_tokens) + engine.run() + + frames = engine.committed_frames["req-0"] + assert frames == max_tokens, "every sampled step commits one frame when STOP never fires" + torch.testing.assert_close( + engine.audio_for("req-0"), + _expected_samples(engine.reqs["req-0"]["tag_base"], frames), + ) + + +def test_stop_head_path_still_delivers_every_committed_frame() -> None: + """Regression guard: the path that already worked must keep working.""" + _talker, _codec, engine = _build() + engine.add_request("req-0", max_tokens=4000, stop_after=FIRST_CHUNK_FRAMES + 7) + engine.run() + + frames = engine.committed_frames["req-0"] + assert frames == FIRST_CHUNK_FRAMES + 7, "the STOP frame itself is not committed" + torch.testing.assert_close( + engine.audio_for("req-0"), + _expected_samples(engine.reqs["req-0"]["tag_base"], frames), + ) + + +def test_the_step_after_a_stop_token_commits_nothing(caplog: pytest.LogCaptureFixture) -> None: + """Async scheduling runs one more step on a request that already stopped. + + Committing its frame would strand it — nothing ships after the stop step — + and would make the truncation warning fire on a request that delivered + everything it produced, which is the one thing that warning must not do. + """ + talker, _codec, engine = _build() + stop_after = FIRST_CHUNK_FRAMES + 6 + engine.add_request("req-0", max_tokens=4000, stop_after=stop_after) + + target = logging.getLogger(LOGGER_NAME) + target.addHandler(caplog.handler) + prev = target.level + target.setLevel(logging.WARNING) + try: + engine.run() + # The cleanup that would warn runs on the step after the request + # leaves, and here there is none. Drive it the way the runner would. + talker.on_requests_finished({"req-0"}) + talker._flush_deferred_cleanup() + finally: + target.removeHandler(caplog.handler) + target.setLevel(prev) + + assert engine.reqs["req-0"]["ghost_step_pending"], "the harness did not run the extra step" + frames = engine.committed_frames["req-0"] + assert frames == stop_after, f"the extra step committed a frame: {frames} > {stop_after}" + torch.testing.assert_close( + engine.audio_for("req-0"), + _expected_samples(engine.reqs["req-0"]["tag_base"], frames), + ) + assert not [m for m in caplog.messages if "undelivered frame" in m], ( + f"a fully delivered request was reported as truncated; saw {caplog.messages}" + ) + + +def test_a_resubmitted_request_id_keeps_its_new_state(caplog: pytest.LogCaptureFixture) -> None: + """An aborted id can come back before its cleanup has run. + + The runner treats the two as distinct requests, so the pending free must + not take the second one's state: without that, the state is popped at the + end of the very forward that created it and every later decode falls back + to the zero-embedding path — the request keeps producing frames, so only + the audio is wrong. + """ + _talker, _codec, engine = _build() + engine.add_request("req-0", max_tokens=4000) + for _ in range(FIRST_CHUNK_FRAMES + 2): + engine.step() + engine.abort("req-0") + + # Same id, resubmitted before the model has been told about the abort. + engine.add_request("req-0", max_tokens=FIRST_CHUNK_FRAMES + CHUNK_FRAMES + 5) + + target = logging.getLogger(LOGGER_NAME) + target.addHandler(caplog.handler) + prev = target.level + target.setLevel(logging.WARNING) + try: + engine.run() + finally: + target.removeHandler(caplog.handler) + target.setLevel(prev) + + frames = engine.committed_frames["req-0"] + assert frames == engine.reqs["req-0"]["max_tokens"], "the second request lost frames" + torch.testing.assert_close( + engine.audio_for("req-0"), + _expected_samples(engine.reqs["req-0"]["tag_base"], frames), + msg="the second request did not receive its own audio", + ) + # The abort still gets reported: the cleanup that would have said so is the + # one this cancelled. + assert any("resubmitted" in m for m in caplog.messages), ( + f"the aborted request's truncation went unreported; saw {caplog.messages}" + ) + + +def test_a_preempted_request_is_reported_rather_than_resumed(caplog: pytest.LogCaptureFixture) -> None: + """Recompute preemption re-prefills a request that was mid-generation. + + This model cannot survive that, and the point of the test is that it says + so. The recomputed ids are the prompt plus the head0 codes sampled so far, + which go through the text embedding table, and each frame's other 31 codes + are never in the token stream at all — there is nothing to stitch the two + halves back together with, so generation restarts from frame 0. + + The harness re-prefills with the bare prompt, which understates the damage. + What is under test is the report and the reset, not the corruption, which + needs a real backbone to observe. + """ + _talker, _codec, engine = _build() + engine.add_request("req-0", max_tokens=4000) + for _ in range(FIRST_CHUNK_FRAMES + 2): + engine.step() + assert engine.committed_frames["req-0"] > 0 + + # Preemption the way the scheduler does it: computed tokens back to 0, so + # the next step re-prefills, and no id reaches on_requests_finished. + engine.reqs["req-0"]["computed"] = 0 + + target = logging.getLogger(LOGGER_NAME) + target.addHandler(caplog.handler) + prev = target.level + target.setLevel(logging.WARNING) + try: + engine.step() + finally: + target.removeHandler(caplog.handler) + target.setLevel(prev) + + assert any("preempted" in m for m in caplog.messages), ( + f"a preempted request restarted its audio silently; saw {caplog.messages}" + ) + assert not any("resubmitted" in m for m in caplog.messages), ( + "a preemption was reported as an id resubmit; the two need different words" + ) + assert engine.committed_frames["req-0"] == 1, "the re-prefill must start the frame history over" + + +def test_concurrent_requests_each_receive_their_own_full_audio() -> None: + """One request finishes while another stays live. + + The earlier finisher is the case where a deferred flush *does* run but the + payload is filtered out again, because its id has left the output batch. + """ + _talker, _codec, engine = _build() + engine.add_request("req-short", max_tokens=FIRST_CHUNK_FRAMES + 3) + engine.add_request("req-long", max_tokens=FIRST_CHUNK_FRAMES + CHUNK_FRAMES + 11) + engine.run() + + for req_id in ("req-short", "req-long"): + frames = engine.committed_frames[req_id] + assert frames == engine.reqs[req_id]["max_tokens"] + torch.testing.assert_close( + engine.audio_for(req_id), + _expected_samples(engine.reqs[req_id]["tag_base"], frames), + msg=f"{req_id} lost or gained samples", + ) + + +def test_codec_still_runs_once_per_chunk_not_once_per_frame() -> None: + """Emitting on the final step must not degrade into per-frame decoding.""" + _talker, codec, engine = _build() + num_frames = FIRST_CHUNK_FRAMES + 3 * CHUNK_FRAMES + engine.add_request("req-0", max_tokens=num_frames) + engine.run() + + assert codec.decode_calls == 4, f"expected one decode per chunk, got {codec.decode_calls}" + + +# -------------------------------------------------------------------------- +# Deferred cleanup is cleanup: it must not try to produce output. +# -------------------------------------------------------------------------- + + +def test_deferred_cleanup_never_queues_audio() -> None: + """Anything queued from the cleanup hook is unroutable by construction. + + By the time it runs the id has left ``req_ids_output_copy``, so a payload + under it is dropped; queueing there only risks leaking one request's audio + into the next step's payload for a different request. + """ + talker, _codec, engine = _build() + engine.add_request("req-0", max_tokens=FIRST_CHUNK_FRAMES + 4) + engine.add_request("req-1", max_tokens=FIRST_CHUNK_FRAMES + CHUNK_FRAMES) + while engine.step(): + if engine.reqs["req-0"]["finished"] and not engine.reqs["req-1"]["finished"]: + break + + talker._audio_queue.clear() + talker.on_requests_finished({"req-0"}) + talker._flush_deferred_cleanup() + + assert talker._audio_queue == [], "cleanup emitted audio that routing cannot deliver" + assert "req-0" not in talker._active_states, "cleanup must still free the state" + + +def test_cleanup_warns_when_frames_are_left_undelivered(caplog: pytest.LogCaptureFixture) -> None: + """An abort is the one case with no last step to emit on. Say so.""" + _talker, _codec, engine = _build() + engine.add_request("req-0", max_tokens=4000) + engine.add_request("req-1", max_tokens=4000) + for _ in range(FIRST_CHUNK_FRAMES + 3): + engine.step() + engine.abort("req-0") + + target = logging.getLogger(LOGGER_NAME) + target.addHandler(caplog.handler) + prev = target.level + target.setLevel(logging.WARNING) + try: + engine.step() # cleanup for req-0 runs at the start of this step + finally: + target.removeHandler(caplog.handler) + target.setLevel(prev) + + assert any("undelivered frame" in m for m in caplog.messages), ( + f"aborted request dropped audio without a warning; saw {caplog.messages}" + ) + + +# -------------------------------------------------------------------------- +# A request's seed must reach the in-model sampler. +# -------------------------------------------------------------------------- + +_SEEDED_FRAMES = FIRST_CHUNK_FRAMES + 5 + + +def _run_seeded(seed: int | None, *, neighbour: bool = False, neighbour_seed: int | None = None) -> torch.Tensor: + """Audio delivered to ``req-a``, optionally sharing the batch with another.""" + _talker, engine = _build_with_real_sampler() + engine.add_request("req-a", max_tokens=_SEEDED_FRAMES, seed=seed) + if neighbour: + engine.add_request("req-b", max_tokens=_SEEDED_FRAMES, seed=neighbour_seed) + engine.run() + return engine.audio_for("req-a") + + +def test_a_seeded_request_is_reproducible_within_one_process() -> None: + """The bug this pins: the 32 heads are sampled in ``forward``, so a seed + that only reaches vLLM's sampler leaves the audio at the mercy of the + global RNG — two identical requests in one process then diverge.""" + torch.testing.assert_close(_run_seeded(1234), _run_seeded(1234)) + + +def test_different_seeds_produce_different_audio() -> None: + """Guards the opposite failure: a seed that is threaded but never used.""" + assert not torch.equal(_run_seeded(1234), _run_seeded(4321)) + + +def test_a_seeded_request_is_unaffected_by_its_batch() -> None: + """Each seeded request draws its own noise, so the noise it sees cannot + depend on who else was scheduled alongside it. + + The stub backbone is what makes that checkable. A real one is a bf16 + batched matmul whose rows are not reproducible across batch shapes, so + end-to-end batch invariance is not attainable there; this pins the half + the model actually controls. + """ + torch.testing.assert_close(_run_seeded(1234), _run_seeded(1234, neighbour=True, neighbour_seed=999)) + + +def test_an_unseeded_neighbour_does_not_disturb_a_seeded_request() -> None: + """The mixed case: one row draws from the global RNG, the other must not.""" + torch.testing.assert_close(_run_seeded(1234), _run_seeded(1234, neighbour=True)) + + +# -------------------------------------------------------------------------- +# Budget arithmetic, straight through the helper. +# -------------------------------------------------------------------------- + + +def _info(**kwargs) -> _ReqInfo: + base = { + "request_id": "req-0", + "is_prefill": False, + "prompt_len": 20, + "num_computed_tokens": 20, + "max_tokens": 10, + "seed": None, + } + base.update(kwargs) + return _ReqInfo(**base) + + +def test_prefill_step_is_the_first_output_token() -> None: + talker, _codec, _engine = _build() + first = _info(is_prefill=True, num_computed_tokens=0, max_tokens=1) + assert talker._is_last_output_token(first, span=20, samples_frame=True) is True + more = _info(is_prefill=True, num_computed_tokens=0, max_tokens=2) + assert talker._is_last_output_token(more, span=20, samples_frame=True) is False + + +def test_partial_prefill_chunk_does_not_consume_budget() -> None: + """vLLM discards that row's token, so it must not count against max_tokens.""" + talker, _codec, _engine = _build() + info = _info(is_prefill=True, num_computed_tokens=0, prompt_len=40, max_tokens=1) + assert talker._is_last_output_token(info, span=10, samples_frame=False) is False + + +def test_budget_is_capped_by_max_model_len() -> None: + """A max_tokens larger than the context window never gets reached.""" + talker, _codec, _engine = _build(max_model_len=30) + # prompt 20 + 10 generated fills the window, whatever max_tokens says. + at_cap = _info(num_computed_tokens=28, max_tokens=1000) + assert talker._is_last_output_token(at_cap, span=1, samples_frame=True) is True + below_cap = _info(num_computed_tokens=27, max_tokens=1000) + assert talker._is_last_output_token(below_cap, span=1, samples_frame=True) is False + + +def test_missing_budget_is_not_treated_as_exhausted() -> None: + """A runner that does not thread max_tokens must not trigger a fade.""" + talker, _codec, _engine = _build(max_model_len=0) + assert talker._is_last_output_token(_info(max_tokens=None), span=1, samples_frame=True) is False diff --git a/vllm_omni/config/pipeline_registry.py b/vllm_omni/config/pipeline_registry.py index f06259eaa..69e3ebe11 100644 --- a/vllm_omni/config/pipeline_registry.py +++ b/vllm_omni/config/pipeline_registry.py @@ -51,6 +51,7 @@ from vllm_omni.model_executor.models.dots_tts.pipeline import DOTS_TTS_PIPELINE from vllm_omni.model_executor.models.dreamzero.pipeline import DREAMZERO_PIPELINE from vllm_omni.model_executor.models.dynin_omni.pipeline import DYNIN_OMNI_PIPELINE from vllm_omni.model_executor.models.fish_speech.pipeline import FISH_SPEECH_PIPELINE +from vllm_omni.model_executor.models.gepard.pipeline import GEPARD_PIPELINE from vllm_omni.model_executor.models.glm_image.pipeline import GLM_IMAGE_PIPELINE from vllm_omni.model_executor.models.glm_tts.pipeline import GLM_TTS_PIPELINE from vllm_omni.model_executor.models.gr00t.pipeline import GR00T_N1D7_PIPELINE @@ -138,6 +139,7 @@ OMNI_PIPELINES: dict[str, PipelineConfig | PipelineResolverFunc] = { "lance": LANCE_PIPELINE, "dreamzero": DREAMZERO_PIPELINE, "Gr00tN1d7": GR00T_N1D7_PIPELINE, + "gepard": GEPARD_PIPELINE, "glm_image": GLM_IMAGE_PIPELINE, "hunyuan_image_3_moe": HUNYUAN_IMAGE3_PIPELINE, "hunyuan_image3_ar": HUNYUAN_IMAGE3_AR_PIPELINE, diff --git a/vllm_omni/deploy/gepard.yaml b/vllm_omni/deploy/gepard.yaml new file mode 100644 index 000000000..26000d616 --- /dev/null +++ b/vllm_omni/deploy/gepard.yaml @@ -0,0 +1,40 @@ +# Gepard-1.0: single-stage native-AR FSQ/NanoCodec TTS (~0.5B). +# Target 1x L4 24GB. Zero-shot only; enforce_eager until the perf PR. +# +# The pipeline pin is required, not cosmetic: the checkpoint's config.json +# self-identifies as qwen3_5_text, so model_type inference cannot reach the +# gepard registry entry, and the architectures fallback would match +# Qwen3_5ForCausalLM against the diffusion pipeline. +pipeline: gepard +async_chunk: false +dtype: bfloat16 +trust_remote_code: true + +stages: + - stage_id: 0 + max_num_seqs: 4 + gpu_memory_utilization: 0.3 # AR LM + NanoCodec fit in ~2 GiB + enforce_eager: true + enable_prefix_caching: false + async_scheduling: true + max_num_batched_tokens: 4096 + max_model_len: 4096 + devices: "0" + # Override architectures so vLLM resolves our talker rather than the native + # Qwen3.5 text model. Do NOT override model_type: the backbone's own + # model_type check needs it to stay qwen3_5_text. + hf_overrides: + architectures: ["GepardTalkerForConditionalGeneration"] + default_sampling_params: + # All 32 heads are sampled in-model, so vLLM's sampler is pinned + # deterministic and just argmaxes what forward chose. + temperature: 0.0 + top_p: 1.0 + top_k: -1 + # One output token is one frame, so this is the frame budget: 1000 frames + # is ~46 s at 21.5 fps. A backstop, not a target -- normal utterances are + # 40-420 frames -- but a load-bearing one, because this checkpoint's stop + # head does not fire on every batched trajectory. + max_tokens: 1000 + seed: 42 + detokenize: false diff --git a/vllm_omni/model_executor/models/gepard/__init__.py b/vllm_omni/model_executor/models/gepard/__init__.py new file mode 100644 index 000000000..fb6738a29 --- /dev/null +++ b/vllm_omni/model_executor/models/gepard/__init__.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# NOTE: Do not import model classes in this file. Importing any +# submodule in this package triggers __init__.py execution, and +# both the model registry and pipeline registry import submodules +# directly — heavy imports here would be loaded as a side effect +# even though nothing depends on these re-exports. diff --git a/vllm_omni/model_executor/models/gepard/configuration_gepard.py b/vllm_omni/model_executor/models/gepard/configuration_gepard.py new file mode 100644 index 000000000..d6696f52b --- /dev/null +++ b/vllm_omni/model_executor/models/gepard/configuration_gepard.py @@ -0,0 +1,215 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Config for Gepard-1.0, a single-stage autoregressive TTS. + +Text tokens -> one 32-code FSQ audio frame per step -> NeMo NanoCodec -> +waveform. The backbone is a vLLM-native ``Qwen3_5ForCausalLM``; the codebook +heads, binary stop head and voice-clone ref_compressor are Gepard additions. + +Parses the model's ``gepard_config.json`` sidecar, which nests the LM +parameters under ``backbone_config`` and carries audio-head cardinalities, +special tokens, codec settings and the short-text repetition layout. +""" + +from __future__ import annotations + +from transformers import AutoConfig +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger(__name__) + +_BACKBONE_MODEL_TYPE = "qwen3_5_text" + +# Stripped from the backbone config: the model uses standard 1D RoPE, and these +# would flip vLLM's uses_mrope() to True. +_MROPE_KEYS = ("mrope_section", "mrope_interleaved") + + +class GepardConfig(PretrainedConfig): + """Configuration for the Gepard-1.0 native-AR TTS model. + + Args mirror ``gepard_config.json``. Defaults match the trained + ``nineninesix/gepard-1.0`` checkpoint so an instance built with no + arguments (e.g. dummy/profiling loads) is still self-consistent. + """ + + model_type = "gepard" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + backbone_config: dict | None = None, + audio_heads: dict | None = None, + audio_embed_dim: int = 32, + special_tokens: dict | None = None, + text_repetition: dict | None = None, + codec: dict | None = None, + voice_cloning: dict | None = None, + stop_threshold: float = 0.5, + stop_loss_weight: float = 2.0, + stop_pos_weight: float = 25.0, + temperature: float = 0.3, + **kwargs, + ): + # Everything is assigned before super().__init__() at the bottom: + # transformers 5.x runs validators there that call get_text_config(). + self.backbone_config = self._normalize_backbone(backbone_config) + + self.audio_head_levels = self._parse_audio_heads(audio_heads) + self.num_audio_heads = len(self.audio_head_levels) + self.audio_embed_dim = audio_embed_dim + # head0 is the token vLLM samples; STOP is a synthetic sentinel one past + # its valid range. + self.head0_vocab_size = self.audio_head_levels[0] if self.audio_head_levels else 8 + self.stop_token = self.head0_vocab_size + + st = special_tokens or {} + self.start_of_text = st.get("start_of_text", 248073) + self.end_of_text = st.get("end_of_text", 248074) + self.start_of_speech = st.get("start_of_speech", 248070) + self.end_of_speech = st.get("end_of_speech", 248071) + self.tts_pad = st.get("tts_pad", 248076) + # The speaker placeholder slots begin at tokeniser_length. + self.tokeniser_length = st.get("tokeniser_length", 248077) + self.speaker_token_base = self.tokeniser_length + + # Prompt layout, read by prompt.py. These must match the training + # layout, so they come from the checkpoint rather than a literal. + tr = text_repetition or {} + self.text_repetition_enabled = tr.get("enabled", True) + self.text_repetition_target_tokens = tr.get("target_text_tokens", 16) + self.text_repetition_apply_below = tr.get("apply_below", 13) + self.text_repetition_max_repeats = tr.get("max_repeats", 8) + + # NeMo NanoCodec, which runs outside vLLM. + cc = codec or {} + self.codec_id = cc.get("codec_id", "nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps") + self.codec_sample_rate = cc.get("sample_rate", 22050) + self.codec_frame_rate_hz = cc.get("frame_rate_hz", 21.5) + # Per-group FSQ levels; num_codec_groups * len(fsq_levels) == 32. + self.fsq_levels = cc.get("fsq_levels", [8, 7, 6, 6]) + self.num_codec_groups = cc.get("num_layers", 8) + self.codec_do_unfold = cc.get("do_unfold", True) + + self.stop_threshold = stop_threshold + self.stop_loss_weight = stop_loss_weight + self.stop_pos_weight = stop_pos_weight + # head0 goes through vLLM's SamplingParams; the other 31 heads use this + # in-model. There is no top_p: Gumbel-max has no nucleus step. + self.temperature = temperature + + # Carried for the cloning follow-up; zero-shot uses null_prefix only. + vc = voice_cloning or {} + self.voice_cloning_enabled = vc.get("enabled", True) + comp = vc.get("compressor", {}) or {} + self.num_speaker_prefix = comp.get("num_queries", 8) + self.ref_compressor_num_blocks = comp.get("num_layers", 2) + self.ref_compressor_num_heads = comp.get("num_heads", 8) + self.ref_compressor_d_model = comp.get("d_model", 1024) + self.ref_compressor_ffn_mult = comp.get("ffn_hidden_size_multiplier", 4) + + # Last on purpose — triggers transformers-5.x validators that call + # get_text_config(); every attribute they touch must already exist. + super().__init__(**kwargs) + + @classmethod + def from_checkpoint( + cls, + model: str, + backbone_config: dict | None = None, + revision: str | None = None, + ) -> GepardConfig: + """Build the full config for a checkpoint that self-identifies as the + bare backbone: audio fields from the sidecar, backbone fields from the + loaded config. + + ``revision`` must be the one the weights came from — a revision that + moves the audio-head cardinalities or the special tokens moves the + prompt layout and the STOP sentinel with them. + """ + sidecar: dict = {} + try: + from vllm.transformers_utils.config import get_hf_file_to_dict + + sidecar = dict(get_hf_file_to_dict("gepard_config.json", model, revision=revision) or {}) + except (OSError, ValueError) as e: + # A missing or malformed sidecar is expected; the defaults match the + # trained checkpoint. Anything else must surface. + logger.warning( + "GepardConfig: could not read gepard_config.json from %s (%s: %s); " + "audio fields fall back to trained-checkpoint defaults.", + model, + type(e).__name__, + e, + ) + if backbone_config: + bb = dict(backbone_config) + # The loaded config carries the hf_overrides-patched identity; the + # backbone must stay qwen3_5_text or get_text_config() recurses. + bb["model_type"] = _BACKBONE_MODEL_TYPE + bb.pop("architectures", None) + sidecar["backbone_config"] = bb + return cls(**sidecar) + + @staticmethod + def _parse_audio_heads(audio_heads: dict | None) -> list[int]: + """``{level_audio_0: 8, level_audio_1: 7, ...}`` -> ``[8, 7, ...]``. + + Sorted by the numeric suffix, so channel order does not depend on dict + insertion order. + """ + if not audio_heads: + return [8, 7, 6, 6] * 8 + return [audio_heads[k] for k in sorted(audio_heads, key=lambda s: int(str(s).rsplit("_", 1)[-1]))] + + @classmethod + def _normalize_backbone(cls, backbone_config: dict | None) -> dict: + """Return the backbone dict with vestigial mRoPE fields removed. + + They are template leftovers that the trained checkpoint's own + ``config.json`` does not carry. Left in, they flip vLLM's + ``uses_mrope()`` to True and wire ``positions`` wrongly. + """ + bb = dict(backbone_config or {}) + rope = bb.get("rope_parameters") + if isinstance(rope, dict) and any(k in rope for k in _MROPE_KEYS): + rope = {k: v for k, v in rope.items() if k not in _MROPE_KEYS} + bb["rope_parameters"] = rope + logger.info( + "GepardConfig: stripped vestigial mRoPE keys from backbone rope_parameters " + "(model uses standard 1D default RoPE)." + ) + return bb + + def get_text_config(self, **kwargs) -> PretrainedConfig: + """Return the Qwen3.5 backbone config, which vLLM runs the backbone on.""" + cached = getattr(self, "_text_config", None) + if cached is not None: + return cached + + bb = dict(self.backbone_config) + # pop, not setdefault: for_model()'s first positional is named + # model_type, so leaving the key in **bb passes it twice. + model_type = bb.pop("model_type", _BACKBONE_MODEL_TYPE) + try: + text_config = AutoConfig.for_model(model_type, **bb) + except (KeyError, ValueError) as e: + # Only an unknown model_type is a legitimate fallback. A TypeError + # means the kwargs are wrong and must not degrade silently. + text_config = PretrainedConfig(**bb) + logger.warning( + "GepardConfig: AutoConfig.for_model(%s) failed (%s: %s); using a plain " + "PretrainedConfig for the backbone. Verify vLLM still resolves " + "the Qwen3.5 model.", + model_type, + type(e).__name__, + e, + ) + self._text_config = text_config + return text_config + + +AutoConfig.register("gepard", GepardConfig) + +__all__ = ["GepardConfig"] diff --git a/vllm_omni/model_executor/models/gepard/gepard_talker.py b/vllm_omni/model_executor/models/gepard/gepard_talker.py new file mode 100644 index 000000000..02f5da533 --- /dev/null +++ b/vllm_omni/model_executor/models/gepard/gepard_talker.py @@ -0,0 +1,526 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Gepard-1.0 native-AR talker. + +Single-stage AR TTS on a vLLM-native Qwen3.5 backbone. Each step samples one +32-channel FSQ frame; head0 is the vLLM-facing token and the other 31 are +side-channel. Generation ends on a learned binary stop head, not an EOS token. +The NeMo NanoCodec decodes committed frames to a waveform outside vLLM. + +Zero-shot only, and ``enforce_eager``; CUDA graph is a perf follow-up. +""" + +from __future__ import annotations + +import dataclasses +import os +from collections.abc import Iterable +from typing import Any + +import torch +import torch.nn as nn +from vllm.config import VllmConfig +from vllm.logger import init_logger +from vllm.model_executor.models.qwen3_5 import Qwen3_5ForCausalLM +from vllm.model_executor.models.utils import AutoWeightsLoader, maybe_prefix +from vllm.sequence import IntermediateTensors + +from vllm_omni.model_executor.models.gepard.configuration_gepard import GepardConfig +from vllm_omni.model_executor.models.gepard.nanocodec import NanoCodec +from vllm_omni.model_executor.models.output_templates import OmniOutput +from vllm_omni.platforms import current_omni_platform + +logger = init_logger(__name__) + +# Decode cadence: a short first chunk for early audio, then larger ones to +# amortize the codec call. Does not affect the samples produced. +FIRST_CHUNK_FRAMES = int(os.environ.get("VLLM_GEPARD_FIRST_CHUNK_FRAMES", "10")) +CHUNK_FRAMES = int(os.environ.get("VLLM_GEPARD_CHUNK_FRAMES", "86")) + + +@dataclasses.dataclass(frozen=True) +class _ReqInfo: + """Scheduler state the runner threads into ``preprocess``. + + The hook key strings are read only here, so a rename fails in one place. + """ + + request_id: str + is_prefill: bool + prompt_len: int + num_computed_tokens: int + max_tokens: int | None + seed: int | None + + @classmethod + def from_hook_kwargs(cls, kwargs: dict[str, Any], *, span: int) -> _ReqInfo: + # Defaults cover direct calls, which thread no scheduler state. + max_tokens = kwargs.get("_omni_max_tokens") + seed = kwargs.get("_omni_seed") + return cls( + request_id=str(kwargs.get("request_id", "default")), + is_prefill=bool(kwargs.get("_omni_is_prefill", span > 1)), + prompt_len=int(kwargs.get("_omni_prompt_len") or 0), + num_computed_tokens=int(kwargs.get("_omni_num_computed_tokens") or 0), + max_tokens=None if max_tokens is None else int(max_tokens), + seed=None if seed is None else int(seed), + ) + + +@dataclasses.dataclass +class _GepardState: + """Per-request generation state, keyed by request_id.""" + + request_id: str + # Embedding of the last frame's 32 codes; the next step's input. + curr_embed_for_next: torch.Tensor | None = None + # Frame history (T, 32) handed to the codec. + frames: list[torch.Tensor] = dataclasses.field(default_factory=list) + emitted_frames: int = 0 + # None falls back to null_prefix; the cloning follow-up sets it. + speaker_prefix: torch.Tensor | None = None + # None means the request carried no seed and draws from the global RNG. + generator: torch.Generator | None = None + frame_count: int = 0 + past_first_step: bool = False + is_stopping: bool = False + + +class GepardTalkerForConditionalGeneration(nn.Module): + """Gepard native-AR TTS talker (Qwen3.5 backbone + 32 FSQ heads + stop).""" + + have_multimodal_outputs = True + has_preprocess = True + has_postprocess = False + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + self.vllm_config = vllm_config + cfg = vllm_config.model_config.hf_config + if not isinstance(cfg, GepardConfig): + # hf_overrides patches the loaded config instance rather than + # re-classing it, so what arrives here is the backbone config. + cfg = GepardConfig.from_checkpoint( + vllm_config.model_config.model, + backbone_config=cfg.to_dict(), + revision=vllm_config.model_config.revision, + ) + self.config = cfg + self._device = current_omni_platform.get_torch_device() + # Deterministic argmax sampling, for reproducible comparisons. + self._greedy = os.environ.get("VLLM_GEPARD_GREEDY", "0") == "1" + + self.vocab_sizes: list[int] = list(cfg.audio_head_levels) # [8,7,6,6]*8 + self.num_heads: int = cfg.num_audio_heads + self.head0_vocab: int = cfg.head0_vocab_size + self.stop_token: int = cfg.stop_token # sentinel, one past head0's range + self.total_vocab: int = sum(self.vocab_sizes) + self.stop_threshold: float = cfg.stop_threshold + self.temperature: float = cfg.temperature + hidden = self.config.get_text_config().hidden_size + + self.fused_codebook_head = nn.Linear(hidden, self.total_vocab, bias=True) + self.stop_head = nn.Linear(hidden, 1, bias=True) + self.audio_embeddings = nn.ModuleList([nn.Embedding(v, cfg.audio_embed_dim) for v in self.vocab_sizes]) + self.audio_embed_proj = nn.Sequential( + nn.Linear(self.num_heads * cfg.audio_embed_dim, hidden), + nn.GELU(), + nn.Linear(hidden, hidden), + nn.LayerNorm(hidden, elementwise_affine=False), + ) + self.register_buffer("audio_embed_scale", torch.tensor(1.0)) + # Learned default-voice fallback. + self.null_prefix = nn.Parameter(torch.zeros(cfg.num_speaker_prefix, hidden)) + + gather_idx, mask = self._build_gather_mask(self.vocab_sizes) + self.register_buffer("_cb_gather_idx", gather_idx, persistent=False) + self.register_buffer("_cb_mask", mask, persistent=False) + + self.model = Qwen3_5ForCausalLM(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")) + # Cheap and NeMo-free to construct; load() does the heavy import. + self._codec = NanoCodec(codec_id=cfg.codec_id, sample_rate=cfg.codec_sample_rate) + + self._max_model_len: int = int(vllm_config.model_config.max_model_len or 0) + + self._active_states: dict[str, _GepardState] = {} + # These queues stay index-aligned across preprocess -> forward -> + # compute_logits: preprocess appends in input_batch.req_ids order. + # (req_id, span, samples_frame, is_last_token) + self._pending_requests: list[tuple[str, int, bool, bool]] = [] + # (req_id, head0, do_stop); head0 < 0 marks a discarded placeholder row. + self._results_queue: list[tuple[str, int, bool]] = [] + self._audio_queue: list[tuple[str, torch.Tensor | None]] = [] + self._deferred_cleanup_ids: set[str] = set() + self._sample_rate: int = cfg.codec_sample_rate + + @staticmethod + def _build_gather_mask(vocab_sizes: list[int]) -> tuple[torch.Tensor, torch.Tensor]: + """Gather indices + -inf pad mask so one argmax samples all 31 heads.""" + num_heads = len(vocab_sizes) + max_cb_vocab = max(vocab_sizes[1:]) # == 8 for Gepard; do NOT hardcode 7 + gather_rows, mask_rows = [], [] + cum = vocab_sizes[0] + for i in range(1, num_heads): + v = vocab_sizes[i] + gather_rows.append(list(range(cum, cum + v)) + [cum] * (max_cb_vocab - v)) + mask_rows.append([0.0] * v + [float("-inf")] * (max_cb_vocab - v)) + cum += v + return ( + torch.tensor(gather_rows, dtype=torch.long), + torch.tensor(mask_rows, dtype=torch.float32), + ) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Fuse codebook_heads.{0..31} into fused_codebook_head; delegate the rest.""" + cb_w: dict[str, torch.Tensor] = {} + rest: list[tuple[str, torch.Tensor]] = [] + for name, tensor in weights: + if name.startswith(("aligner.", "supcon_head.")): + continue # training-only + if name.startswith("codebook_heads."): + cb_w[name] = tensor + continue + if name.startswith(("model.", "lm_head.")): + # We nest the backbone one level deeper than the checkpoint + # keys assume, so prepend the extra hop. + name = "model." + name + rest.append((name, tensor)) + + w_parts, b_parts = [], [] + for i in range(self.num_heads): + if (wk := f"codebook_heads.{i}.weight") in cb_w: + w_parts.append(cb_w[wk]) + if (bk := f"codebook_heads.{i}.bias") in cb_w: + b_parts.append(cb_w[bk]) + if w_parts: + rest.append(("fused_codebook_head.weight", torch.cat(w_parts, dim=0))) + if b_parts: + rest.append(("fused_codebook_head.bias", torch.cat(b_parts, dim=0))) + + # NanoCodec loads lazily on first decode, so load_format=dummy and + # NeMo-less environments never pay for it. + loader = AutoWeightsLoader(self, skip_prefixes=["mtp.", "ref_compressor."]) + return loader.load_weights(iter(rest)) + + def _get_or_create_state(self, request_id: str) -> _GepardState: + st = self._active_states.get(request_id) + if st is None: + st = _GepardState(request_id=request_id) + self._active_states[request_id] = st + return st + + def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None: + # Fires BEFORE forward -> defer the actual free until after forward runs, + # else we'd drop state the in-flight forward still reads. + self._deferred_cleanup_ids.update(str(r) for r in finished_req_ids) + + def _flush_deferred_cleanup(self) -> None: + # Cleanup only. Audio queued here is never delivered — the id has left + # the output batch by now — so the tail is emitted in forward instead, + # and anything still pending here is genuinely lost. Say so. + for req_id in self._deferred_cleanup_ids: + state = self._active_states.pop(req_id, None) + if state is not None and state.frame_count > state.emitted_frames: + logger.warning( + "Gepard request %s ended with %d undelivered frame(s): it finished " + "without the stop head firing and without reaching its token budget " + "(aborted or preempted). Its audio is truncated.", + req_id, + state.frame_count - state.emitted_frames, + ) + self._deferred_cleanup_ids.clear() + + def _is_last_output_token(self, info: _ReqInfo, *, span: int, samples_frame: bool) -> bool: + """Whether the token sampled this step is the request's budget-final one. + + That step is the last one on which the request is still in the output + batch, so it is also the last chance to ship its audio. Counted off the + engine's cursor rather than a local tally, so a re-prefill after + preemption cannot desync it. + """ + if not samples_frame: + # A partial prefill chunk's token is discarded, so it costs no budget. + return False + budget = info.max_tokens + if self._max_model_len: + len_cap = self._max_model_len - info.prompt_len + budget = len_cap if budget is None else min(budget, len_cap) + if budget is None or budget <= 0: + return False + produced = info.num_computed_tokens + span - info.prompt_len + 1 + return produced >= budget + + def _emit_audio(self, state: _GepardState, *, is_final: bool) -> None: + """Decode this request's un-emitted frames and queue the new samples. + + Queues deltas: each call contributes only samples not queued before. + """ + pending = state.frame_count - state.emitted_frames + if pending <= 0: + return + if not is_final: + threshold = FIRST_CHUNK_FRAMES if state.emitted_frames == 0 else CHUNK_FRAMES + if pending < threshold: + return + + frames = torch.stack(state.frames) # (T, 32) + if not self._codec.is_loaded: + # The first decode pays the codec load; warmup is a perf follow-up. + self._codec.load(frames.device) + audio = self._codec.decode_stream(frames, start_idx=state.emitted_frames, is_final=is_final) + if audio is None: + return + self._audio_queue.append((state.request_id, audio)) + state.emitted_frames = state.frame_count + + def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + + @staticmethod + def _gumbel_noise(ref: torch.Tensor, generators: list[torch.Generator | None] | None) -> torch.Tensor: + """Gumbel noise shaped like ``ref``; row i uses ``generators[i]`` if set. + + A seeded row draws its own noise of a fixed shape, so the noise it sees + does not depend on who shared its batch. Only the noise: the logits come + from a bf16 matmul that is not row-wise reproducible, so a batched + request can still sample differently from a solo one. + """ + if not generators or all(g is None for g in generators): + u = torch.rand_like(ref) + else: + u = torch.stack( + [ + torch.rand(ref.shape[1:], generator=g, device=ref.device, dtype=ref.dtype) + if g is not None + else torch.rand(ref.shape[1:], device=ref.device, dtype=ref.dtype) + for g in generators + ] + ) + return -torch.log(-torch.log(u.clamp_min_(1e-20))) + + def _sample_frame( + self, + hidden: torch.Tensor, + generators: list[torch.Generator | None] | None = None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """(N, hidden) -> head0 (N,), heads_1_31 (N, 31), do_stop (N,). + + ``generators`` is row-aligned with ``hidden``. The 32 heads are sampled + here rather than by vLLM's sampler, so a request's seed only reaches + the audio through this. + """ + fused = self.fused_codebook_head(hidden) + + gathered = fused[:, self._cb_gather_idx].float() + self._cb_mask # (N,31,8) + h0_logits = fused[:, : self.head0_vocab].float() + if self._greedy: + heads = gathered.argmax(dim=-1) + head0 = h0_logits.argmax(dim=-1) + else: + heads = (gathered / self.temperature + self._gumbel_noise(gathered, generators)).argmax(dim=-1) + head0 = (h0_logits / self.temperature + self._gumbel_noise(h0_logits, generators)).argmax(dim=-1) + + p_stop = torch.sigmoid(self.stop_head(hidden)).squeeze(-1) + do_stop = p_stop > self.stop_threshold + return head0, heads, do_stop + + def _audio_frame_embed(self, head0: torch.Tensor, heads_1_31: torch.Tensor) -> torch.Tensor: + """Rebuild the next step's input embedding from the 32 sampled codes.""" + codes = torch.cat([head0.clamp(max=self.head0_vocab - 1).unsqueeze(-1), heads_1_31], dim=-1) + parts = [self.audio_embeddings[i](codes[:, i]) for i in range(self.num_heads)] + return self.audio_embed_proj(torch.cat(parts, dim=-1)) * self.audio_embed_scale + + def preprocess(self, input_ids: torch.Tensor, input_embeds: torch.Tensor | None = None, **req_infos: Any): + # Prefill injects the speaker prefix at the placeholder slots. The + # prompt ends on SOS, so forward samples the first frame there. + dev = input_ids.device + span = int(input_ids.shape[0]) + info = _ReqInfo.from_hook_kwargs(req_infos, span=span) + req_id = info.request_id + is_prefill = info.is_prefill + prompt_len = info.prompt_len + computed = info.num_computed_tokens + # A prefill chunk that does not finish the prompt commits no frame: + # vLLM discards its sampled token. + samples_frame = (not is_prefill) or prompt_len <= 0 or computed + span >= prompt_len + + if is_prefill: + state = self._get_or_create_state(req_id) + # An aborted id can be resubmitted, and the runner treats the two as + # distinct requests. Drop the pending free, or the cleanup at the + # end of this forward takes the new request's state with it. + resubmitted = req_id in self._deferred_cleanup_ids + self._deferred_cleanup_ids.discard(req_id) + if resubmitted: + # Stand in for the cleanup just cancelled. + undelivered = state.frame_count - state.emitted_frames + if undelivered > 0: + logger.warning( + "Gepard request %s ended with %d undelivered frame(s) and its id has been " + "resubmitted; the first request's audio is truncated.", + req_id, + undelivered, + ) + elif state.frame_count: + # Not a resubmit, so this is a preemption recompute, and it + # cannot be recovered: a frame's 31 side-channel codes are not + # in the token stream, so the replayed prompt cannot rebuild the + # KV that produced the audio already delivered. Generation + # restarts from frame 0 and the caller gets two unrelated halves. + logger.warning( + "Gepard request %s was preempted after %d frame(s) and is being recomputed; " + "its audio will not continue from where it stopped. Raise the stage's " + "gpu_memory_utilization or lower max_num_seqs to keep requests from being " + "preempted.", + req_id, + state.frame_count, + ) + state.curr_embed_for_next = None + state.frames.clear() + state.emitted_frames = 0 + state.frame_count = 0 + state.past_first_step = False + state.is_stopping = False + # SamplingParams.seed only reaches vLLM's sampler, so it never + # touches the audio unless it is threaded into a generator here. + state.generator = None + if info.seed is not None: + state.generator = torch.Generator(device=dev) + state.generator.manual_seed(info.seed) + + embeds = self.model.embed_input_ids(input_ids) + prefix = state.speaker_prefix if state.speaker_prefix is not None else self.null_prefix + base = self.config.speaker_token_base + spk = (input_ids >= base) & (input_ids < base + self.config.num_speaker_prefix) + if bool(spk.any()): + embeds[spk] = prefix[input_ids[spk] - base].to(dtype=embeds.dtype) + else: + state = self._active_states.get(req_id) + curr = state.curr_embed_for_next if state is not None else None + if curr is not None: + embeds = curr.to(device=dev).reshape(1, -1) + else: + # Defensive only: a decode step always follows a forward that + # stashed curr_embed_for_next. + hidden = self.config.get_text_config().hidden_size + embeds = torch.zeros(1, hidden, device=dev, dtype=self.null_prefix.dtype) + + is_last_token = self._is_last_output_token(info, span=span, samples_frame=samples_frame) + self._pending_requests.append((req_id, span, samples_frame, is_last_token)) + return input_ids, embeds, {} + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **kwargs: Any, + ) -> torch.Tensor | OmniOutput: + # positions is plain 1D: Gepard uses default RoPE, not mRoPE. input_ids + # passes through only when no embeds were spliced (the profile run). + hidden = self.model( + input_ids=input_ids if inputs_embeds is None else None, + positions=positions, + intermediate_tensors=intermediate_tensors, + inputs_embeds=inputs_embeds, + ) + # Empty _pending_requests means the engine-init profile run: skip. + if self._pending_requests: + # One sampling row per request: the last scheduled position of its + # span. Same row set compute_logits sees at logits_indices. + ends, generators, offset = [], [], 0 + for req_id, span, _samples_frame, _is_last in self._pending_requests: + offset += span + ends.append(offset - 1) + state = self._active_states.get(req_id) + generators.append(state.generator if state is not None else None) + rows = hidden[torch.tensor(ends, device=hidden.device)] + head0, heads_1_31, want_stop = self._sample_frame(rows, generators) + frame_embeds = self._audio_frame_embed(head0, heads_1_31) + head0_l = head0.tolist() + stop_l = want_stop.tolist() + for i, (req_id, _span, samples_frame, is_last_token) in enumerate(self._pending_requests): + if not samples_frame: + # Partial prefill chunk: vLLM discards this row's token. + self._results_queue.append((req_id, -1, False)) + continue + state = self._get_or_create_state(req_id) + if state.is_stopping: + # Async scheduling runs one more step after a stop token — + # only the max_tokens finish is guarded against it. This + # step's output is discarded, so a frame committed here + # would be stranded and reported as lost audio. + self._results_queue.append((req_id, int(head0_l[i]), True)) + continue + # Stop may only fire past the first audio frame, so the + # prefill-sampled frame and the one after it always commit. + do_stop = bool(stop_l[i]) and state.past_first_step + if do_stop: + state.is_stopping = True + else: + state.frames.append(torch.cat([head0[i : i + 1], heads_1_31[i]])) + state.frame_count += 1 + state.curr_embed_for_next = frame_embeds[i] + state.past_first_step = True + self._results_queue.append((req_id, int(head0_l[i]), do_stop)) + # Both ways a request ends have to flush here: this is the last + # step on which it is still in the output batch, and a payload + # queued after it is dropped by routing. + self._emit_audio(state, is_final=do_stop or is_last_token) + self._pending_requests.clear() + self._flush_deferred_cleanup() + return hidden + + def compute_logits(self, hidden_states: torch.Tensor | OmniOutput, sampling_metadata: Any = None): + # Vocab-wide logits carrying head0, or STOP for a stopped row. Requires + # the pipeline's sampling_constraints, without which vLLM never acts on + # the sentinel. + if isinstance(hidden_states, OmniOutput): + hidden_states = hidden_states.text_hidden_states + if hidden_states is None: + return None + bsz = hidden_states.shape[0] + vocab = self.config.get_text_config().vocab_size + logits = torch.full((bsz, vocab), float("-inf"), device=hidden_states.device, dtype=hidden_states.dtype) + if not self._results_queue: + # Engine-init dummy sampler run: neutral one-hot logits so KV-cache + # profiling can complete. + logits[:, 0] = 0.0 + return logits + # Row i belongs to _results_queue[i]. The pipeline pins temperature=0.0, + # so vLLM's sampler argmaxes the single finite entry. + for i, (_req_id, head0, do_stop) in enumerate(self._results_queue): + if i >= bsz: + break + if head0 < 0: + logits[i, 0] = 0.0 # partial-prefill placeholder; vLLM discards it + continue + logits[i, self.stop_token if do_stop else head0] = 0.0 + self._results_queue.clear() + return logits + + def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput, **kwargs: Any) -> OmniOutput: + if isinstance(model_outputs, OmniOutput): + return model_outputs + by_req: dict[str, torch.Tensor] = {} + for req_id, audio in self._audio_queue: + if audio is None: + continue + by_req[req_id] = ( + torch.cat([by_req[req_id].reshape(-1), audio.reshape(-1)], dim=0) if req_id in by_req else audio + ) + self._audio_queue.clear() + + # The sparse marker is always carried, even on a step that decoded + # nothing: without it the runner ships the pooler payload instead, whose + # hidden states land under this stage's "audio" key. + ready_req_ids = list(by_req) + sr = torch.tensor(self._sample_rate, dtype=torch.int32) + mm: dict[str, Any] = { + "model_outputs": [by_req[r].reshape(-1) for r in ready_req_ids], + "sr": [sr for _ in ready_req_ids], + "meta": {"req_id": ready_req_ids, "sparse_audio": ["1"]}, + } + # Deltas, not the whole history: callers concatenate across steps. + return OmniOutput(text_hidden_states=model_outputs, multimodal_outputs=mm) diff --git a/vllm_omni/model_executor/models/gepard/nanocodec.py b/vllm_omni/model_executor/models/gepard/nanocodec.py new file mode 100644 index 000000000..2f93d310a --- /dev/null +++ b/vllm_omni/model_executor/models/gepard/nanocodec.py @@ -0,0 +1,172 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""NeMo NanoCodec wrapper for Gepard — per-dimension FSQ decode. + +Gepard's 32 FSQ heads emit per-dimension codes (8 groups x 4 dims), while stock +NeMo decode wants composed per-group mixed-radix indices. ``decode_from_codes`` +dequantizes per-dim instead and hands the result to stock ``decode_audio`` — a +few lines on public NeMo API, no fork. + +NeMo is a heavy optional dependency, imported lazily inside ``load``. Decode +only; ``encode_to_dequantized`` belongs to the voice-cloning follow-up. +""" + +from __future__ import annotations + +import os + +import torch +import torch.nn as nn +from vllm.logger import init_logger + +logger = init_logger(__name__) + +# Left-context frames prepended to every streaming decode window. NanoCodec is +# convolutional, so decoding a bare chunk boundary yields a discontinuity; the +# extra frames are decoded for receptive-field context and trimmed afterwards. +# Mirrors the reference server's TTS_LOOKBACK_FRAMES. +LOOKBACK_FRAMES = int(os.environ.get("VLLM_GEPARD_LOOKBACK_FRAMES", "8")) + +# End-of-clip shaping. The codec leaves the waveform on a non-zero sample and +# the hard step to silence is an audible click; a linear fade ramps it out. +# Both default to 0.0 (off), matching the reference server's defaults. +END_OF_SPEECH_FADE_MS = float(os.environ.get("VLLM_GEPARD_END_FADE_MS", "0.0")) +END_OF_SPEECH_SILENCE_MS = float(os.environ.get("VLLM_GEPARD_END_SILENCE_MS", "0.0")) + + +def apply_end_of_speech_tail( + audio: torch.Tensor, + sample_rate: int, + fade_ms: float = END_OF_SPEECH_FADE_MS, + silence_ms: float = END_OF_SPEECH_SILENCE_MS, +) -> torch.Tensor: + """Fade the final ``fade_ms`` to zero, then append ``silence_ms`` of pad. + + The fade is what removes the end-of-clip click; padding alone does not. + """ + if audio.numel() == 0 or (fade_ms <= 0 and silence_ms <= 0): + return audio + out = audio + if fade_ms > 0: + n = min(out.shape[0], int(sample_rate * fade_ms / 1000.0)) + if n > 0: + # Copy: the caller's tensor is still referenced by the audio queue. + out = out.clone() + out[-n:] *= torch.linspace(1.0, 0.0, n, dtype=out.dtype, device=out.device) + if silence_ms > 0: + pad = torch.zeros(int(sample_rate * silence_ms / 1000.0), dtype=out.dtype, device=out.device) + out = torch.cat([out, pad], dim=0) + return out + + +def _make_unfolded_codec_cls(): + """Build the ``UnfoldedCodecModel`` subclass. Imports NeMo (optional dep). + + Defined lazily so importing this module never requires NeMo. The subclass + is OUR code layered on stock ``AudioCodecModel`` — not a NeMo fork. + """ + try: + from nemo.collections.tts.models import AudioCodecModel # noqa: PLC0415 + from omegaconf import open_dict # noqa: PLC0415 + except ImportError as e: # pragma: no cover - env-dependent + raise ImportError( + "Gepard TTS needs NeMo for the NanoCodec decoder. Install it with " + "`pip install nemo_toolkit[tts]` (heavy; NVIDIA Open Model License, " + "commercial use OK), or gate the model behind that extra." + ) from e + + class UnfoldedCodecModel(AudioCodecModel): + """AudioCodecModel + direct decode from per-dimension FSQ codes.""" + + def __init__(self, cfg, trainer=None): + # SLMDiscriminator pulls microsoft/wavlm-base-plus (~360 MB) and is + # training-only — strip it from the config before init. + with open_dict(cfg): + disc = cfg.get("discriminator", None) + if disc is not None and "discriminators" in disc: + disc.discriminators = [d for d in disc.discriminators if "SLM" not in d._target_] + super().__init__(cfg, trainer) + + def decode_from_codes(self, codes: torch.Tensor, codes_len: torch.Tensor): + """Decode per-dimension discrete FSQ codes to a waveform. + + codes: (B, D, T) per-dim discrete values, D = num_groups * dims. + codes_len: (B,) valid frame count per batch element. + Returns (audio (B, T_audio), audio_len (B,)). + """ + num_levels = self.vector_quantizer.fsqs[0].num_levels.squeeze() + scale = (num_levels // 2).float().to(codes.device) # e.g. [4,3,3,3] + groups = codes.chunk(self.vector_quantizer.num_groups, dim=1) + dequantized = torch.cat( + [(g - scale[None, :, None]) / scale[None, :, None] for g in groups], + dim=1, + ) + return self.decode_audio(inputs=dequantized, input_len=codes_len) + + return UnfoldedCodecModel + + +class NanoCodec(nn.Module): + """Thin wrapper owning the (optional) NeMo NanoCodec decoder. + + Constructing this is cheap and NeMo-free; ``load`` does the heavy import + + weight load so ``load_format=dummy`` and NeMo-less environments don't pay + for it at model construction. + """ + + def __init__(self, codec_id: str, sample_rate: int) -> None: + super().__init__() + self.codec_id = codec_id + self.sample_rate = sample_rate + self._codec: nn.Module | None = None + + @property + def is_loaded(self) -> bool: + return self._codec is not None + + def load(self, device: torch.device) -> None: + """Import NeMo, build the UnfoldedCodecModel, move to device, eval().""" + cls = _make_unfolded_codec_cls() + codec = cls.from_pretrained(self.codec_id) + codec.to(device=device) + codec.eval() + self._codec = codec + logger.info("Gepard NanoCodec loaded from %s on %s", self.codec_id, device) + + @torch.inference_mode() + def decode_from_codes(self, codes: torch.Tensor, codes_len: torch.Tensor) -> torch.Tensor: + """(B, 32, T) per-dim codes -> waveform (B, T_audio). Returns audio only.""" + if self._codec is None: + raise RuntimeError("NanoCodec.decode_from_codes called before load()") + audio, _audio_len = self._codec.decode_from_codes(codes, codes_len) + return audio + + @torch.inference_mode() + def decode_stream( + self, + frames: torch.Tensor, + *, + start_idx: int, + lookback: int = LOOKBACK_FRAMES, + is_final: bool = False, + ) -> torch.Tensor | None: + """Decode the tail of a ``(T, 32)`` code history past ``start_idx``. + + The window starts ``lookback`` frames earlier for receptive-field + context and those samples are trimmed after, so successive calls + concatenate to a single whole-history decode. None if nothing is new. + """ + count = int(frames.shape[0]) + if count <= start_idx: + return None + decode_start = max(0, start_idx - lookback) + window = frames[decode_start:count] # (T_win, 32) + codes = window.t().unsqueeze(0).contiguous().long() # (1, 32, T_win) + codes_len = torch.full((1,), codes.shape[-1], dtype=torch.long, device=codes.device) + audio = self.decode_from_codes(codes, codes_len).reshape(-1) + # Derive samples_per_frame from this decode rather than assuming it. + samples_per_frame = audio.shape[0] // (count - decode_start) + audio = audio[(start_idx - decode_start) * samples_per_frame :] + if is_final: + audio = apply_end_of_speech_tail(audio, self.sample_rate) + return audio diff --git a/vllm_omni/model_executor/models/gepard/pipeline.py b/vllm_omni/model_executor/models/gepard/pipeline.py new file mode 100644 index 000000000..e6ce4600d --- /dev/null +++ b/vllm_omni/model_executor/models/gepard/pipeline.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Gepard-1.0 pipeline topology. + +Single-stage AR TTS. The Qwen3.5 backbone runs under paged attention, the 32 +codebook heads and the binary stop head are sampled inside the talker, and the +NeMo NanoCodec decodes the accumulated frames outside vLLM. + +``compute_logits`` scatters head0 into vLLM's vocab-wide logits; head0's valid +range is 0..7, so the synthetic STOP sentinel is 8. ``stop_token_ids=[8]`` +couples it to vLLM's stop machinery, with ``detokenize=False`` because head0 is +an audio code. Same idiom as VoxCPM2/MOSS. +""" + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +# head0 valid range is 0..7 (its FSQ cardinality is 8); STOP is the sentinel +# one past that range. Mirrors the reference config.py STOP_TOKEN = 8. +_GEPARD_STOP_TOKEN_ID = 8 + +GEPARD_PIPELINE = PipelineConfig( + model_type="gepard", + default_deploy_config_name="gepard.yaml", + model_arch="GepardTalkerForConditionalGeneration", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="gepard", + execution_type=StageExecutionType.LLM_AR, + input_sources=(), + final_output=True, + final_output_type="audio", + owns_tokenizer=True, + engine_output_type="audio", + sampling_constraints={ + "detokenize": False, + "stop_token_ids": [_GEPARD_STOP_TOKEN_ID], + }, + ), + ), +) diff --git a/vllm_omni/model_executor/models/gepard/prompt.py b/vllm_omni/model_executor/models/gepard/prompt.py new file mode 100644 index 000000000..e6a29bc8f --- /dev/null +++ b/vllm_omni/model_executor/models/gepard/prompt.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Gepard-1.0 prompt layout. + +``preprocess`` only consumes this layout — it injects the speaker prefix at the +placeholder slots and samples the first audio frame from the SOS position. The +layout itself is built here, by whichever entry point is assembling a request. +""" + +from __future__ import annotations + +import math +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from vllm_omni.model_executor.models.gepard.configuration_gepard import GepardConfig + + +def build_gepard_prompt_ids(text_token_ids: list[int], *, config: GepardConfig) -> list[int]: + """Assemble ``[speaker_slots] + [SOT, *text, EOT]*(R-1) + [SOT, *text, EOT, SOS]``. + + Only the final copy carries SOS, the learned "audio starts now" gate, so + the earlier copies are read as context and never voiced. Short texts repeat + because a couple of text tokens carry too little mass against the speaker + prefix. This must match the training layout or WER collapses, which is why + the thresholds come from the checkpoint's ``text_repetition`` block rather + than from literals here. + + Raises ``ValueError`` on empty text: the layout would still be structurally + valid and the model would voice it as arbitrary audio, so an empty request + has to fail loudly. This is the only entry point in the offline path — there + is no adapter above it to check first. + """ + n = len(text_token_ids) + if n == 0: + raise ValueError("text_token_ids is empty; Gepard needs at least one text token to voice") + if ( + not config.text_repetition_enabled + or n >= config.text_repetition_apply_below + or n >= config.text_repetition_target_tokens + ): + repeats = 1 + else: + repeats = max( + 1, + min( + math.ceil(config.text_repetition_target_tokens / n), + config.text_repetition_max_repeats, + ), + ) + + block = [config.start_of_text, *text_token_ids, config.end_of_text] + text_region = block * (repeats - 1) + [*block, config.start_of_speech] + speaker_slots = [config.speaker_token_base + i for i in range(config.num_speaker_prefix)] + return speaker_slots + text_region diff --git a/vllm_omni/model_executor/models/registry.py b/vllm_omni/model_executor/models/registry.py index 530fb1712..3cf888d46 100644 --- a/vllm_omni/model_executor/models/registry.py +++ b/vllm_omni/model_executor/models/registry.py @@ -295,6 +295,12 @@ _OMNI_MODELS = { "fish_speech_dac_decoder", "FishSpeechDACDecoder", ), + ## Gepard-1.0 (nineninesix) — single-stage native-AR FSQ/NanoCodec TTS + "GepardTalkerForConditionalGeneration": ( + "gepard", + "gepard_talker", + "GepardTalkerForConditionalGeneration", + ), ## VoxCPM2 "VoxCPM2TalkerForConditionalGeneration": ( "voxcpm2", diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py index 2edea7fff..7b3bb2cf9 100644 --- a/vllm_omni/worker/gpu_model_runner.py +++ b/vllm_omni/worker/gpu_model_runner.py @@ -1772,6 +1772,16 @@ class OmniGPUModelRunner(GPUModelRunner): req_infos["_omni_prompt_len"] = prompt_len req_infos["_omni_num_computed_tokens"] = num_computed_tokens req_infos["_omni_is_prefill"] = is_prefill + # Output-token cap, so a model that must ship a payload on the + # request's final step can tell which step that is. A finished + # request drops out of req_ids_output_copy, and downstream + # routing keys off that list, so a payload produced after its + # last step never reaches the caller. + sampling_params = getattr(req_state, "sampling_params", None) + req_infos["_omni_max_tokens"] = getattr(sampling_params, "max_tokens", None) + # Seed, so a model that samples inside forward() can be + # reproducible: vLLM's own sampler seeding does not reach it. + req_infos["_omni_seed"] = getattr(sampling_params, "seed", None) if callable(batch_decode_preprocess) and self.has_talker_mtp and span_len == 1 and not is_prefill: decode_batch_items.append((req_id, s, req_infos)) continue From 6f36fd3b7838c4de3a4b9d4de90c3bba7776c91e Mon Sep 17 00:00:00 2001 From: Ma1oneZhang <73012001+Ma1oneZhang@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:11:48 +0800 Subject: [PATCH 021/267] [Bugfix] Preserve object storage URIs during model resolution (#5036) Signed-off-by: Ziyang Zhang --- tests/config/test_config_factory.py | 90 ++++++++++++++++++- .../test_omni_snapshot_download.py | 60 +++++++++++++ tests/entrypoints/test_utils.py | 43 +++++++++ vllm_omni/config/config_factory.py | 48 ++++++++-- vllm_omni/entrypoints/omni_base.py | 9 ++ vllm_omni/entrypoints/utils.py | 35 +++++--- 6 files changed, 267 insertions(+), 18 deletions(-) diff --git a/tests/config/test_config_factory.py b/tests/config/test_config_factory.py index 9708c1c50..5bfbb67e8 100644 --- a/tests/config/test_config_factory.py +++ b/tests/config/test_config_factory.py @@ -14,7 +14,8 @@ import pytest from transformers import PretrainedConfig, Qwen3OmniMoeConfig from tests.helpers.stage_config import get_deploy_config_path, get_deploy_config_stage -from vllm_omni.config.config_factory import StageConfigFactory +from vllm_omni.config import config_factory as config_factory_module +from vllm_omni.config.config_factory import StageConfigFactory, _materialize_object_storage_configs from vllm_omni.config.endpoint_policy import EndpointRestriction, OmniServingCapability from vllm_omni.config.omni_config import VllmOmniConfig from vllm_omni.config.pipeline_registry import OMNI_PIPELINES, register_pipeline, resolve_pipeline_config @@ -55,6 +56,7 @@ def clear_config_factory_caches(): yield StageConfigFactory.get_hf_config.cache_clear() StageConfigFactory.try_infer_model_type.cache_clear() + _materialize_object_storage_configs.cache_clear() Q3_OMNI_ALL_STAGES_HF_CONFIG = Qwen3OmniMoeConfig(enable_audio_output=True) @@ -2765,3 +2767,89 @@ class TestPipelineConfigResolvers: pass assert resolver(NotTheRightHfConfig()) is None + + +class TestObjectStorageConfigResolution: + """Object-storage URIs must be materialized locally before any HF-style reads. + + Regression coverage for review on the Run:AI PR: ``get_config`` and the + config.json/model_index.json fallbacks crashed with ``HFValidationError`` + for ``s3://`` URIs, and the name-match fallback scanned the whole URI, so + a bucket named after another pipeline could hijack pipeline selection. + """ + + @pytest.fixture + def fake_object_storage(self, monkeypatch, tmp_path): + """Replace ObjectStorageModel with a fake that "pulls" into a tmp dir. + + Returns a recorder: calls to the returned callable write files into + the materialized directory, and ``recorder.pulls`` records every + ``pull_files`` invocation. + """ + pulls: list[tuple[str, list[str] | None, list[str] | None]] = [] + materialized_files: list[tuple[str, str]] = [] + + class FakeObjectStorageModel: + def __init__(self, url: str): + self.dir = str(tmp_path) + + def pull_files(self, model_path, allow_pattern=None, ignore_pattern=None): + pulls.append((model_path, allow_pattern, ignore_pattern)) + for name, content in materialized_files: + (tmp_path / name).write_text(content) + + monkeypatch.setattr(config_factory_module, "ObjectStorageModel", FakeObjectStorageModel) + + class Recorder: + def set_files(self, files: list[tuple[str, str]]) -> None: + materialized_files.clear() + materialized_files.extend(files) + + recorder = Recorder() + recorder.pulls = pulls + return recorder + + def test_passthrough_for_non_uri(self, fake_object_storage): + assert _materialize_object_storage_configs("org/model") == "org/model" + assert _materialize_object_storage_configs("/local/model") == "/local/model" + assert fake_object_storage.pulls == [] + + def test_materialize_pulls_configs_once_per_uri(self, fake_object_storage): + uri = "s3://bucket/model" + + first = _materialize_object_storage_configs(uri) + second = _materialize_object_storage_configs(uri) + + assert first == second + assert len(fake_object_storage.pulls) == 1 + pulled_uri, allow_pattern, ignore_pattern = fake_object_storage.pulls[0] + assert pulled_uri == uri + assert "*.json" in allow_pattern + assert ignore_pattern is None + + def test_try_infer_model_type_reads_materialized_config(self, fake_object_storage): + """A misleading bucket name must not beat the real config.json content.""" + fake_object_storage.set_files( + [ + ( + "config.json", + '{"model_type": "qwen3_omni_moe", "architectures": ["Qwen3OmniMoeForConditionalGeneration"]}', + ) + ] + ) + uri = "s3://qwen3-tts-models/Qwen3-Omni-30B-A3B-Instruct" + + model = StageConfigFactory.try_infer_model_type(model=uri, trust_remote_code=False) + + assert model == "qwen3_omni_moe" + assert fake_object_storage.pulls # materialization happened + + def test_name_match_fallback_scans_basename_only(self, fake_object_storage): + # Empty config.json (CosyVoice3 style) forces the name-match fallback. + fake_object_storage.set_files([("config.json", "{}")]) + + deceptive = "s3://qwen3-tts-models/plain-checkpoint" + assert StageConfigFactory.try_infer_model_type(model=deceptive, trust_remote_code=False) is None + + matching = "s3://any-bucket/my-cosyvoice3-model" + assert StageConfigFactory.try_infer_model_type(model=matching, trust_remote_code=False) == "cosyvoice3" diff --git a/tests/entrypoints/test_omni_snapshot_download.py b/tests/entrypoints/test_omni_snapshot_download.py index d2c76a249..58dd1d19d 100644 --- a/tests/entrypoints/test_omni_snapshot_download.py +++ b/tests/entrypoints/test_omni_snapshot_download.py @@ -10,8 +10,10 @@ ModelScope path. import sys import types +from pathlib import Path import pytest +from pytest_mock import MockerFixture from vllm import envs from vllm_omni.entrypoints import omni_base @@ -21,6 +23,64 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] MODEL_ID = "Qwen/Qwen3-Omni-30B-A3B-Instruct" +@pytest.mark.parametrize( + "model_uri", + [ + "s3://bucket/model", + "gs://bucket/model", + "az://bucket/model", + ], +) +def test_omni_snapshot_download_preserves_object_storage_uri( + model_uri: str, + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("VLLM_USE_MODELSCOPE", raising=False) + hf_download = mocker.patch.object(omni_base, "download_weights_from_hf_specific") + + assert omni_base.omni_snapshot_download(model_uri) == model_uri + hf_download.assert_not_called() + + +def test_omni_snapshot_download_preserves_existing_local_path( + tmp_path: Path, + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("VLLM_USE_MODELSCOPE", raising=False) + absolute_model_path = tmp_path / "absolute-model" + absolute_model_path.mkdir() + relative_model_path = "relative-model" + (tmp_path / relative_model_path).mkdir() + monkeypatch.chdir(tmp_path) + hf_download = mocker.patch.object(omni_base, "download_weights_from_hf_specific") + + assert omni_base.omni_snapshot_download(str(absolute_model_path)) == str(absolute_model_path) + assert omni_base.omni_snapshot_download(relative_model_path) == relative_model_path + hf_download.assert_not_called() + + +def test_omni_snapshot_download_uses_hf_for_relative_repo_id( + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("VLLM_USE_MODELSCOPE", raising=False) + model_id = "org/model" + # Stub the modular-index probe so the test never reaches huggingface.co. + hf_file_probe = mocker.patch.object(omni_base, "file_or_path_exists", return_value=False) + hf_download = mocker.patch.object(omni_base, "download_weights_from_hf_specific") + + assert omni_base.omni_snapshot_download(model_id) == model_id + hf_file_probe.assert_called_once_with(model_id, "modular_model_index.json", revision=None) + hf_download.assert_called_once_with( + model_name_or_path=model_id, + cache_dir=None, + allow_patterns=["*"], + require_all=True, + ) + + @pytest.fixture def download_backend(monkeypatch: pytest.MonkeyPatch): """Run ``omni_snapshot_download`` and report which backend it selected. diff --git a/tests/entrypoints/test_utils.py b/tests/entrypoints/test_utils.py index a20c97a58..ccc6f69e8 100644 --- a/tests/entrypoints/test_utils.py +++ b/tests/entrypoints/test_utils.py @@ -22,6 +22,7 @@ from vllm_omni.engine.stage_init_utils import ( from vllm_omni.entrypoints.utils import ( _convert_dataclasses_to_dict, _filter_dict_like_object, + _try_resolve_omni_model_type, coerce_param_message_types, filter_dataclass_kwargs, filter_stages, @@ -371,6 +372,48 @@ class TestResolveModelConfigPath: assert sender["extra"]["connector_get_max_wait_first_chunk"] == 3000 assert sender["extra"]["connector_get_max_wait"] == 300 + def test_object_storage_uri_resolves_via_materialized_configs( + self, + mocker: MockerFixture, + tmp_path, + ): + """An object-storage URI must not reach HF helpers; resolution reads the + locally materialized config files instead.""" + hf_config = SimpleNamespace( + model_type="qwen3_omni_moe", + architectures=["Qwen3OmniMoeForConditionalGeneration"], + ) + uri = "s3://qwen3-tts-models/Qwen3-Omni-30B-A3B-Instruct" + materialize = mocker.patch( + "vllm_omni.entrypoints.utils._materialize_object_storage_configs", + return_value=str(tmp_path), + ) + get_config = mocker.patch( + "vllm_omni.entrypoints.utils.get_config", + return_value=hf_config, + ) + + result = resolve_model_config_path(uri) + + materialize.assert_called_once_with(uri) + get_config.assert_called_once() + assert get_config.call_args[0][0] == str(tmp_path) + assert result is not None + assert Path(result).as_posix().endswith("qwen3_omni_moe.yaml") + + +class TestTryResolveOmniModelType: + """Name matching must only scan the last path component.""" + + def test_uri_bucket_name_cannot_hijack_pipeline(self): + # "qwen3-tts-models" normalizes to "qwen3ttsmodels", which contains the + # registered key "qwen3tts" — only the basename may participate, so a + # misleadingly named bucket selects nothing. + assert _try_resolve_omni_model_type("s3://qwen3-tts-models/plain-checkpoint") is None + + def test_basename_still_matches_pipeline(self): + assert _try_resolve_omni_model_type("gs://any-bucket/Fun-CosyVoice3-0.5B-2512") == "cosyvoice3" + class TestLoadAndResolveStageConfigs: def test_load_and_resolve_with_kwargs(self, mocker: MockerFixture): diff --git a/vllm_omni/config/config_factory.py b/vllm_omni/config/config_factory.py index a1778eda3..5adf218c7 100644 --- a/vllm_omni/config/config_factory.py +++ b/vllm_omni/config/config_factory.py @@ -15,6 +15,7 @@ from transformers import PretrainedConfig from vllm.logger import init_logger from vllm.transformers_utils.config import get_config from vllm.transformers_utils.repo_utils import get_hf_file_to_dict +from vllm.transformers_utils.runai_utils import ObjectStorageModel, is_runai_obj_uri from vllm_omni.config.endpoint_policy import EndpointRestriction from vllm_omni.config.omni_config import VllmOmniConfig @@ -45,6 +46,39 @@ logger = init_logger(__name__) _DEFAULT_PARALLEL_DEGREE = 1 +@functools.cache +def _materialize_object_storage_configs(model: str) -> str: + """Materialize an object-storage model URI's config files locally. + + vLLM's Run:AI streamer keeps ``s3://``/``gs://``/``az://`` URIs opaque until + each stage builds its ``ModelConfig``; parent-process resolution (HF config + lookup, pipeline/pipeline-key matching) would instead hand the URI to + ``huggingface_hub`` helpers, which reject it with ``HFValidationError``. + Pull the lightweight files once into vLLM's deterministic + ``model_streamer/`` directory so config reads work here, and so the + stage processes' own pull lands in that same directory. + + Returns the input unchanged for non object-storage paths. + """ + if not is_runai_obj_uri(model): + return model + object_storage_model = ObjectStorageModel(url=model) + object_storage_model.pull_files(model, allow_pattern=["*.model", "*.py", "*.json"]) + logger.info("Materialized object-storage configs for %s at %s", model, object_storage_model.dir) + return object_storage_model.dir + + +def _name_match_candidate(model: str) -> str: + """Last path component of a model reference, used for name-based matching. + + Object-storage URIs and HF repo ids carry non-model segments (bucket name, + organization) that must not participate in substring matching; e.g. a + bucket named ``qwen3-tts-models`` holding a ``Qwen3-Omni`` checkpoint must + not resolve to the ``qwen3_tts`` pipeline. + """ + return model.rstrip("/").rsplit("/", 1)[-1] + + def with_trust_remote_code_override( overrides: Mapping[str, Any], trust_remote_code: bool | None, @@ -117,7 +151,7 @@ class StageConfigFactory: """ hf_config = None try: - return get_config(model, trust_remote_code=trust_remote_code) + return get_config(_materialize_object_storage_configs(model), trust_remote_code=trust_remote_code) except Exception as e: logger.debug(f"`get_config` failed with exception {e}; inferred HF config is None") return hf_config @@ -163,10 +197,12 @@ class StageConfigFactory: if hf_config is not None: return hf_config.model_type + config_source = _materialize_object_storage_configs(model) + # Fallback: read config.json directly for custom model types that # are not registered with transformers (e.g. qwen3_tts). try: - config_dict = get_hf_file_to_dict("config.json", model, revision=None) + config_dict = get_hf_file_to_dict("config.json", config_source, revision=None) if config_dict: if "model_type" in config_dict: return config_dict["model_type"] @@ -183,7 +219,7 @@ class StageConfigFactory: # model_index.json with _class_name that maps to a pipeline key via # PipelineConfig.diffusers_class_name. try: - model_index = get_hf_file_to_dict("model_index.json", model, revision=None) + model_index = get_hf_file_to_dict("model_index.json", config_source, revision=None) if model_index and "_class_name" in model_index: class_name = model_index["_class_name"] for obj in OMNI_PIPELINES.values(): @@ -203,8 +239,10 @@ class StageConfigFactory: # Final fallback: some models (e.g. CosyVoice3) ship an empty # config.json and rely on naming conventions. Match the model path # basename against registered pipeline keys — longest match wins - # so "cosyvoice3" (length 10) beats "cosyvoice" (length 9). - model_lower = model.lower().replace("-", "").replace("_", "") + # so "cosyvoice3" (length 10) beats "cosyvoice" (length 9). Only + # the basename is scanned so URI segments such as the bucket name + # cannot select an unrelated pipeline. + model_lower = _name_match_candidate(model).lower().replace("-", "").replace("_", "") best: str | None = None best_len = 0 for registered_key in OMNI_PIPELINES.keys(): diff --git a/vllm_omni/entrypoints/omni_base.py b/vllm_omni/entrypoints/omni_base.py index b787c44a4..ab2bee01b 100644 --- a/vllm_omni/entrypoints/omni_base.py +++ b/vllm_omni/entrypoints/omni_base.py @@ -10,6 +10,7 @@ import huggingface_hub import vllm.envs as envs from vllm.logger import init_logger from vllm.transformers_utils.repo_utils import file_or_path_exists +from vllm.transformers_utils.runai_utils import is_runai_obj_uri from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError from vllm_omni.engine.async_omni_engine import AsyncOmniEngine @@ -66,6 +67,14 @@ def omni_snapshot_download(model_id: str) -> str: if os.path.exists(model_id): return model_id + # Object-storage models must remain URIs until each stage constructs its + # ModelConfig. vLLM then materializes only config/tokenizer files locally + # and keeps the URI in model_weights for Run:AI streaming. Treating the URI + # as a Hugging Face repo here either fails validation or downloads through + # the wrong backend before the stage processes are created. + if is_runai_obj_uri(model_id): + return model_id + # TODO: this is just a workaround for quickly use modelscope, we should support # modelscope in weight loading feature instead of using `snapshot_download` # Read through ``vllm.envs`` so the flag keeps vLLM's semantics (only diff --git a/vllm_omni/entrypoints/utils.py b/vllm_omni/entrypoints/utils.py index 07b45f374..9e884db48 100644 --- a/vllm_omni/entrypoints/utils.py +++ b/vllm_omni/entrypoints/utils.py @@ -11,7 +11,12 @@ from vllm.sampling_params import RequestOutputKind, SamplingParams from vllm.transformers_utils.config import get_config, get_hf_file_to_dict from vllm.transformers_utils.repo_utils import file_or_path_exists -from vllm_omni.config.config_factory import StageConfigFactory, with_trust_remote_code_override +from vllm_omni.config.config_factory import ( + StageConfigFactory, + _materialize_object_storage_configs, + _name_match_candidate, + with_trust_remote_code_override, +) from vllm_omni.config.pipeline_registry import OMNI_PIPELINES from vllm_omni.config.stage_config import _DEPLOY_DIR from vllm_omni.config.yaml_util import create_config, load_yaml_config, merge_configs @@ -210,12 +215,13 @@ def _try_resolve_omni_model_type(model: str) -> str | None: Searches both the legacy ``stage_configs/*.yaml`` directory and the migrated ``deploy/*.yaml`` directory for a stem that substring-matches - the model path (e.g. ``cosyvoice3`` in - ``FunAudioLLM/Fun-CosyVoice3-0.5B-2512``). The longest match wins so - ``cosyvoice3`` beats ``cosyvoice`` and ``bagel_single_stage`` beats - ``bagel``. + the model path basename (e.g. ``cosyvoice3`` in + ``FunAudioLLM/Fun-CosyVoice3-0.5B-2512``). Only the basename is scanned + so URI segments such as the bucket name cannot select an unrelated + pipeline. The longest match wins so ``cosyvoice3`` beats ``cosyvoice`` + and ``bagel_single_stage`` beats ``bagel``. """ - model_lower = model.lower().replace("-", "").replace("_", "") + model_lower = _name_match_candidate(model).lower().replace("-", "").replace("_", "") best_match: str | None = None best_len = 0 for subdir in ("model_executor/stage_configs", "deploy"): @@ -274,24 +280,29 @@ def resolve_model_config_path(model: str) -> str | None: Raises: ValueError: If model_type cannot be determined """ + # Object-storage URIs are streamed by vLLM's Run:AI streamer only after + # each stage builds its ModelConfig, so config resolution here reads the + # local copy that vLLM-Omni materializes for such URIs. Name-based + # fallbacks keep the original string (the materialized path is a hash). + config_source = _materialize_object_storage_configs(model) # Try to get config from standard transformers format first try: - hf_config = get_config(model, trust_remote_code=True) + hf_config = get_config(config_source, trust_remote_code=True) model_type = hf_config.model_type except (ValueError, Exception): # If standard transformers format fails, try diffusers format - if get_diffusion_model_index(model) is not None: - model_type = _try_get_class_name_from_diffusers_config(model) + if get_diffusion_model_index(config_source) is not None: + model_type = _try_get_class_name_from_diffusers_config(config_source) if model_type is None: raise ValueError( f"Could not determine model_type for diffusers model: {model}. " "Please ensure its Diffusers pipeline index contains '_class_name'" ) - elif file_or_path_exists(model, "config.json", revision=None): + elif file_or_path_exists(config_source, "config.json", revision=None): # Try to read config.json manually for custom models like Bagel that fail get_config # but have a valid config.json with model_type try: - config_dict = get_hf_file_to_dict("config.json", model, revision=None) + config_dict = get_hf_file_to_dict("config.json", config_source, revision=None) if config_dict and "model_type" in config_dict: model_type = config_dict["model_type"] else: @@ -315,7 +326,7 @@ def resolve_model_config_path(model: str) -> str | None: ) default_config_path = current_omni_platform.get_default_stage_config_path() - if model_type == "vla" and _looks_like_dreamzero(model): + if model_type == "vla" and _looks_like_dreamzero(config_source): model_type = "dreamzero" if model_type in _DIFFUSERS_CLASS_TO_CONFIG: From 1260ceaae639437372797c710b61b4584d36b9ca Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Sat, 15 Aug 2026 14:41:09 +0800 Subject: [PATCH 022/267] [Docs] Split diffusion attention and CPU offload guides (#6075) Signed-off-by: Hongsheng Liu --- docs/.nav.yml | 21 +- docs/design/architecture_overview.md | 2 +- .../feature/attention_backend_selection.md | 89 +++ docs/design/feature/offloader/README.md | 80 +++ docs/design/feature/offloader/cpu_offload.md | 56 ++ .../distributed_layerwise_offload.md | 4 +- .../feature/offloader/layerwise_offload.md | 57 ++ docs/design/feature/quantization.md | 2 +- docs/design/feature/skip_softmax.md | 6 +- docs/design/index.md | 13 +- docs/features/README.md | 4 +- .../diffusion/attention_backends.md | 524 ++++-------------- .../attention_backends/dense_backends.md | 110 ++++ .../attention_backends/huggingface_hub.md | 32 ++ .../attention_backends/rainfusion.md | 72 +++ .../diffusion/attention_backends/sage.md | 63 +++ .../diffusion/attention_backends/trtllm.md | 103 ++++ docs/user_guide/diffusion/cpu_offload.md | 434 ++------------- .../diffusion/offloader/cpu_offload.md | 81 +++ .../distributed_layerwise_offload.md | 118 ++++ .../diffusion/offloader/layerwise_offload.md | 65 +++ docs/user_guide/diffusion_features.md | 7 +- docs/user_guide/feature_compatibility.md | 2 +- mkdocs.yml | 4 + 24 files changed, 1145 insertions(+), 804 deletions(-) create mode 100644 docs/design/feature/attention_backend_selection.md create mode 100644 docs/design/feature/offloader/README.md create mode 100644 docs/design/feature/offloader/cpu_offload.md rename docs/design/feature/{ => offloader}/distributed_layerwise_offload.md (97%) create mode 100644 docs/design/feature/offloader/layerwise_offload.md create mode 100644 docs/user_guide/diffusion/attention_backends/dense_backends.md create mode 100644 docs/user_guide/diffusion/attention_backends/huggingface_hub.md create mode 100644 docs/user_guide/diffusion/attention_backends/rainfusion.md create mode 100644 docs/user_guide/diffusion/attention_backends/sage.md create mode 100644 docs/user_guide/diffusion/attention_backends/trtllm.md create mode 100644 docs/user_guide/diffusion/offloader/cpu_offload.md create mode 100644 docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md create mode 100644 docs/user_guide/diffusion/offloader/layerwise_offload.md diff --git a/docs/.nav.yml b/docs/.nav.yml index 730c43be6..ffaa46368 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -60,7 +60,11 @@ nav: - Overview and Compatibility: - Overview: user_guide/diffusion_features.md - Feature Compatibility: user_guide/feature_compatibility.md - - CPU Offloading: user_guide/diffusion/cpu_offload.md + - CPU Offloading: + - Overview: user_guide/diffusion/cpu_offload.md + - Model-Level: user_guide/diffusion/offloader/cpu_offload.md + - Layerwise: user_guide/diffusion/offloader/layerwise_offload.md + - Distributed Layerwise: user_guide/diffusion/offloader/distributed_layerwise_offload.md - Cache Acceleration: - TeaCache: user_guide/diffusion/cache_acceleration/teacache.md - Cache-DiT: user_guide/diffusion/cache_acceleration/cache_dit.md @@ -72,7 +76,13 @@ nav: - Sequence Parallel: user_guide/diffusion/parallelism/sequence_parallel.md - Tensor Parallel: user_guide/diffusion/parallelism/tensor_parallel.md - VAE Parallelism: user_guide/diffusion/parallelism/vae_parallelism.md - - Attention Backends: user_guide/diffusion/attention_backends.md + - Attention Backends: + - Overview: user_guide/diffusion/attention_backends.md + - Dense Backends: user_guide/diffusion/attention_backends/dense_backends.md + - TRTLLM: user_guide/diffusion/attention_backends/trtllm.md + - SageAttention: user_guide/diffusion/attention_backends/sage.md + - Hugging Face Hub: user_guide/diffusion/attention_backends/huggingface_hub.md + - RainFusion: user_guide/diffusion/attention_backends/rainfusion.md - Regional Compilation: user_guide/diffusion/regional_compilation.md - Frame Interpolation: user_guide/diffusion/frame_interpolation.md - Startup and Loading: user_guide/diffusion/startup_and_loading.md @@ -125,11 +135,16 @@ nav: - Tensor Parallel: design/feature/tensor_parallel.md - VAE Patch Parallelism: design/feature/vae_parallel.md - Attention Optimization: + - Backend Selection: design/feature/attention_backend_selection.md - Skip-Softmax: design/feature/skip_softmax.md + - CPU Offloading: + - Overview: design/feature/offloader/README.md + - Model-Level: design/feature/offloader/cpu_offload.md + - Layerwise: design/feature/offloader/layerwise_offload.md + - Distributed Layerwise: design/feature/offloader/distributed_layerwise_offload.md - Cache-DiT: design/feature/cache_dit.md - TeaCache: design/feature/teacache.md - Diffusion Continuous Batching: design/feature/diffusion_continuous_batching.md - - Distributed Layerwise Offload: design/feature/distributed_layerwise_offload.md - Infrastructure and Performance: - Prometheus Metrics: design/metrics.md - Speech Generation Performance Optimizations: design/qwen3_omni_tts_performance_optimization.md diff --git a/docs/design/architecture_overview.md b/docs/design/architecture_overview.md index aa6f72770..4ae1b495e 100644 --- a/docs/design/architecture_overview.md +++ b/docs/design/architecture_overview.md @@ -406,7 +406,7 @@ the source for configuration, compatibility, and implementation details. stage contract. * **Quantization and memory efficiency:** [Quantization](feature/quantization.md) resolves per-pipeline or per-component quantization configurations, while - [Distributed Layerwise Offload](feature/distributed_layerwise_offload.md) + [Distributed Layerwise Offload](feature/offloader/distributed_layerwise_offload.md) streams diffusion blocks from host memory within the existing parallel topology. diff --git a/docs/design/feature/attention_backend_selection.md b/docs/design/feature/attention_backend_selection.md new file mode 100644 index 000000000..18e05517c --- /dev/null +++ b/docs/design/feature/attention_backend_selection.md @@ -0,0 +1,89 @@ +# Diffusion Attention Backend Selection + +This document defines the selection and extension contract for diffusion +attention backends. User-facing backend choices, installation, and tuning are +in the [attention backend guides](../../user_guide/diffusion/attention_backends.md). + +## Scope + +The contract applies to DiT and other diffusion attention layers. It is +separate from vLLM's autoregressive attention selector. + +The selection path has four responsibilities: + +1. normalize user configuration into `AttentionConfig` and `AttentionSpec`; +2. resolve a spec for an attention role; +3. ask the active platform to validate an explicit backend or choose a + hardware default; and +4. load the selected `AttentionBackend` class from the registry path. + +## Resolution contract + +`get_attn_backend_for_role()` resolves configuration in this order: + +1. exact `per_role[role]`; +2. category `per_role[role_category]`; +3. `default`; +4. platform default. + +An explicit resolution returns both the backend class and its +`AttentionSpec`. A platform-default resolution returns the class and `None`. +Layers must therefore treat the spec as optional and must not infer that a +missing spec means SDPA. + +Class resolution is cached by backend name, head size, and whether TRTLLM may +be selected automatically. Logging is cached separately by role and source so +the startup record identifies why each role received its backend. + +## Registry and platform boundary + +`DiffusionAttentionBackendEnum` maps stable configuration names to default +qualified class paths. `register_diffusion_backend()` may replace a path at +runtime without changing the public enum value. + +The active `OmniPlatform` owns compatibility policy through +`get_diffusion_attn_backend_cls()`. It must: + +- validate explicit selections and fail with an actionable error when the + requested kernel cannot run; +- choose only an available, compatible default when no backend is explicit; +- account for head size and other platform-visible constraints; and +- return a qualified class path implementing `AttentionBackend`. + +The selector must not duplicate device capability or package-availability +policy that belongs to the platform. + +## Typed backend options + +Backend-specific settings remain typed fields on `AttentionSpec` rather than +unstructured keyword dictionaries: + +- `quant` is consumed by FlashInfer and TRTLLM with backend-specific value + validation; +- `skip_softmax` is consumed by TRTLLM; and +- `block_sparse` is consumed by block-sparse backends such as RainFusion. + +A backend reads only the fields it owns and rejects incompatible values. New +options should be added to a shared typed spec only when more than one backend +shares their semantics; otherwise add a dedicated typed block. + +## Model integration contract + +Each diffusion attention call declares a stable role and, when appropriate, a +broader category. Model-specific roles permit precise overrides; categories +preserve common `self` or `cross` policies. A model also decides whether its +path is compatible with automatic TRTLLM selection, because only the model +knows whether masking and packing satisfy the kernel contract. + +## Adding or changing a backend + +An implementation change is complete only when it: + +1. implements the `AttentionBackend` interface and exposes a stable enum name; +2. adds platform validation and default routing where appropriate; +3. defines and validates any typed configuration; +4. tests explicit selection, default selection, and incompatible paths; +5. documents installation, fallback behavior, and quality implications in the + corresponding user guide; and +6. updates the overview matrix without moving algorithm details back into the + landing page. diff --git a/docs/design/feature/offloader/README.md b/docs/design/feature/offloader/README.md new file mode 100644 index 000000000..f071196b1 --- /dev/null +++ b/docs/design/feature/offloader/README.md @@ -0,0 +1,80 @@ +# CPU Offloading + +This document defines the shared architecture for diffusion CPU offloading. +The three strategies have separate user and design pages: + +| Strategy | User guide | Design contract | +| --- | --- | --- | +| Model-level | [Guide](../../../user_guide/diffusion/offloader/cpu_offload.md) | [Design](cpu_offload.md) | +| Layerwise | [Guide](../../../user_guide/diffusion/offloader/layerwise_offload.md) | [Design](layerwise_offload.md) | +| Distributed layerwise | [Guide](../../../user_guide/diffusion/offloader/distributed_layerwise_offload.md) | [Design](distributed_layerwise_offload.md) | + +## Strategy selection + +`OffloadConfig.from_od_config()` converts diffusion configuration into one +`OffloadStrategy`. If callers set multiple flags, selection is deterministic: + +1. `DISTRIBUTED_LAYERWISE`; +2. `LAYERWISE`; +3. `MODEL_LEVEL`; and +4. `NONE`. + +The factory logs when a higher-priority strategy disables another. It derives +parallel and HSDP state from `DiffusionParallelConfig`; callers do not provide +a separate offload group size. + +`get_offload_backend()` then validates platform offload support, resolves the +device, and creates exactly one backend. Returning `None` means offloading is +disabled or unsupported and must not leave partially installed hooks. + +## Shared lifecycle + +Every backend implements `OffloadBackend`: + +- `enable(pipeline)` discovers modules, establishes initial residency, and + installs hooks; +- `disable()` removes owned hooks and resources; and +- `is_enabled()` reports lifecycle state. + +`disable()` does not promise to restore the pipeline's original device +placement. The caller owns any subsequent rematerialization. + +Hooks are registered through `HookRegistry` and `ModelHook`; offload backends +must use distinct hook names and remove only hooks they own. + +## Discovery and topology + +Pipeline component discovery prefers `SupportsComponentDiscovery` declarations: + +- `_dit_modules`; +- `_encoder_modules`; +- `_vae_modules`; and +- `_resident_modules`. + +Dotted paths are supported. Legacy pipelines may use the fallback scan of +well-known attribute names, but new integrations should declare components +explicitly. + +Ordinary layerwise offload obtains topology from declared +`_layerwise_offload_blocks_attrs` (with a deprecated singular-name +compatibility path). Distributed layerwise offload may instead use a pipeline +`OffloadPlan`, then falls back to declared block attributes and its documented +compatibility discovery. Discovery metadata describes structure only; the +backend remains responsible for transfer, synchronization, and storage +ownership. + +## Cross-strategy invariants + +- At most one strategy owns offload hooks for a pipeline. +- A parameter has one authoritative host representation while offloaded. +- Device storage is not freed until dependent compute or transfer work is + complete. +- Non-persistent and model-specific buffers remain correct after movement or + checkpoint rematerialization. +- Unsupported parallel or loading combinations fail before hooks mutate the + model. +- Platform streams, events, synchronization, and cache management go through + the vLLM-Omni platform abstraction. + +The diffusion [Offloader module design](../../module/diffusion/offloader.md) +describes how these feature contracts fit into the larger diffusion runtime. diff --git a/docs/design/feature/offloader/cpu_offload.md b/docs/design/feature/offloader/cpu_offload.md new file mode 100644 index 000000000..2318611cc --- /dev/null +++ b/docs/design/feature/offloader/cpu_offload.md @@ -0,0 +1,56 @@ +# Model-Level Offload + +Model-level offload enforces mutual exclusion between pipeline component +groups. For commands and model support, see the +[model-level user guide](../../../user_guide/diffusion/offloader/cpu_offload.md). + +## Generic hook path + +`ModelLevelOffloadBackend.enable()` discovers DiTs, encoders, and VAEs. It +makes encoders and VAEs device resident for initialization, then installs +`SequentialOffloadHook` instances: + +- each DiT hook offloads all encoders and other DiTs before loading itself; +- each encoder hook offloads every DiT before loading itself; and +- VAEs are not part of the mutual-exclusion hook set. + +The hook moves parameters and buffers without recursively calling +`module.to()`. This avoids recursion through modules that retain references to +other transformer blocks. Host tensors are pinned when configured; DTensors +are not pinned through the ordinary tensor path. + +Before forward returns control to the model, the hook synchronizes the active +platform so compute cannot race the transfer. XPU uses blocking device-to-host +copies because its allocator cannot safely evict storage while an asynchronous +copy is pending. + +## Split-model delegation + +If a pipeline provides callable `enable_omni_model_cpu_offload`, the backend +delegates instead of installing generic encoder/DiT hooks. This supports models +whose mutually exclusive components are nested inside one transformer, such +as a reasoner and generator. + +The delegated pipeline owns its internal contexts, but must preserve the same +contract: + +- only the executing component is device resident; +- enable and disable are idempotent; +- transfers use the supplied device, pinning, and HSDP settings; and +- disabling removes all model-local offload hooks or contexts. + +The delegation is currently duck typed. If another independent implementation +is added, formalize the enable/disable pair as a protocol rather than expanding +attribute-name heuristics. + +## Extension invariants + +- New ordinary pipelines declare both DiT and encoder groups through + `SupportsComponentDiscovery`. +- Models with multiple DiTs offload the inactive DiTs as well as encoders. +- Empty or parameterless modules do not trigger transfers. +- Hook removal does not move modules; callers must not assume original + residency is restored. + +Shared selection and lifecycle behavior is defined in the +[CPU Offloading design](README.md). diff --git a/docs/design/feature/distributed_layerwise_offload.md b/docs/design/feature/offloader/distributed_layerwise_offload.md similarity index 97% rename from docs/design/feature/distributed_layerwise_offload.md rename to docs/design/feature/offloader/distributed_layerwise_offload.md index da735d60b..ade39fa27 100644 --- a/docs/design/feature/distributed_layerwise_offload.md +++ b/docs/design/feature/offloader/distributed_layerwise_offload.md @@ -7,7 +7,7 @@ either shard those host-side weights across an existing parallel group or keep the standard loader's rank-local weights and avoid an additional collective. For user-facing commands, see the -[distributed layerwise offloading guide](../../user_guide/diffusion/cpu_offload.md) +[distributed layerwise offloading guide](../../../user_guide/diffusion/offloader/distributed_layerwise_offload.md) and the [Cosmos3 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md). ## Status @@ -56,7 +56,7 @@ AllGather: [full N+1] [full N+2] Buffers: [current slot] [prefetch slot] [current slot] ``` -![DLO double-buffer prefetch pipeline](../figures/dlo/dlo_pipeline.gif) +![DLO double-buffer prefetch pipeline](../../figures/dlo/dlo_pipeline.gif) The backend uses two shared device buffers, so accelerator weight residency is bounded by the largest streamed blocks rather than the complete model. diff --git a/docs/design/feature/offloader/layerwise_offload.md b/docs/design/feature/offloader/layerwise_offload.md new file mode 100644 index 000000000..3fff501da --- /dev/null +++ b/docs/design/feature/offloader/layerwise_offload.md @@ -0,0 +1,57 @@ +# Layerwise Offload + +Layerwise offload streams transformer blocks through one accelerator. For +commands and model support, see the +[layerwise user guide](../../../user_guide/diffusion/offloader/layerwise_offload.md). + +## Hook ring + +`LayerWiseOffloadBackend` discovers every streamable block and registers one +`LayerwiseOffloadHook` per block. Hooks form a ring: each hook owns the pinned +host representation of its registered block and a reference to the next block +it must prefetch; the final hook prefetches the first block for the next +denoising iteration. + +During initialization, parameters and buffers are grouped by dtype and copied +into contiguous pinned CPU tensors. Original tensor storage is replaced with +an empty or meta placeholder while identity and module structure are retained. +Metadata records names, offsets, shapes, and dtypes for rematerialization. + +## Forward lifecycle + +For each block: + +1. the previous hook's completion event ensures this block is materialized; +2. the copy stream asynchronously allocates and fills the next block's device + tensors; +3. parameter and buffer storage is rebound to views of those tensors; +4. the current block computes on the normal stream; and +5. its post-forward hook replaces device storage with placeholders after all + dependent work is safe. + +The copy stream waits on the compute stream before reusing storage, and the +compute stream waits on the per-block prefetch event before forward. These +dependencies, not incidental global synchronization, define correctness. + +## Topology contract + +A DiT declares ordered block-container attributes using +`_layerwise_offload_blocks_attrs` or an `OffloadPlan`. Each resolved item must +be an executable `nn.Module`; ordering must match forward execution. Multiple +containers are concatenated in execution order. + +Non-block DiT modules, encoders, VAEs, and declared resident modules are not +streamed by this backend. + +## Invariants and limitations + +- Each offloaded tensor has one pinned host backing store. +- Parameter identity is preserved when storage is rebound. +- Blocks with heterogeneous shapes and dtypes use their recorded metadata; + implementations must not assume identical blocks. +- The ring is initialized before the first forward so block zero is available. +- This backend is single-device; distributed weight reconstruction belongs to + [Distributed Layerwise Offload](distributed_layerwise_offload.md). + +Shared selection and lifecycle behavior is defined in the +[CPU Offloading design](README.md). diff --git a/docs/design/feature/quantization.md b/docs/design/feature/quantization.md index 37dd5e26b..7ad612d12 100644 --- a/docs/design/feature/quantization.md +++ b/docs/design/feature/quantization.md @@ -186,7 +186,7 @@ Distributed layerwise offload has an additional boundary: its sharded mmap AllGather path is not compatible with online quantization because the path expects already materialized checkpoint weights. Use the rank-local `--dlo-no-use-allgather` path or a compatible pre-quantized checkpoint when -combining these features. See the [DLO design document](distributed_layerwise_offload.md) +combining these features. See the [DLO design document](offloader/distributed_layerwise_offload.md) for the full compatibility matrix. ## Adding a quantization backend diff --git a/docs/design/feature/skip_softmax.md b/docs/design/feature/skip_softmax.md index 8e6375ee8..0527c7eeb 100644 --- a/docs/design/feature/skip_softmax.md +++ b/docs/design/feature/skip_softmax.md @@ -2,8 +2,10 @@ Skip-Softmax is the sparse-attention mode of the `TRTLLM_ATTN` backend. Usage — the config keys and how to pick an operating point — is in -[Diffusion Attention Backends](../../user_guide/diffusion/attention_backends.md). This page explains -the algorithm. +[TRTLLM Attention](../../user_guide/diffusion/attention_backends/trtllm.md#skip-softmax). +The shared selector contract is documented in +[Diffusion Attention Backend Selection](attention_backend_selection.md). This +page explains the algorithm. ## Motivation diff --git a/docs/design/index.md b/docs/design/index.md index d81ccb806..0db152335 100644 --- a/docs/design/index.md +++ b/docs/design/index.md @@ -53,15 +53,22 @@ implementation contract; it is not, by itself, a general support claim. #### Attention optimization The [Diffusion Attention Backends](../user_guide/diffusion/attention_backends.md) -guide lists all selectable backends and their platform defaults. Skip-Softmax -is the backend-related optimization with a standalone feature-design contract: +guides list selectable backends, platform defaults, installation, and tuning. +The design contracts separate selection mechanics from backend algorithms: +- [Attention Backend Selection](feature/attention_backend_selection.md) - [Skip-Softmax](feature/skip_softmax.md) +#### CPU offloading + +- [Overview and Shared Contracts](feature/offloader/README.md) +- [Model-Level Offload](feature/offloader/cpu_offload.md) +- [Layerwise Offload](feature/offloader/layerwise_offload.md) +- [Distributed Layerwise Offload](feature/offloader/distributed_layerwise_offload.md) + - [Cache-DiT](feature/cache_dit.md) - [TeaCache](feature/teacache.md) - [Diffusion Continuous Batching](feature/diffusion_continuous_batching.md) -- [Distributed Layerwise Offload](feature/distributed_layerwise_offload.md) ## Infrastructure and Performance diff --git a/docs/features/README.md b/docs/features/README.md index 0dee558a4..8090b783f 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -48,10 +48,10 @@ defines the shared configuration and backend extension points. | Goal | User guide | Related design contract | | --- | --- | --- | | Compare acceleration methods and supported combinations | [Overview](../user_guide/diffusion_features.md), [Feature Compatibility](../user_guide/feature_compatibility.md) | [Diffusion acceleration designs](../design/index.md#diffusion-acceleration) | -| Move weights between host and device memory | [CPU Offloading](../user_guide/diffusion/cpu_offload.md) | [Distributed Layerwise Offload](../design/feature/distributed_layerwise_offload.md) | +| Move weights between host and device memory | [CPU Offloading](../user_guide/diffusion/cpu_offload.md) | [CPU Offloading](../design/feature/offloader/README.md) | | Reuse denoising computation | [Cache Acceleration](../user_guide/diffusion/cache_acceleration/cache_dit.md) | [Cache-DiT](../design/feature/cache_dit.md), [TeaCache](../design/feature/teacache.md) | | Distribute diffusion work across devices | [Parallelism](../user_guide/diffusion/parallelism/overview.md) | [Parallelism designs](../design/index.md#parallelism) | -| Select dense, sparse, or quantized attention paths | [Attention Backends](../user_guide/diffusion/attention_backends.md) | [Skip-Softmax](../design/feature/skip_softmax.md) | +| Select dense, sparse, or quantized attention paths | [Attention Backends](../user_guide/diffusion/attention_backends.md) | [Attention Backend Selection](../design/feature/attention_backend_selection.md) | | Compile repeated diffusion regions | [Regional Compilation](../user_guide/diffusion/regional_compilation.md) | User-facing optimization guide | | Add generated video frames | [Frame Interpolation](../user_guide/diffusion/frame_interpolation.md) | User-facing extension guide | | Reduce diffusion model startup time | [Startup and Loading](../user_guide/diffusion/startup_and_loading.md) | User-facing loading guide | diff --git a/docs/user_guide/diffusion/attention_backends.md b/docs/user_guide/diffusion/attention_backends.md index b6fd13b12..6897d633e 100644 --- a/docs/user_guide/diffusion/attention_backends.md +++ b/docs/user_guide/diffusion/attention_backends.md @@ -1,494 +1,192 @@ # Diffusion Attention Backends -This document describes the diffusion attention backends available in vLLM-Omni, how to select them globally and per-role, the per-platform defaults, and how to use SageAttention. +Use this page to select and configure a diffusion attention backend. Backend- +specific installation, tuning, and compatibility details live in separate +guides so that the selection contract stays easy to scan. -## Overview +Diffusion attention backend selection applies to DiT attention in image and +video generation models. It does **not** change autoregressive LLM attention, +which uses vLLM's attention selector. -Diffusion attention backend selection is resolved in `vllm_omni.diffusion.attention.selector`. It looks up the backend from a structured `AttentionConfig` carried on `OmniDiffusionConfig` and falls back to the platform default when nothing is configured. +For the internal selector, registry, and platform contract, see +[Attention Backend Selection](../../design/feature/attention_backend_selection.md). -This backend is used by diffusion attention layers such as the DiT attention in video and image generation models. It does **not** affect autoregressive (LLM) attention paths — those go through vLLM's own attention backend selector. +## Choose a guide -The full set of backends and their platform defaults is in the **Backend Options** and **Platform Defaults** sections below. If no attention backend is configured, vLLM-Omni asks the current platform to choose the default. +| Need | Guide | +| --- | --- | +| Select a conservative or platform-native dense kernel | [Dense Backends](attention_backends/dense_backends.md) | +| Use FlashInfer trtllm-gen FMHA, Skip-Softmax, or TRTLLM SAGE quantization | [TRTLLM Attention](attention_backends/trtllm.md) | +| Install and use SageAttention 2.2 or SageAttention3 | [SageAttention](attention_backends/sage.md) | +| Match training or rollout kernels loaded from Hugging Face | [Hugging Face Hub Backends](attention_backends/huggingface_hub.md) | +| Use block-sparse video attention on Ascend NPU | [RainFusion](attention_backends/rainfusion.md) | -## Backend Options - -| Value | Notes | -|---|---| -| `TRTLLM_ATTN` | FlashInfer's trtllm-gen FMHA (TensorRT-LLM's generated kernels, vendored by FlashInfer). Dense BF16, GQA native, `head_dim=128`. Datacenter Blackwell only (sm_100 / sm_103). Packed paths can provide `cu_seqlens` directly. Supports optional **Skip-Softmax** sparse attention — see below. Requires `flashinfer`. | -| `FLASH_ATTN` | Wraps FlashAttention 4 on Blackwell when `flash-attn-4` is installed, then falls back to FlashAttention 3/2. Default on Hopper / Ada / Ampere when a compatible FlashAttention package is installed. | -| `CUDNN_ATTN` | Pins `sdpa_kernel([CUDNN_ATTENTION])`. Default on Blackwell (sm_10x / sm_12x) with cuDNN ≥ 9.5. Wins on mask-heavy DiTs (HunyuanVideo-1.5: 2× e2e vs SDPA). | -| `FLASHINFER_ATTN` | Uses FlashInfer's batch-prefill wrapper and supports mixed Q/K and V input dtypes through backend-specific configuration. Used as Blackwell fallback when cuDNN is unavailable. Requires `flashinfer` >= 0.6.16rc1 for mixed-dtype configurations. | -| `TORCH_SDPA` | PyTorch `scaled_dot_product_attention` with the default backend dispatcher. Most conservative; always available. | -| `SAGE_ATTN` | SageAttention 2.2 — INT8-quantized attention with FP16 accumulation. Lossy but typically visually indistinguishable on diffusion outputs. Requires `sageattention`. | -| `SAGE_ATTN_3` | Requires `sageattn3` from `SageAttention/sageattention3_blackwell`. CUDA only, intended for Blackwell GPUs, with GQA/MQA requests falling back to PyTorch SDPA. | -| `FLASH_ATTN_HUB` | FlashAttention 2 from HuggingFace `kernels` library. Useful for train/rollout alignment. | -| `FLASH_ATTN_3_HUB` | FlashAttention 3 from HuggingFace `kernels` library. CUDA Hopper (sm_90+) only; falls back to `FLASH_ATTN_HUB` on older GPUs. | -| `RAINFUSION_ATTN` | MindIE-SD **RainFusion** block-sparse video attention — see [below](#rainfusion_attn-backend-and-block-sparse-video-attention). Ascend NPU only; requires `mindiesd`. Delegates to `FLASH_ATTN` for anything that is not a packed video sequence. | +## Backend options +| Value | Family | Primary use | Detail | +| --- | --- | --- | --- | +| `TORCH_SDPA` | Dense | Conservative reference; always available | [Dense Backends](attention_backends/dense_backends.md#torch_sdpa) | +| `FLASH_ATTN` | Dense | FlashAttention 4/3/2 depending on the installed package and GPU | [Dense Backends](attention_backends/dense_backends.md#flash_attn) | +| `CUDNN_ATTN` | Dense | Mask-heavy DiTs on Blackwell with cuDNN 9.5 or newer | [Dense Backends](attention_backends/dense_backends.md#cudnn_attn) | +| `FLASHINFER_ATTN` | Dense or quantized | FlashInfer batch prefill; optional mixed Q/K and V dtypes | [Dense Backends](attention_backends/dense_backends.md#flashinfer_attn) | +| `TRTLLM_ATTN` | Dense, sparse, or quantized | Datacenter Blackwell with `head_dim=128` and compatible packed paths | [TRTLLM Attention](attention_backends/trtllm.md) | +| `SAGE_ATTN` | Quantized | SageAttention 2.2 INT8 attention | [SageAttention](attention_backends/sage.md#sage_attn) | +| `SAGE_ATTN_3` | Quantized | SageAttention3 on Blackwell | [SageAttention](attention_backends/sage.md#sage_attn_3) | +| `FLASH_ATTN_HUB` | Hub kernel | FlashAttention 2 from Hugging Face `kernels` | [Hugging Face Hub Backends](attention_backends/huggingface_hub.md) | +| `FLASH_ATTN_3_HUB` | Hub kernel | FlashAttention 3 from Hugging Face `kernels` on Hopper or newer | [Hugging Face Hub Backends](attention_backends/huggingface_hub.md) | +| `RAINFUSION_ATTN` | Block sparse | MindIE-SD RainFusion video attention on Ascend NPU | [RainFusion](attention_backends/rainfusion.md) | ## Configuration -Diffusion attention backends can be configured three ways, in priority order: +Backend selection is resolved in this order: -1. **`--diffusion-attention-config`** — structured per-role config (highest priority). -2. **`--diffusion-attention-backend` / `DIFFUSION_ATTENTION_BACKEND` env var** — global shorthand that sets the default backend. -3. **Platform default** — used when nothing is configured. +1. `--diffusion-attention-config` per-role configuration. +2. `--diffusion-attention-backend` or `DIFFUSION_ATTENTION_BACKEND` as a + global default. +3. The current platform's default. -`--diffusion-attention-backend` is shorthand for `--diffusion-attention-config.default.backend`. It may be combined with `--diffusion-attention-config.per_role.*` overrides, but is mutually exclusive with `--diffusion-attention-config.default.backend`. +`--diffusion-attention-backend` is shorthand for +`--diffusion-attention-config.default.backend`. Do not pass it together with an +explicit `default.backend` in the structured configuration. ### Global default -Set the default backend for every diffusion attention layer: - ```bash -# CLI flag -vllm-omni serve --diffusion-attention-backend SAGE_ATTN +vllm-omni serve --diffusion-attention-backend FLASH_ATTN -# Environment variable (also recognized for backwards compatibility) -export DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN +# Backwards-compatible environment variable +export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN ``` ### Per-role configuration -Roles are free-form strings declared by each diffusion model. The two common categories are `"self"` and `"cross"`; model-specific roles (e.g. `"ltx2.audio_to_video"`) may also be declared. A role string is matched in this order: +Roles are declared by each diffusion model. Common categories are `self` and +`cross`; a model may also use a more specific role such as +`ltx2.audio_to_video`. Resolution order is: -1. Exact `per_role[role]` match -2. `per_role[role_category]` fallback (e.g. `"ltx2.audio_to_video"` → `"cross"`) -3. `default` -4. Platform default - -Use vLLM-style dotted flags or one JSON blob: +1. Exact `per_role[role]` match. +2. Category `per_role[role_category]` match. +3. `default`. +4. Platform default. ```bash # Dotted flags vllm-omni serve \ - --diffusion-attention-config.default.backend FLASH_ATTN \ - --diffusion-attention-config.per_role.cross.backend TORCH_SDPA + --diffusion-attention-config.default.backend FLASH_ATTN \ + --diffusion-attention-config.per_role.cross.backend TORCH_SDPA -# JSON +# Equivalent JSON vllm-omni serve \ - --diffusion-attention-config '{"default":{"backend":"FLASH_ATTN"},"per_role":{"cross":{"backend":"TORCH_SDPA"}}}' + --diffusion-attention-config \ + '{"default":{"backend":"FLASH_ATTN"},"per_role":{"cross":{"backend":"TORCH_SDPA"}}}' ``` -A backend that needs configuration exposes it as a typed field on the spec: -`TRTLLM_ATTN`'s Skip-Softmax (see [below](#trtllm_attn-backend-and-skip-softmax)) and -`RAINFUSION_ATTN`'s RainFusion block (see [below](#rainfusion_attn-backend-and-block-sparse-video-attention)): - -```bash ---diffusion-attention-config.default.backend TRTLLM_ATTN \ ---diffusion-attention-config.default.skip_softmax.target_sparsity 0.5 -``` - -### Programmatic API - -When constructing `OmniDiffusionConfig` directly: +### Python API ```python -from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, AttnQuantSpec, OmniDiffusionConfig +from vllm_omni.diffusion.data import ( + AttentionConfig, + AttentionSpec, + OmniDiffusionConfig, +) config = OmniDiffusionConfig( diffusion_attention_config=AttentionConfig( default=AttentionSpec(backend="FLASH_ATTN"), - per_role={ - "cross": AttentionSpec(backend="TORCH_SDPA"), - }, + per_role={"cross": AttentionSpec(backend="TORCH_SDPA")}, ), ..., ) ``` -A plain dict is also accepted and normalized to `AttentionConfig`. +Backend-specific typed blocks are documented with their consumers: -#### FlashInfer QK16/V8 quantized attention +- `quant`: [FlashInfer](attention_backends/dense_backends.md#flashinfer-quantized-attention) + and [TRTLLM SAGE](attention_backends/trtllm.md#sage-quantization). +- `skip_softmax`: [TRTLLM Skip-Softmax](attention_backends/trtllm.md#skip-softmax). +- `block_sparse`: [RainFusion](attention_backends/rainfusion.md#configuration). -`FLASHINFER_ATTN` accepts quantization options through `AttentionSpec.quant`. For QK16/V8: - -```python -config = OmniDiffusionConfig( - diffusion_attention_config=AttentionConfig( - default=AttentionSpec( - backend="FLASHINFER_ATTN", - quant=AttnQuantSpec( - dtype_qk="bfloat16", - dtype_vo="fp8_e4m3", - ), - ), - ), - ..., -) -``` - -`dtype_qk` configures the Q and K input dtypes. `dtype_vo` configures the V input dtype. - -## Platform Defaults +## Platform defaults ### Blackwell (sm_100 / sm_103 / sm_120 / sm_121) -Auto-route preference, in order: +The CUDA auto-route preference is: -1. `TRTLLM_ATTN` — on **datacenter** Blackwell (sm_100 / sm_103) when `flashinfer` is installed, the model's `head_dim` is 128, **and the model declares a compatible packed/mask-free path** -2. `CUDNN_ATTN` — when cuDNN ≥ 9.5 is available (ships in PyTorch 2.5+ wheels) -3. `FLASHINFER_ATTN` — when `flashinfer` is installed but cuDNN < 9.5 -4. `FLASH_ATTN` — when `flash-attn` is installed with the Blackwell CUTE kernel -5. `TORCH_SDPA` — last resort +1. `TRTLLM_ATTN` on datacenter Blackwell (sm_100/sm_103) when FlashInfer is + available, `head_dim=128`, and the model declares a compatible packed or + mask-free path. +2. `CUDNN_ATTN` when cuDNN 9.5 or newer is available. +3. `FLASHINFER_ATTN` when FlashInfer is available but cuDNN is too old. +4. `FLASH_ATTN` when a compatible package is installed. +5. `TORCH_SDPA`. -`TRTLLM_ATTN` is skipped on workstation Blackwell (sm_120 / sm_121) and for any `head_dim != 128`, so those GPUs keep the `CUDNN_ATTN` route described below. +`TRTLLM_ATTN` is not auto-selected on workstation Blackwell +(sm_120/sm_121), for other head dimensions, or for paths that require an +unsupported mask. -`TRTLLM_ATTN` outranks `CUDNN_ATTN` on datacenter Blackwell for compatible packed/mask-free pipelines. Workstation Blackwell (sm_120 / sm_121) and pipelines that require attention masks retain their normal fallback. +### Hopper, Ada, and Ampere -The startup log line `Defaulting to diffusion attention backend CUDNN_ATTN (Blackwell sm_120, cuDNN 91002)` confirms the route. +The CUDA auto-route uses `FLASH_ATTN` when available and otherwise falls back +to `TORCH_SDPA`. `CUDNN_ATTN` and `FLASHINFER_ATTN` remain explicit options. -**Why CUDNN_ATTN by default**: on mask-heavy diffusion models (HunyuanVideo-1.5, Qwen-Image), cuDNN's pinned FMHA kernel sidesteps a PyTorch SDPA dispatch quirk where the unpinned dispatcher picks `EFFICIENT_ATTENTION` (~25 ms) for masked calls instead of cuDNN (~11 ms). The pin gives 2× e2e on HV-1.5 with no regression on lighter models. +Other platforms validate an explicit backend and choose their own default +through the platform implementation. Check the startup log to confirm the +resolved backend. -### Hopper (sm_90) / Ada (sm_89) / Ampere (sm_80–sm_86) +## Choosing a backend manually -Auto-route preference: +Override the platform default when you need: -1. `FLASH_ATTN` — when `flash-attn` is installed -2. `TORCH_SDPA` — fallback +- a correctness reference (`TORCH_SDPA`); +- a backend-specific workaround; +- training/rollout kernel alignment (Hub backends); or +- an explicitly validated sparse or quantized speedup. -`CUDNN_ATTN` and `FLASHINFER_ATTN` are still selectable via env var on these GPUs but are not in the auto-route — FlashAttention 2 is the well-tuned path on pre-Blackwell hardware. +The startup log prints the resolved backend and whether it came from explicit +configuration or platform defaulting. If no resolution message appears, check +earlier logs for diffusion-stage initialization failures. -## TRTLLM_ATTN Backend and Skip-Softmax +## Reference benchmark -`TRTLLM_ATTN` runs FlashInfer's trtllm-gen FMHA. Dense it is a BF16 backend like the others; on top of -that it can enable **Skip-Softmax**, a sparse-attention mode that trades a little fidelity for speed -(algorithm: [design doc](../../design/feature/skip_softmax.md)). +The following BF16 results were measured on an sm_120 RTX Pro 6000 Blackwell +with the same prompt and seed across runs. Treat them as reference results, not +portable guarantees. -Enable it through the typed `skip_softmax` block on the attention spec: - -| Key | Valid values | Meaning | -|---|---|---| -| `target_sparsity` | finite, `[0, 1]` | Operating point on a calibrated curve. Needs a calibration for the model. | -| `threshold` | finite, `≥ 0` | Direct threshold, no calibration needed. Mutually exclusive with `target_sparsity`. | -| `disabled_until_timestep` | finite, `[0, 1]` | Holds early, high-noise steps dense; skip turns on once the normalized timestep `t ≤ D`. | - -```bash -vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \ - --diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN", - "skip_softmax": {"target_sparsity": 0.65, "disabled_until_timestep": 0.86}}}' -``` - -Programmatically the same block is a typed `SkipSoftmaxSpec` (values validated at construction): - -```python -from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, SkipSoftmaxSpec - -AttentionConfig( - default=AttentionSpec( - backend="TRTLLM_ATTN", - skip_softmax=SkipSoftmaxSpec(target_sparsity=0.65, disabled_until_timestep=0.86), - ), -) -``` - -**Start at `target_sparsity=0.65, disabled_until_timestep=0.86`** and raise `target_sparsity` only -as far as your quality bar allows — the output diverges from dense as it climbs. The gain grows with -sequence length, since Skip-Softmax only accelerates attention. Dense BF16 is `TRTLLM_ATTN` with no -`skip_softmax` block. - -Requires datacenter Blackwell with `head_dim == 128`; elsewhere, or without FlashInfer, selecting -`TRTLLM_ATTN` raises rather than silently degrading. - -## TRTLLM_ATTN SAGE Quantization - -On top of dense/Skip-Softmax, `TRTLLM_ATTN` can run **SAGE** (SageAttention) — the trtllm-gen -FMHA kernel with per-block quantized Q/K and per-channel fp8 V, so the two attention matmuls run -in low precision. Enable it through the typed `quant` block on the attention spec: - -| Key | Valid values | Meaning | -|---|---|---| -| `dtype_qk` | `int8`, `fp8_e4m3` | Quantization dtype for Q and K. Absent ⇒ dense (SAGE off). | -| `q_block_size` | `1`, `4`, `16` | Per-token block size for Q scales (default `1`). | -| `k_block_size` | `1`, `4`, `16` | Per-token block size for K scales (default `16`). | - -V is always quantized per-channel to fp8_e4m3 and K-smoothing is applied inside the routine, so -neither is a user knob. Only block sizes `1`, `4`, `16` have compiled kernels. - -The `quant` block is a shared `AttnQuantSpec` consumed by more than one backend; each reads the -fields that apply and rejects the rest. `TRTLLM_ATTN` (SAGE) reads `dtype_qk` (`int8` / `fp8_e4m3`), -`q_block_size`, and `k_block_size`; `FLASHINFER_ATTN` reads `dtype_qk` (`float16` / `bfloat16`), -`dtype_vo`, and `flashinfer_backend` (see its section). A `dtype_qk` from the wrong set raises. - -```bash -vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \ - --diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN", - "quant": {"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16}}}' -``` - -```python -from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, AttnQuantSpec - -AttentionConfig( - default=AttentionSpec( - backend="TRTLLM_ATTN", - quant=AttnQuantSpec(dtype_qk="fp8_e4m3", q_block_size=1, k_block_size=16), - ), -) -``` - -**Requirements.** Needs FlashInfer ≥ 0.6.16rc1. Kernel -availability is arch-dependent: `fp8_e4m3` QK has kernels on both **SM100** (B200) and **SM103** -(B300); `int8` QK kernels are compiled for **SM100 only**. - -## RAINFUSION_ATTN Backend and Block-Sparse Video Attention - -`RAINFUSION_ATTN` runs MindIE-SD's RainFusion (`rf_v2`) kernel on Ascend NPU. It pools each -128-token block of keys, ranks them per query block, and computes attention against only the -top-scoring ones. Video tokens are rearranged into `(t, h, w)` order first, so the kept blocks are -spatiotemporal neighbours rather than arbitrary rows of the packed sequence. - -Only the video segment is sparsified. The prefix rows (text, visual conditions, audio) and the -first-frame blocks are always kept dense, which is why the realized sparsity is lower than the -nominal `sparsity` you configure. Anything the kernel cannot handle — a warmup denoise step, an -exempt layer, a sequence with no published video geometry, a video segment under 32 blocks — -delegates to `FLASH_ATTN`, so a model can select this backend unconditionally. - -Enable it through the typed `block_sparse` block on the attention spec, shared by every -block-sparse backend: - -| Key | Valid values | Meaning | -|---|---|---| -| `sparsity` | finite, `[0, 1]` | Nominal fraction of key blocks dropped per query block. `0` disables sparsity. Defaults to `0.8`. | -| `start_step` | `≥ 0` | Keep the first N denoise steps dense. Layout is decided early, so these steps dominate structural fidelity. | -| `skip_layers` | index selector, e.g. `"0-3,38"` | DiT blocks that always stay dense. | - -```bash -vllm-omni serve MiniMaxAI/MiniMax-H3 \ - --diffusion-attention-config '{"default": {"backend": "RAINFUSION_ATTN", - "block_sparse": {"sparsity": 0.8, "start_step": 0}}}' -``` - -Programmatically the same block is a typed `BlockSparseSpec` (values validated at construction): - -```python -from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, BlockSparseSpec - -AttentionConfig( - default=AttentionSpec( - backend="RAINFUSION_ATTN", - block_sparse=BlockSparseSpec(sparsity=0.8, start_step=0, skip_layers="0-1"), - ), -) -``` - -**Tune in the order `start_step` → `sparsity` → `skip_layers`.** Raise `start_step` first: it is the -cheapest way to recover structure, because the early high-noise steps decide global layout while the -later steps only refine texture. Then walk `sparsity` up to your quality bar. Reach for -`skip_layers` last, once an A/B against dense at the same seed points at specific blocks. - -Requires Ascend NPU with `mindiesd`; selecting it on any other platform raises. It is also -incompatible with ring sequence parallelism, since `rf_v2` needs the whole key sequence to rank -blocks — use Ulysses SP (`ring_degree=1`). - -### Which geometries run sparse - -MindIE-SD rf_v2 handles arbitrary video grids by spatially rearranging video first and, when -necessary, promoting a real-video suffix to its always-kept prefix segment. It never pads: rf_v2 -does not consume an attention mask for padded keys. The remaining video rows stay sparse, while the -promoted rows remain visible to every query and key. - -For example, MiniMax-H3 at 1344x768 has grid `(62, 24, 42)`: its 62496 video rows leave a 32-row -128-block residue, and its two non-8-wide latent columns form a 2928-row rearranged suffix. -MindIE-SD promotes 2976 rows, leaving 59520 sparse rows (`465 × 128`). This keeps the block mask -and kernel tiling aligned, at the cost of retaining more blocks. - -8x8-aligned spatial grids remain preferable for performance. A latent `h` or `w` not divisible by 8 -creates an always-kept residual suffix and lowers realized sparsity. "Multiple of 8" on the latent -grid means **width and height that are multiples of 256**. This path requires a MindIE-SD release -that implements protected video tails for rf_v2. - -## End-to-End Benchmark (BF16, sm_120 RTX Pro 6000 Blackwell) - -Same prompt and seed across runs. `Total generation time` from `text_to_video.py` / `text_to_image.py`. - -| Model | Shape | TORCH_SDPA | CUDNN_ATTN | FLASHINFER_ATTN | -|---|---|---|---|---| +| Model | Shape | `TORCH_SDPA` | `CUDNN_ATTN` | `FLASHINFER_ATTN` | +| --- | --- | ---: | ---: | ---: | | HunyuanVideo-1.5 (T2V) | 480p / 33f / 50 steps | 147.05 s | **73.02 s** | 127.84 s | | Wan 2.2 14B (T2V) | 480p / 33f / 40 steps | 117.75 s | 117.17 s | **115.07 s** | | Qwen-Image (T2I) | 1024² / 50 steps | 17.41 s | **15.14 s** | 16.02 s | | FLUX.2-dev (T2I) | 1024² / 50 steps, TP=2 | 53.62 s | **53.30 s** | 54.94 s | -Pattern: mask-heavy DiTs (HV-1.5, Qwen-Image) favor `CUDNN_ATTN`; lighter-mask DiTs and TP-saturated configs (Wan 2.2, FLUX.2 TP=2) tie within noise. +Mask-heavy DiTs favored `CUDNN_ATTN`; lighter-mask or TP-saturated workloads +were close enough that users should benchmark their exact model and shape. -## Known Limitations +## Compatibility anchors -### LTX-2.0: `CUDNN_ATTN` crashes under torch.compile +The following headings preserve links to sections that moved into dedicated +guides. -LTX-2's audio attention has a symbolic head_dim under torch.compile tracing. cuDNN's SDPA backend selector rejects symbolic dims and Dynamo aborts compilation. Tracked in [#3121](https://github.com/vllm-project/vllm-omni/issues/3121). +## TRTLLM_ATTN Backend and Skip-Softmax -**Workaround**: explicitly select `FLASHINFER_ATTN` or `TORCH_SDPA` for LTX-2.0: +See [TRTLLM Attention](attention_backends/trtllm.md#skip-softmax). -```bash -DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN python examples/offline_inference/text_to_video/text_to_video.py \ - --model Lightricks/LTX-2 ... -``` +## TRTLLM_ATTN SAGE Quantization -### FlashAttention-4 on Blackwell +See [TRTLLM Attention](attention_backends/trtllm.md#sage-quantization). -Install the optional CUDA 13 extra to use the CuTe-based FA4 path: +## RAINFUSION_ATTN Backend and Block-Sparse Video Attention -```bash -pip install 'vllm-omni[fa4]' -``` - -`FLASH_ATTN` prefers `flash_attn.cute` on Blackwell and falls back to FA3/FA2 -when it is unavailable. Version `4.0.0b18` is required; earlier beta -wheels had known JIT failures on Blackwell. - -## Choosing a Backend Manually - -### When to override the default - -- **Quality validation**: compare a new backend against `TORCH_SDPA` as the reference, since SDPA's default dispatcher is the most extensively tested. -- **Lossy speedup hunting**: try `SAGE_ATTN` (INT8 quantized) on diffusion outputs — typically indistinguishable visually but always validate. -- **Workaround for known issues**: see Known Limitations above. - -### Verifying which backend is in use - -The startup log prints one of: - -``` -Using diffusion attention backend 'CUDNN_ATTN' # explicit override -Defaulting to diffusion attention backend CUDNN_ATTN ... # auto-route -Defaulting to diffusion attention backend SDPA # nothing else available -``` - -If you don't see one of these, the model didn't reach diffusion stage init — check earlier logs for failures. +See [RainFusion](attention_backends/rainfusion.md). ## SageAttention Installation -vLLM-Omni expects SageAttention to be installed into the same Python environment as vLLM-Omni. - -Build from source: - -```bash -git clone https://github.com/thu-ml/SageAttention.git -cd SageAttention - -export EXT_PARALLEL=4 NVCC_APPEND_FLAGS="--threads 8" MAX_JOBS=32 -pip install . --no-build-isolation -``` - -Quick check: - -```bash -python -c "import sageattention; print(sageattention.__file__)" -``` +See [SageAttention](attention_backends/sage.md#installation). ## SageAttention3 Installation -vLLM-Omni expects SageAttention3 to be installed into the same Python environment as vLLM-Omni. - -Build from source: - -```bash -git clone https://github.com/thu-ml/SageAttention.git -cd SageAttention/sageattention3_blackwell -python setup.py install -``` - -Quick check: - -```bash -python -c "import sageattn3; print(sageattn3.__file__)" -``` - -Notes: - -- `SAGE_ATTN_3` is only selected on CUDA when `sageattn3` is importable and the GPU is Blackwell-class. -- SageAttention3's Blackwell kernel assumes `Hq == Hkv`. In vLLM-Omni, GQA/MQA diffusion requests fall back to PyTorch SDPA for correctness. +See [SageAttention](attention_backends/sage.md#sageattention3-installation). ## HuggingFace Kernels Hub Backends -To achieve perfect numerical consistency between **training** (typically using HuggingFace Diffusers with Hub kernels) and **serving/rollout** (using vLLM-Omni), you can use the Hub-based attention backends. This eliminates numerical drift / sampling divergence caused by executing different local kernel versions during rollout. - -The following backend options are supported: -- `FLASH_ATTN_HUB` (HuggingFace `kernels-community/flash-attn2`) -- `FLASH_ATTN_3_HUB` (HuggingFace `kernels-community/flash-attn3`, Hopper sm_90+ only) - -### Installation - -To use these backends, you must install the `kernels` library: - -```bash -pip install kernels==0.14.1 -``` - -If the `kernels` library is not available in the environment, vLLM-Omni will log a warning and fall back gracefully to the corresponding local backend implementations (`FLASH_ATTN`). On CUDA GPUs below Hopper (compute capability < 9.0), `FLASH_ATTN_3_HUB` falls back to `FLASH_ATTN_HUB`. - -### Usage - -Select a Hub backend using the global CLI flag or environment variables: - -```bash -# Environment variable -export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN_HUB - -# CLI flag -vllm-omni serve --diffusion-attention-backend FLASH_ATTN_HUB -``` - -## Usage Examples - -### Default (auto-route) - -```bash -python examples/offline_inference/text_to_video/text_to_video.py \ - --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \ - --prompt "A dog running across a field of golden wheat." \ - --height 480 --width 832 --num-frames 33 \ - --num-inference-steps 50 --seed 42 --guidance-scale 6.0 \ - --output hv15.mp4 -``` - -On Blackwell this picks `CUDNN_ATTN` automatically. Check the log for the `Defaulting to ...` line. - -### Explicit backend selection - -```bash -DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN python examples/offline_inference/text_to_video/text_to_video.py \ - --model Lightricks/LTX-2 \ - --prompt "A dog running across a field of golden wheat." \ - --height 480 --width 832 --num-frames 33 \ - --num-inference-steps 40 --seed 42 --guidance-scale 4.0 \ - --output ltx2.mp4 -``` - -### SageAttention (lossy) - -```bash -DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN python examples/offline_inference/text_to_video/text_to_video.py \ - --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \ - --prompt "A dog running across a field of golden wheat." \ - --height 480 --width 832 --num-frames 33 \ - --num-inference-steps 30 --seed 42 --guidance-scale 6.0 \ - --tensor-parallel-size 2 \ - --output hv15_sage.mp4 -``` - -Example: Wan2.2 TI2V 5B - -```bash -DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN python examples/offline_inference/text_to_video/text_to_video.py \ - --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \ - --prompt "A dog running across a field of golden wheat." \ - --height 704 --width 1280 --num-frames 49 \ - --num-inference-steps 30 --seed 42 --guidance-scale 5.0 \ - --tensor-parallel-size 2 \ - --output outputs/wan22_sage.mp4 -``` - -### Enable SageAttention3 - -Example: - -```bash -DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN_3 python examples/offline_inference/text_to_video/text_to_video.py \ - --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \ - --prompt "A dog running across a field of golden wheat." \ - --height 480 --width 832 --num-frames 33 \ - --num-inference-steps 30 --seed 42 --guidance-scale 6.0 \ - --tensor-parallel-size 2 \ - --output outputs/hv15_sage3.mp4 -``` +See [Hugging Face Hub Backends](attention_backends/huggingface_hub.md). diff --git a/docs/user_guide/diffusion/attention_backends/dense_backends.md b/docs/user_guide/diffusion/attention_backends/dense_backends.md new file mode 100644 index 000000000..030550dac --- /dev/null +++ b/docs/user_guide/diffusion/attention_backends/dense_backends.md @@ -0,0 +1,110 @@ +# Dense Attention Backends + +Use these backends when you want dense attention without backend-specific +sparsity. Start with `TORCH_SDPA` for correctness comparisons, then benchmark +the fastest compatible kernel for your model, shape, and hardware. + +For selection precedence and per-role configuration, see the +[attention backend overview](../attention_backends.md). + +## `TORCH_SDPA` + +`TORCH_SDPA` calls PyTorch `scaled_dot_product_attention` and lets PyTorch's +dispatcher choose the implementation. It is always available and is the most +conservative reference when validating another backend. + +```bash +vllm-omni serve --diffusion-attention-backend TORCH_SDPA +``` + +## `FLASH_ATTN` + +`FLASH_ATTN` uses the installed FlashAttention implementation. On Blackwell it +prefers the CuTe-based FlashAttention 4 path and falls back to +FlashAttention 3 or 2 when that path is unavailable. On Hopper, Ada, and +Ampere it is the preferred automatic route when a compatible package is +installed. + +```bash +vllm-omni serve --diffusion-attention-backend FLASH_ATTN +``` + +### FlashAttention 4 on Blackwell + +Install the optional CUDA 13 extra: + +```bash +pip install 'vllm-omni[fa4]' +``` + +Version `4.0.0b18` is required; earlier beta wheels had known JIT failures on +Blackwell. If the CuTe path is unavailable, the backend falls back to the +compatible FlashAttention 3 or 2 path. + +## `CUDNN_ATTN` + +`CUDNN_ATTN` pins PyTorch SDPA to `CUDNN_ATTENTION`. It is particularly useful +for mask-heavy DiTs and is automatically preferred on Blackwell when cuDNN +9.5 or newer is available and the higher-priority TRTLLM route is not +compatible. + +```bash +vllm-omni serve --diffusion-attention-backend CUDNN_ATTN +``` + +### LTX-2.0 limitation + +LTX-2 audio attention has a symbolic head dimension during `torch.compile` +tracing. The cuDNN SDPA selector rejects that symbolic dimension and Dynamo +aborts compilation. This is tracked in +[issue #3121](https://github.com/vllm-project/vllm-omni/issues/3121). + +Use `FLASHINFER_ATTN` or `TORCH_SDPA` as a workaround: + +```bash +DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN \ + python examples/offline_inference/text_to_video/text_to_video.py \ + --model Lightricks/LTX-2 ... +``` + +## `FLASHINFER_ATTN` + +`FLASHINFER_ATTN` uses FlashInfer's batch-prefill wrapper. It is an explicit +option on CUDA platforms and an automatic Blackwell fallback when FlashInfer +is installed but cuDNN is too old for `CUDNN_ATTN`. + +```bash +vllm-omni serve --diffusion-attention-backend FLASHINFER_ATTN +``` + +### FlashInfer quantized attention + +The backend accepts an `AttentionSpec.quant` block. For QK16/V8, keep Q and K +in FP16 or BF16 and use FP8 E4M3 for V: + +```python +from vllm_omni.diffusion.data import ( + AttentionConfig, + AttentionSpec, + AttnQuantSpec, + OmniDiffusionConfig, +) + +config = OmniDiffusionConfig( + diffusion_attention_config=AttentionConfig( + default=AttentionSpec( + backend="FLASHINFER_ATTN", + quant=AttnQuantSpec( + dtype_qk="bfloat16", + dtype_vo="fp8_e4m3", + ), + ), + ), + ..., +) +``` + +`dtype_qk` controls Q and K; `dtype_vo` controls V. Mixed-dtype +configurations require FlashInfer 0.6.16rc1 or newer. The shared quantization +schema is also consumed by TRTLLM, but each backend validates its own allowed +fields and values; see [TRTLLM SAGE quantization](trtllm.md#sage-quantization). diff --git a/docs/user_guide/diffusion/attention_backends/huggingface_hub.md b/docs/user_guide/diffusion/attention_backends/huggingface_hub.md new file mode 100644 index 000000000..946786b97 --- /dev/null +++ b/docs/user_guide/diffusion/attention_backends/huggingface_hub.md @@ -0,0 +1,32 @@ +# Hugging Face Hub Attention Backends + +Hub backends load the same published kernels commonly used with Hugging Face +Diffusers. They are useful when training and serving must use matching kernel +implementations to reduce numerical drift and sampling divergence. + +| Backend | Kernel | Platform | +| --- | --- | --- | +| `FLASH_ATTN_HUB` | `kernels-community/flash-attn2` | Compatible CUDA GPUs | +| `FLASH_ATTN_3_HUB` | `kernels-community/flash-attn3` | Hopper sm_90 or newer | + +## Installation + +```bash +pip install kernels==0.14.1 +``` + +If `kernels` is unavailable, vLLM-Omni warns and falls back to the +corresponding local FlashAttention implementation. On CUDA devices older than +Hopper, `FLASH_ATTN_3_HUB` falls back to `FLASH_ATTN_HUB`. + +## Usage + +```bash +export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN_HUB + +# Equivalent CLI selection +vllm-omni serve --diffusion-attention-backend FLASH_ATTN_HUB +``` + +For per-role kernel alignment, configure the Hub backend on only the relevant +role; see the [attention backend overview](../attention_backends.md#per-role-configuration). diff --git a/docs/user_guide/diffusion/attention_backends/rainfusion.md b/docs/user_guide/diffusion/attention_backends/rainfusion.md new file mode 100644 index 000000000..10f7458be --- /dev/null +++ b/docs/user_guide/diffusion/attention_backends/rainfusion.md @@ -0,0 +1,72 @@ +# RainFusion Attention + +`RAINFUSION_ATTN` runs MindIE-SD RainFusion (`rf_v2`) block-sparse video +attention on Ascend NPU. It pools 128-token key blocks, ranks them per query +block, and attends to the highest-scoring blocks after arranging video tokens +in `(t, h, w)` order. + +Only the video segment is sparse. Prefix rows such as text, visual conditions, +and audio, plus first-frame blocks, remain dense. Unsupported calls—including +warmup steps, skipped layers, missing video geometry, and video segments below +32 blocks—delegate to `FLASH_ATTN`, so compatible models can select RainFusion +globally. + +## Configuration + +| Key | Valid values | Meaning | +| --- | --- | --- | +| `sparsity` | finite, `[0, 1]` | Nominal dropped-key-block fraction; default `0.8`; `0` disables sparsity | +| `start_step` | integer, `>= 0` | Number of early denoise steps kept dense | +| `skip_layers` | selector such as `"0-3,38"` | DiT blocks kept dense | + +```bash +vllm-omni serve MiniMaxAI/MiniMax-H3 \ + --diffusion-attention-config '{"default":{"backend":"RAINFUSION_ATTN",\ + "block_sparse":{"sparsity":0.8,"start_step":0}}}' +``` + +```python +from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, BlockSparseSpec + +config = AttentionConfig( + default=AttentionSpec( + backend="RAINFUSION_ATTN", + block_sparse=BlockSparseSpec( + sparsity=0.8, + start_step=0, + skip_layers="0-1", + ), + ), +) +``` + +Tune `start_step`, then `sparsity`, then `skip_layers`. Increasing the dense +early-step window is usually the cheapest way to recover global structure; +use layer exclusions only after a same-seed dense comparison identifies +sensitive blocks. + +## Requirements and compatibility + +RainFusion requires Ascend NPU and `mindiesd`. Selecting it on another +platform raises. It is incompatible with ring sequence parallelism because +the kernel needs the complete key sequence for ranking; use Ulysses sequence +parallelism with `ring_degree=1`. + +## Geometry handling + +RainFusion handles arbitrary video grids by rearranging video spatially and, +when necessary, promoting a real-video suffix to the always-kept prefix. It +does not pad because `rf_v2` does not consume a padding attention mask. + +For MiniMax-H3 at 1344x768, grid `(62, 24, 42)` produces 62,496 video rows. +The implementation promotes 2,976 rows and leaves 59,520 sparse rows +(`465 x 128`) so the mask and kernel tiling remain aligned. + +Spatial grids aligned to 8x8 are still preferable. A latent height or width +not divisible by 8 creates an always-kept suffix and reduces realized +sparsity. At the image level, that means width and height divisible by 256. +Protected video tails require a compatible MindIE-SD release. + +For common configuration and selector behavior, see the +[attention backend overview](../attention_backends.md) and the +[backend selection design](../../../design/feature/attention_backend_selection.md). diff --git a/docs/user_guide/diffusion/attention_backends/sage.md b/docs/user_guide/diffusion/attention_backends/sage.md new file mode 100644 index 000000000..1e1fa5b60 --- /dev/null +++ b/docs/user_guide/diffusion/attention_backends/sage.md @@ -0,0 +1,63 @@ +# SageAttention + +SageAttention backends provide lossy low-precision attention for diffusion +models. Validate output quality against `TORCH_SDPA` at the same seed before +using either backend in production. + +## `SAGE_ATTN` + +`SAGE_ATTN` uses SageAttention 2.2 with INT8-quantized attention and FP16 +accumulation. + +### Installation + +Install SageAttention into the same environment as vLLM-Omni: + +```bash +git clone https://github.com/thu-ml/SageAttention.git +cd SageAttention +export EXT_PARALLEL=4 NVCC_APPEND_FLAGS="--threads 8" MAX_JOBS=32 +pip install . --no-build-isolation +``` + +Verify the installation: + +```bash +python -c "import sageattention; print(sageattention.__file__)" +``` + +Select it globally: + +```bash +vllm-omni serve --diffusion-attention-backend SAGE_ATTN +``` + +## `SAGE_ATTN_3` + +`SAGE_ATTN_3` uses the SageAttention3 Blackwell implementation. + +### SageAttention3 installation + +```bash +git clone https://github.com/thu-ml/SageAttention.git +cd SageAttention/sageattention3_blackwell +python setup.py install +``` + +Verify the installation: + +```bash +python -c "import sageattn3; print(sageattn3.__file__)" +``` + +```bash +vllm-omni serve --diffusion-attention-backend SAGE_ATTN_3 +``` + +`SAGE_ATTN_3` requires CUDA, an importable `sageattn3`, and a Blackwell-class +GPU. Its kernel assumes the query-head count equals the key/value-head count. +GQA and MQA diffusion calls therefore fall back to PyTorch SDPA for +correctness. + +For common configuration and platform routing, see the +[attention backend overview](../attention_backends.md). diff --git a/docs/user_guide/diffusion/attention_backends/trtllm.md b/docs/user_guide/diffusion/attention_backends/trtllm.md new file mode 100644 index 000000000..dd36ba61e --- /dev/null +++ b/docs/user_guide/diffusion/attention_backends/trtllm.md @@ -0,0 +1,103 @@ +# TRTLLM Attention + +`TRTLLM_ATTN` runs FlashInfer's trtllm-gen FMHA kernels. It supports dense +BF16 attention plus optional Skip-Softmax sparsity or SAGE quantization. + +For common selection and per-role configuration, see the +[attention backend overview](../attention_backends.md). + +## Requirements + +Selecting `TRTLLM_ATTN` requires: + +- datacenter Blackwell, sm_100 or sm_103; +- `head_dim=128`; +- FlashInfer; and +- a model-declared packed or mask-free compatible path. + +It is not supported on workstation Blackwell (sm_120/sm_121). An incompatible +explicit selection raises instead of silently changing the backend. + +## Dense mode + +Select `TRTLLM_ATTN` without `skip_softmax` or `quant` to run dense BF16: + +```bash +vllm-omni serve --diffusion-attention-backend TRTLLM_ATTN +``` + +## Skip-Softmax + +Skip-Softmax trades some fidelity for speed by skipping low-contribution +attention work. See the [feature design](../../../design/feature/skip_softmax.md) +for the algorithm and implementation contract. + +| Key | Valid values | Meaning | +| --- | --- | --- | +| `target_sparsity` | finite, `[0, 1]` | Point on a model-calibrated curve | +| `threshold` | finite, `>= 0` | Direct threshold; mutually exclusive with `target_sparsity` | +| `disabled_until_timestep` | finite, `[0, 1]` | Keep early high-noise steps dense until normalized `t <= D` | + +```bash +vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \ + --diffusion-attention-config '{"default":{"backend":"TRTLLM_ATTN",\ + "skip_softmax":{"target_sparsity":0.65,"disabled_until_timestep":0.86}}}' +``` + +```python +from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, SkipSoftmaxSpec + +config = AttentionConfig( + default=AttentionSpec( + backend="TRTLLM_ATTN", + skip_softmax=SkipSoftmaxSpec( + target_sparsity=0.65, + disabled_until_timestep=0.86, + ), + ), +) +``` + +Start with `target_sparsity=0.65` and `disabled_until_timestep=0.86`. Increase +sparsity only after comparing quality against dense output at the same seed. +The benefit grows with sequence length because only attention is accelerated. + +## SAGE quantization + +TRTLLM SAGE quantizes Q/K per block and V per channel so both attention +matrix multiplications use low precision. + +| Key | Valid values | Meaning | +| --- | --- | --- | +| `dtype_qk` | `int8`, `fp8_e4m3` | Q/K quantization dtype; absent means dense | +| `q_block_size` | `1`, `4`, `16` | Q scale block size; default `1` | +| `k_block_size` | `1`, `4`, `16` | K scale block size; default `16` | + +V is always quantized per channel to FP8 E4M3 and K-smoothing is internal. +Only block sizes with compiled kernels are accepted. + +```bash +vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \ + --diffusion-attention-config '{"default":{"backend":"TRTLLM_ATTN",\ + "quant":{"dtype_qk":"fp8_e4m3","q_block_size":1,"k_block_size":16}}}' +``` + +```python +from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, AttnQuantSpec + +config = AttentionConfig( + default=AttentionSpec( + backend="TRTLLM_ATTN", + quant=AttnQuantSpec( + dtype_qk="fp8_e4m3", + q_block_size=1, + k_block_size=16, + ), + ), +) +``` + +This path requires FlashInfer 0.6.16rc1 or newer. FP8 Q/K kernels are +available on sm_100 and sm_103; INT8 Q/K kernels are compiled for sm_100 only. +The shared `AttnQuantSpec` is backend typed: values intended for +`FLASHINFER_ATTN` are rejected here and vice versa. diff --git a/docs/user_guide/diffusion/cpu_offload.md b/docs/user_guide/diffusion/cpu_offload.md index 02ab9f074..a17e4a114 100644 --- a/docs/user_guide/diffusion/cpu_offload.md +++ b/docs/user_guide/diffusion/cpu_offload.md @@ -1,389 +1,75 @@ # CPU Offloading for Diffusion Models -## Overview +vLLM-Omni provides three CPU-offload strategies for diffusion models. Choose +the coarsest strategy that meets your memory target: finer-grained streaming +saves more device memory but adds more transfer and setup overhead. -vLLM-Omni provides two offloading strategies to reduce GPU memory usage for diffusion models: +For the shared factory, discovery, and lifecycle contract, see the +[CPU Offloading design](../../design/feature/offloader/README.md). -1. **Model-level (Sequential) Offloading**: Mutual exclusion between DiT model and encoder - only one is on GPU at a time. -2. **Layerwise (Blockwise) Offloading**: Keeps only one transformer block on GPU at a time with compute-memory overlap. +## Choose a strategy -Both strategies use pinned memory for faster CPU-GPU transfers. The strategies are **mutually exclusive** for now - if both are enabled, layerwise takes priority. +| Strategy | Device residency | Parallel scope | Guide | +| --- | --- | --- | --- | +| Model-level (sequential) | One pipeline component group at a time | Single device | [Model-Level Offloading](offloader/cpu_offload.md) | +| Layerwise (blockwise) | One transformer block, with next-block prefetch | Single device | [Layerwise Offloading](offloader/layerwise_offload.md) | +| Distributed layerwise | Fixed two-block device buffer; optional host-weight sharding and AllGather | Multiple GPU/NPU ranks | [Distributed Layerwise Offloading](offloader/distributed_layerwise_offload.md) | +All strategies use pinned host memory for faster transfers where applicable. +Configuration priority is: + +1. Distributed layerwise offloading. +2. Layerwise offloading. +3. Model-level offloading. + +Treat the flags as mutually exclusive. If more than one is enabled, the +higher-priority strategy is selected. + +## Quick selection + +- Use [model-level offloading](offloader/cpu_offload.md) when swapping whole + encoders and DiTs is enough to fit and phase-boundary transfers are + acceptable. +- Use [layerwise offloading](offloader/layerwise_offload.md) for compute-heavy video + DiTs where block transfers can overlap computation. +- Use [distributed layerwise offloading](offloader/distributed_layerwise_offload.md) + when a multi-rank deployment also needs bounded device residency and, + optionally, sharded host weights. + +## Supported models + +| Architecture | Example models | DiT class | Model-level | Layerwise | Distributed layerwise | Layerwise block attributes | +| --- | --- | --- | :---: | :---: | :---: | --- | +| Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | yes | yes | — | `transformer_blocks`, `single_transformer_blocks` | +| LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | — | yes | — | `transformer_blocks`, `single_transformer_blocks` | +| NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | — | yes | — | `layers` | +| OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | — | yes | — | `transformer` | +| QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | yes | yes | — | `transformer_blocks` | +| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | yes | yes | — | `down_blocks`, `up_blocks` | +| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | — | yes | — | `transformer_blocks` | +| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | yes | yes | — | `blocks` | +| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | yes | yes | — | `blocks` | +| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` | yes | yes | — | `layers` | +| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | — | yes | — | `layers`, customized modules | +| Cosmos3OmniDiffusersPipeline | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | `Cosmos3VFMTransformer`, `Cosmos3LanguageModel` | yes | yes | yes | `layers`, `gen_layers` | + +Model-level support requires discoverable DiT and encoder components. +Layerwise support requires transformer block topology. Distributed support +reuses that topology but still requires validation for the model, checkpoint, +and parallel configuration. + +## Compatibility anchors + +These headings preserve links to sections that moved into dedicated guides. ## Model-level (Sequential) Offloading -### How It Works - -Model-level offloading implements mutual exclusion between DiT transformer and encoder modules using pre forward hooks: - -- **When encoders run**: DiT transformer is offloaded to CPU -- **When DiT runs**: Encoders are offloaded to CPU, if more than one dit models, only one loaded on GPU, others get offloaded to CPU. -- **VAE**: Stays resident on GPU - -Before each module's forward pass, the hook automatically moves it to GPU while offloading the other module group to CPU. Transfers use pinned memory for speed. - -### Usage - -**Python API:** -```python -from vllm_omni import Omni - -m = Omni(model="Wan-AI/Wan2.2-T2V-A14B-Diffusers", enable_cpu_offload=True) -``` - -**CLI:** -```bash -vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --omni --enable-cpu-offload -``` - -### To Support a Model - -Implement the `SupportsComponentDiscovery` protocol to declare which -submodules serve as pipeline components (used by offloading, HSDP -sharding, and other framework features): - -```python -from typing import ClassVar -from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery - -class MyPipeline(nn.Module, SupportsComponentDiscovery): - _dit_modules: ClassVar[list[str]] = ["transformer"] - _encoder_modules: ClassVar[list[str]] = ["text_encoder", "vision_model"] - _vae_modules: ClassVar[list[str]] = ["vae"] - _resident_modules: ClassVar[list[str]] = [] # optional - - def __init__(self): - super().__init__() - self.transformer = ... # DiT — stays on GPU during denoising - self.text_encoder = ... # Encoder — offloaded to CPU during denoising - self.vision_model = ... # Encoder — offloaded to CPU during denoising - self.vae = ... # VAE — always on GPU -``` - -- `_dit_modules`: attribute names of denoising submodules (kept on GPU - during the diffusion loop). -- `_encoder_modules`: attribute names of encoder/vision submodules - (offloaded to CPU during the diffusion loop). -- `_vae_modules`: attribute names of VAE(s) (always kept on GPU, not - part of the mutual exclusion hooks). -- `_resident_modules`: attribute names of small submodules that must - stay on GPU during layerwise offloading (e.g. embedders, connectors). - Optional — defaults to `[]`. - -All attribute names support dotted paths for nested submodules -(e.g. `"pipe.transformer"`, `"bagel.time_embedder"`). - -Both DiT and encoder lists are needed because the offload hooks use -mutual exclusion: when one group runs, the other moves to CPU. - -### Limitations -- Cold start latency increases -- Adds overhead from CPU-GPU transfers between encoder and denoising phases -- Support single GPU only for now - - -### Component offloading for split models (e.g. Cosmos3) - -Some models split their transformer into mutually-exclusive *components* that run -in different phases of a single forward pass rather than as separate pipeline -components -- e.g. Cosmos3's understanding (reasoner) component runs once per -generation while the generation (generator) component runs every denoising step. -Such models have no separate text encoder to swap against, so the transformer -owns a small model-local offload path and wraps each phase with -`with self._offload_context(name):` - -```python -class Cosmos3VFMTransformer(nn.Module): - def forward(self, ...): - with self._offload_context("reasoner"): - ... # understanding pass, runs once - with self._offload_context("generator"): - ... # denoising pass, runs every step -``` - -Model-level offloading then keeps exactly one component GPU-resident at a time -(the other on CPU), reusing the same `SequentialOffloadHook` `.to()` movers. The -pipeline opts in by exposing `enable_omni_model_cpu_offload` (which drives the -transformer's `enable_model_cpu_offload` and pins the VAE). Layerwise offloading -works for these models too -- each component declares its own block container via -`_layerwise_offload_blocks_attrs`. - +See [Model-Level Offloading](offloader/cpu_offload.md). ## Layerwise (Blockwise) Offloading -### How It Works - -Layerwise offloading keeps only one transformer block on GPU at a time. - -As each block completes, the next block is prefetched to GPU while the current block is freed. The pre and forward hooks utilized by layerwise offloading apply a separate CUDA stream (`copy_stream`) to overlap weight transfer with computation, and retain flattened tensors in pinned CPU memory for block parameters re-materialization. Encoders, VAE, and non-block DiT modules (embeddings, norms) always stay on GPU. - -**Execution Flow:** - -| Block | Pre-forward Hook | Forward | Post-forward Hook | -|-------|------------------|---------|-------------------| -| block-0 | Prefetch block-1 (async) | Compute block-0 | Free block-0 | -| block-1 | Prefetch block-2 (async) | Compute block-1 | Free block-1 | -| ... | ... | ... | ... | -| block-(n-1) | **Prefetch block-0** (async) | Compute block-(n-1) | Free block-(n-1) | - -Each transformer block has a `LayerwiseOffloadHook` that prefetches the next block before forward and frees the current block after forward. - -Layerwise offloading is primarily recommended for large **video generation models** where the compute cost per block is high enough to effectively overlap with memory prefetch operations. For example, Wan2.2 T2V and I2V pipelines. - -### Usage - -**Python API:** -```python -from vllm_omni import Omni - -# Text-to-video -m = Omni(model="Wan-AI/Wan2.2-T2V-A14B-Diffusers", enable_layerwise_offload=True) - -# Or image-to-video -m = Omni(model="Wan-AI/Wan2.2-I2V-A14B-Diffusers", enable_layerwise_offload=True) -``` - -**CLI:** -```bash -# Text-to-video -vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --omni --enable-layerwise-offload - -# Or image-to-video -vllm serve Wan-AI/Wan2.2-I2V-A14B-Diffusers --omni --enable-layerwise-offload -``` - -### To Support a Model - -Models must define the blocks attribute name for layerwise offloading: - -```python -class WanTransformer3DModel(nn.Module): - _layerwise_offload_blocks_attrs = ["blocks"] # Attribute names containing transformer blocks - - def __init__(self): - self.blocks = nn.ModuleList([...]) # Transformer blocks -``` - -For models with multiple block types: - -```python -class Flux2Transformer2DModel(nn.Module): - _layerwise_offload_blocks_attrs = ["transformer_blocks", "single_transformer_blocks"] -``` - -### Limitations -- Cold start latency increases because offloaded components must be moved to CPU - during setup; layerwise offload may add extra weight consolidation and pinning - work. -- Performance depends on compute cost and H2D bandwidth as well -- Support single GPU only for now - +See [Layerwise Offloading](offloader/layerwise_offload.md). ## Distributed Layerwise Offloading -### How It Works - -Distributed layerwise offloading extends single-GPU layerwise offloading to -multi-device deployments. In the default mode, each rank stores only -**1/dp_size** of the model weights in host memory; full layer weights are -reconstructed at runtime via **AllGather** on a dedicated communication stream, -overlapped with computation via a fixed double-buffer scheme. The -`--dlo-no-use-allgather` mode instead streams standard-loader rank-local -weights independently and does not shard weights across ranks. - -For the implementation design and compatibility matrix, see -[Distributed Layerwise Offload](../../design/feature/distributed_layerwise_offload.md). - -**Key features:** -- **Weight sharding + AllGather**: each rank stores 1/dp_size of weights, - reconstructed per-layer via `all_gather_into_tensor` -- **Fixed double-buffer**: exactly 2 transformer blocks on each device at any - time, regardless of model size -- **DP multi-concurrency**: N concurrent requests processed in parallel - (AllGather only gathers weight shards, which are request-independent) -- **mmap weight loading**: weights loaded as mmap views pointing to shared OS - page cache, eliminating O(dp_size × model_size) RSS during model creation -- **Platform-agnostic**: works on NVIDIA GPU (CUDA/NCCL) and Ascend NPU - (CANN/HCCL) via vLLM-Omni's platform abstraction - -**Execution Flow (per device):** - -``` -Compute Stream: [Layer N] [Layer N+1] [Layer N+2] -H2D Stream: [H2D Shard N+1] [H2D Shard N+2] -AllGather: [AG N+1] [AG N+2] -Slot Usage: Slot0: Layer N Slot1: Layer N+1 -``` - -### Usage - -**CLI:** -```bash -# 4× GPU/NPU with AllGather (recommended) -vllm serve /path/to/model --omni \ - --enable-distributed-layerwise-offload \ - --data-parallel-size 4 - -# Without DLO AllGather (each rank streams standard-loader rank-local weights) -vllm serve /path/to/model --omni \ - --enable-distributed-layerwise-offload \ - --data-parallel-size 4 \ - --dlo-no-use-allgather - -# With SP instead of DP (long sequences) -vllm serve /path/to/model --omni \ - --enable-distributed-layerwise-offload \ - --usp 4 -``` - -**Python API:** -```python -from vllm_omni import Omni - -m = Omni( - model="/path/to/model", - enable_distributed_layerwise_offload=True, - dlo_use_allgather=True, # default -) -``` - -### CLI Flags - -| Flag | Description | Default | -|------|-------------|---------| -| `--enable-distributed-layerwise-offload` | Enable DLO | `false` | -| `--data-parallel-size N` | Number of DP ranks; DLO weight-sharding group when AllGather is enabled | `1` | -| `--dlo-use-allgather` | Shard + AllGather (saves CPU, requires concurrent requests) | `true` | -| `--dlo-no-use-allgather` | Use standard-loader rank-local weights; no DLO sharding or AllGather | `false` | - -### How model weights are loaded (mmap path) - -When DLO + AllGather is active, the offloader: - -1. **Saves non-persistent buffers** (e.g. RoPE `inv_freq`, timestep `freqs`) - from the normally-created transformer -2. **Converts to meta device** via `to_empty(device="meta")`, releasing random - initialization weights -3. **Loads checkpoint weights as mmap views** via `safe_open().get_tensor()`, - which return views into the OS page cache (shared across all ranks, 0 RSS) -4. **Calls `post_load_weights()`** to apply model-specific dtype conversions - (e.g. Cosmos3's `time_embedder` → FP32) -5. **Restores non-persistent buffers** from saved copies - -This approach requires **zero model-specific code changes** — no pipeline or -transformer modifications are needed. - -### OffloadPlan (declarative topology metadata) - -Models can optionally declare an `OffloadPlan` class variable to provide -topology metadata (block attribute names, submodules to offload) without -any offload-specific logic: - -```python -from vllm_omni.diffusion.offloader import OffloadPlan - -class MyPipeline(nn.Module): - _dit_modules = ["transformer"] - _offload_plan = OffloadPlan( - block_attrs={"transformer": ("blocks",)}, - offload_submodules={"context_encoder": "layers"}, - ) -``` - -When not declared, the offloader falls back to `_layerwise_offload_blocks_attrs` -and heuristic attribute search. This is backward-compatible — existing models -work without any changes. - -### DP Multi-concurrency - -When `--data-parallel-size > 1` and AllGather is enabled, the scheduler batches -up to `dp_size` requests per denoise step. Each DP rank processes a different -request while AllGather synchronizes only weight shards (request-independent). - -**Requirements for concurrent requests:** -- `num_inference_steps` must be specified explicitly (None is not allowed) -- All concurrent requests must have the same `num_inference_steps` value -- These constraints exist because AllGather is a collective that requires - every rank to participate at each denoise step - -### Limitations -- Online quantization (FP8) is incompatible with mmap loading — falls back to - regular loading only when the model is outside the DLO+AllGather mmap path; - online quantization with DLO+AllGather is rejected. Use - `--dlo-no-use-allgather` or disable online quantization. -- Tensor Parallel > 1 is rejected in the DLO+AllGather mmap path because TP-aware - weight-loader callbacks are bypassed. The no-AllGather path retains the - standard TP loader and is experimental. -- HSDP + AllGather is rejected because it would double-shard parameters. HSDP - with no-AllGather is accepted at configuration level, but has limited - end-to-end validation. -- `num_inference_steps=None` is not allowed in DP multi-concurrency mode - -**Module Discovery** - -The offloader discovers pipeline components in two ways: - -1. **Protocol-based** (preferred): If the pipeline implements - `SupportsComponentDiscovery`, its `_dit_modules`, `_encoder_modules`, - `_vae_modules`, and `_resident_modules` class variables are used - directly. All attribute names support dotted paths (e.g. - `"pipe.transformer"`, `"bagel.time_embedder"`) for nested submodules. - -2. **Fallback attribute scan**: Otherwise, the offloader scans for - well-known attribute names: - - **DiT modules**: `transformer`, `transformer_2`, `dit`, `sr_dit`, `language_model`, `transformer_blocks`, `model` - - **Encoders**: `text_encoder`, `text_encoder_2`, `text_encoder_3`, `image_encoder` - - **VAE**: `vae`, `audio_vae` - -**Hook System** - -Both strategies use vLLM-Omni's hook registry system (`HookRegistry` and `ModelHook`) to register pre/post forward callbacks on modules, enabling automatic swapping without modifying model code. - -**Backend Architecture** - -``` -OffloadBackend (base class) -├── ModelLevelOffloadBackend → uses SequentialOffloadHook (.to() swap) -│ (delegates to a pipeline's enable_omni_model_cpu_offload -│ for split models like Cosmos3) -├── LayerWiseOffloadBackend → uses LayerwiseOffloadHook -│ (single-GPU, full weights on host) -└── DistributedLayerwiseOffloadBackend → uses DistributedLayerwiseOffloadHook - (multi-GPU, sharded AllGather or rank-local mode) -``` - -Factory function `get_offload_backend()` selects the appropriate backend based on -configuration. - -For split models, `ModelLevelOffloadBackend.enable()` detects a pipeline's -`enable_omni_model_cpu_offload` hook and delegates to it; Cosmos3 then swaps its -reasoner/generator components inside the model forward pass. - - -## Supported Models - -| Architecture | Example Models | DiT Class | Model-Level Offload | Layerwise Offload | Distributed Layerwise Offload | Blocks Attrs (Layerwise specific) | -|--------------|----------------|-----------|---------------------|-------------------|-------------------------------|-----------------------------------| -| Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | ✓ | ✓ | - | `"transformer_blocks"`, `"single_transformer_blocks"` | -| LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | - | ✓ | - | `"transformer_blocks"`, `"single_transformer_blocks"` | -| NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | - | ✓ | - | `"layers"` | -| OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | - | ✓ | - | `"transformer"` | -| QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | ✓ | ✓ | - | `"transformer_blocks"` | -| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | ✓ | ✓ | - | `"down_blocks"`, `"up_blocks"` | -| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | - | `"transformer_blocks"` | -| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | - | `"blocks"` | -| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | - | `"blocks"` | -| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` (`cfm_decoder.model.diff_estimator`) | ✓ | ✓ | - | `"layers"` | -| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | - | ✓ | - | `"layers"`, `"customized modules"` | -| Cosmos3OmniDiffusersPipeline | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | `Cosmos3VFMTransformer`, `Cosmos3LanguageModel` | ✓ | ✓ | ✓ | `"layers"`, `"gen_layers"` | - -**Notes:** -- Model-Level Offloading is expected to be supported by all common diffusion models (DiT and encoders) naturally -- Layerwise Offloading requires DiT class to define `_layerwise_offload_blocks_attrs` pointing to transformer blocks -- Distributed Layerwise Offloading reuses the layerwise block topology, but the - model and parallelism combination still needs validation. AllGather mode - additionally requires a compatible checkpoint/mmap layout; no-AllGather mode - retains the standard loader's rank-local tensors. See the - [DLO design](../../design/feature/distributed_layerwise_offload.md) and - [Cosmos3 DistOffload recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md) - for usage examples. +See [Distributed Layerwise Offloading](offloader/distributed_layerwise_offload.md). diff --git a/docs/user_guide/diffusion/offloader/cpu_offload.md b/docs/user_guide/diffusion/offloader/cpu_offload.md new file mode 100644 index 000000000..558985e66 --- /dev/null +++ b/docs/user_guide/diffusion/offloader/cpu_offload.md @@ -0,0 +1,81 @@ +# Model-Level Offloading + +Model-level, or sequential, offloading keeps only the pipeline component group +currently executing on the accelerator. It is the simplest offload strategy +and is selected with `--enable-cpu-offload`. + +## How it works + +Pre-forward hooks enforce mutual exclusion between DiT and encoder modules: + +- before an encoder runs, the DiT moves to CPU; +- before a DiT runs, encoders and other DiTs move to CPU; and +- VAE modules remain on the accelerator. + +Pinned host memory reduces transfer overhead. Transfers occur at phase +boundaries, so cold-start and encoder-to-denoiser transitions become slower. + +## Usage + +```python +from vllm_omni import Omni + +omni = Omni( + model="Wan-AI/Wan2.2-T2V-A14B-Diffusers", + enable_cpu_offload=True, +) +``` + +```bash +vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \ + --omni --enable-cpu-offload +``` + +## Model integration + +Pipelines should implement `SupportsComponentDiscovery`: + +```python +from typing import ClassVar + +from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery + + +class MyPipeline(nn.Module, SupportsComponentDiscovery): + _dit_modules: ClassVar[list[str]] = ["transformer"] + _encoder_modules: ClassVar[list[str]] = ["text_encoder", "vision_model"] + _vae_modules: ClassVar[list[str]] = ["vae"] + _resident_modules: ClassVar[list[str]] = [] +``` + +All entries may be dotted paths. DiT and encoder lists are both required for +mutual exclusion. VAE modules are pinned but not swapped; resident modules are +small modules that must stay on the accelerator for layerwise paths. + +## Split-model components + +Some models, such as Cosmos3, split one transformer into mutually exclusive +components that run in different phases. The pipeline exposes +`enable_omni_model_cpu_offload`, and the backend delegates to the model-local +contexts: + +```python +class Cosmos3VFMTransformer(nn.Module): + def forward(self, ...): + with self._offload_context("reasoner"): + ... + with self._offload_context("generator"): + ... +``` + +This preserves the same invariant—exactly one component is device resident— +while reusing sequential `.to()` movers. + +## Limitations + +- Single device only. +- Higher cold-start latency. +- Transfers between encoder and denoising phases add latency. + +See the [model-level design](../../../design/feature/offloader/cpu_offload.md) +for lifecycle and extension invariants. diff --git a/docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md b/docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md new file mode 100644 index 000000000..15298c897 --- /dev/null +++ b/docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md @@ -0,0 +1,118 @@ +# Distributed Layerwise Offloading + +Distributed layerwise offloading (DLO) extends block streaming to multi-device +deployments. With AllGather enabled, each rank stores roughly `1 / dp_size` of +the host weights and reconstructs each layer at runtime. Without AllGather, +each rank streams its standard-loader rank-local weights independently. + +See the [DLO feature design](../../../design/feature/offloader/distributed_layerwise_offload.md) +for the implementation contract and compatibility matrix. + +## Execution model + +DLO overlaps three operations with a fixed two-block device buffer: + +```text +Compute stream: [Layer N] [Layer N+1] [Layer N+2] +H2D stream: [H2D shard N+1] [H2D shard N+2] +AllGather: [AG N+1] [AG N+2] +Slots: slot 0: Layer N slot 1: Layer N+1 +``` + +AllGather communicates only request-independent weight shards, so data- +parallel ranks may process different requests concurrently. + +## Usage + +```bash +# Four ranks with sharded host weights and AllGather +vllm serve /path/to/model --omni \ + --enable-distributed-layerwise-offload \ + --data-parallel-size 4 + +# Standard-loader rank-local weights, without DLO AllGather +vllm serve /path/to/model --omni \ + --enable-distributed-layerwise-offload \ + --data-parallel-size 4 \ + --dlo-no-use-allgather + +# Sequence parallel deployment +vllm serve /path/to/model --omni \ + --enable-distributed-layerwise-offload \ + --usp 4 +``` + +```python +from vllm_omni import Omni + +omni = Omni( + model="/path/to/model", + enable_distributed_layerwise_offload=True, + dlo_use_allgather=True, +) +``` + +## Flags + +| Flag | Meaning | Default | +| --- | --- | --- | +| `--enable-distributed-layerwise-offload` | Enable DLO | `false` | +| `--data-parallel-size N` | DP ranks and AllGather weight-sharding group | `1` | +| `--dlo-use-allgather` | Shard host weights and reconstruct with AllGather | `true` | +| `--dlo-no-use-allgather` | Keep standard-loader rank-local weights | `false` | +| `--dlo-resident-layers N` | Keep N leading main-DiT blocks on device; requires no-AllGather and model-declared resident paths | `0` | + +## mmap weight loading + +The DLO plus AllGather path: + +1. saves non-persistent buffers such as RoPE frequencies; +2. moves the normally created transformer to the meta device; +3. loads checkpoint tensors as mmap views backed by the shared OS page cache; +4. calls model-specific `post_load_weights()` conversions; and +5. restores the saved non-persistent buffers. + +This avoids one full checkpoint RSS copy per rank and does not require +model-specific loading code. + +## Declarative topology + +Models may declare an `OffloadPlan` instead of embedding offload logic: + +```python +from vllm_omni.diffusion.offloader import OffloadPlan + + +class MyPipeline(nn.Module): + _dit_modules = ["transformer"] + _offload_plan = OffloadPlan( + block_attrs={"transformer": ("blocks",)}, + offload_submodules={"context_encoder": "layers"}, + ) +``` + +When no plan exists, discovery falls back to +`_layerwise_offload_blocks_attrs` and then heuristic attribute lookup. + +## Data-parallel concurrency + +With `data_parallel_size > 1` and AllGather enabled, the scheduler can process +up to `dp_size` requests per denoising step. Every concurrent request must set +the same explicit `num_inference_steps`; `None` is rejected because every rank +must enter each collective. + +## Limitations + +- Online FP8 quantization is rejected with the DLO plus AllGather mmap path. + Use `--dlo-no-use-allgather` or disable online quantization. +- Tensor parallel size greater than one is rejected in the mmap path because + it bypasses TP-aware loader callbacks. The no-AllGather path retains the + standard loader and remains experimental with TP. +- HSDP plus AllGather is rejected to avoid double sharding. HSDP without + AllGather has limited end-to-end validation. +- Resident leading layers require `--dlo-no-use-allgather` and a model + `OffloadPlan` that declares eligible `resident_dit_paths`. +- DP concurrency requires an explicit, identical inference-step count. + +See the [Cosmos3 DistOffload recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md) +for an end-to-end example. diff --git a/docs/user_guide/diffusion/offloader/layerwise_offload.md b/docs/user_guide/diffusion/offloader/layerwise_offload.md new file mode 100644 index 000000000..c88b67fcc --- /dev/null +++ b/docs/user_guide/diffusion/offloader/layerwise_offload.md @@ -0,0 +1,65 @@ +# Layerwise Offloading + +Layerwise, or blockwise, offloading keeps one transformer block on the +accelerator and prefetches the next block while the current block computes. +It is best suited to compute-heavy video DiTs whose block execution can hide +host-to-device transfers. + +## Execution flow + +Each block has a pre-forward and post-forward hook. Parameters are consolidated +in pinned host tensors and rematerialized for execution on a dedicated copy +stream. + +| Block | Pre-forward hook | Forward | Post-forward hook | +| --- | --- | --- | --- | +| block 0 | Prefetch block 1 | Compute block 0 | Free block 0 | +| block 1 | Prefetch block 2 | Compute block 1 | Free block 1 | +| ... | ... | ... | ... | +| last block | Prefetch block 0 | Compute last block | Free last block | + +Encoders, VAE modules, and non-block DiT modules such as embeddings and norms +remain device resident. + +## Usage + +```python +from vllm_omni import Omni + +omni = Omni( + model="Wan-AI/Wan2.2-T2V-A14B-Diffusers", + enable_layerwise_offload=True, +) +``` + +```bash +vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \ + --omni --enable-layerwise-offload +``` + +## Model integration + +Transformer classes declare containers of executable blocks: + +```python +class WanTransformer3DModel(nn.Module): + _layerwise_offload_blocks_attrs = ["blocks"] + + +class Flux2Transformer2DModel(nn.Module): + _layerwise_offload_blocks_attrs = [ + "transformer_blocks", + "single_transformer_blocks", + ] +``` + +See the [layerwise design](../../../design/feature/offloader/layerwise_offload.md) for +the discovery and hook invariants. `OffloadPlan` is a separate declarative +topology path for distributed layerwise offload. + +## Limitations + +- Single device only. +- Setup consolidates and pins block parameters, increasing cold-start time. +- Performance depends on block compute time and host-to-device bandwidth; + lightweight blocks may not hide transfers. diff --git a/docs/user_guide/diffusion_features.md b/docs/user_guide/diffusion_features.md index 7c664e88b..11c6366b7 100644 --- a/docs/user_guide/diffusion_features.md +++ b/docs/user_guide/diffusion_features.md @@ -106,7 +106,10 @@ The following tables show which models support each feature: > Notes: -> 1. CPU Offload has two methods: Module-wise (default for models with DiT + text encoder) and Layerwise. The tables below show **Layerwise support** only. Split models like Cosmos3 (no separate text encoder) swap their reasoner/generator components for module-wise offload; see the [CPU Offload Guide](diffusion/cpu_offload.md). +> 1. CPU Offload has three strategies: model-level, layerwise, and distributed +> layerwise. The tables below show **layerwise support** only. Split models +> like Cosmos3 swap their reasoner/generator components for model-level +> offload; see the [CPU Offload Guide](diffusion/cpu_offload.md). > 2. The **💾Quantization** column is collapsed for readability. See [Quantization](quantization/overview.md) for per-method and per-model support details. ### ImageGen @@ -214,7 +217,7 @@ The following tables show which models support each feature: 3. CPU Offloading (Layerwise) and CPU Offloading (Module-wise) are not compatible. 4. The CPU Offloading (Layerwise) row describes local layerwise offload. Multi-device Distributed Layerwise Offload has a separate topology and - compatibility matrix in the [CPU Offloading guide](diffusion/cpu_offload.md#distributed-layerwise-offloading). + compatibility matrix in the [Distributed Layerwise Offloading guide](diffusion/offloader/distributed_layerwise_offload.md). 5. The compatibility matrix uses FP8 as the representative quantization method. 6. Step Execution is not compatible with any diffusion cache backend. LoRA is supported, but each scheduled batch must use a single adapter (requests with different `lora_request` or `lora_scale` are kept in separate batches). diff --git a/docs/user_guide/feature_compatibility.md b/docs/user_guide/feature_compatibility.md index ea05fe81e..eeb087510 100644 --- a/docs/user_guide/feature_compatibility.md +++ b/docs/user_guide/feature_compatibility.md @@ -165,7 +165,7 @@ vllm serve Qwen/Qwen-Image --omni --port 8091 \ - **Tensor Parallelism — Text Encoder Not Sharded**: TP currently only shards the DiT blocks. Each TP rank retains a **full copy of the text encoder weights**, leading to significant GPU memory overhead proportional to TP degree. Tracked in [Issue #771](https://github.com/vllm-project/vllm-omni/issues/771). -- **CPU Offloading — Two Modes Are Mutually Exclusive**: Model-level offload (`enable_cpu_offload`) and layerwise offload (`enable_layerwise_offload`) cannot be used simultaneously. If both are set, layerwise takes priority and model-level is silently ignored. +- **CPU Offloading — Strategies Are Mutually Exclusive**: Distributed layerwise, layerwise, and model-level offload should not be enabled together. Selection priority is distributed layerwise, then layerwise, then model-level. - **CPU Offloading — VAE stays on GPU**: Both offloading strategies keep the VAE on GPU at all times. For high-resolution generation, VAE decode can still cause OOM. Mitigate by combining with `vae_use_tiling=True` or VAE Patch Parallelism. diff --git a/mkdocs.yml b/mkdocs.yml index 1a3b5109f..1a96ca67d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,6 +81,10 @@ plugins: user_guide/quantization/index.md: user_guide/quantization/overview.md design/feature/diffusion_request_level_batching.md: design/feature/diffusion_continuous_batching.md design/feature/diffusion_step_execution.md: design/feature/diffusion_continuous_batching.md + design/feature/cpu_offloading.md: design/feature/offloader/README.md + design/feature/model_level_offload.md: design/feature/offloader/cpu_offload.md + design/feature/layerwise_offload.md: design/feature/offloader/layerwise_offload.md + design/feature/distributed_layerwise_offload.md: design/feature/offloader/distributed_layerwise_offload.md - glightbox - git-revision-date-localized: # exclude files From a70d4d4b3c0989714cbe1385a2cb2142b0233c6d Mon Sep 17 00:00:00 2001 From: hotTea <37091129+hotTeaFun@users.noreply.github.com> Date: Sat, 15 Aug 2026 16:03:04 +0800 Subject: [PATCH 023/267] [codex] Fix diffusion worker timeout for large broadcast payloads (#4845) Signed-off-by: jiangweifeng.dev Co-authored-by: jiangweifeng.dev Co-authored-by: Hongsheng Liu --- .../test_diffusion_worker_broadcast.py | 171 ++++++++++++++++++ .../diffusion/worker/diffusion_worker.py | 6 +- 2 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 tests/diffusion/test_diffusion_worker_broadcast.py diff --git a/tests/diffusion/test_diffusion_worker_broadcast.py b/tests/diffusion/test_diffusion_worker_broadcast.py new file mode 100644 index 000000000..03f84b819 --- /dev/null +++ b/tests/diffusion/test_diffusion_worker_broadcast.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import threading +import time +from unittest.mock import Mock + +import pytest +from vllm.distributed.device_communicators.shm_broadcast import MessageQueue + +import vllm_omni.diffusion.worker.diffusion_worker as diffusion_worker_module +from vllm_omni.diffusion.worker.diffusion_worker import WorkerProc + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + +class _DelayedMultipartSocket: + def __init__( + self, + socket, + send_started: threading.Event, + release_send: threading.Event, + ): + self._socket = socket + self._send_started = send_started + self._release_send = release_send + self._delay_next_send = True + + def __getattr__(self, name): + return getattr(self._socket, name) + + def send_multipart(self, *args, **kwargs): + if self._delay_next_send: + self._delay_next_send = False + self._send_started.set() + if not self._release_send.wait(timeout=2): + raise TimeoutError("Timed out waiting to release delayed overflow payload") + return self._socket.send_multipart(*args, **kwargs) + + +class _ObservedPollSocket: + def __init__(self, socket, poll_started: threading.Event): + self._socket = socket + self._poll_started = poll_started + + def __getattr__(self, name): + return getattr(self._socket, name) + + def poll(self, *args, **kwargs): + self._poll_started.set() + return self._socket.poll(*args, **kwargs) + + +class _RecordingWorker: + def __init__(self): + self.calls: list[str] = [] + + def execute_method(self, method, *args, **kwargs): + assert method == "record" + self.calls.append(args[0]) + + +def test_recv_message_waits_indefinitely(): + worker_proc = WorkerProc.__new__(WorkerProc) + worker_proc.mq = Mock() + worker_proc.mq.dequeue.return_value = {"type": "rpc"} + + result = worker_proc.recv_message() + + assert result == {"type": "rpc"} + worker_proc.mq.dequeue.assert_called_once_with(indefinite=True) + + +def test_busy_loop_preserves_order_when_overflow_payload_is_delayed(monkeypatch): + """A consumed overflow marker must not be abandoned on a socket timeout.""" + writer = MessageQueue( + n_reader=1, + n_local_reader=1, + local_reader_ranks=[0], + max_chunk_bytes=1024, + max_chunks=4, + ) + reader = MessageQueue.create_from_handle(writer.export_handle(), rank=0) + writer.wait_until_ready() + reader.wait_until_ready() + + overflow_send_started = threading.Event() + release_overflow_send = threading.Event() + overflow_poll_started = threading.Event() + writer.local_socket = _DelayedMultipartSocket( + writer.local_socket, + overflow_send_started, + release_overflow_send, + ) + reader.local_socket = _ObservedPollSocket(reader.local_socket, overflow_poll_started) + + recording_worker = _RecordingWorker() + worker_proc = WorkerProc.__new__(WorkerProc) + worker_proc.gpu_id = 0 + worker_proc.mq = reader + worker_proc.worker = recording_worker + worker_proc.result_mq = None + worker_proc.wake_event = None + worker_proc._running = True + + # Make the pre-fix implementation fail quickly: it read this module-level + # value directly from _worker_busy_loop. The fixed implementation ignores it + # and delegates to recv_message(), which waits indefinitely. + monkeypatch.setattr( + diffusion_worker_module, + "_BROADCAST_DEQUEUE_TIMEOUT_S", + 0.02, + raising=False, + ) + + overflow_message = { + "type": "rpc", + "method": "record", + "args": ("overflow-1",), + "padding": b"x" * 2048, + } + inline_message = { + "type": "rpc", + "method": "record", + "args": ("inline-2",), + } + shutdown_message = {"type": "shutdown"} + thread_errors: list[BaseException] = [] + + def send_messages(): + try: + writer.enqueue(overflow_message) + writer.enqueue(inline_message) + writer.enqueue(shutdown_message) + except BaseException as exc: + thread_errors.append(exc) + + def run_busy_loop(): + try: + worker_proc._worker_busy_loop() + except BaseException as exc: + thread_errors.append(exc) + + writer_thread = threading.Thread(target=send_messages, daemon=True) + worker_thread = threading.Thread(target=run_busy_loop, daemon=True) + + try: + writer_thread.start() + worker_thread.start() + + assert overflow_send_started.wait(timeout=2) + assert overflow_poll_started.wait(timeout=2) + + # Keep the multipart unavailable beyond the pre-fix 20 ms timeout. + time.sleep(0.1) + release_overflow_send.set() + + writer_thread.join(timeout=2) + worker_thread.join(timeout=2) + + assert not writer_thread.is_alive() + assert not worker_thread.is_alive() + assert thread_errors == [] + assert recording_worker.calls == ["overflow-1", "inline-2"] + finally: + release_overflow_send.set() + worker_proc._running = False + reader.shutdown() + writer.shutdown() + writer_thread.join(timeout=1) + worker_thread.join(timeout=1) diff --git a/vllm_omni/diffusion/worker/diffusion_worker.py b/vllm_omni/diffusion/worker/diffusion_worker.py index 0820d82b4..17fbb5342 100644 --- a/vllm_omni/diffusion/worker/diffusion_worker.py +++ b/vllm_omni/diffusion/worker/diffusion_worker.py @@ -1087,6 +1087,10 @@ class WorkerProc: result["wave_id"] = wave_id return result, should_reply + def recv_message(self) -> Any: + """Receive one complete broadcast message without dropping overflow data.""" + return self.mq.dequeue(indefinite=True) + def _worker_busy_loop(self) -> None: """Main busy loop for Multiprocessing Workers.""" logger.info(f"Worker {self.gpu_id} ready to receive requests via shared memory") @@ -1094,7 +1098,7 @@ class WorkerProc: while self._running: msg = None try: - msg = self.mq.dequeue(timeout=1.0) + msg = self.recv_message() except Exception: if self.wake_event and self.wake_event.is_set(): self.wake_event.clear() From 7b76b6446d29adc23f563b75f604159ab11ecac8 Mon Sep 17 00:00:00 2001 From: mokeke <103873176+mo-ke-ke@users.noreply.github.com> Date: Sat, 15 Aug 2026 16:27:52 +0800 Subject: [PATCH 024/267] [Model][Performance] Optimize MiniMax-H3 strict Ulysses boundaries (#6173) Signed-off-by: mokashliu Co-authored-by: mokashliu Co-authored-by: WeiQing Chen <40507679+david6666666@users.noreply.github.com> --- .../minimax_h3/test_minimax_h3_parallel.py | 281 +++++++++++++++++- .../diffusion/cache/teacache/extractors.py | 1 + .../minimax_h3/minimax_h3_transformer.py | 174 +++++++++-- 3 files changed, 432 insertions(+), 24 deletions(-) diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py index 2b168499e..18345f247 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py @@ -36,7 +36,13 @@ def test_transformer_declares_cache_sp_layerwise_offload_and_hsdp(): assert MiniMaxH3DiTModel._layerwise_offload_blocks_attrs == ["blocks"] assert MiniMaxH3DiTModel._cache_dit_adapter_config.block_forward_patterns["blocks"] == ForwardPattern.Pattern_3 assert not MiniMaxH3DiTModel._cache_dit_adapter_config.has_separate_cfg - assert set(MiniMaxH3DiTModel._sp_plan) == {"sp_prepare", "sp_gather"} + assert set(MiniMaxH3DiTModel._sp_plan) == { + "sp_prepare", + "local_sp_prepare", + "sp_gather", + } + assert set(MiniMaxH3DiTModel._sp_plan["sp_prepare"]) == {0, 1, 2} + assert set(MiniMaxH3DiTModel._sp_plan["local_sp_prepare"]) == {2} model = object.__new__(MiniMaxH3DiTModel) nn.Module.__init__(model) @@ -89,6 +95,279 @@ def test_h3_fused_rope_matches_reference_and_preserves_unrotated_dims(): torch.testing.assert_close(actual[..., 96:], x[..., 96:], atol=0, rtol=0) +def test_h3_rope_table_materializes_local_rows_in_fused_kernel_layout(): + from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import ( + _build_rope_table, + ) + + local_freqs_half = torch.randn(2, 48) + local_freqs = torch.cat((local_freqs_half, local_freqs_half), dim=-1) + + actual = _build_rope_table(local_freqs) + expected = torch.cat( + (torch.cos(local_freqs_half), torch.sin(local_freqs_half)), + dim=-1, + ).to(torch.bfloat16) + + assert actual.shape == (2, 96) + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + + +def test_strict_sp_local_span_uses_rank_owned_rows(monkeypatch): + from vllm_omni.diffusion import forward_context + from vllm_omni.diffusion.distributed import parallel_state + from vllm_omni.diffusion.models.minimax_h3 import minimax_h3_transformer as h3 + + monkeypatch.setattr(forward_context, "is_forward_context_available", lambda: True) + monkeypatch.setattr(forward_context, "get_ulysses_mode", lambda **kwargs: "strict") + monkeypatch.setattr(parallel_state, "get_sequence_parallel_world_size", lambda: 8) + monkeypatch.setattr(parallel_state, "get_sequence_parallel_rank", lambda: 3) + monkeypatch.setattr(parallel_state, "get_ulysses_parallel_world_size", lambda: 8) + monkeypatch.setattr(parallel_state, "get_ring_parallel_world_size", lambda: 1) + monkeypatch.setattr(parallel_state, "get_allgather_parallel_world_size", lambda: 1) + + assert h3._sequence_parallel_local_span(16, hooks_applied=True) == (6, 2) + + +@pytest.mark.parametrize( + ("hooks_applied", "mode", "seq_len", "ulysses", "ring", "allgather"), + [ + (False, "strict", 16, 8, 1, 1), + (True, "advanced_uaa", 16, 8, 1, 1), + (True, "strict", 17, 8, 1, 1), + (True, "strict", 16, 4, 2, 1), + (True, "strict", 16, 1, 1, 8), + ], +) +def test_local_span_falls_back_when_local_embedding_is_unsafe( + monkeypatch, + hooks_applied, + mode, + seq_len, + ulysses, + ring, + allgather, +): + from vllm_omni.diffusion import forward_context + from vllm_omni.diffusion.distributed import parallel_state + from vllm_omni.diffusion.models.minimax_h3 import minimax_h3_transformer as h3 + + monkeypatch.setattr(forward_context, "is_forward_context_available", lambda: True) + monkeypatch.setattr(forward_context, "get_ulysses_mode", lambda **kwargs: mode) + monkeypatch.setattr(parallel_state, "get_sequence_parallel_world_size", lambda: 8) + monkeypatch.setattr(parallel_state, "get_sequence_parallel_rank", lambda: 3) + monkeypatch.setattr( + parallel_state, + "get_ulysses_parallel_world_size", + lambda: ulysses, + ) + monkeypatch.setattr(parallel_state, "get_ring_parallel_world_size", lambda: ring) + monkeypatch.setattr( + parallel_state, + "get_allgather_parallel_world_size", + lambda: allgather, + ) + + assert h3._sequence_parallel_local_span(seq_len, hooks_applied=hooks_applied) == ( + 0, + seq_len, + ) + + +def test_local_embedding_spans_reassemble_multimodal_rows(): + from vllm_omni.diffusion.models.minimax_h3 import minimax_h3_transformer as h3 + + class _IdentityProjection(nn.Module): + def forward(self, x): + return x, None + + class _IdentityRefiner(nn.Module): + def forward(self, x, **kwargs): + return x + + model = object.__new__(h3.MiniMaxH3DiTModel) + nn.Module.__init__(model) + model.hidden_size = 2 + model.video_patch_proj = _IdentityProjection() + model.audio_patch_proj = _IdentityProjection() + model.condition_proj = _IdentityProjection() + model.token_refiner = _IdentityRefiner() + model.time_embedder = nn.Identity() + + seq_len = 16 + x = torch.arange(seq_len * 2, dtype=torch.float32).reshape(1, seq_len, 2) + audio_x = x + 100 + text = torch.tensor( + [[200.0, 201.0], [202.0, 203.0], [204.0, 205.0], [206.0, 207.0]], + dtype=torch.bfloat16, + ) + common = { + "x": x, + "audio_x": audio_x, + "text_embeddings_selected": text, + "unique_timesteps": torch.tensor([0.2, 0.3]), + "img_pos": torch.tensor([1, 2, 6, 7, 11, 12]), + "audio_pos": torch.tensor([3, 4, 8, 9, 13, 14]), + "text_pos": torch.tensor([0, 5, 10, 15]), + "refiner_cu_seqlens": torch.tensor([0, 4, 4], dtype=torch.int32), + "refiner_max_seqlen": 4, + "seq_len": seq_len, + "device": torch.device("cpu"), + "local_span": (0, seq_len), + } + + full, full_t_emb = model._embed(**common) + local_spans = [] + for rank in range(8): + local_span = (rank * (seq_len // 8), seq_len // 8) + local, local_t_emb = model._embed( + **{ + **common, + "local_span": local_span, + } + ) + assert local.shape == (2, 2) + torch.testing.assert_close(local_t_emb, full_t_emb) + local_spans.append(local) + + torch.testing.assert_close(torch.cat(local_spans), full, atol=0, rtol=0) + + +@pytest.mark.parametrize( + "local_projection", + [False, True], + ids=["fallback", "strict-local"], +) +def test_sp_output_boundary_preserves_logits_and_minimizes_gather( + monkeypatch, + local_projection, +): + from vllm_omni.diffusion.models.minimax_h3 import minimax_h3_transformer as h3 + + class _StaticRope(nn.Module): + input_rows = None + + def forward(self, position_ids): + self.input_rows = position_ids.shape[1] + return torch.zeros(position_ids.shape[1], 6) + + class _Prepare(nn.Module): + def __init__(self, shard): + super().__init__() + self.shard = shard + + def forward(self, hidden, rope, combined): + local_len = hidden.shape[0] // 2 if self.shard else hidden.shape[0] + return hidden[:local_len], rope[:local_len], combined[:local_len] + + class _Gather(nn.Module): + def __init__(self, remote_rows, replace): + super().__init__() + self.remote_rows = remote_rows + self.replace = replace + self.input_width = None + + def forward(self, tensor): + self.input_width = tensor.shape[-1] + if self.replace: + return self.remote_rows + return torch.cat((tensor, self.remote_rows), dim=0) + + class _FinalLayer(nn.Module): + def forward(self, hidden, *, t_emb, inverse_indices): + del t_emb + inverse = inverse_indices.to(torch.float32).unsqueeze(-1) + return ( + hidden[:, :2].to(torch.float32) + inverse, + hidden[:, 2:3].to(torch.float32) - inverse, + ) + + seq_len = 8 + full_hidden = torch.arange(seq_len * 4, dtype=torch.bfloat16).reshape(seq_len, 4) + inverse_indices = torch.tensor([0, 1, 0, 1, 1, 0, 1, 0]) + final_layer = _FinalLayer() + remote_video, remote_audio = final_layer( + full_hidden[seq_len // 2 :], + t_emb=torch.empty(0), + inverse_indices=inverse_indices[seq_len // 2 :], + ) + if local_projection: + remote_rows = torch.cat((remote_video, remote_audio), dim=-1) + else: + remote_rows = full_hidden + gather = _Gather(remote_rows, replace=not local_projection) + + model = object.__new__(h3.MiniMaxH3DiTModel) + nn.Module.__init__(model) + model.arch = h3.MiniMaxH3DiTArchConfig( + latents_dim=2, + audio_latents_dim=1, + patch_size=(1, 1, 1), + ) + rope = _StaticRope() + model.rope = rope + model.sp_prepare = _Prepare(shard=True) + model.local_sp_prepare = _Prepare(shard=False) + model.sp_gather = gather + model.blocks = nn.ModuleList() + model.final_layer = final_layer + monkeypatch.setattr( + model, + "_embed", + lambda **kwargs: ( + full_hidden[: seq_len // 2] if local_projection else full_hidden, + torch.tensor([[0.5]], dtype=torch.float32), + ), + ) + + def _local_span(*args, **kwargs): + del args, kwargs + return (0, seq_len // 2) if local_projection else (0, seq_len) + + monkeypatch.setattr( + h3, + "_sequence_parallel_local_span", + _local_span, + ) + + img_pos = torch.tensor([0, 2, 4, 6]) + audio_pos = torch.tensor([1, 3, 5, 7]) + video, audio = model( + x=torch.zeros(1, seq_len, 2), + audio_x=torch.zeros(1, seq_len, 1), + img_position_ids=torch.zeros(1, seq_len, 3, dtype=torch.long), + unique_timesteps=torch.tensor([0.5]), + inverse_indices=inverse_indices, + update_mask=torch.ones(img_pos.numel()), + token_tags=torch.zeros(seq_len, dtype=torch.long), + prompt_embeds=torch.empty(0, 2), + img_pos_info={"position_ids": img_pos}, + audio_pos_info={"position_ids": audio_pos}, + text_pos_info={"position_ids": torch.empty(0, dtype=torch.long)}, + img_pos_for_infer_output_info={"position_ids": img_pos}, + packed_seq_params={ + "cu_seqlens_q": torch.tensor([0, seq_len, seq_len], dtype=torch.int32), + "max_seqlen_q": seq_len, + }, + refiner_packed_seq_params={ + "cu_seqlens_q": torch.tensor([0, 0, 0], dtype=torch.int32), + "max_seqlen_q": 0, + }, + ) + + expected_video, expected_audio = final_layer( + full_hidden, + t_emb=torch.empty(0), + inverse_indices=inverse_indices, + ) + torch.testing.assert_close(video, expected_video.index_select(0, img_pos)) + torch.testing.assert_close(audio, expected_audio.index_select(0, audio_pos)) + expected_gather_width = 3 if local_projection else full_hidden.shape[-1] + expected_rope_rows = seq_len // 2 if local_projection else seq_len + assert gather.input_width == expected_gather_width + assert rope.input_rows == expected_rope_rows + + @pytest.mark.parametrize( ("tp_size", "message"), [ diff --git a/vllm_omni/diffusion/cache/teacache/extractors.py b/vllm_omni/diffusion/cache/teacache/extractors.py index 8689c2c54..93e306ebb 100644 --- a/vllm_omni/diffusion/cache/teacache/extractors.py +++ b/vllm_omni/diffusion/cache/teacache/extractors.py @@ -1333,6 +1333,7 @@ def extract_minimax_h3_context( refiner_max_seqlen=refiner_max, seq_len=seq_len, device=device, + local_span=(0, seq_len), ) combined_indices = (inverse_indices * MINIMAX_H3_ADALN_MODALITY_NUM + token_tags.clamp(min=0)).to(device) diff --git a/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py b/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py index 659b64bc4..52f9ab589 100644 --- a/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py +++ b/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py @@ -107,6 +107,7 @@ MINIMAX_H3_FP32_BUFFER_NAMES = frozenset({"rope.inv_freq"}) # video/text/audio tokens (padding is clamped to 0 before the embedding # lookup and masked out afterwards). MINIMAX_H3_ADALN_MODALITY_NUM = 3 +_LOCAL_SP_PREPARE_HOOK = "sp_input---local_sp_prepare" # Opt-in fp16-range protection for the NPU ascend_laser_attention kernel # (consumed only via the "laser_input_scale" extra key; other backends and @@ -212,6 +213,51 @@ def _modulate_gate( return (x + gate.index_select(0, indices) * other).to(dtype) +def _sequence_parallel_local_span( + seq_len: int, + *, + hooks_applied: bool, +) -> tuple[int, int]: + """Return the packed-row span owned by this sequence-parallel rank.""" + from vllm_omni.diffusion.forward_context import ( + get_ulysses_mode, + is_forward_context_available, + ) + + if not hooks_applied or not is_forward_context_available(): + return 0, seq_len + if get_ulysses_mode(default="strict") != "strict": + return 0, seq_len + + try: + from vllm_omni.diffusion.distributed.parallel_state import ( + get_allgather_parallel_world_size, + get_ring_parallel_world_size, + get_sequence_parallel_rank, + get_sequence_parallel_world_size, + get_ulysses_parallel_world_size, + ) + + world_size = int(get_sequence_parallel_world_size()) + rank = int(get_sequence_parallel_rank()) + ulysses_world_size = int(get_ulysses_parallel_world_size()) + ring_world_size = int(get_ring_parallel_world_size()) + allgather_world_size = int(get_allgather_parallel_world_size()) + except AssertionError: + return 0, seq_len + + if world_size <= 1 or ulysses_world_size != world_size: + return 0, seq_len + if ring_world_size != 1 or allgather_world_size != 1: + return 0, seq_len + if seq_len < world_size or seq_len % world_size: + return 0, seq_len + + chunk_size = seq_len // world_size + start = rank * chunk_size + return start, chunk_size + + class MiniMaxH3Rope(nn.Module): """3D rope over (t, h, w); rotates 96 of 128 head dims (rotary_percent 0.75). @@ -888,6 +934,13 @@ class MiniMaxH3DiTModel(nn.Module): split_output=True, ), }, + "local_sp_prepare": { + 2: SequenceParallelInput( + split_dim=0, + expected_dims=1, + split_output=True, + ), + }, "sp_gather": SequenceParallelOutput(gather_dim=0, expected_dims=2), } # The checkpoint already stores qkv and the MLP gate/up as single tensors @@ -995,6 +1048,7 @@ class MiniMaxH3DiTModel(nn.Module): ] ) self.sp_prepare = MiniMaxH3SPPrepare() + self.local_sp_prepare = MiniMaxH3SPPrepare() self.sp_gather = MiniMaxH3SPGather() self.final_layer = MiniMaxH3FinalLayer( arch, @@ -1087,16 +1141,39 @@ class MiniMaxH3DiTModel(nn.Module): refiner_max_seqlen: int, seq_len: int, device: torch.device, + local_span: tuple[int, int], ) -> tuple[torch.Tensor, torch.Tensor]: - """Build packed multimodal embeddings for the TP=1/SP=1 inference path. + """Build this rank's packed multimodal embedding rows. - Returns (decoder_input [S, H] bf16, t_emb [M, t_dim] fp32). + Returns (decoder_input [S_local, H] bf16, t_emb [M, t_dim] fp32). """ + local_start, local_len = local_span + local_end = local_start + local_len + local_only = local_len != seq_len + if local_only: + img_mask = (img_pos >= local_start) & (img_pos < local_end) + audio_mask = (audio_pos >= local_start) & (audio_pos < local_end) + text_mask = (text_pos >= local_start) & (text_pos < local_end) + img_global_pos = img_pos[img_mask] + audio_global_pos = audio_pos[audio_mask] + img_local_pos = img_global_pos - local_start + audio_local_pos = audio_global_pos - local_start + text_local_pos = text_pos[text_mask] - local_start + text_local_indices = torch.nonzero(text_mask, as_tuple=False).view(-1) + else: + img_global_pos = img_pos + audio_global_pos = audio_pos + img_local_pos = img_pos + audio_local_pos = audio_pos + text_local_pos = text_pos + text_local_indices = None + # Latent embedders stay fp32 in and out; their outputs are cast to the # bf16 sequence dtype only during indexed scattering. - x_rows = x.view(-1, x.shape[-1]).index_select(0, img_pos).to(_FP32_DTYPE) + x_rows = x.view(-1, x.shape[-1]).index_select(0, img_global_pos).to(_FP32_DTYPE) video_embed, _ = self.video_patch_proj(x_rows) - audio_rows = audio_x.view(-1, audio_x.shape[-1]).index_select(0, audio_pos).to(_FP32_DTYPE) + audio_rows = audio_x.view(-1, audio_x.shape[-1]) + audio_rows = audio_rows.index_select(0, audio_global_pos).to(_FP32_DTYPE) audio_embed, _ = self.audio_patch_proj(audio_rows) text_rows = text_embeddings_selected.to(device=device, dtype=_BF16_DTYPE) @@ -1106,11 +1183,29 @@ class MiniMaxH3DiTModel(nn.Module): cu_seqlens=refiner_cu_seqlens, max_seqlen=refiner_max_seqlen, ) + if text_local_indices is not None: + text_embed = text_embed.index_select(0, text_local_indices) - embeddings = torch.zeros((seq_len, self.hidden_size), device=device, dtype=_BF16_DTYPE) - embeddings.index_add_(0, text_pos, text_embed.to(_BF16_DTYPE)[: text_pos.shape[0]]) - embeddings.index_add_(0, img_pos, video_embed.to(_BF16_DTYPE)[: img_pos.shape[0]]) - embeddings.index_add_(0, audio_pos, audio_embed.to(_BF16_DTYPE)[: audio_pos.shape[0]]) + embeddings = torch.zeros( + (local_len, self.hidden_size), + device=device, + dtype=_BF16_DTYPE, + ) + embeddings.index_add_( + 0, + text_local_pos, + text_embed.to(_BF16_DTYPE)[: text_local_pos.shape[0]], + ) + embeddings.index_add_( + 0, + img_local_pos, + video_embed.to(_BF16_DTYPE)[: img_local_pos.shape[0]], + ) + embeddings.index_add_( + 0, + audio_local_pos, + audio_embed.to(_BF16_DTYPE)[: audio_local_pos.shape[0]], + ) t_emb = self.time_embedder(unique_timesteps) return embeddings, t_emb @@ -1140,7 +1235,6 @@ class MiniMaxH3DiTModel(nn.Module): update_mask = _required_kwarg(kwargs, "update_mask") token_tags = _required_kwarg(kwargs, "token_tags").view(-1).to(torch.long) skip_mask_out_condition = bool(kwargs.get("skip_mask_out_condition", False)) - text_selected = _required_kwarg(kwargs, "prompt_embeds") img_pos = self._pos_ids(_required_kwarg(kwargs, "img_pos_info"), "img_pos_info") @@ -1170,8 +1264,18 @@ class MiniMaxH3DiTModel(nn.Module): if inverse_indices.shape[0] != seq_len: raise ValueError(f"inverse_indices must be [{seq_len}], got {list(inverse_indices.shape)}") device = x.device - # Compute the packed RoPE table once for all transformer blocks. - rope_table = _build_rope_table(self.rope(img_position_ids).to(device)) + local_sp_registry = getattr(self.local_sp_prepare, "_hook_registry", None) + hooks_applied = local_sp_registry is not None + if local_sp_registry is not None: + local_sp_hook = local_sp_registry.get_hook(_LOCAL_SP_PREPARE_HOOK) + hooks_applied = local_sp_hook is not None + local_span = _sequence_parallel_local_span( + seq_len, + hooks_applied=hooks_applied, + ) + local_start, local_len = local_span + rope_position_ids = img_position_ids.narrow(1, local_start, local_len) + rope_table = _build_rope_table(self.rope(rope_position_ids).to(device)) decoder_input, t_emb = self._embed( x=x, @@ -1185,6 +1289,7 @@ class MiniMaxH3DiTModel(nn.Module): refiner_max_seqlen=refiner_max, seq_len=seq_len, device=device, + local_span=local_span, ) combined_indices = (inverse_indices * MINIMAX_H3_ADALN_MODALITY_NUM + token_tags.clamp(min=0)).to(device) @@ -1195,11 +1300,18 @@ class MiniMaxH3DiTModel(nn.Module): block_rope = rope_table block_combined = combined_indices - hidden, block_rope, block_combined = self.sp_prepare( - hidden, - block_rope, - block_combined, - ) + if local_len == seq_len: + hidden, block_rope, block_combined = self.sp_prepare( + hidden, + block_rope, + block_combined, + ) + else: + hidden, block_rope, block_combined = self.local_sp_prepare( + hidden, + block_rope, + block_combined, + ) for block in self.blocks: hidden = block( hidden, @@ -1211,13 +1323,29 @@ class MiniMaxH3DiTModel(nn.Module): packed_total=seq_len, video_layout=video_layout, ) - hidden = self.sp_gather(hidden) - - video_logits, audio_logits = self.final_layer( - hidden, - t_emb=t_emb, - inverse_indices=inverse_indices, - ) + if local_len == seq_len: + hidden = self.sp_gather(hidden) + video_logits, audio_logits = self.final_layer( + hidden, + t_emb=t_emb, + inverse_indices=inverse_indices, + ) + else: + local_inverse_indices = inverse_indices.narrow( + 0, + local_start, + local_len, + ) + video_logits, audio_logits = self.final_layer( + hidden, + t_emb=t_emb, + inverse_indices=local_inverse_indices, + ) + compact_logits = torch.cat((video_logits, audio_logits), dim=-1) + compact_logits = self.sp_gather(compact_logits) + video_width = self.arch.latents_dim * math.prod(self.arch.patch_size) + video_logits = compact_logits[..., :video_width] + audio_logits = compact_logits[..., video_width:] # Select target and condition rows at inference-output positions, then # zero the condition rows. From e57574ae290b42a1a341c86f0d1cecdb563e3187 Mon Sep 17 00:00:00 2001 From: MrlixiangWE <102979255+MrlixiangWE@users.noreply.github.com> Date: Sat, 15 Aug 2026 16:37:27 +0800 Subject: [PATCH 025/267] [Bugfix] Carry ec_transfer_params and num_cache_creation_tokens on OmniRequestOutput (#6152) Signed-off-by: MrlixiangWE --- tests/engine/test_omni_request_output.py | 53 ++++++++++++++++++++++++ vllm_omni/outputs/__init__.py | 4 ++ 2 files changed, 57 insertions(+) diff --git a/tests/engine/test_omni_request_output.py b/tests/engine/test_omni_request_output.py index fa25bba9e..eb1bdfd41 100644 --- a/tests/engine/test_omni_request_output.py +++ b/tests/engine/test_omni_request_output.py @@ -270,3 +270,56 @@ class TestMsgpackRoundTrip: # Content merged from legacy nested key assert result.prompt == "legacy prompt" assert result.prompt_token_ids == [1, 2] + + +class TestRequestOutputFieldParity: + """``OmniRequestOutput`` must expose everything ``RequestOutput`` sets. + + ``RequestOutput`` is a plain class, so the dataclass-generated ``__init__`` + replaces its ``__init__`` and the inherited attributes only exist because + they are redeclared as fields. Any vLLM field that is not redeclared simply + does not exist on an omni output, and vLLM's own serving code reads those + attributes unconditionally — ``/v1/completions`` returned HTTP 500 with + ``AttributeError: 'OmniRequestOutput' object has no attribute + 'ec_transfer_params'`` for exactly that reason. This test fails on the next + vLLM bump that adds a field, instead of leaving it for an endpoint to hit. + """ + + def test_carries_every_attribute_request_output_sets(self) -> None: + reference = _make_text_request_output() + omni = OmniRequestOutput(request_id="req-parity") + + missing = sorted(name for name in vars(reference) if not name.startswith("_") and not hasattr(omni, name)) + assert not missing, ( + f"OmniRequestOutput is missing RequestOutput attributes {missing}; " + "redeclare them as dataclass fields in vllm_omni/outputs/__init__.py" + ) + + def test_transfer_param_fields_default_to_none(self) -> None: + omni = OmniRequestOutput(request_id="req-defaults") + assert omni.kv_transfer_params is None + assert omni.ec_transfer_params is None + assert omni.num_cached_tokens is None + assert omni.num_cache_creation_tokens is None + + def test_transfer_param_fields_are_copied_from_the_stage_output(self) -> None: + """Declaring the fields is not enough — they must also be copied. + + ``from_stage_output`` only carries the names listed in + ``_REQUEST_OUTPUT_CONTENT_ATTRS``. With the fields declared but absent + from that list the endpoint stops raising ``AttributeError`` while + silently dropping external-connector metadata and cache-creation + accounting, which is harder to notice than the crash it replaced. + """ + source = _make_text_request_output() + source.num_cache_creation_tokens = 7 + source.ec_transfer_params = {"connector": "shm", "handle": "abc"} + source.num_cached_tokens = 3 + source.kv_transfer_params = {"role": "producer"} + + omni = OmniRequestOutput.from_stage_output(source, stage_id=1) + + assert omni.num_cache_creation_tokens == 7 + assert omni.ec_transfer_params == {"connector": "shm", "handle": "abc"} + assert omni.num_cached_tokens == 3 + assert omni.kv_transfer_params == {"role": "producer"} diff --git a/vllm_omni/outputs/__init__.py b/vllm_omni/outputs/__init__.py index 105d0222d..90197f6e6 100644 --- a/vllm_omni/outputs/__init__.py +++ b/vllm_omni/outputs/__init__.py @@ -85,7 +85,9 @@ _REQUEST_OUTPUT_CONTENT_ATTRS = ( "encoder_prompt", "encoder_prompt_token_ids", "num_cached_tokens", + "num_cache_creation_tokens", "kv_transfer_params", + "ec_transfer_params", ) # Omni-specific content copied when the wrapped stage output is itself an @@ -147,7 +149,9 @@ class OmniRequestOutput(RequestOutput): encoder_prompt: str | None = None encoder_prompt_token_ids: list[int] | None = None num_cached_tokens: int | None = None + num_cache_creation_tokens: int | None = None kv_transfer_params: dict[str, Any] | None = None + ec_transfer_params: dict[str, Any] | None = None # --- Pipeline stage fields --- stage_id: int | None = None From f39b8684eb6f463dcd48f3c24d88be78108dbe09 Mon Sep 17 00:00:00 2001 From: NumberWan Date: Sat, 15 Aug 2026 17:38:30 +0800 Subject: [PATCH 026/267] [BugFix][Nightly CI] Adjust Qwen-Image accuracy thresholds for stable FA-deterministic mode (#5963) Signed-off-by: NumberWan --- tests/e2e/accuracy/test_qwen_image.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/e2e/accuracy/test_qwen_image.py b/tests/e2e/accuracy/test_qwen_image.py index 6f3ed7e71..6bb985050 100644 --- a/tests/e2e/accuracy/test_qwen_image.py +++ b/tests/e2e/accuracy/test_qwen_image.py @@ -29,14 +29,14 @@ HEIGHT = 512 NUM_INFERENCE_STEPS = 20 TRUE_CFG_SCALE = 4.0 SEED = 42 -SSIM_THRESHOLD = 0.97 -# The FA3-hub-matched comparison clears 0.97 with margin. The SDPA-matched -# fallback (hub kernel unavailable for the image's torch) landed at 0.9687 in -# build 2954 — the attention math matches but the surrounding stacks (vLLM -# compiled path vs eager diffusers) still differ slightly. Keep the strict -# gate for the FA3 path and a slightly looser one for the fallback only. +SSIM_THRESHOLD = 0.94 +# With --fa-deterministic, H100 nightlies stably land near SSIM 0.958 / PSNR 27.8 +# vs Diffusers (see #5734 / #5963), so the primary gates are 0.94 / 27.0. +# The SDPA-matched fallback (hub kernel unavailable for the image's torch) +# landed at SSIM 0.9687 in build 2954 — keep a slightly tighter SSIM gate for +# that path only. SSIM_THRESHOLD_SDPA_FALLBACK = 0.96 -PSNR_THRESHOLD = 30.0 +PSNR_THRESHOLD = 27.0 MODEL_2512_ID = "Qwen/Qwen-Image-2512" MODEL_2512_ENV_VAR = "QWEN_IMAGE_2512_MODEL" From ce6ab0ca4666350329261adaf0ce6198764ca7b2 Mon Sep 17 00:00:00 2001 From: SYLAR <125541396+lishunyang12@users.noreply.github.com> Date: Sat, 15 Aug 2026 18:27:23 +0800 Subject: [PATCH 027/267] [Model] Add LTX-2.5 support (#6070) Signed-off-by: lishunyang12 --- .buildkite/cuda/test-nightly.yml | 13 + .gitignore | 1 + docs/models/supported_models.md | 7 +- docs/user_guide/diffusion_features.md | 3 +- .../image_to_video/image_to_video.py | 9 +- .../text_to_video/text_to_video.py | 2 + recipes/LTX/LTX-2.5.md | 175 ++++ recipes/LTX/LTX-2.md | 23 +- recipes/README.md | 1 + requirements/common.txt | 10 +- tests/diffusion/attention/test_cudnn_attn.py | 43 + tests/diffusion/cache/test_cache_dit.py | 8 +- .../model_loader/test_hub_prefetch.py | 72 ++ .../models/ltx2/test_ltx25_pipeline.py | 881 ++++++++++++++++++ .../models/ltx2/test_ltx2_conditioning.py | 292 +++++- .../models/ltx2/test_ltx2_latents.py | 65 ++ .../models/ltx2/test_ltx2_parallel.py | 9 + .../models/ltx2/test_ltx2_pipeline.py | 97 +- .../models/ltx2/test_ltx2_transformer.py | 371 +++++++- tests/e2e/accuracy/ltx/run_ltx25_reference.py | 591 ++++++++++++ .../ltx/test_ltx25_official_similarity.py | 753 +++++++++++++++ tests/e2e/online_serving/test_ltx25.py | 89 ++ .../openai_api/test_video_server.py | 47 +- .../test_resolve_dreamzero_config.py | 4 +- tests/helpers/__init__.py | 13 +- tests/model_extras/test_model_extras.py | 52 ++ tests/model_tests/diffusion/test_alignment.py | 2 + .../attention/backends/cudnn_attn.py | 20 +- vllm_omni/diffusion/data.py | 28 +- .../diffusion/model_loader/hub_prefetch.py | 6 +- vllm_omni/diffusion/models/ltx2/__init__.py | 4 + .../diffusion/models/ltx2/ltx2_components.py | 239 ++++- .../models/ltx2/ltx2_conditioning.py | 136 ++- .../diffusion/models/ltx2/ltx2_denoise.py | 151 ++- .../diffusion/models/ltx2/ltx2_guidance.py | 37 +- .../diffusion/models/ltx2/ltx2_latents.py | 36 +- .../diffusion/models/ltx2/ltx2_recipes.py | 102 +- .../diffusion/models/ltx2/ltx2_request.py | 69 +- .../diffusion/models/ltx2/ltx2_runtime.py | 55 +- .../diffusion/models/ltx2/ltx2_transformer.py | 105 ++- .../diffusion/models/ltx2/pipeline_ltx2.py | 15 +- .../models/ltx2/pipeline_ltx2_two_stage.py | 8 +- vllm_omni/diffusion/registry.py | 12 + vllm_omni/entrypoints/openai/serving_video.py | 24 +- vllm_omni/model_extras/__init__.py | 4 + vllm_omni/model_extras/ltx2.py | 28 + vllm_omni/model_extras/registry.py | 64 +- 47 files changed, 4558 insertions(+), 218 deletions(-) create mode 100644 recipes/LTX/LTX-2.5.md create mode 100644 tests/diffusion/model_loader/test_hub_prefetch.py create mode 100644 tests/diffusion/models/ltx2/test_ltx25_pipeline.py create mode 100644 tests/e2e/accuracy/ltx/run_ltx25_reference.py create mode 100644 tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py create mode 100644 tests/e2e/online_serving/test_ltx25.py diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index ea74847b9..ed112b84b 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -459,6 +459,12 @@ steps: - pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx.py -m "full_model and cuda" --run-level "full_model" mirror_hardwares: h100_2 + - label: ":full_moon: Diffusion X2V · LTX-2.5 Online Test" + timeout_in_minutes: 60 + commands: + - pytest -s -v tests/e2e/online_serving/test_ltx25.py -m "full_model and cuda" --run-level "full_model" + mirror_hardwares: h100_1 + - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test" timeout_in_minutes: 180 commands: @@ -487,6 +493,13 @@ steps: - pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_matches_official[ltx23_two_stage_layer_fused_i2v]' -m full_model --run-level full_model mirror_hardwares: h100_1 + - label: ":full_moon: Diffusion X2V · LTX-2.5 Official Parity Test" + timeout_in_minutes: 180 + commands: + - pytest -s -v tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py::test_ltx25_distilled_two_stage_matches_official -m full_model --run-level full_model + - pytest -s -v tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py::test_ltx25_full_matches_official -m full_model --run-level full_model + mirror_hardwares: h100_1 + - label: ":full_moon: Diffusion X2V · Perf Test" key: nightly-diffusion-x2v-performance timeout_in_minutes: 180 diff --git a/.gitignore b/.gitignore index f11b57388..7502df8a0 100644 --- a/.gitignore +++ b/.gitignore @@ -254,6 +254,7 @@ outputs/ results/ generated/ output_*/ +/tests/e2e/accuracy/artifacts/ # Configuration overrides configs/local.yaml diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index e6a38640f..5097c83fa 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -43,7 +43,12 @@ th { | `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | | — | | `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | — | | `LTX2TwoStagePipeline` | LTX-2 / LTX-2.3 ordinary two-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` + matching Lightricks LoRA and upsampler | ✅︎ | ✅︎ | | | — | -| `LTX2DistilledPipeline` | LTX-2 / LTX-2.3 full-distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` + `Lightricks/LTX-2.3` upsampler | ✅︎ | ✅︎ | | | — | +| `LTX2DistilledOneStagePipeline` | LTX-2 / LTX-2.3 merged-distilled one-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` | ✅︎ | ✅︎ | | | — | +| `LTX2DistilledTwoStagePipeline` | LTX-2 / LTX-2.3 merged-distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` + matching Lightricks upsampler | ✅︎ | ✅︎ | | | — | +| `LTX2Pipeline` | LTX-2.5 Full/SFT one-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) | +| `LTX2TwoStagePipeline` | LTX-2.5 Full/SFT two-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` + `Lightricks/LTX-2.5` LoRA | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) | +| `LTX2DistilledOneStagePipeline` | LTX-2.5 merged-distilled one-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) | +| `LTX2DistilledTwoStagePipeline` | LTX-2.5 merged-distilled two-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) | | `LingBotVideoPipeline` | LingBot-Video dense and MoE T2I, T2V, TI2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | | — | | `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | ✅︎ | | | [Published](https://recipes.vllm.ai/MiniMaxAI/MiniMax-H3) | | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | diff --git a/docs/user_guide/diffusion_features.md b/docs/user_guide/diffusion_features.md index 11c6366b7..5844d1114 100644 --- a/docs/user_guide/diffusion_features.md +++ b/docs/user_guide/diffusion_features.md @@ -161,6 +161,7 @@ The following tables show which models support each feature: | **Wan2.1-VACE** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ | | **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ | | **LTX-2.3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ | +| **LTX-2.5** | ❌ | ❓ (one-stage) | ❓ (Ulysses only) | ✅ (Full only) | ❓ | ❌ | ✅ | ✅ | ✅ (decode) | ❓ (FP8) | ❌ | | **Helios** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅* | | **HunyuanVideo-1.5 T2V I2V** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (encode/decode) | ✅ | ❌ | | **DreamID-Omni** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | @@ -174,7 +175,7 @@ The following tables show which models support each feature: **Frame Interpolation Support** - **Supported**: Wan2.2 text-to-video, image-to-video, and TI2V pipelines -- **Not supported**: Wan2.1-VACE, LTX-2, LTX-2.3, Helios, HunyuanVideo-1.5, DreamID-Omni +- **Not supported**: Wan2.1-VACE, LTX-2, LTX-2.3, LTX-2.5, Helios, HunyuanVideo-1.5, DreamID-Omni ### AudioGen diff --git a/examples/offline_inference/image_to_video/image_to_video.py b/examples/offline_inference/image_to_video/image_to_video.py index 222502e8a..28c98990a 100644 --- a/examples/offline_inference/image_to_video/image_to_video.py +++ b/examples/offline_inference/image_to_video/image_to_video.py @@ -68,6 +68,7 @@ from vllm_omni.model_extras import ( from vllm_omni.model_extras import ( get_extra_body_params, get_model_class_name, + should_preserve_reference_image_size, ) from vllm_omni.outputs import OmniRequestOutput from vllm_omni.platforms import current_omni_platform @@ -454,7 +455,13 @@ def main(): media_inputs: dict[str, Any] = {} if image is not None: - media_inputs["image"] = image.resize((width, height), PIL.Image.Resampling.LANCZOS) + preserve_image_size = should_preserve_reference_image_size( + model_class_name, + model=args.model, + ) + media_inputs["image"] = ( + image if preserve_image_size else image.resize((width, height), PIL.Image.Resampling.LANCZOS) + ) if last_image is not None: media_inputs["last_image"] = last_image.resize((width, height), PIL.Image.Resampling.LANCZOS) if mask_image is not None: diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index 129ca860a..406aab39e 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -570,6 +570,8 @@ def main(): num_frames=args.num_frames, extra_args=extra_args, ) + if args.frame_rate is not None: + sampling_kwargs["frame_rate"] = args.frame_rate if args.guidance_scale_high is not None: sampling_kwargs["guidance_scale_2"] = args.guidance_scale_high diff --git a/recipes/LTX/LTX-2.5.md b/recipes/LTX/LTX-2.5.md new file mode 100644 index 000000000..4ff5f6dab --- /dev/null +++ b/recipes/LTX/LTX-2.5.md @@ -0,0 +1,175 @@ +# LTX-2.5 + +> Text-to-video and first-frame image-to-video generation with synchronized audio + +## Overview + +vLLM-Omni supports the gated +[`Lightricks/LTX-2.5-Diffusers`](https://huggingface.co/Lightricks/LTX-2.5-Diffusers) +checkpoint for T2V and first-frame I2V. Every output includes synchronized +48 kHz stereo audio. + +The raw [`Lightricks/LTX-2.5`](https://huggingface.co/Lightricks/LTX-2.5) +repository is not directly loadable with `--model`. It supplies the official +upsampler and LoRA sidecars used by the Full/SFT two-stage pipeline, so accept +both model licenses and authenticate before first use. + +## Choose a pipeline + +| Pipeline | Mode | Default size | Schedule | +|---|---|---:|---:| +| `LTX2Pipeline` | Full/SFT one-stage | 960x544 | 30 steps | +| `LTX2TwoStagePipeline` | Full/SFT two-stage | 1920x1088 | 30 + 3 steps | +| `LTX2DistilledOneStagePipeline` | Distilled one-stage | 960x544 | 8 steps | +| `LTX2DistilledTwoStagePipeline` | Distilled two-stage | 1920x1088 | 8 + 3 steps | + +Both two-stage pipelines generate at half resolution, apply the official x2 +latent upsampler, and run a three-step refinement stage. Select the class with +`--model-class-name`; no `--task-type` flag is required. Supplying one initial +image selects I2V, while omitting it selects T2V. + +## Prerequisites + +```bash +hf auth login +export MODEL=Lightricks/LTX-2.5-Diffusers +``` + +Install matching vLLM and vLLM-Omni versions, and ensure `ffmpeg` and +`ffprobe` are on `PATH`. I2V requires PyAV backed by an FFmpeg build with the +`libx264` encoder. + +## Hardware + +| GPU | Status | Recommended scope | +|---|---|---| +| NVIDIA B300 | Verified | All four canonical pipelines | +| NVIDIA B200 or H200 | Capacity-based recommendation; not yet verified | All four canonical pipelines | +| NVIDIA GB200 or GB300 | Capacity-based recommendation; not yet verified | All four canonical pipelines | +| NVIDIA H100 80 GB | FP8 recipe | Distilled one-stage at 960x544 | + +The 1920x1088 two-stage examples require about 114 GB of peak GPU memory. +80 GB GPUs do not have enough safety margin for the canonical two-stage +configuration; use a larger GPU or reduce the output configuration. + +### H100 80 GB FP8 + +Use the distilled one-stage pipeline with FP8 and cuDNN attention: + +```bash +vllm serve Lightricks/LTX-2.5-Diffusers \ + --omni \ + --model-class-name LTX2DistilledOneStagePipeline \ + --quantization fp8 \ + --diffusion-attention-backend CUDNN_ATTN \ + --host 0.0.0.0 \ + --port 8000 +``` + +Use the T2V or I2V request below after the server is ready. + +## Offline inference + +Choose values from the pipeline table. For example, the distilled two-stage +path uses: + +```bash +export PIPELINE=LTX2DistilledTwoStagePipeline +export WIDTH=1920 +export HEIGHT=1088 +export STEPS=8 +``` + +### Text-to-video + +```bash +python examples/offline_inference/text_to_video/text_to_video.py \ + --model "${MODEL}" \ + --model-class-name "${PIPELINE}" \ + --prompt "A cinematic shot of a red fox walking through a snowy forest at dawn, the camera tracking alongside, snow crunching underfoot." \ + --width "${WIDTH}" \ + --height "${HEIGHT}" \ + --num-frames 121 \ + --num-inference-steps "${STEPS}" \ + --frame-rate 24 \ + --fps 24 \ + --seed 42 \ + --output ltx25-t2v.mp4 +``` + +### First-frame image-to-video + +```bash +python examples/offline_inference/image_to_video/image_to_video.py \ + --model "${MODEL}" \ + --model-class-name "${PIPELINE}" \ + --image /absolute/path/to/first-frame.png \ + --prompt "The red fox walks forward while the camera tracks alongside." \ + --width "${WIDTH}" \ + --height "${HEIGHT}" \ + --num-frames 121 \ + --num-inference-steps "${STEPS}" \ + --frame-rate 24 \ + --fps 24 \ + --seed 42 \ + --output ltx25-i2v.mp4 +``` + +LTX-2.5 uses the official CRF-18 first-frame conditioning path by default. + +## Online serving + +Start one server for the selected pipeline: + +```bash +vllm serve "${MODEL}" \ + --omni \ + --model-class-name "${PIPELINE}" \ + --host 0.0.0.0 \ + --port 8000 \ + --stage-init-timeout 900 +``` + +T2V request: + +```bash +curl -sS --fail-with-body \ + -X POST http://127.0.0.1:8000/v1/videos/sync \ + -F 'prompt=A cinematic shot of a red fox walking through a snowy forest at dawn, the camera tracking alongside, snow crunching underfoot.' \ + -o ltx25-online-t2v.mp4 +``` + +For I2V, add exactly one first frame: + +```bash +export FIRST_FRAME=/absolute/path/to/first-frame.png + +curl -sS --fail-with-body \ + -X POST http://127.0.0.1:8000/v1/videos/sync \ + -F "input_reference=@${FIRST_FRAME};type=image/png" \ + -F 'prompt=The red fox walks forward while the camera tracks alongside.' \ + -o ltx25-online-i2v.mp4 +``` + +Restart the server after changing `PIPELINE` because each class selects a +different weight layout and execution topology. + +## Constraints + +- Online `width`, `height`, `num_frames`, `fps`, and `num_inference_steps` + are optional and use the selected pipeline defaults when omitted. `seed` is + optional and random when omitted. +- `num_frames` must be `8k+1` when overridden. +- One-stage width and height must be divisible by 32; two-stage final + dimensions must be divisible by 64. +- Full/SFT pipelines accept negative prompts. Distilled pipelines are + positive-only and reject negative prompts. +- I2V accepts exactly one initial image per prompt. +- The supported `--model` value is `Lightricks/LTX-2.5-Diffusers`. + +## References + +- [LTX-2.5-Diffusers model card](https://huggingface.co/Lightricks/LTX-2.5-Diffusers) +- [Official LTX-2 repository](https://github.com/Lightricks/LTX-2) +- [Generic T2V example](../../examples/offline_inference/text_to_video/text_to_video.py) +- [Generic I2V example](../../examples/offline_inference/image_to_video/image_to_video.py) diff --git a/recipes/LTX/LTX-2.md b/recipes/LTX/LTX-2.md index 123880d09..ec4e8fb3b 100644 --- a/recipes/LTX/LTX-2.md +++ b/recipes/LTX/LTX-2.md @@ -8,10 +8,12 @@ |---|---|---| | `LTX2Pipeline` | LTX-2 one-stage T2V/I2V | `Lightricks/LTX-2` | | `LTX2TwoStagePipeline` | LTX-2 ordinary two-stage T2V/I2V | `Lightricks/LTX-2` | -| `LTX2DistilledPipeline` | LTX-2 full-distilled two-stage T2V/I2V | `rootonchair/LTX-2-19b-distilled` | +| `LTX2DistilledOneStagePipeline` | LTX-2 merged-distilled one-stage T2V/I2V | `rootonchair/LTX-2-19b-distilled` | +| `LTX2DistilledTwoStagePipeline` | LTX-2 merged-distilled two-stage T2V/I2V | `rootonchair/LTX-2-19b-distilled` | | `LTX2Pipeline` | LTX-2.3 one-stage T2V/I2V | `diffusers/LTX-2.3-Diffusers` | | `LTX2TwoStagePipeline` | LTX-2.3 ordinary two-stage T2V/I2V | `diffusers/LTX-2.3-Diffusers`
`Lightricks/LTX-2.3` | -| `LTX2DistilledPipeline` | LTX-2.3 full-distilled two-stage T2V/I2V | `diffusers/LTX-2.3-Distilled-Diffusers`
`Lightricks/LTX-2.3` | +| `LTX2DistilledOneStagePipeline` | LTX-2.3 merged-distilled one-stage T2V/I2V | `diffusers/LTX-2.3-Distilled-Diffusers` | +| `LTX2DistilledTwoStagePipeline` | LTX-2.3 merged-distilled two-stage T2V/I2V | `diffusers/LTX-2.3-Distilled-Diffusers`
`Lightricks/LTX-2.3` | Repositories in the table are download units. A full pipeline repository contains the Transformer, text encoder, connectors, VAEs, vocoder, scheduler, @@ -26,9 +28,12 @@ image selects I2V. Both one-stage repositories declare this class, so the raw `Lightricks/LTX-2.3` safetensors repository is not directly loadable. `LTX2TwoStagePipeline` samples the regular model at half resolution, upsamples, -then refines with the distilled LoRA. `LTX2DistilledPipeline` uses a fully -merged distilled Transformer in both stages. Both entries support T2V and I2V; -select their class explicitly. +then refines with the distilled LoRA. `LTX2DistilledOneStagePipeline` uses a +merged distilled Transformer without upsampling, while +`LTX2DistilledTwoStagePipeline` uses it in both stages. All entries support +T2V and I2V; select their class explicitly. The deprecated +`LTX2DistilledPipeline` name remains an alias for +`LTX2DistilledTwoStagePipeline`. ## API Migration @@ -52,8 +57,8 @@ The consolidation also removes these registry names without aliases: | `LTX23Pipeline` | `LTX2Pipeline`; checkpoint metadata selects LTX-2.3 | | `LTX2ImageToVideoPipeline` | `LTX2Pipeline` with `image=` | | `LTX23ImageToVideoPipeline` | `LTX2Pipeline` with `image=`; checkpoint metadata selects LTX-2.3 | -| `LTX2TwoStagesPipeline` | `LTX2DistilledPipeline` | -| `LTX2ImageToVideoTwoStagesPipeline` | `LTX2DistilledPipeline` with `image=` | +| `LTX2TwoStagesPipeline` | `LTX2DistilledTwoStagePipeline` | +| `LTX2ImageToVideoTwoStagesPipeline` | `LTX2DistilledTwoStagePipeline` with `image=` | Passing any second positional argument now raises `TypeError`. These changes affect direct Python callers and explicit `--model-class-name` overrides; @@ -112,10 +117,10 @@ Start a two-stage checkpoint with its explicit class: ```bash vllm serve rootonchair/LTX-2-19b-distilled --omni \ - --model-class-name LTX2DistilledPipeline --stage-init-timeout 600 + --model-class-name LTX2DistilledTwoStagePipeline --stage-init-timeout 600 # LTX-2.3 full-distilled; the v1.1 x2 upsampler is resolved when absent vllm serve diffusers/LTX-2.3-Distilled-Diffusers --omni \ - --model-class-name LTX2DistilledPipeline --stage-init-timeout 600 + --model-class-name LTX2DistilledTwoStagePipeline --stage-init-timeout 600 # Ordinary LTX-2.3 two-stage vllm serve diffusers/LTX-2.3-Diffusers --omni \ --model-class-name LTX2TwoStagePipeline \ diff --git a/recipes/README.md b/recipes/README.md index e010ade67..6a3b23770 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -55,6 +55,7 @@ recipes/ | [`MiniMaxAI/MiniMax-H3-MUSA.md`](./MiniMaxAI/MiniMax-H3-MUSA.md) | MiniMax H3 T2VA, FL2VA, and Ref2VA on MUSA | 4x MTT S5000 (TP4 Ref2VA profile) | | [`krea/Krea-2.md`](./krea/Krea-2.md) | Text-to-image (Turbo + Raw), offline + online, with LoRA | 1x H100 80GB | | [`LTX/LTX-2.md`](./LTX/LTX-2.md) | LTX-2/LTX-2.3 text-to-video and image-to-video with synchronized audio | H200 141GB / 96GB-class GPU | +| [`LTX/LTX-2.5.md`](./LTX/LTX-2.5.md) | LTX-2.5-Diffusers: Full/SFT one-stage and distilled two-stage T2V/I2V with synchronized audio | NVIDIA B300; cuDNN-qualified | | [`MammothModa2/MammothModa2.md`](./MammothModa2/MammothModa2.md) | Preview and Dev text-to-image (AR → DiT); Dev text/image understanding | Preview: 1x L40S 48GB / 1x ≥40GB GPU; Dev: 1x NVIDIA GPU with sufficient cache headroom | | [`meituan-longcat/LongCat-Video-Avatar-1.5.md`](./meituan-longcat/LongCat-Video-Avatar-1.5.md) | Audio-driven avatar video generation (AT2V / AI2V, single- and multi-speaker, AVC continuation) | 1x H100 80GB | | [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB | diff --git a/requirements/common.txt b/requirements/common.txt index e0233bd8d..368cbe8be 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -1,15 +1,15 @@ # Common dependencies for all platforms -# transformers: kept exactly aligned with the upstream vLLM constraint. -# vLLM pins ">= 4.56.0, != 5.0.*, != 5.1.*, != 5.2.*, != 5.3.*, != 5.4.*, != 5.5.0". -# The previous extra "< 5.9.0" exclusion was removed when re-aligning with -# upstream: transformers 5.9 changed _adjust_missing_and_unexpected_keys from +# transformers: required unified multimodal model classes are unavailable +# before 5.10.1; 5.10.0 is yanked, and 5.15 has a known construction +# regression. transformers 5.9 also changed +# _adjust_missing_and_unexpected_keys from # list+list to set|set, breaking models that declare # _keys_to_ignore_on_load_unexpected as a list (e.g. OpenMOSS-Team/MOSS-TTS-Nano) # with "TypeError: unsupported operand type(s) for |: 'list' and 'set'". # That breakage is now handled in code instead of by pinning: # MossTTSNanoForGeneration wraps weight loading in # transformers_keys_to_ignore_compat (vllm_omni/model_executor/models/utils.py). -transformers >= 5.5.3 +transformers >= 5.10.1, < 5.15 av>=14.0.0 omegaconf>=2.3.0 diffusers==0.38.0 diff --git a/tests/diffusion/attention/test_cudnn_attn.py b/tests/diffusion/attention/test_cudnn_attn.py index de1adf8c7..1ff3baafb 100644 --- a/tests/diffusion/attention/test_cudnn_attn.py +++ b/tests/diffusion/attention/test_cudnn_attn.py @@ -5,6 +5,7 @@ from contextlib import nullcontext import pytest import torch +from torch.nn.attention import SDPBackend from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata from vllm_omni.diffusion.attention.backends.cudnn_attn import CuDNNAttentionImpl @@ -12,6 +13,48 @@ from vllm_omni.diffusion.attention.backends.cudnn_attn import CuDNNAttentionImpl pytestmark = [pytest.mark.diffusion, pytest.mark.cpu, pytest.mark.core_model] +@pytest.mark.parametrize( + ("query_sequence_length", "key_sequence_length", "expected_backend"), + [ + (1, 1, SDPBackend.MATH), + (8, 1, SDPBackend.MATH), + (1, 8, SDPBackend.MATH), + (8, 8, SDPBackend.CUDNN_ATTENTION), + ], +) +def test_cudnn_routes_single_token_attention_to_math( + monkeypatch, query_sequence_length, key_sequence_length, expected_backend +): + selected_backends = [] + + def fake_sdpa_kernel(backends): + selected_backends.append(backends) + return nullcontext() + + monkeypatch.setattr( + "vllm_omni.diffusion.attention.backends.cudnn_attn.sdpa_kernel", + fake_sdpa_kernel, + ) + monkeypatch.setattr( + torch.nn.functional, + "scaled_dot_product_attention", + lambda query, _key, _value, **_kwargs: query, + ) + impl = CuDNNAttentionImpl( + num_heads=2, + head_size=4, + softmax_scale=0.5, + ) + query = torch.randn(1, query_sequence_length, 2, 4) + key = torch.randn(1, key_sequence_length, 2, 4) + value = torch.randn_like(key) + + output = impl.forward_cuda(query, key, value) + + assert output.shape == query.shape + assert selected_backends == [[expected_backend]] + + def test_cudnn_slices_valid_kv_prefix_without_padding_mask(monkeypatch): observed = {} diff --git a/tests/diffusion/cache/test_cache_dit.py b/tests/diffusion/cache/test_cache_dit.py index 2799f1607..0723c1d62 100644 --- a/tests/diffusion/cache/test_cache_dit.py +++ b/tests/diffusion/cache/test_cache_dit.py @@ -36,7 +36,6 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] SEPARATE_CFG_TRANSFORMERS = [ DreamIdOmniModel, - LTX2VideoTransformer3DModel, HeliosTransformer3DModel, LongCatImageTransformer2DModel, Cosmos3VFMTransformer, @@ -124,6 +123,13 @@ def test_cache_dit_configs_have_separate_cfg(transformer_model): assert transformer_model._cache_dit_adapter_config.has_separate_cfg is True +def test_ltx2_cache_dit_uses_one_forward_per_denoise_step(): + """LTX batches all guidance passes into one Transformer invocation.""" + adapter_config = LTX2VideoTransformer3DModel._cache_dit_adapter_config + + assert adapter_config.has_separate_cfg is False + + @patch("vllm_omni.diffusion.cache.cachedit.model_specific.BlockAdapter") @patch("vllm_omni.diffusion.cache.cachedit.model_specific.cache_dit") def test_separate_wan22_custom_enabler_has_separate_cfg(mock_cache_dit, mock_block_adapter): diff --git a/tests/diffusion/model_loader/test_hub_prefetch.py b/tests/diffusion/model_loader/test_hub_prefetch.py new file mode 100644 index 000000000..39eb2b277 --- /dev/null +++ b/tests/diffusion/model_loader/test_hub_prefetch.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import contextlib + +import huggingface_hub +import pytest + +from vllm_omni.diffusion.model_loader import hub_prefetch + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + +def test_prefetch_subfolders_propagates_revision(monkeypatch): + calls = [] + + def fake_snapshot_download(**kwargs): + calls.append(kwargs) + + monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download) + monkeypatch.setattr(hub_prefetch, "_repo_prefetch_lock", lambda _model: contextlib.nullcontext()) + + hub_prefetch.prefetch_subfolders( + "org/model", + ["transformer"], + revision="pinned-revision", + ) + + assert calls == [ + { + "repo_id": "org/model", + "revision": "pinned-revision", + "allow_patterns": ["transformer/*", "transformer/**", "*.json", "*.txt"], + } + ] + + +def test_from_pretrained_retry_preserves_revision(monkeypatch): + factory_calls = [] + prefetch_calls = [] + + def fake_factory(model, **kwargs): + factory_calls.append((model, kwargs)) + if len(factory_calls) == 1: + raise OSError("partial cache") + return "loaded" + + def fake_prefetch(model, subfolders, **kwargs): + prefetch_calls.append((model, tuple(subfolders), kwargs)) + + monkeypatch.setattr(hub_prefetch, "prefetch_subfolders", fake_prefetch) + monkeypatch.setattr(hub_prefetch.time, "sleep", lambda _seconds: None) + + result = hub_prefetch.from_pretrained_with_prefetch( + fake_factory, + "org/model", + subfolder="transformer", + prefetch_list=["transformer", "vae"], + revision="pinned-revision", + max_attempts=2, + ) + + assert result == "loaded" + assert len(factory_calls) == 2 + assert all(call[1]["revision"] == "pinned-revision" for call in factory_calls) + assert prefetch_calls == [ + ( + "org/model", + ("transformer", "vae"), + {"local_files_only": False, "revision": "pinned-revision"}, + ) + ] diff --git a/tests/diffusion/models/ltx2/test_ltx25_pipeline.py b/tests/diffusion/models/ltx2/test_ltx25_pipeline.py new file mode 100644 index 000000000..cbe52e997 --- /dev/null +++ b/tests/diffusion/models/ltx2/test_ltx25_pipeline.py @@ -0,0 +1,881 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Focused LTX-2.5 pipeline correctness tests.""" + +import json +from dataclasses import replace +from types import SimpleNamespace + +import pytest +import torch + +from vllm_omni.diffusion.data import DiffusionOutput +from vllm_omni.diffusion.models.ltx2 import ltx2_components, ltx2_latents, ltx2_runtime +from vllm_omni.diffusion.models.ltx2.ltx2_components import ( + LTX25_DISTILLED_COMPONENT_PROFILE, + LTX25_DISTILLED_ONE_STAGE_COMPONENT_PROFILE, + LTX25_FULL_COMPONENT_PROFILE, + LTX25_TWO_STAGE_COMPONENT_PROFILE, + detect_ltx_model_version, + resolve_ltx_checkpoint_kind, + resolve_ltx_component_profile, +) +from vllm_omni.diffusion.models.ltx2.ltx2_denoise import ( + LTXPhaseResult, + _first_frame_keyframes_mask, + _official_ltx_sigmas, +) +from vllm_omni.diffusion.models.ltx2.ltx2_guidance import ( + LTX_GUIDANCE_EXECUTOR, + LTXGuidanceSpec, + LTXModalityGuidance, + velocity_from_x0, + x0_from_velocity, +) +from vllm_omni.diffusion.models.ltx2.ltx2_latents import LTXAVState +from vllm_omni.diffusion.models.ltx2.ltx2_recipes import ( + LTX2_DISTILLED_TWO_STAGE_RECIPE, + LTX2_ONE_STAGE_RECIPE, + LTX23_ONE_STAGE_RECIPE, + LTX25_DISTILLED_ONE_STAGE_RECIPE, + LTX25_DISTILLED_TWO_STAGE_RECIPE, + LTX25_FULL_RECIPE, + LTX25_TWO_STAGE_RECIPE, + LTX_DISTILLED_SIGMAS, + LTX_POSITIVE_ONLY_RECIPE, + resolve_ltx_pipeline_recipe, +) +from vllm_omni.diffusion.models.ltx2.ltx2_request import LTXRequestInputs +from vllm_omni.diffusion.models.ltx2.ltx2_runtime import LTXRuntime +from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import ( + LTX2DistilledOneStagePipeline, + LTX2Pipeline, +) +from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_two_stage import ( + LTX2DistilledTwoStagePipeline, + LTX2TwoStagePipeline, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +@pytest.mark.parametrize("model_version", ["2", "2.3", "2.5"]) +def test_ltx_timestep_adaln_expands_per_token_like_official_without_sp(model_version): + pipe = object.__new__(LTX2Pipeline) + pipe.model_version = model_version + pipe.od_config = SimpleNamespace(parallel_config=SimpleNamespace(sequence_parallel_size=1)) + ts = torch.tensor([750.0, 500.0]) + + kwargs = LTXRuntime._denoise_timestep_kwargs( + pipe, + ts, + SimpleNamespace(), + SimpleNamespace(), + video_token_count=4, + audio_token_count=3, + ) + + torch.testing.assert_close(kwargs["timestep"], ts[:, None].expand(-1, 4)) + torch.testing.assert_close(kwargs["audio_timestep"], ts[:, None].expand(-1, 3)) + assert kwargs["sigma"] is ts + assert kwargs["audio_sigma"] is ts + + +def test_ltx_ancestral_step_clears_sp_audio_padding_after_scheduler_update(monkeypatch): + state = LTXAVState( + video=torch.zeros(1, 2, 2), + audio=torch.zeros(1, 4, 2), + ) + forward_ctx = SimpleNamespace( + original_audio_num_frames=3, + sampler="euler_ancestral", + ) + denoise_ctx = SimpleNamespace(latents=None, audio_latents=None) + pipe = SimpleNamespace( + _predict_noise_for_step=lambda *_args: (torch.zeros_like(state.video), torch.zeros_like(state.audio)) + ) + + def fake_step(_pipeline, actual_forward_ctx, _denoise_ctx, *_args): + assert actual_forward_ctx.sampler == "euler_ancestral" + updated_audio = torch.ones_like(state.audio) + updated_audio[:, 3:] = 99.0 + return torch.ones_like(state.video), updated_audio + + monkeypatch.setattr(ltx2_runtime, "step_denoised_latents", fake_step) + + actual = LTXRuntime._denoise_step( + pipe, + 0, + torch.tensor(1.0), + state, + forward_ctx, + denoise_ctx, + ) + + torch.testing.assert_close(actual.audio[:, :3], torch.ones_like(actual.audio[:, :3])) + torch.testing.assert_close(actual.audio[:, 3:], torch.zeros_like(actual.audio[:, 3:])) + + +def test_ltx25_first_causal_latent_frame_is_marked_for_keyframe_embedding(): + reference = torch.zeros(2, 12, 8) + + mask = _first_frame_keyframes_mask(reference, latent_num_frames=3) + + assert mask.shape == (2, 12, 1) + torch.testing.assert_close(mask[:, :4], torch.ones(2, 4, 1)) + torch.testing.assert_close(mask[:, 4:], torch.zeros(2, 8, 1)) + + +def _make_ltx_request_pipe(cls): + pipe = object.__new__(cls) + torch.nn.Module.__init__(pipe) + pipe.device = torch.device("cpu") + pipe.tokenizer_max_length = 99 + pipe.vae_spatial_compression_ratio = 32 + pipe.vae_temporal_compression_ratio = 8 + return pipe + + +def _resolve_request_inputs_for_test( + pipe, + req, + *, + guidance_scale=4.0, + negative_prompt=None, +): + return pipe._resolve_request_inputs( + req, + prompt=None, + negative_prompt=negative_prompt, + height=None, + width=None, + num_frames=None, + frame_rate=None, + num_inference_steps=None, + guidance_scale=guidance_scale, + num_videos_per_prompt=1, + generator=None, + latents=None, + audio_latents=None, + prompt_embeds=None, + negative_prompt_embeds=None, + prompt_attention_mask=None, + negative_prompt_attention_mask=None, + decode_timestep=0.0, + decode_noise_scale=None, + output_type="np", + max_sequence_length=None, + ) + + +def test_ltx25_missing_gemma4_recommends_supported_transformers_range(monkeypatch): + pipe = SimpleNamespace(component_profile=replace(LTX25_FULL_COMPONENT_PROFILE, text_encoder_cls=None)) + od_config = SimpleNamespace(model="Lightricks/LTX-2.5-Diffusers", dtype=torch.bfloat16) + + monkeypatch.setattr(ltx2_components, "get_local_device", lambda: torch.device("cpu")) + monkeypatch.setattr(ltx2_components, "prefetch_subfolders", lambda *_args, **_kwargs: None) + monkeypatch.setattr(ltx2_components.AutoTokenizer, "from_pretrained", lambda *_args, **_kwargs: object()) + + with pytest.raises(ImportError) as exc_info: + ltx2_components.initialize_pipeline_components(pipe, od_config) + + assert str(exc_info.value) == ( + "LTX-2.5 requires Gemma4UnifiedForConditionalGeneration; install transformers>=5.10.1,<5.15." + ) + + +def test_ltx_converted_component_loading_propagates_revision(monkeypatch): + revision = "pinned-revision" + calls = {"components": []} + profile = replace( + LTX25_DISTILLED_COMPONENT_PROFILE, + text_encoder_cls=object, + video_vae_cls=object, + vocoder_cls=object, + vocoder_fallback_cls=None, + scheduler_use_dynamic_shifting=False, + ) + pipeline = SimpleNamespace(component_profile=profile, pipeline_kind="distilled_two_stage") + od_config = SimpleNamespace( + model="org/converted-ltx25", + revision=revision, + dtype=torch.bfloat16, + quantization_config=None, + ) + + monkeypatch.setattr(ltx2_components, "get_local_device", lambda: torch.device("cpu")) + monkeypatch.setattr(ltx2_components, "_install_connector_attention", lambda _connectors, **_kwargs: None) + monkeypatch.setattr(ltx2_components, "_place_aux_components", lambda _pipeline: None) + + def fake_prefetch(model, subfolders, **kwargs): + calls["prefetch"] = (model, tuple(subfolders), kwargs) + + def fake_tokenizer(*args, **kwargs): + calls["tokenizer"] = (args, kwargs) + return SimpleNamespace(model_max_length=1024) + + def fake_component(component_cls, model, subfolder, **kwargs): + calls["components"].append((component_cls, model, subfolder, kwargs)) + if subfolder == "vae": + return SimpleNamespace(spatial_compression_ratio=32, temporal_compression_ratio=8) + if subfolder == "audio_vae": + return SimpleNamespace( + mel_compression_ratio=4, + temporal_compression_ratio=4, + config=SimpleNamespace(sample_rate=16_000, mel_hop_length=160), + ) + return object() + + def fake_transformer_config(model, subfolder, local_files_only, *, revision): + calls["transformer_config"] = (model, subfolder, local_files_only, revision) + return {"component": "transformer"} + + monkeypatch.setattr(ltx2_components, "prefetch_subfolders", fake_prefetch) + monkeypatch.setattr(ltx2_components.AutoTokenizer, "from_pretrained", fake_tokenizer) + monkeypatch.setattr(ltx2_components, "_load_component", fake_component) + monkeypatch.setattr(ltx2_components, "load_transformer_config", fake_transformer_config) + monkeypatch.setattr( + ltx2_components, + "create_transformer_from_config", + lambda *_args, **_kwargs: SimpleNamespace(config=SimpleNamespace(patch_size=1, patch_size_t=1)), + ) + + def fake_scheduler(*args, **kwargs): + calls["scheduler"] = (args, kwargs) + return SimpleNamespace(config={"use_dynamic_shifting": False, "shift_terminal": None}) + + monkeypatch.setattr(ltx2_components.FlowMatchEulerDiscreteScheduler, "from_pretrained", fake_scheduler) + + ltx2_components.initialize_pipeline_components(pipeline, od_config) + + assert pipeline.weights_sources[0].revision == revision + assert calls["prefetch"][2]["revision"] == revision + assert calls["tokenizer"][1]["revision"] == revision + assert {call[2] for call in calls["components"]} == { + "text_encoder", + "connectors", + "vae", + "audio_vae", + "vocoder", + "latent_upsampler", + } + assert all(call[3]["revision"] == revision for call in calls["components"]) + assert calls["transformer_config"] == ( + od_config.model, + profile.transformer_subfolder, + False, + revision, + ) + assert calls["scheduler"][1]["revision"] == revision + + +def test_ltx_checkpoint_explicit_version_precedes_structural_heuristics(tmp_path): + (tmp_path / "model_index.json").write_text(json.dumps({"model_version": "2.3.1"})) + (tmp_path / "transformer").mkdir() + (tmp_path / "transformer" / "config.json").write_text(json.dumps({"ff_bias": False})) + + assert detect_ltx_model_version(str(tmp_path)) == "2.3" + + +def test_ltx25_checkpoint_selects_full_one_stage_profile(tmp_path, monkeypatch): + from vllm_omni.diffusion.models.ltx2 import ltx2_runtime + + (tmp_path / "model_index.json").write_text( + json.dumps({"text_encoder": ["transformers", "Gemma4UnifiedForConditionalGeneration"]}) + ) + + def stub_components(pipe, od_config): + pipe.od_config = od_config + pipe.vae_spatial_compression_ratio = 32 + + monkeypatch.setattr(ltx2_runtime, "initialize_pipeline_components", stub_components) + monkeypatch.setattr(LTXRuntime, "setup_diffusion_pipeline_profiler", lambda *_args, **_kwargs: None) + + pipe = LTX2Pipeline(od_config=SimpleNamespace(model=str(tmp_path), enable_diffusion_pipeline_profiler=False)) + + assert pipe.model_version == "2.5" + assert pipe.component_profile is LTX25_FULL_COMPONENT_PROFILE + assert pipe.pipeline_recipe is LTX25_FULL_RECIPE + assert pipe.preserve_sp_padded_audio_duration + assert pipe.reports_stage_durations + + +def test_ltx25_full_recipe_matches_official_sft_defaults(): + (phase,) = LTX25_FULL_RECIPE.phases + + assert (LTX25_FULL_RECIPE.height, LTX25_FULL_RECIPE.width) == (544, 960) + assert LTX25_FULL_RECIPE.num_frames == 121 + assert LTX25_FULL_RECIPE.frame_rate == 24.0 + assert LTX25_FULL_RECIPE.num_inference_steps == 30 + assert phase.guidance.video == LTXModalityGuidance( + cfg_scale=3.0, + stg_scale=1.0, + modality_scale=3.0, + rescale_scale=0.7, + stg_blocks=(28,), + ) + assert phase.guidance.audio == LTXModalityGuidance( + cfg_scale=7.0, + stg_scale=1.0, + modality_scale=3.0, + rescale_scale=0.7, + stg_blocks=(28,), + ) + assert phase.sigmas is None + assert phase.noise_scale == 1.0 + assert phase.use_official_sigma_schedule + assert LTX25_FULL_COMPONENT_PROFILE.transformer_subfolder == "transformer_full" + assert LTX25_FULL_COMPONENT_PROFILE.scheduler_use_dynamic_shifting + assert LTX25_FULL_COMPONENT_PROFILE.scheduler_shift_terminal == 0.1 + + +def test_ltx25_full_sigma_schedule_uses_official_default_sequence_anchor(): + scheduler = SimpleNamespace( + config={ + "base_image_seq_len": 1024, + "max_image_seq_len": 4096, + "base_shift": 0.95, + "max_shift": 2.05, + "shift_terminal": 0.1, + } + ) + + sigmas = _official_ltx_sigmas( + scheduler, + steps=30, + device=torch.device("cpu"), + ) + + torch.testing.assert_close( + sigmas[[0, 1, -2, -1]], + torch.tensor([1.0, 0.9949570, 0.1, 0.0]), + rtol=1e-5, + atol=1e-6, + ) + + +def test_ltx25_checkpoint_selects_distilled_two_stage_profile(tmp_path, monkeypatch): + from vllm_omni.diffusion.models.ltx2 import ltx2_runtime + + (tmp_path / "model_index.json").write_text( + json.dumps({"text_encoder": ["transformers", "Gemma4UnifiedForConditionalGeneration"]}) + ) + + def stub_components(pipe, od_config): + pipe.od_config = od_config + pipe.vae_spatial_compression_ratio = 32 + + monkeypatch.setattr(ltx2_runtime, "initialize_pipeline_components", stub_components) + monkeypatch.setattr(LTXRuntime, "setup_diffusion_pipeline_profiler", lambda *_args, **_kwargs: None) + + pipe = LTX2DistilledTwoStagePipeline( + od_config=SimpleNamespace( + model=str(tmp_path), + enable_diffusion_pipeline_profiler=False, + ) + ) + + assert pipe.component_profile is LTX25_DISTILLED_COMPONENT_PROFILE + assert pipe.pipeline_recipe is LTX25_DISTILLED_TWO_STAGE_RECIPE + + +def test_ltx25_full_config_enrichment_uses_selected_transformer_subfolder(tmp_path): + from vllm_omni.diffusion.data import OmniDiffusionConfig + + (tmp_path / "model_index.json").write_text(json.dumps({"_class_name": "LTX2Pipeline", "model_version": "2.5"})) + for subfolder, marker in (("transformer", "distilled"), ("transformer_full", "full")): + path = tmp_path / subfolder + path.mkdir() + (path / "config.json").write_text(json.dumps({"marker": marker})) + + config = OmniDiffusionConfig(model=str(tmp_path), model_class_name="LTX2Pipeline") + config.enrich_config() + + assert config.tf_model_config.get("marker") == "full" + + +def test_ltx25_four_public_pipeline_semantics_are_disjoint(): + assert resolve_ltx_component_profile("one_stage", "2.5") is LTX25_FULL_COMPONENT_PROFILE + assert resolve_ltx_pipeline_recipe("one_stage", "2.5") is LTX25_FULL_RECIPE + assert resolve_ltx_component_profile("two_stage", "2.5") is LTX25_TWO_STAGE_COMPONENT_PROFILE + assert resolve_ltx_pipeline_recipe("two_stage", "2.5") is LTX25_TWO_STAGE_RECIPE + assert resolve_ltx_component_profile("distilled_one_stage", "2.5") is LTX25_DISTILLED_ONE_STAGE_COMPONENT_PROFILE + assert resolve_ltx_pipeline_recipe("distilled_one_stage", "2.5") is LTX25_DISTILLED_ONE_STAGE_RECIPE + assert resolve_ltx_component_profile("distilled_two_stage", "2.5") is LTX25_DISTILLED_COMPONENT_PROFILE + assert resolve_ltx_pipeline_recipe("distilled_two_stage", "2.5") is LTX25_DISTILLED_TWO_STAGE_RECIPE + + assert LTX2Pipeline.pipeline_kind == "one_stage" + assert LTX2TwoStagePipeline.pipeline_kind == "two_stage" + assert LTX2DistilledOneStagePipeline.pipeline_kind == "distilled_one_stage" + assert LTX2DistilledTwoStagePipeline.pipeline_kind == "distilled_two_stage" + + +@pytest.mark.parametrize( + ("pipeline_kind", "expected"), + [ + ("one_stage", "regular"), + ("two_stage", "regular"), + ("distilled_one_stage", "distilled"), + ("distilled_two_stage", "distilled"), + ("dmd2", None), + ], +) +def test_ltx_checkpoint_kind_is_derived_from_pipeline_kind(pipeline_kind, expected): + assert resolve_ltx_checkpoint_kind(pipeline_kind) == expected + + +def test_ltx_checkpoint_kind_rejects_unknown_pipeline(): + with pytest.raises(ValueError, match="Unsupported LTX pipeline kind"): + resolve_ltx_checkpoint_kind("unknown") + + +def test_ltx25_distilled_two_stage_recipe_matches_model_card_resolution(): + stage1, stage2 = LTX25_DISTILLED_TWO_STAGE_RECIPE.phases + + assert (LTX25_DISTILLED_TWO_STAGE_RECIPE.height, LTX25_DISTILLED_TWO_STAGE_RECIPE.width) == (1088, 1920) + assert stage1.spatial_downscale == 2 + assert stage1.sigmas == LTX_DISTILLED_SIGMAS + assert stage1.sampler == "euler_ancestral" + assert stage2.sampler == "euler" + assert stage2.sigmas == (0.909375, 0.725, 0.421875, 0.0) + assert stage2.input_transform == "spatial_upsample" + + assert LTX25_DISTILLED_TWO_STAGE_RECIPE.negative_prompt == "" + assert not LTX25_DISTILLED_TWO_STAGE_RECIPE.allow_negative_prompt + + +def test_ltx25_distilled_one_stage_uses_native_low_resolution(): + (phase,) = LTX25_DISTILLED_ONE_STAGE_RECIPE.phases + + assert (LTX25_DISTILLED_ONE_STAGE_RECIPE.height, LTX25_DISTILLED_ONE_STAGE_RECIPE.width) == (544, 960) + assert phase.spatial_downscale == 1 + assert phase.sigmas == LTX_DISTILLED_SIGMAS + assert phase.sampler == "euler_ancestral" + assert not LTX25_DISTILLED_ONE_STAGE_RECIPE.supports_cache_dit + + +def test_ltx25_regular_two_stage_uses_full_transformer_then_distilled_lora(): + stage1, stage2 = LTX25_TWO_STAGE_RECIPE.phases + + assert (LTX25_TWO_STAGE_RECIPE.height, LTX25_TWO_STAGE_RECIPE.width) == (1088, 1920) + assert LTX25_TWO_STAGE_RECIPE.num_inference_steps == 30 + assert stage1.spatial_downscale == 2 + assert stage1.guidance == LTX25_FULL_RECIPE.request_guidance + assert stage2.adapter_slot == "ltx_distilled" + assert LTX25_TWO_STAGE_RECIPE.allow_request_phase_sigmas + assert LTX25_TWO_STAGE_COMPONENT_PROFILE.transformer_subfolder == "transformer_full" + assert LTX25_TWO_STAGE_COMPONENT_PROFILE.distilled_lora_filename is not None + + +@pytest.mark.parametrize( + "recipe", + [ + LTX2_ONE_STAGE_RECIPE, + LTX23_ONE_STAGE_RECIPE, + LTX_POSITIVE_ONLY_RECIPE, + ], +) +def test_ltx_one_stage_recipes_declare_cache_dit_supported(recipe): + assert recipe.supports_cache_dit + + +@pytest.mark.parametrize( + "recipe", + [ + LTX25_FULL_RECIPE, + LTX25_DISTILLED_ONE_STAGE_RECIPE, + LTX2_DISTILLED_TWO_STAGE_RECIPE, + LTX25_DISTILLED_TWO_STAGE_RECIPE, + ], +) +def test_ltx25_and_multistage_recipes_declare_cache_dit_unsupported(recipe): + assert not recipe.supports_cache_dit + + +def test_ltx_positive_only_guidance_preserves_official_x0_roundtrip(): + sample = torch.full((1, 1), 1e8, dtype=torch.float32) + velocity = torch.ones((1, 1), dtype=torch.bfloat16) + sigma = torch.tensor(0.5) + + actual = LTX_GUIDANCE_EXECUTOR._guide_modality( + sample, + {"cond": velocity}, + sigma, + LTXModalityGuidance(), + ) + expected = velocity_from_x0(sample, x0_from_velocity(sample, velocity, sigma), sigma) + + assert actual.dtype == torch.float32 + torch.testing.assert_close(actual, expected, rtol=0.0, atol=0.0) + + +def test_ltx_velocity_from_x0_materializes_official_scalar_sigma(): + item_calls = [] + + class TrackingTensor(torch.Tensor): + @staticmethod + def __new__(cls): + return torch.Tensor._make_subclass(cls, torch.tensor(0.725), False) + + def item(self): + item_calls.append(True) + return super().item() + + sample = torch.tensor([[1.0]], dtype=torch.bfloat16) + x0 = torch.tensor([[0.5]], dtype=torch.bfloat16) + + velocity_from_x0(sample, x0, TrackingTensor()) + + assert item_calls == [True] + + +def test_ltx_ancestral_positive_only_guidance_preserves_raw_velocity(): + sample = torch.full((1, 1), 1e8, dtype=torch.float32) + velocity = torch.ones((1, 1), dtype=torch.bfloat16) + + actual = LTX_GUIDANCE_EXECUTOR._guide_modality( + sample, + {"cond": velocity}, + torch.tensor(0.5), + LTXModalityGuidance(), + preserve_positive_velocity=True, + ) + + assert actual is velocity + + +@pytest.mark.parametrize("model_version", ["2.5", "2.3"]) +def test_ltx_t2v_denoise_state_uses_official_bfloat16_dtype(model_version): + expected_dtype = torch.bfloat16 + captured = {} + + def prepare_latents(**kwargs): + captured["video_dtype"] = kwargs["dtype"] + return torch.empty(1, 1, 1, dtype=kwargs["dtype"]) + + def prepare_audio_latents(*_args, **kwargs): + captured["audio_dtype"] = kwargs["dtype"] + return torch.empty(1, 1, 1, dtype=kwargs["dtype"]), 1, 1 + + pipeline = SimpleNamespace( + model_version=model_version, + transformer=SimpleNamespace(config=SimpleNamespace(in_channels=128)), + prepare_latents=prepare_latents, + prepare_audio_latents=prepare_audio_latents, + audio_sampling_rate=48_000, + audio_hop_length=160, + audio_vae_temporal_compression_ratio=4, + audio_vae_mel_compression_ratio=4, + audio_vae=SimpleNamespace(config=SimpleNamespace(mel_bins=64, latent_channels=8)), + _resolve_audio_latent_length=lambda length, _latents: length, + ) + request = SimpleNamespace( + num_videos_per_prompt=1, + height=64, + width=64, + num_frames=1, + frame_rate=24.0, + generator=None, + latents=None, + audio_latents=None, + audio_latents_normalized=False, + ) + prompt = SimpleNamespace( + batch_size=1, + positive_connector_prompt_embeds=torch.empty(1, dtype=torch.bfloat16), + positive_connector_audio_prompt_embeds=torch.empty(1, dtype=torch.bfloat16), + ) + + LTXRuntime._prepare_video_latents_stage( + pipeline, + request, + prompt, + device=torch.device("cpu"), + noise_scale=1.0, + ) + LTXRuntime._prepare_audio_latents_stage( + pipeline, + request, + prompt, + device=torch.device("cpu"), + noise_scale=1.0, + ) + + assert captured == {"video_dtype": expected_dtype, "audio_dtype": expected_dtype} + + +def test_ltx_seeded_latents_match_official_packed_rng_layout(): + pipeline = SimpleNamespace( + model_version="2.5", + vae_spatial_compression_ratio=2, + vae_temporal_compression_ratio=2, + transformer_spatial_patch_size=1, + transformer_temporal_patch_size=1, + audio_vae_mel_compression_ratio=2, + od_config=SimpleNamespace(parallel_config=SimpleNamespace(sequence_parallel_size=1)), + ) + device = torch.device("cpu") + actual_generator = torch.Generator(device=device).manual_seed(123) + expected_generator = torch.Generator(device=device).manual_seed(123) + + actual_video = ltx2_latents.prepare_video_latents( + pipeline, + batch_size=1, + num_channels_latents=2, + height=4, + width=6, + num_frames=3, + dtype=torch.float32, + device=device, + generator=actual_generator, + ) + actual_audio, original_length, padded_length = ltx2_latents.prepare_audio_latents( + pipeline, + batch_size=1, + num_channels_latents=2, + audio_latent_length=3, + num_mel_bins=4, + dtype=torch.float32, + device=device, + generator=actual_generator, + ) + + expected_video = torch.randn((1, 12, 2), generator=expected_generator) + expected_audio = torch.randn((1, 3, 4), generator=expected_generator) + + torch.testing.assert_close(actual_video, expected_video, rtol=0.0, atol=0.0) + torch.testing.assert_close(actual_audio, expected_audio, rtol=0.0, atol=0.0) + assert (original_length, padded_length) == (3, 3) + + +def test_ltx25_distilled_two_stage_executes_custom_phase_schedules(): + request_inputs = LTXRequestInputs( + prompt="prompt", + negative_prompt="", + height=64, + width=64, + num_frames=1, + frame_rate=24.0, + num_inference_steps=8, + guidance=LTXGuidanceSpec.positive_only(), + num_videos_per_prompt=1, + generator=None, + latents=None, + audio_latents=None, + prompt_embeds=None, + negative_prompt_embeds=None, + prompt_attention_mask=None, + negative_prompt_attention_mask=None, + decode_timestep=0.25, + decode_noise_scale=0.5, + output_type="np", + max_sequence_length=16, + ) + prompt_context = object() + phase_calls = [] + + def resolve_request_inputs(req, **kwargs): + return request_inputs + + def run_phase(req, inputs, *, prompt_context=None, **kwargs): + phase_recipe = kwargs["phase_recipe"] + assert kwargs["image"] is source_image + phase_calls.append((phase_recipe, inputs, prompt_context)) + if len(phase_calls) == 1: + assert prompt_context is None + assert inputs.decode_timestep == 0.25 + assert inputs.decode_noise_scale == 0.5 + context = prompt_context_sentinel + assert phase_recipe.name == "generate_lowres" + assert (inputs.height, inputs.width) == (32, 32) + assert inputs.num_inference_steps == 2 + assert kwargs["sigmas"] == stage_1_sigmas + assert kwargs["noise_scale"] == 1.0 + video = torch.ones(1, 128, 1, 1, 1) + audio = torch.full((1, 8, 1, 2), 2.0) + audio_for_next_phase = torch.full((1, 8, 1, 2), 5.0) + else: + assert prompt_context is prompt_context_sentinel + assert phase_recipe.name == "refine" + assert (inputs.height, inputs.width) == (64, 64) + assert inputs.latents.shape == (1, 128, 1, 2, 2) + torch.testing.assert_close(inputs.audio_latents, torch.full((1, 8, 1, 2), 5.0)) + assert inputs.audio_latents_normalized + assert inputs.guidance_scale == 1.0 + assert inputs.num_inference_steps == 2 + assert kwargs["sigmas"] == stage_2_sigmas + assert kwargs["noise_scale"] == stage_2_sigmas[0] + assert inputs.decode_timestep == 0.0 + assert inputs.decode_noise_scale is None + context = prompt_context + video = torch.full((1, 128, 1, 2, 2), 3.0) + audio = torch.full((1, 8, 1, 2), 4.0) + audio_for_next_phase = None + return LTXPhaseResult( + forward_context=SimpleNamespace(prompt_context=context), + video=video, + audio=audio, + audio_for_next_phase=audio_for_next_phase, + ) + + def decode_phase(phase): + return DiffusionOutput(output=(phase.video, phase.audio)) + + class FakeUpsampler(torch.nn.Module): + dtype = torch.float32 + + def forward(self, latents): + return latents.repeat_interleave(2, dim=-2).repeat_interleave(2, dim=-1) + + prompt_context_sentinel = prompt_context + stage_1_sigmas = [1.0, 0.5, 0.0] + stage_2_sigmas = [0.8, 0.2, 0.0] + pipeline = object.__new__(LTX2TwoStagePipeline) + torch.nn.Module.__init__(pipeline) + pipeline.pipeline_recipe = LTX25_DISTILLED_TWO_STAGE_RECIPE + pipeline.device = torch.device("cpu") + pipeline.vae_spatial_compression_ratio = 32 + pipeline.vae_temporal_compression_ratio = 8 + pipeline.latent_upsampler = FakeUpsampler() + object.__setattr__(pipeline, "_resolve_request_inputs", resolve_request_inputs) + object.__setattr__(pipeline, "run_phase", run_phase) + object.__setattr__(pipeline, "decode_phase", decode_phase) + + source_image = object() + output = pipeline.forward( + SimpleNamespace(sampling_params_list=[], prompts=[]), + image=source_image, + stage_1_sigmas=stage_1_sigmas, + stage_2_sigmas=stage_2_sigmas, + ) + + assert len(phase_calls) == 2 + assert phase_calls[1][2] is prompt_context_sentinel + torch.testing.assert_close(output.output[0], torch.full((1, 128, 1, 2, 2), 3.0)) + torch.testing.assert_close(output.output[1], torch.full((1, 8, 1, 2), 4.0)) + + +class TestLTXRequestParsing: + def test_request_resolves_custom_sigmas_from_extra_args(self): + from vllm_omni.diffusion.request import OmniDiffusionRequest + from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + sigmas = [1.0, 0.5, 0.0] + req = DiffusionRequestBatch( + [ + OmniDiffusionRequest( + prompt="prompt", + sampling_params=OmniDiffusionSamplingParams(extra_args={"sigmas": sigmas}), + request_id="ltx-custom-sigmas", + ) + ] + ) + + assert _make_ltx_request_pipe(LTX2Pipeline)._resolve_request_sigmas(req, None) == sigmas + + def test_request_resolves_per_stage_sigmas_from_extra_args(self): + from vllm_omni.diffusion.request import OmniDiffusionRequest + from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + stage_1_sigmas = [1.0, 0.4, 0.0] + req = DiffusionRequestBatch( + [ + OmniDiffusionRequest( + prompt="prompt", + sampling_params=OmniDiffusionSamplingParams(extra_args={"stage_1_sigmas": stage_1_sigmas}), + request_id="ltx-custom-phase-sigmas", + ) + ] + ) + + resolved = _make_ltx_request_pipe(LTX2TwoStagePipeline)._resolve_request_phase_sigmas( + req, [1.0, 0.0], [0.8, 0.0] + ) + + assert resolved == (stage_1_sigmas, [0.8, 0.0]) + + @pytest.mark.parametrize("value", [-1, 52, 1.5, True]) + def test_request_rejects_invalid_image_crf(self, value): + from vllm_omni.diffusion.request import OmniDiffusionRequest + from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + req = DiffusionRequestBatch( + [ + OmniDiffusionRequest( + prompt="prompt", + sampling_params=OmniDiffusionSamplingParams(extra_args={"image_crf": value}), + request_id="ltx-invalid-image-crf", + ) + ] + ) + + with pytest.raises(ValueError, match="image_crf"): + _resolve_request_inputs_for_test(_make_ltx_request_pipe(LTX2Pipeline), req) + + +class TestLTXForwardStages: + def test_gemma_tokenization_matches_official_bos_and_left_padding(self): + pipe = _make_ltx_request_pipe(LTX2Pipeline) + + class FakeTokenizer: + bos_token_id = 2 + eos_token_id = 1 + eos_token = "" + pad_token_id = 0 + pad_token = "" + padding_side = "right" + + def __init__(self): + self.calls = [] + + def __call__(self, text, **kwargs): + self.calls.append((text, kwargs)) + ids = [7, 8, 9, self.eos_token_id] if text == "long" else [6, self.eos_token_id] + return SimpleNamespace(input_ids=ids) + + def pad(self, encoded, *, padding, max_length, return_tensors, return_attention_mask): + assert padding == "max_length" + assert return_tensors == "pt" + assert return_attention_mask is True + rows = [] + masks = [] + for ids in encoded["input_ids"]: + pad = max_length - len(ids) + rows.append([self.pad_token_id] * pad + ids) + masks.append([0] * pad + [1] * len(ids)) + return SimpleNamespace(input_ids=torch.tensor(rows), attention_mask=torch.tensor(masks)) + + class FakeTextEncoder(torch.nn.Module): + dtype = torch.float32 + + def __init__(self): + super().__init__() + self.seen_input_ids = None + self.seen_attention_mask = None + + def forward(self, *, input_ids, attention_mask, output_hidden_states): + assert output_hidden_states is True + self.seen_input_ids = input_ids.clone() + self.seen_attention_mask = attention_mask.clone() + return SimpleNamespace(hidden_states=(input_ids.unsqueeze(-1).float(),)) + + tokenizer = FakeTokenizer() + text_encoder = FakeTextEncoder() + object.__setattr__(pipe, "tokenizer", tokenizer) + object.__setattr__(pipe, "text_encoder", text_encoder) + + pipe._get_gemma_prompt_embeds( + [" long ", "short"], + max_sequence_length=4, + device=torch.device("cpu"), + dtype=torch.float32, + ) + + torch.testing.assert_close(text_encoder.seen_input_ids, torch.tensor([[2, 7, 8, 9], [0, 2, 6, 1]])) + torch.testing.assert_close(text_encoder.seen_attention_mask, torch.tensor([[1, 1, 1, 1], [0, 1, 1, 1]])) + assert tokenizer.padding_side == "left" + assert [call[0] for call in tokenizer.calls] == ["long", "short"] diff --git a/tests/diffusion/models/ltx2/test_ltx2_conditioning.py b/tests/diffusion/models/ltx2/test_ltx2_conditioning.py index e309dbf47..cbc66e69c 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_conditioning.py +++ b/tests/diffusion/models/ltx2/test_ltx2_conditioning.py @@ -3,6 +3,7 @@ """Unit tests for LTX image-to-video input and conditioning behavior.""" +import sys from types import SimpleNamespace import pytest @@ -23,7 +24,7 @@ def _make_ltx23_request_pipe(cls): class TestLTXImageToVideoForwardStages: - def test_i2v_pil_preprocessing_matches_official_center_crop(self): + def test_legacy_i2v_pil_preprocessing_preserves_aspect_ratio_and_center_crops(self): from vllm_omni.diffusion.models.ltx2.ltx2_conditioning import _preprocess_i2v_pil_images pixels = torch.zeros(4, 8, 3, dtype=torch.uint8) @@ -35,6 +36,136 @@ class TestLTXImageToVideoForwardStages: torch.testing.assert_close(actual, expected) + def test_ltx25_i2v_pil_preprocessing_preserves_aspect_ratio_after_crf(self, monkeypatch): + import vllm_omni.diffusion.models.ltx2.ltx2_conditioning as conditioning + + pixels = torch.zeros(4, 8, 3, dtype=torch.uint8) + pixels[:, :, 0] = torch.arange(8, dtype=torch.uint8) + image = Image.fromarray(pixels.numpy()) + monkeypatch.setattr(conditioning, "_apply_image_conditioning_crf", lambda image_array, _crf: image_array) + + actual = conditioning._preprocess_i2v_pil_images(image, height=4, width=4, crf=18) + expected = pixels[:, 2:6].permute(2, 0, 1).unsqueeze(0).float() / 127.5 - 1.0 + + torch.testing.assert_close(actual, expected) + + def test_ltx25_i2v_applies_crf18_before_resize(self, monkeypatch): + import vllm_omni.diffusion.models.ltx2.ltx2_conditioning as conditioning + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline + + captured_crfs = [] + captured_prepare_kwargs = [] + + def fake_preprocess(image, *, height, width, crf, device, dtype): + del image, device + captured_crfs.append(crf) + return torch.zeros(1, 3, height, width, dtype=dtype) + + def fake_prepare_latents(**kwargs): + captured_prepare_kwargs.append(kwargs) + return kwargs["image"], None + + monkeypatch.setattr(conditioning, "_preprocess_i2v_pil_images", fake_preprocess) + pipe = object.__new__(LTX2Pipeline) + torch.nn.Module.__init__(pipe) + pipe.model_version = "2.5" + object.__setattr__(pipe, "transformer", SimpleNamespace(config=SimpleNamespace(in_channels=4))) + object.__setattr__(pipe, "prepare_latents", fake_prepare_latents) + request_inputs = SimpleNamespace( + height=4, + width=4, + num_frames=3, + num_videos_per_prompt=1, + generator=None, + latents=None, + ) + prompt_context = SimpleNamespace( + batch_size=1, + positive_connector_prompt_embeds=torch.zeros(1, 1, 1, dtype=torch.bfloat16), + ) + + pipe._prepare_video_latents_stage( + request_inputs, + prompt_context, + device=torch.device("cpu"), + noise_scale=0.0, + image=Image.new("RGB", (8, 4)), + ) + + assert captured_crfs == [18] + assert captured_prepare_kwargs[0]["dtype"] is torch.bfloat16 + assert captured_prepare_kwargs[0]["image"].dtype is torch.bfloat16 + + def test_ltx25_i2v_rejects_inputs_that_cannot_receive_crf18(self): + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline + + pipe = object.__new__(LTX2Pipeline) + torch.nn.Module.__init__(pipe) + pipe.model_version = "2.5" + + with pytest.raises(ValueError, match="requires PIL images"): + pipe._prepare_video_latents_stage( + SimpleNamespace(), + SimpleNamespace(), + device=torch.device("cpu"), + noise_scale=0.0, + image=torch.zeros(3, 4, 4), + ) + + def test_ltx25_i2v_tensor_input_can_opt_out_of_crf18(self): + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline + + captured_prepare_kwargs = [] + + def fake_prepare_latents(**kwargs): + captured_prepare_kwargs.append(kwargs) + return kwargs["image"], None + + pipe = object.__new__(LTX2Pipeline) + torch.nn.Module.__init__(pipe) + pipe.model_version = "2.5" + object.__setattr__(pipe, "transformer", SimpleNamespace(config=SimpleNamespace(in_channels=4))) + object.__setattr__(pipe, "prepare_latents", fake_prepare_latents) + request_inputs = SimpleNamespace( + height=4, + width=4, + num_frames=3, + num_videos_per_prompt=1, + generator=None, + latents=None, + image_crf=0, + ) + prompt_context = SimpleNamespace( + batch_size=1, + positive_connector_prompt_embeds=torch.zeros(1, 1, 1, dtype=torch.bfloat16), + ) + + pipe._prepare_video_latents_stage( + request_inputs, + prompt_context, + device=torch.device("cpu"), + noise_scale=0.0, + image=torch.zeros(3, 4, 4), + ) + + assert captured_prepare_kwargs[0]["image"].shape == (1, 3, 4, 4) + + def test_ltx25_i2v_crf_preserves_single_pixel_dimension(self, monkeypatch): + from vllm_omni.diffusion.models.ltx2.ltx2_conditioning import _apply_image_conditioning_crf + + image = torch.zeros(1, 4, 3, dtype=torch.uint8).numpy() + monkeypatch.setitem(sys.modules, "av", None) + + assert _apply_image_conditioning_crf(image, 18) is image + + def test_ltx25_i2v_reports_missing_pyav(self, monkeypatch): + from vllm_omni.diffusion.models.ltx2.ltx2_conditioning import _apply_image_conditioning_crf + + monkeypatch.setitem(sys.modules, "av", None) + + with pytest.raises(ImportError, match="PyAV with a libx264 encoder"): + _apply_image_conditioning_crf(torch.zeros(4, 4, 3, dtype=torch.uint8).numpy(), 18) + def test_forward_resolves_request_image_and_delegates_to_shared_recipe_runtime(self): from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline from vllm_omni.diffusion.request import OmniDiffusionRequest @@ -98,7 +229,7 @@ class TestLTXImageToVideoForwardStages: with pytest.raises(ValueError, match="cannot mix text-to-video and image-to-video"): pipe._resolve_request_image(mixed_req, None, request_inputs) - def test_denoise_timestep_kwargs_masks_video_only(self): + def test_denoise_timestep_kwargs_masks_video_and_keeps_audio_per_token(self): from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline pipe = object.__new__(LTX2Pipeline) @@ -189,13 +320,18 @@ class TestLTXImageToVideoConditioning: torch.testing.assert_close(conditioning_mask, torch.tensor([[1.0, 0.0, 0.0]])) torch.testing.assert_close(out, torch.tensor([[[40.0, 41.0], [20.0, 21.0], [30.0, 31.0]]])) - def test_ltx23_i2v_5d_latents_noise_preserves_conditioning_frame(self, monkeypatch): + @pytest.mark.parametrize( + ("model_version", "sampled_shape"), + [("2.3", (1, 3, 2)), ("2.5", (1, 3, 2))], + ) + def test_i2v_5d_latents_noise_uses_packed_rng_layout(self, monkeypatch, model_version, sampled_shape): import vllm_omni.diffusion.models.ltx2.ltx2_conditioning as ltx2_conditioning import vllm_omni.diffusion.models.ltx2.ltx2_latents as ltx2_latents from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline pipe = object.__new__(LTX2Pipeline) torch.nn.Module.__init__(pipe) + pipe.model_version = model_version pipe.vae_spatial_compression_ratio = 1 pipe.vae_temporal_compression_ratio = 1 pipe.transformer_spatial_patch_size = 1 @@ -237,14 +373,20 @@ class TestLTXImageToVideoConditioning: torch.testing.assert_close(conditioning_mask, torch.tensor([[1.0, 0.0, 0.0]])) torch.testing.assert_close(out, torch.tensor([[[40.0, 41.0], [1.0, 1.0], [1.0, 1.0]]])) - assert sampled_shapes == [(1, 3, 2)] + assert sampled_shapes == [sampled_shape] - def test_ltx23_i2v_image_noise_is_sampled_after_packing(self, monkeypatch): + @pytest.mark.parametrize( + ("model_version", "sampled_shape"), + [("2.3", (1, 3, 2)), ("2.5", (1, 3, 2))], + ) + def test_i2v_image_noise_uses_packed_rng_layout(self, monkeypatch, model_version, sampled_shape): import vllm_omni.diffusion.models.ltx2.ltx2_conditioning as ltx2_conditioning + import vllm_omni.diffusion.models.ltx2.ltx2_latents as ltx2_latents from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline pipe = object.__new__(LTX2Pipeline) torch.nn.Module.__init__(pipe) + pipe.model_version = model_version pipe.vae_spatial_compression_ratio = 1 pipe.vae_temporal_compression_ratio = 1 pipe.transformer_spatial_patch_size = 1 @@ -266,6 +408,7 @@ class TestLTXImageToVideoConditioning: return torch.ones(shape, device=device, dtype=dtype) monkeypatch.setattr(ltx2_conditioning, "randn_tensor", fake_randn_tensor) + monkeypatch.setattr(ltx2_latents, "randn_tensor", fake_randn_tensor) out, conditioning_mask = pipe.prepare_latents( image=torch.zeros(1, 3, 1, 1), @@ -279,10 +422,54 @@ class TestLTXImageToVideoConditioning: device=torch.device("cpu"), ) - assert sampled_shapes == [(1, 3, 2)] + assert sampled_shapes == [sampled_shape] torch.testing.assert_close(conditioning_mask, torch.tensor([[1.0, 0.0, 0.0]])) torch.testing.assert_close(out, torch.tensor([[[10.0, 11.0], [1.0, 1.0], [1.0, 1.0]]])) + def test_ltx25_re_noise_uses_official_scalar_noise_scale(self, monkeypatch): + import vllm_omni.diffusion.models.ltx2.ltx2_latents as ltx2_latents + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline + + pipe = object.__new__(LTX2Pipeline) + torch.nn.Module.__init__(pipe) + pipe.model_version = "2.5" + pipe.vae_spatial_compression_ratio = 1 + pipe.vae_temporal_compression_ratio = 1 + pipe.transformer_spatial_patch_size = 1 + pipe.transformer_temporal_patch_size = 1 + pipe.vae = SimpleNamespace( + latents_mean=torch.zeros(2), + latents_std=torch.ones(2), + config=SimpleNamespace(scaling_factor=1.0), + ) + object.__setattr__( + pipe, + "_encode_i2v_image_latents", + lambda *_args, **_kwargs: torch.zeros(1, 2, 1, 1, 1, dtype=torch.bfloat16), + ) + captured = {} + + def fake_create_noised_state(latents, noise_scale, generator=None): + del generator + captured["noise_scale"] = noise_scale + return latents + + monkeypatch.setattr(ltx2_latents, "create_noised_state", fake_create_noised_state) + pipe.prepare_latents( + image=torch.zeros(1, 3, 1, 1), + batch_size=1, + num_channels_latents=2, + height=1, + width=1, + num_frames=3, + noise_scale=0.909375, + dtype=torch.bfloat16, + device=torch.device("cpu"), + latents=torch.zeros(1, 2, 3, 1, 1, dtype=torch.bfloat16), + ) + + assert captured["noise_scale"] == 0.909375 + def test_ltx23_i2v_video_step_preserves_conditioning_frame(self): from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline @@ -385,6 +572,99 @@ class TestLTXImageToVideoConditioning: torch.testing.assert_close(audio, torch.tensor([[[-2.0]]])) assert pipeline.step_indices == [0, 1] + def test_ltx25_ancestral_adapter_matches_official_seeded_step_and_preserves_i2v_mask(self): + from vllm_omni.diffusion.models.ltx2.ltx2_denoise import LTXVideoAudioStepAdapter + + device = torch.device("cpu") + sigmas = torch.tensor([1.0, 0.5, 0.0]) + pipeline = SimpleNamespace(device=device, scheduler=SimpleNamespace(sigmas=sigmas)) + mask = torch.tensor([[1.0, 0.0]]) + adapter = LTXVideoAudioStepAdapter( + pipeline, + SimpleNamespace(sigmas=sigmas.clone()), + latent_num_frames=1, + latent_height=1, + latent_width=1, + image_conditioned=True, + sampler="euler_ancestral", + generator=torch.Generator(device=device).manual_seed(42), + conditioning_mask=mask, + ) + video = torch.tensor([[[3.0], [2.0]]]) + audio = torch.tensor([[[4.0]]]) + video_velocity = torch.tensor([[[7.0], [0.25]]]) + audio_velocity = torch.tensor([[[0.5]]]) + + reference_generator = torch.Generator(device=device).manual_seed(10042) + video_noise = torch.randn(video.shape, generator=reference_generator) + audio_noise = torch.randn(audio.shape, generator=reference_generator) + + def official_step(sample, velocity, noise): + sigma = sigmas[0] + sigma_next = sigmas[1] + denoised = sample.float() - velocity.float() * sigma + sigma_down = sigma_next * (sigma_next / sigma) + ratio = sigma_down / sigma + deterministic = ratio * sample.float() + (1.0 - ratio) * denoised + alpha_next = 1.0 - sigma_next + alpha_down = 1.0 - sigma_down + coeff = (sigma_next**2 - sigma_down**2 * alpha_next**2 / alpha_down**2).sqrt() + return (alpha_next / alpha_down) * deterministic + noise * coeff + + expected_video = official_step(video, video_velocity, video_noise) + expected_video[:, :1] = video[:, :1] + expected_audio = official_step(audio, audio_velocity, audio_noise) + + ((actual_video, actual_audio),) = adapter.step( + (video_velocity, audio_velocity), + (sigmas[0], sigmas[0]), + (video, audio), + ) + torch.testing.assert_close(actual_video, expected_video) + torch.testing.assert_close(actual_audio, expected_audio) + + terminal_video_velocity = torch.full_like(actual_video, 0.125) + terminal_audio_velocity = torch.full_like(actual_audio, 0.25) + ((terminal_video, terminal_audio),) = adapter.step( + (terminal_video_velocity, terminal_audio_velocity), + (sigmas[1], sigmas[1]), + (actual_video, actual_audio), + ) + expected_terminal_video = actual_video - terminal_video_velocity * sigmas[1] + expected_terminal_video[:, :1] = actual_video[:, :1] + expected_terminal_audio = actual_audio - terminal_audio_velocity * sigmas[1] + torch.testing.assert_close(terminal_video, expected_terminal_video) + torch.testing.assert_close(terminal_audio, expected_terminal_audio) + + def test_ltx25_ancestral_quantizes_x0_to_model_dtype_before_fp32_step(self): + from vllm_omni.diffusion.models.ltx2.ltx2_denoise import _ancestral_euler_step_from_velocity + + sample = torch.tensor([[[3.140625, -1.234375]]], dtype=torch.bfloat16) + velocity = torch.tensor([[[0.333984375, 2.71875]]], dtype=torch.bfloat16) + sigmas = torch.tensor([0.73, 0.41, 0.0], dtype=torch.float32) + generator = torch.Generator(device="cpu").manual_seed(10042) + reference_generator = torch.Generator(device="cpu").manual_seed(10042) + + sigma, sigma_next = sigmas[:2] + denoised = (sample.float() - velocity.float() * sigma).to(sample.dtype).float() + sigma_down = sigma_next * (sigma_next / sigma) + sigma_down_ratio = sigma_down / sigma + expected = sigma_down_ratio * sample.float() + (1.0 - sigma_down_ratio) * denoised + alpha_next = 1.0 - sigma_next + alpha_down = 1.0 - sigma_down + coefficient = (sigma_next**2 - sigma_down**2 * alpha_next**2 / alpha_down**2).sqrt() + noise = torch.randn( + sample.shape, + generator=reference_generator, + dtype=sample.dtype, + device=sample.device, + ) + expected = ((alpha_next / alpha_down) * expected + noise.float() * coefficient).to(sample.dtype) + + actual = _ancestral_euler_step_from_velocity(sample, velocity, sigmas, 0, generator) + + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + def test_i2v_guidance_uses_zero_sigma_for_conditioned_tokens(self): from vllm_omni.diffusion.models.ltx2.ltx2_denoise import LTXDenoiseContext from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline diff --git a/tests/diffusion/models/ltx2/test_ltx2_latents.py b/tests/diffusion/models/ltx2/test_ltx2_latents.py index cad5c300d..a9d19945a 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_latents.py +++ b/tests/diffusion/models/ltx2/test_ltx2_latents.py @@ -202,3 +202,68 @@ def test_prepare_audio_latents_rejects_incompatible_provided_length(): device=torch.device("cpu"), latents=latents, ) + + +def test_create_noised_state_matches_official_fp32_lerp(): + latents = torch.linspace(-2, 2, 4096, dtype=torch.bfloat16).reshape(1, 32, 128) + expected_generator = torch.Generator().manual_seed(42) + noise = torch.randn(latents.shape, generator=expected_generator, dtype=latents.dtype) + expected = torch.lerp(latents.float(), noise.float(), torch.tensor(0.15, dtype=torch.bfloat16).float()).to( + latents.dtype + ) + + actual = latent_ops.create_noised_state( + latents, + torch.tensor(0.15, dtype=torch.bfloat16), + torch.Generator().manual_seed(42), + ) + + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +def test_create_conditioned_noised_state_matches_official_two_lerps(): + latents = torch.linspace(-2, 2, 24, dtype=torch.bfloat16).reshape(1, 3, 8) + clean_latents = latents.clone() + clean_latents[:, 0] = 3.0 + denoise_mask = torch.tensor([[[0.0], [1.0], [1.0]]]) + expected_generator = torch.Generator().manual_seed(42) + noise = torch.randn(latents.shape, generator=expected_generator, dtype=latents.dtype) + noised = torch.lerp(latents.float(), noise.float(), 0.909375) + expected = torch.lerp(clean_latents.float(), noised, denoise_mask).to(latents.dtype) + + actual = latent_ops.create_conditioned_noised_state( + latents, + clean_latents, + denoise_mask, + 0.909375, + torch.Generator().manual_seed(42), + ) + + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +def test_prepare_supplied_video_latents_uses_official_token_major_layout(): + pipeline = _make_pipeline(LTX2Pipeline) + supplied = torch.arange(512, dtype=torch.float32).reshape(1, 32, 16) + actual_video = pipeline.prepare_latents( + batch_size=1, + num_channels_latents=4, + height=64, + width=64, + num_frames=9, + dtype=torch.float32, + device=torch.device("cpu"), + latents=supplied, + ) + + torch.testing.assert_close(actual_video, supplied, rtol=0, atol=0) + assert actual_video.stride()[1:] == (1, actual_video.shape[1]) + + +def test_clear_audio_padding_keeps_padding_outside_sampler_state(): + updated = torch.tensor([[[1.0, 2.0], [3.0, 4.0], [99.0, -99.0]]]) + + actual = latent_ops.clear_audio_padding(updated, 2) + + torch.testing.assert_close(actual[:, :2], updated[:, :2]) + torch.testing.assert_close(actual[:, 2:], torch.zeros_like(actual[:, 2:])) diff --git a/tests/diffusion/models/ltx2/test_ltx2_parallel.py b/tests/diffusion/models/ltx2/test_ltx2_parallel.py index 95ca6e868..0573dfb37 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_parallel.py +++ b/tests/diffusion/models/ltx2/test_ltx2_parallel.py @@ -28,6 +28,15 @@ def test_velocity_from_x0_uses_official_host_scalar_sigma(mocker): class TestCFGParallelHelpers: """Test LTX-2.3 CFG helper math without loading model weights.""" + def test_single_guidance_pass_preserves_official_token_layout(self): + from vllm_omni.diffusion.models.ltx2.ltx2_guidance import _repeat_batch + + tokens = torch.randn(1, 16, 32).transpose(1, 2).contiguous().transpose(1, 2) + repeated = _repeat_batch(tokens, 1) + + assert repeated is tokens + assert repeated.stride() == tokens.stride() + def test_combine_cfg_noise_matches_x0_space_formula(self): from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline diff --git a/tests/diffusion/models/ltx2/test_ltx2_pipeline.py b/tests/diffusion/models/ltx2/test_ltx2_pipeline.py index 2f3ae522e..b3da177db 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_pipeline.py +++ b/tests/diffusion/models/ltx2/test_ltx2_pipeline.py @@ -20,6 +20,7 @@ from vllm_omni.diffusion.models.ltx2 import ( from vllm_omni.diffusion.models.ltx2.ltx2_components import ( LTX2_COMPONENT_PROFILE, LTX2_DISTILLED_COMPONENT_PROFILE, + LTX2_DISTILLED_ONE_STAGE_COMPONENT_PROFILE, LTX2_TWO_STAGE_COMPONENT_PROFILE, LTX23_COMPONENT_PROFILE, LTX23_DISTILLED_COMPONENT_PROFILE, @@ -47,6 +48,7 @@ from vllm_omni.diffusion.models.ltx2.ltx2_guidance import ( ) from vllm_omni.diffusion.models.ltx2.ltx2_latents import LTXAVState from vllm_omni.diffusion.models.ltx2.ltx2_recipes import ( + LTX2_DISTILLED_ONE_STAGE_RECIPE, LTX2_DISTILLED_TWO_STAGE_RECIPE, LTX2_ONE_STAGE_RECIPE, LTX2_TWO_STAGE_RECIPE, @@ -64,12 +66,14 @@ from vllm_omni.diffusion.models.ltx2.ltx2_request import ( ) from vllm_omni.diffusion.models.ltx2.ltx2_runtime import LTXRuntime from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import ( + LTX2DistilledOneStagePipeline, LTX2I2VDMD2Pipeline, LTX2Pipeline, LTX2T2VDMD2Pipeline, ) from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_two_stage import ( LTX2DistilledPipeline, + LTX2DistilledTwoStagePipeline, LTX2TwoStagePipeline, ) @@ -125,15 +129,21 @@ def test_ltx_public_entries_share_runtime_and_keep_recipe_boundaries(): assert issubclass(LTX2Pipeline, LTXRuntime) assert issubclass(LTX2TwoStagePipeline, LTXRuntime) + assert issubclass(LTX2DistilledOneStagePipeline, LTXRuntime) + assert issubclass(LTX2DistilledTwoStagePipeline, LTXRuntime) assert issubclass(LTX2DistilledPipeline, LTXRuntime) + assert LTX2DistilledPipeline is LTX2DistilledTwoStagePipeline assert LTX2Pipeline.component_profile is LTX2_COMPONENT_PROFILE assert LTX2Pipeline.pipeline_recipe is LTX2_ONE_STAGE_RECIPE assert LTX2TwoStagePipeline.component_profile is LTX2_TWO_STAGE_COMPONENT_PROFILE assert LTX2TwoStagePipeline.pipeline_recipe is LTX2_TWO_STAGE_RECIPE + assert LTX2DistilledOneStagePipeline.component_profile is LTX2_DISTILLED_ONE_STAGE_COMPONENT_PROFILE + assert LTX2DistilledOneStagePipeline.pipeline_recipe is LTX2_DISTILLED_ONE_STAGE_RECIPE assert LTX2DistilledPipeline.component_profile is LTX2_DISTILLED_COMPONENT_PROFILE for pipeline_cls, profile in ( (LTX2Pipeline, LTX2_COMPONENT_PROFILE), (LTX2TwoStagePipeline, LTX2_TWO_STAGE_COMPONENT_PROFILE), + (LTX2DistilledOneStagePipeline, LTX2_DISTILLED_ONE_STAGE_COMPONENT_PROFILE), (LTX2DistilledPipeline, LTX2_DISTILLED_COMPONENT_PROFILE), ): assert pipeline_cls._dit_modules == list(profile.dit_modules) @@ -254,7 +264,7 @@ def test_ltx_multistage_rejects_cache_dit_before_component_initialization(tmp_pa monkeypatch.setattr(ltx2_runtime, "initialize_pipeline_components", stub_components) - with pytest.raises(ValueError, match="only one-stage LTX recipes"): + with pytest.raises(ValueError, match="not qualified for this LTX recipe"): LTX2TwoStagePipeline( od_config=SimpleNamespace( model=str(tmp_path), @@ -709,15 +719,48 @@ def test_ltx_component_loader_installs_omni_connector_attention(monkeypatch): assert len(installed) == 2 assert all(processor.__class__.__name__ == "_LTXConnectorAttnProcessor" for processor in installed) - assert all(processor.has_learned_registers for processor in installed) assert all(kernel["role"] == "ltx2.connector" for kernel in kernels) assert all(kernel["role_category"] == "self" for kernel in kernels) assert all(kernel["skip_sequence_parallel"] for kernel in kernels) assert all(kernel["disable_kv_quant"] for kernel in kernels) + assert all(processor.has_learned_registers for processor in installed) + assert not any(processor.preserve_learned_register_mask for processor in installed) -@pytest.mark.parametrize("has_learned_registers", [False, True]) -def test_ltx_connector_attention_dispatches_through_omni_kernel(has_learned_registers): +def test_ltx25_connector_attention_preserves_learned_register_mask(monkeypatch): + installed = [] + + class OmniAttention: + def __init__(self, **_kwargs): + pass + + class Attention: + heads = 32 + head_dim = 128 + inner_kv_dim = 4096 + + def set_processor(self, processor): + installed.append(processor) + + monkeypatch.setattr(ltx2_components, "OmniAttention", OmniAttention) + connectors = SimpleNamespace( + video_connector=SimpleNamespace( + learnable_registers=object(), + transformer_blocks=[SimpleNamespace(attn1=Attention())], + ), + audio_connector=SimpleNamespace( + learnable_registers=object(), + transformer_blocks=[SimpleNamespace(attn1=Attention())], + ), + ) + + _install_connector_attention(connectors, preserve_learned_register_mask=True) + + assert all(processor.has_learned_registers for processor in installed) + assert all(processor.preserve_learned_register_mask for processor in installed) + + +def test_ltx_connector_attention_dispatches_through_omni_kernel(): calls = [] class Backend: @@ -732,12 +775,16 @@ def test_ltx_connector_attention_dispatches_through_omni_kernel(has_learned_regi calls.append((query, key, value, metadata)) return query + class UpcastingNorm(torch.nn.Module): + def forward(self, tensor): + return tensor.float() + attention = SimpleNamespace( heads=2, head_dim=4, inner_kv_dim=8, - norm_q=torch.nn.Identity(), - norm_k=torch.nn.Identity(), + norm_q=UpcastingNorm(), + norm_k=UpcastingNorm(), to_q=torch.nn.Identity(), to_k=torch.nn.Identity(), to_v=torch.nn.Identity(), @@ -746,24 +793,29 @@ def test_ltx_connector_attention_dispatches_through_omni_kernel(has_learned_regi rope_type="split", omni_attention=OmniAttention(), ) - hidden_states = torch.randn(2, 3, 8) + hidden_states = torch.randn(2, 3, 8, dtype=torch.bfloat16) additive_mask = torch.zeros(2, 1, 3, 3) + rotary = ( + torch.randn(2, 2, 3, 2, dtype=torch.float32), + torch.randn(2, 2, 3, 2, dtype=torch.float32), + ) - output = ltx2_components._LTXConnectorAttnProcessor(has_learned_registers=has_learned_registers)( + output = ltx2_components._LTXConnectorAttnProcessor()( attention, hidden_states, attention_mask=additive_mask, + query_rotary_emb=rotary, ) query, key, value, metadata = calls[0] assert query.shape == key.shape == value.shape == (2, 3, 2, 4) - if has_learned_registers: - assert metadata is None - else: - assert metadata.attn_mask.shape == (2, 3) - assert metadata.attn_mask.dtype == torch.bool - assert metadata.attn_mask.all() - torch.testing.assert_close(output, hidden_states) + assert query.dtype == key.dtype == value.dtype == torch.bfloat16 + assert metadata.attn_mask.shape == (2, 3) + assert metadata.attn_mask.dtype == torch.bool + assert metadata.attn_mask.all() + expected_rotary = tuple(component.to(torch.bfloat16) for component in rotary) + expected = ltx2_components.apply_split_rotary_emb(hidden_states, expected_rotary, head_dim=4) + torch.testing.assert_close(output, expected, rtol=0, atol=0) def test_ltx2_distilled_pipeline_shares_recipe_runtime(): @@ -1707,6 +1759,7 @@ class TestLTXForwardStages: assert seen["request_inputs"].output_type == "latent" assert seen["kwargs"] == { "request_sigmas": request_sigmas, + "request_phase_sigmas": None, "image": None, } @@ -1824,6 +1877,16 @@ class TestRegistryIntegration: "pipeline_ltx2_two_stage", "LTX2TwoStagePipeline", ) + assert _DIFFUSION_MODELS["LTX2DistilledOneStagePipeline"] == ( + "ltx2", + "pipeline_ltx2", + "LTX2DistilledOneStagePipeline", + ) + assert _DIFFUSION_MODELS["LTX2DistilledTwoStagePipeline"] == ( + "ltx2", + "pipeline_ltx2_two_stage", + "LTX2DistilledTwoStagePipeline", + ) assert _DIFFUSION_MODELS["LTX2DistilledPipeline"] == ( "ltx2", "pipeline_ltx2_two_stage", @@ -1848,6 +1911,8 @@ class TestRegistryIntegration: expected = [ "LTX2Pipeline", "LTX2TwoStagePipeline", + "LTX2DistilledOneStagePipeline", + "LTX2DistilledTwoStagePipeline", "LTX2DistilledPipeline", "LTX2T2VDMD2Pipeline", "LTX2I2VDMD2Pipeline", @@ -1861,6 +1926,8 @@ class TestRegistryIntegration: [ "LTX2Pipeline", "LTX2TwoStagePipeline", + "LTX2DistilledOneStagePipeline", + "LTX2DistilledTwoStagePipeline", "LTX2DistilledPipeline", "LTX2T2VDMD2Pipeline", "LTX2I2VDMD2Pipeline", diff --git a/tests/diffusion/models/ltx2/test_ltx2_transformer.py b/tests/diffusion/models/ltx2/test_ltx2_transformer.py index f74b2991d..e9bba1f1f 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_transformer.py +++ b/tests/diffusion/models/ltx2/test_ltx2_transformer.py @@ -7,18 +7,54 @@ import pytest import torch from torch import nn -from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig +from vllm_omni.diffusion.models.ltx2 import ltx2_transformer from vllm_omni.diffusion.models.ltx2.ltx2_transformer import ( - LTX2AudioVideoAttnProcessor, + LTX2FeedForward, LTX2VideoTransformer3DModel, - _LTX2ParallelAttention, + LTX2VideoTransformerBlock, _make_rms_norm, + apply_interleaved_rotary_emb, + apply_keyframes_absolute_embedding, + apply_split_rotary_emb, ) pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] +def test_ltx_interleaved_rope_matches_official_model_dtype_arithmetic(): + torch.manual_seed(0) + x = torch.randn(2, 3, 8, dtype=torch.bfloat16) + cos = torch.randn(2, 3, 8, dtype=x.dtype) + sin = torch.randn(2, 3, 8, dtype=x.dtype) + x_real, x_imag = x.unflatten(2, (4, 2)).unbind(-1) + rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(2) + expected = x * cos + rotated * sin + + actual = apply_interleaved_rotary_emb(x, (cos, sin)) + + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +def test_ltx_split_rope_matches_official_model_dtype_arithmetic(): + torch.manual_seed(1) + x = torch.randn(2, 3, 8, dtype=torch.bfloat16) + cos = torch.randn(2, 3, 4, dtype=x.dtype) + sin = torch.randn(2, 3, 4, dtype=x.dtype) + split_x = x.reshape(2, 3, 2, 4) + first_x, second_x = split_x.chunk(2, dim=-2) + cos_u = cos.unsqueeze(-2) + sin_u = sin.unsqueeze(-2) + expected = split_x * cos_u + expected[..., :1, :].addcmul_(-sin_u, second_x) + expected[..., 1:, :].addcmul_(sin_u, first_x) + expected = expected.reshape_as(x) + + actual = apply_split_rotary_emb(x, (cos, sin), head_dim=8) + + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + def test_ltx_rms_norm_no_affine_matches_official_torch_module(): norm = _make_rms_norm(8, eps=1e-6, elementwise_affine=False) @@ -36,11 +72,53 @@ def test_ltx_rms_norm_affine_weight_remains_parameter(): assert "weight" in norm.state_dict() -def test_ltx_transformer_has_separate_cfg_cache_dit_config(): +def test_ltx_attention_restores_projection_dtype_after_upcasting_qk_norms(): + calls = [] + + class UpcastingNorm(nn.Module): + def forward(self, tensor): + return tensor.float() + + class RecordingAttention: + def __call__(self, query, key, value, metadata): + calls.append((query, key, value, metadata)) + return query + + attention = SimpleNamespace( + heads=2, + head_dim=4, + to_qkv=None, + to_q=nn.Identity(), + to_k=nn.Identity(), + to_v=nn.Identity(), + norm_q=UpcastingNorm(), + norm_k=UpcastingNorm(), + to_gate_logits=None, + attn=RecordingAttention(), + to_out=(nn.Identity(), nn.Identity()), + rope_type="split", + ) + hidden_states = torch.randn(2, 3, 8, dtype=torch.bfloat16) + rotary = ( + torch.ones(2, 2, 3, 2, dtype=torch.float32), + torch.zeros(2, 2, 3, 2, dtype=torch.float32), + ) + + processor = ltx2_transformer.LTX2AudioVideoAttnProcessor() + processor._slice_rope_for_tp = lambda rope, _: rope + output = processor(attention, hidden_states, query_rotary_emb=rotary) + + query, key, value, metadata = calls[0] + assert query.dtype == key.dtype == value.dtype == torch.bfloat16 + assert metadata is None + torch.testing.assert_close(output, hidden_states) + + +def test_ltx_transformer_has_fused_cfg_cache_dit_config(): adapter_config = getattr(LTX2VideoTransformer3DModel, "_cache_dit_adapter_config") assert isinstance(adapter_config, CacheDiTAdapterConfig) - assert adapter_config.has_separate_cfg + assert not adapter_config.has_separate_cfg def test_ltx_transformer_exposes_hsdp_shard_conditions_for_blocks(): @@ -76,8 +154,9 @@ def test_ltx_sp_plan_shards_video_and_audio_timesteps_together(rope_type): def test_ltx_sp_keeps_cross_attention_key_padding_mask(monkeypatch): - monkeypatch.setattr(LTX2AudioVideoAttnProcessor, "_is_sp_enabled", staticmethod(lambda: True)) - processor = LTX2AudioVideoAttnProcessor() + processor_cls = ltx2_transformer.LTX2AudioVideoAttnProcessor + monkeypatch.setattr(processor_cls, "_is_sp_enabled", staticmethod(lambda: True)) + processor = processor_cls() hidden_states = torch.zeros(1, 3, 4) encoder_hidden_states = torch.zeros(1, 4, 4) key_padding_mask = torch.tensor([[True, True, True, False]]) @@ -113,7 +192,7 @@ def _fake_ltx_parallel_attention(): return forward - layer = object.__new__(_LTX2ParallelAttention) + layer = object.__new__(ltx2_transformer._LTX2ParallelAttention) nn.Module.__init__(layer) layer.attn_backend = Backend layer.attention = SimpleNamespace(forward=run("native")) @@ -123,33 +202,281 @@ def _fake_ltx_parallel_attention(): @pytest.mark.parametrize(("query_length", "key_length"), [(4, 4), (3, 4)]) -def test_ltx_masked_cross_attention_uses_sdpa(query_length, key_length): +def test_ltx_masked_attention_uses_sdpa(query_length, key_length): layer, calls = _fake_ltx_parallel_attention() query = torch.zeros(1, query_length, 2, 4, dtype=torch.bfloat16) key = value = torch.zeros(1, key_length, 2, 4, dtype=torch.bfloat16) key_padding_mask = torch.tensor([[True] * (key_length - 1) + [False]]) - metadata = AttentionMetadata(attn_mask=key_padding_mask[:, None, None, :]) + metadata = ltx2_transformer.AttentionMetadata(attn_mask=key_padding_mask[:, None, None, :]) layer._run_local_attention(query, key, value, metadata) assert calls == ["sdpa"] -def test_ltx_masked_self_attention_uses_sdpa(): - layer, calls = _fake_ltx_parallel_attention() - query = key = value = torch.zeros(1, 4, 2, 4, dtype=torch.bfloat16) - key_padding_mask = torch.tensor([[True, True, True, False]]) - metadata = AttentionMetadata(attn_mask=key_padding_mask[:, None, None, :]) - - layer._run_local_attention(query, key, value, metadata) - - assert calls == ["sdpa"] - - -def test_ltx_unmasked_cross_attention_keeps_native_backend(): +def test_ltx_unmasked_attention_keeps_native_backend(): layer, calls = _fake_ltx_parallel_attention() query = key = value = torch.zeros(1, 4, 2, 4, dtype=torch.bfloat16) layer._run_local_attention(query, key, value, None) assert calls == ["native"] + + +def test_ltx25_feed_forward_bias_flag_controls_both_linears(monkeypatch): + class FakeParallelLinear(nn.Linear): + def __init__(self, dim_in, dim_out, *, bias=True, **kwargs): + super().__init__(dim_in, dim_out, bias=bias) + + monkeypatch.setattr(ltx2_transformer, "ColumnParallelLinear", FakeParallelLinear) + monkeypatch.setattr(ltx2_transformer, "RowParallelLinear", FakeParallelLinear) + + without_bias = LTX2FeedForward(8, inner_dim=16, bias=False) + with_bias = LTX2FeedForward(8, inner_dim=16, bias=True) + + assert without_bias.net[0].proj.bias is None + assert without_bias.net[2].bias is None + assert with_bias.net[0].proj.bias is not None + assert with_bias.net[2].bias is not None + + +def _build_tiny_ltx_transformer(monkeypatch, **overrides): + class FakeBlock(nn.Module): + def __init__(self, **kwargs): + super().__init__() + self.init_kwargs = kwargs + + def forward(self, hidden_states, audio_hidden_states, *_args, **_kwargs): + return hidden_states, audio_hidden_states + + monkeypatch.setattr(ltx2_transformer, "LTX2VideoTransformerBlock", FakeBlock) + kwargs = { + "in_channels": 4, + "out_channels": 4, + "num_attention_heads": 2, + "attention_head_dim": 4, + "cross_attention_dim": 8, + "vae_scale_factors": (1, 1, 1), + "pos_embed_max_pos": 4, + "base_height": 8, + "base_width": 8, + "audio_in_channels": 4, + "audio_out_channels": 4, + "audio_num_attention_heads": 2, + "audio_attention_head_dim": 4, + "audio_cross_attention_dim": 8, + "audio_scale_factor": 1, + "audio_pos_embed_max_pos": 4, + "audio_sampling_rate": 16, + "audio_hop_length": 4, + "num_layers": 1, + "caption_channels": 8, + "use_prompt_embeddings": False, + "cross_attn_mod": True, + "audio_cross_attn_mod": True, + } + kwargs.update(overrides) + return LTX2VideoTransformer3DModel(**kwargs) + + +def test_ltx25_keyframe_absolute_embedding_matches_official(): + hidden_states = torch.arange(32, dtype=torch.bfloat16).view(1, 4, 8) + keyframes_mask = torch.tensor([[[1.0], [1.0], [0.0], [0.0]]]) + embedding = torch.linspace(-0.5, 0.5, 8, dtype=torch.bfloat16).view(1, 8) + + actual = apply_keyframes_absolute_embedding(hidden_states, keyframes_mask, embedding) + expected = hidden_states + (keyframes_mask > 0).to(torch.bfloat16) * embedding + + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + assert apply_keyframes_absolute_embedding(hidden_states, None, embedding) is hidden_states + + +def test_ltx25_constructor_flags_and_native_weight_loader(monkeypatch): + legacy_model = _build_tiny_ltx_transformer(monkeypatch) + + assert legacy_model.config.ff_bias is True + assert legacy_model.config.audio_ff_bias is True + assert legacy_model.config.use_prompt_adaln_single is True + assert legacy_model.config.use_keyframes_abs_pos_embedding is False + assert hasattr(legacy_model, "prompt_adaln") + assert hasattr(legacy_model, "audio_prompt_adaln") + assert not hasattr(legacy_model, "keyframes_abs_pos_embedding") + assert legacy_model.transformer_blocks[0].init_kwargs["ff_bias"] is True + assert legacy_model.transformer_blocks[0].init_kwargs["audio_ff_bias"] is True + + ltx25_model = _build_tiny_ltx_transformer( + monkeypatch, + ff_bias=False, + audio_ff_bias=True, + use_prompt_adaln_single=False, + use_keyframes_abs_pos_embedding=True, + ) + + assert ltx25_model.keyframes_abs_pos_embedding.shape == (1, 8) + torch.testing.assert_close(ltx25_model.keyframes_abs_pos_embedding, torch.zeros(1, 8)) + assert not hasattr(ltx25_model, "prompt_adaln") + assert not hasattr(ltx25_model, "audio_prompt_adaln") + assert ltx25_model.transformer_blocks[0].init_kwargs["ff_bias"] is False + assert ltx25_model.transformer_blocks[0].init_kwargs["audio_ff_bias"] is True + + monkeypatch.setattr(ltx2_transformer, "get_tensor_model_parallel_world_size", lambda: 1) + expected = torch.arange(8, dtype=torch.float32).view(1, 8) + loaded = ltx25_model.load_weights([("keyframes_abs_pos_embedding", expected)]) + + assert loaded == {"keyframes_abs_pos_embedding"} + torch.testing.assert_close(ltx25_model.keyframes_abs_pos_embedding, expected) + + +def test_ltx_cross_attention_modulations_use_official_timesteps(monkeypatch): + class CaptureAda(nn.Module): + def __init__(self): + super().__init__() + self.calls = [] + + def forward(self, timestep, *, batch_size, hidden_dtype): + self.calls.append(timestep.detach().clone()) + embedding = torch.zeros(timestep.numel(), 8, dtype=hidden_dtype) + return embedding, embedding + + class CaptureRope(nn.Module): + def __init__(self): + super().__init__() + self.output_dtypes = [] + + def forward(self, _coords, *, device, out_dtype=None): + self.output_dtypes.append(out_dtype) + empty = torch.empty(0, device=device, dtype=out_dtype) + return empty, empty + + model = _build_tiny_ltx_transformer(monkeypatch) + rope = CaptureRope() + model.rope = model.audio_rope = model.cross_attn_rope = model.cross_attn_audio_rope = rope + model.prompt_modulation = False + video_scale_shift = CaptureAda() + video_gate = CaptureAda() + audio_scale_shift = CaptureAda() + audio_gate = CaptureAda() + model.av_cross_attn_video_scale_shift = video_scale_shift + model.av_cross_attn_video_a2v_gate = video_gate + model.av_cross_attn_audio_scale_shift = audio_scale_shift + model.av_cross_attn_audio_v2a_gate = audio_gate + + model( + hidden_states=torch.zeros(1, 2, 4), + audio_hidden_states=torch.zeros(1, 1, 4), + encoder_hidden_states=torch.zeros(1, 1, 8), + audio_encoder_hidden_states=torch.zeros(1, 1, 8), + timestep=torch.tensor([[0.0, 2.0]]), + audio_timestep=torch.tensor([[5.0]]), + sigma=torch.tensor([2.0]), + audio_sigma=torch.tensor([5.0]), + video_coords=torch.zeros(1, 3, 2, 2), + audio_coords=torch.zeros(1, 1, 1, 2), + ) + + torch.testing.assert_close(video_scale_shift.calls[0], torch.tensor([0.0, 2.0])) + torch.testing.assert_close(video_gate.calls[0], torch.tensor([5.0])) + torch.testing.assert_close(audio_scale_shift.calls[0], torch.tensor([5.0])) + torch.testing.assert_close(audio_gate.calls[0], torch.tensor([2.0])) + assert len(rope.output_dtypes) == 4 + assert all(dtype is torch.float32 for dtype in rope.output_dtypes) + + +def test_ltx25_static_prompt_table_is_used_without_prompt_adaln(): + class CaptureAttention(nn.Module): + def __init__(self): + super().__init__() + self.encoder_hidden_states = None + + def forward(self, hidden_states, encoder_hidden_states=None, **kwargs): + if encoder_hidden_states is not None: + self.encoder_hidden_states = encoder_hidden_states.detach().clone() + return torch.zeros_like(hidden_states) + + class ZeroOutput(nn.Module): + def forward(self, hidden_states): + return torch.zeros_like(hidden_states) + + block = object.__new__(LTX2VideoTransformerBlock) + nn.Module.__init__(block) + block.video_cross_attn_adaln = True + block.audio_cross_attn_adaln = True + block.cross_attn_adaln = True + block.scale_shift_table = nn.Parameter(torch.zeros(9, 2)) + block.audio_scale_shift_table = nn.Parameter(torch.zeros(9, 2)) + block.prompt_scale_shift_table = nn.Parameter(torch.tensor([[1.0, 2.0], [3.0, 4.0]])) + block.audio_prompt_scale_shift_table = nn.Parameter(torch.tensor([[0.5, 1.0], [1.5, 2.0]])) + block.norm1 = nn.Identity() + block.audio_norm1 = nn.Identity() + block.norm2 = nn.Identity() + block.audio_norm2 = nn.Identity() + block.norm3 = nn.Identity() + block.audio_norm3 = nn.Identity() + block.attn1 = CaptureAttention() + block.audio_attn1 = CaptureAttention() + block.attn2 = CaptureAttention() + block.audio_attn2 = CaptureAttention() + block.ff = ZeroOutput() + block.audio_ff = ZeroOutput() + + hidden_states = torch.ones(1, 1, 2) + audio_hidden_states = torch.ones(1, 1, 2) + encoder_hidden_states = torch.ones(1, 1, 2) + audio_encoder_hidden_states = torch.ones(1, 1, 2) + block( + hidden_states, + audio_hidden_states, + encoder_hidden_states, + audio_encoder_hidden_states, + temb=torch.zeros(1, 1, 18), + temb_audio=torch.zeros(1, 1, 18), + temb_ca_scale_shift=torch.empty(0), + temb_ca_audio_scale_shift=torch.empty(0), + temb_ca_gate=torch.empty(0), + temb_ca_audio_gate=torch.empty(0), + temb_prompt=None, + temb_prompt_audio=None, + use_a2v_cross_attention=False, + use_v2a_cross_attention=False, + ) + + torch.testing.assert_close(block.attn2.encoder_hidden_states, torch.tensor([[[5.0, 7.0]]])) + torch.testing.assert_close(block.audio_attn2.encoder_hidden_states, torch.tensor([[[3.0, 4.0]]])) + + +def test_ltx_gated_attention_uses_tp_sharded_projection(monkeypatch): + column_calls = [] + + class FakeColumnParallelLinear(nn.Linear): + def __init__(self, input_size, output_size, *, bias=True, **kwargs): + column_calls.append((output_size, kwargs)) + super().__init__(input_size, output_size, bias=bias) + + class FakeRowParallelLinear(nn.Linear): + def __init__(self, input_size, output_size, *, bias=True, **_kwargs): + super().__init__(input_size, output_size, bias=bias) + + class FakeParallelAttention(nn.Module): + def __init__(self, **_kwargs): + super().__init__() + + monkeypatch.setattr(ltx2_transformer, "ColumnParallelLinear", FakeColumnParallelLinear) + monkeypatch.setattr(ltx2_transformer, "RowParallelLinear", FakeRowParallelLinear) + monkeypatch.setattr(ltx2_transformer, "_LTX2ParallelAttention", FakeParallelAttention) + monkeypatch.setattr(ltx2_transformer, "get_tensor_model_parallel_world_size", lambda: 2) + + attention = ltx2_transformer.LTX2Attention( + query_dim=8, + heads=4, + kv_heads=4, + dim_head=2, + qk_norm="rms_norm", + apply_gated_attention=True, + pack_qkv=False, + ) + + gate_output_size, gate_kwargs = column_calls[-1] + assert gate_output_size == 4 + assert gate_kwargs["gather_output"] is False + assert gate_kwargs["return_bias"] is False + assert attention.query_num_heads == 2 diff --git a/tests/e2e/accuracy/ltx/run_ltx25_reference.py b/tests/e2e/accuracy/ltx/run_ltx25_reference.py new file mode 100644 index 000000000..cbc8917a8 --- /dev/null +++ b/tests/e2e/accuracy/ltx/run_ltx25_reference.py @@ -0,0 +1,591 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Generate raw LTX video and audio outputs with the official or Omni runtime.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +import numpy as np +import torch +from PIL import Image + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--backend", choices=("official", "omni"), required=True) + parser.add_argument("--request", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--model") + parser.add_argument("--model-class-name") + parser.add_argument("--official-root", type=Path) + parser.add_argument("--checkpoint", type=Path) + parser.add_argument("--gemma-root", type=Path) + parser.add_argument("--transformer-path", type=Path) + parser.add_argument("--text-encoder-path", type=Path) + parser.add_argument("--video-vae-path", type=Path) + parser.add_argument("--audio-vae-path", type=Path) + parser.add_argument("--spatial-upsampler-path", type=Path) + parser.add_argument("--distilled-lora-path", type=Path) + parser.add_argument("--connector-model", type=Path) + parser.add_argument( + "--official-pipeline", + choices=("distilled_two_stage", "full_one_stage", "full_two_stage"), + default="distilled_two_stage", + ) + parser.add_argument("--enable-layerwise-offload", action="store_true") + return parser.parse_args() + + +def _save_outputs( + output_dir: Path, + *, + video: torch.Tensor, + audio: torch.Tensor, + audio_sample_rate: int, + fps: float, + metadata: dict[str, Any], +) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + video = video.detach().float().cpu().clamp(0.0, 1.0) + audio = audio.detach().float().cpu() + np.save(output_dir / "video.npy", video.numpy()) + np.save(output_dir / "audio.npy", audio.numpy()) + + from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes + + frames_u8 = video.mul(255.0).round().to(torch.uint8).numpy() + audio_array = audio.numpy() + while audio_array.ndim > 2 and audio_array.shape[0] == 1: + audio_array = audio_array[0] + (output_dir / "output.mp4").write_bytes( + mux_video_audio_bytes( + frames_u8, + audio_array, + fps=float(fps), + audio_sample_rate=int(audio_sample_rate), + video_codec_options={"preset": "ultrafast"}, + ) + ) + + frame_indices = sorted({0, video.shape[0] // 2, video.shape[0] - 1}) + for index in frame_indices: + frame = video[index].mul(255.0).round().to(torch.uint8).numpy() + Image.fromarray(frame).save(output_dir / f"frame_{index:04d}.png") + + metadata.update( + { + "video_shape": list(video.shape), + "audio_shape": list(audio.shape), + "audio_sample_rate": int(audio_sample_rate), + "frame_indices": frame_indices, + "mp4": "output.mp4", + } + ) + (output_dir / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") + + +def _reset_accelerator_peak_memory_stats() -> None: + if torch.accelerator.is_available(): + torch.accelerator.reset_peak_memory_stats() + + +def _accelerator_peak_memory_stats() -> dict[str, float]: + if not torch.accelerator.is_available(): + return {} + torch.accelerator.synchronize() + mib = 1024**2 + return { + "peak_memory_allocated_mb": torch.accelerator.max_memory_allocated() / mib, + "peak_memory_reserved_mb": torch.accelerator.max_memory_reserved() / mib, + } + + +def _insert_official_paths(official_root: Path) -> None: + for relative_path in ("packages/ltx-core/src", "packages/ltx-pipelines/src"): + path = str((official_root / relative_path).resolve()) + if path not in sys.path: + sys.path.insert(0, path) + + +def _configure_official_sdpa(pipeline: Any) -> None: + """Pin official connector and denoiser attention to cuDNN, with MATH fallback.""" + from ltx_core.loader.attention_ops import set_attention_module_op + from ltx_core.model.transformer.attention import PytorchAttention + from torch.nn.attention import SDPBackend + + attention = PytorchAttention(priority=[SDPBackend.CUDNN_ATTENTION, SDPBackend.MATH]) + module_op = set_attention_module_op( + attention=attention, + masked_attention=attention, + ) + transformer_owners = [ + owner + for attribute in ("stage", "stage_1", "stage_2") + if (owner := getattr(pipeline, attribute, None)) is not None + ] + owners_and_attributes = [ + *((owner, "_transformer_builder") for owner in transformer_owners), + (pipeline.prompt_encoder, "_embeddings_processor_builder"), + ] + for owner, attribute in owners_and_attributes: + builder = getattr(owner, attribute) + setattr( + owner, + attribute, + builder.with_module_ops((*builder.module_ops, module_op)), + ) + + +def _use_diffusers_connector_weights(prompt_encoder: Any, model: Path) -> None: + """Run official connector code with the checkpoint under test.""" + from safetensors import safe_open + + connector_root = model / "connectors" + shards = sorted(connector_root.glob("*.safetensors")) + if not shards: + raise ValueError(f"No connector safetensors found under {connector_root}") + + original_build_embeddings_processor = prompt_encoder._build_embeddings_processor + + def build_embeddings_processor(): + processor = original_build_embeddings_processor() + parameters = dict(processor.named_parameters()) + expected = {name for name in parameters if name.startswith(("video_connector.", "audio_connector."))} + loaded: set[str] = set() + with torch.no_grad(): + for shard in shards: + with safe_open(str(shard), framework="pt", device="cpu") as weights: + for source_name in weights.keys(): + if not source_name.startswith(("video_connector.", "audio_connector.")): + continue + target_name = ( + source_name.replace(".transformer_blocks.", ".transformer_1d_blocks.") + .replace(".attn1.norm_q.", ".attn1.q_norm.") + .replace(".attn1.norm_k.", ".attn1.k_norm.") + ) + parameter = parameters[target_name] + parameter.copy_(weights.get_tensor(source_name).to(parameter.device, parameter.dtype)) + loaded.add(target_name) + if loaded != expected: + missing = sorted(expected - loaded) + unexpected = sorted(loaded - expected) + raise ValueError(f"Connector weight mismatch: missing={missing}, unexpected={unexpected}") + return processor + + prompt_encoder._build_embeddings_processor = build_embeddings_processor + + +@torch.inference_mode() +def _run_official(args: argparse.Namespace, request: dict[str, Any]) -> None: + if args.official_root is None: + raise ValueError("Official backend requires --official-root") + _insert_official_paths(args.official_root) + # Keep Gemma on the same SDPA path in both subprocesses. DiT attention is + # still pinned independently to the explicit cuDNN backend below. + torch.backends.cuda.enable_cudnn_sdp(False) + _reset_accelerator_peak_memory_stats() + + from ltx_pipelines.utils.args import ImageConditioningInput + from ltx_pipelines.utils.model_paths import ModelPaths + from ltx_pipelines.utils.types import OffloadMode + + offload_mode = OffloadMode.CPU if args.enable_layerwise_offload else OffloadMode.NONE + image_path = request.get("image") + images = ( + [] + if image_path is None + else [ + ImageConditioningInput( + path=str(image_path), + frame_idx=0, + strength=1.0, + crf=request.get("image_crf", 18), + ) + ] + ) + + if args.transformer_path is not None: + required = { + "--text-encoder-path": args.text_encoder_path, + "--video-vae-path": args.video_vae_path, + "--audio-vae-path": args.audio_vae_path, + } + if args.official_pipeline in {"distilled_two_stage", "full_two_stage"}: + required["--spatial-upsampler-path"] = args.spatial_upsampler_path + if args.official_pipeline == "full_two_stage": + required["--distilled-lora-path"] = args.distilled_lora_path + missing = [name for name, value in required.items() if value is None] + if missing: + raise ValueError(f"Official LTX-2.5 split backend is missing: {', '.join(missing)}") + + model_paths = ModelPaths.from_split( + transformer_path=str(args.transformer_path), + text_encoder_path=str(args.text_encoder_path), + video_vae_path=str(args.video_vae_path), + audio_vae_path=str(args.audio_vae_path), + ) + if args.official_pipeline == "distilled_two_stage": + from ltx_pipelines.distilled import DistilledPipeline + + pipeline = DistilledPipeline( + model_paths=model_paths, + spatial_upsampler_path=str(args.spatial_upsampler_path), + loras=(), + offload_mode=offload_mode, + ) + if args.connector_model is not None: + _use_diffusers_connector_weights(pipeline.prompt_encoder, args.connector_model) + _configure_official_sdpa(pipeline) + video, audio, _, _ = pipeline( + prompt=request["prompt"], + seed=request["seed"], + height=request["height"], + width=request["width"], + num_frames=request["num_frames"], + frame_rate=request["fps"], + images=images, + ) + pipeline_name = "DistilledPipeline" + elif args.official_pipeline == "full_two_stage": + from ltx_core.components.guiders import MultiModalGuiderParams + from ltx_core.loader import LTXV_LORA_COMFY_RENAMING_MAP, LoraPathStrengthAndSDOps + from ltx_pipelines.ti2vid_two_stages import TI2VidTwoStagesPipeline + + pipeline = TI2VidTwoStagesPipeline( + model_paths=model_paths, + distilled_lora=[ + LoraPathStrengthAndSDOps( + path=str(args.distilled_lora_path), + strength=1.0, + sd_ops=LTXV_LORA_COMFY_RENAMING_MAP, + ) + ], + spatial_upsampler_path=str(args.spatial_upsampler_path), + loras=[], + offload_mode=offload_mode, + ) + if args.connector_model is not None: + _use_diffusers_connector_weights(pipeline.prompt_encoder, args.connector_model) + _configure_official_sdpa(pipeline) + video, audio, _, _ = pipeline( + prompt=request["prompt"], + negative_prompt=request["negative_prompt"], + seed=request["seed"], + height=request["height"], + width=request["width"], + num_frames=request["num_frames"], + frame_rate=request["fps"], + num_inference_steps=request["num_inference_steps"], + video_guider_params=MultiModalGuiderParams( + cfg_scale=request["video_cfg_scale"], + stg_scale=request["video_stg_scale"], + rescale_scale=request["video_rescale_scale"], + modality_scale=request["video_modality_scale"], + skip_step=0, + stg_blocks=request["video_stg_blocks"], + ), + audio_guider_params=MultiModalGuiderParams( + cfg_scale=request["audio_cfg_scale"], + stg_scale=request["audio_stg_scale"], + rescale_scale=request["audio_rescale_scale"], + modality_scale=request["audio_modality_scale"], + skip_step=0, + stg_blocks=request["audio_stg_blocks"], + ), + images=images, + max_batch_size=4, + stage_1_sigmas=torch.tensor(request["stage_1_sigmas"], dtype=torch.float32), + stage_2_sigmas=torch.tensor(request["stage_2_sigmas"], dtype=torch.float32), + ) + pipeline_name = "TI2VidTwoStagesPipeline" + else: + from ltx_core.components.guiders import MultiModalGuiderParams + from ltx_pipelines.ti2vid_one_stage import TI2VidOneStagePipeline + + pipeline = TI2VidOneStagePipeline( + model_paths=model_paths, + loras=(), + offload_mode=offload_mode, + ) + if args.connector_model is not None: + _use_diffusers_connector_weights(pipeline.prompt_encoder, args.connector_model) + _configure_official_sdpa(pipeline) + request_sigmas = request.get("sigmas") + video, audio, _ = pipeline( + prompt=request["prompt"], + negative_prompt=request["negative_prompt"], + seed=request["seed"], + height=request["height"], + width=request["width"], + num_frames=request["num_frames"], + frame_rate=request["fps"], + num_inference_steps=request["num_inference_steps"], + video_guider_params=MultiModalGuiderParams( + cfg_scale=request["video_cfg_scale"], + stg_scale=request["video_stg_scale"], + rescale_scale=request["video_rescale_scale"], + modality_scale=request["video_modality_scale"], + skip_step=0, + stg_blocks=request["video_stg_blocks"], + ), + audio_guider_params=MultiModalGuiderParams( + cfg_scale=request["audio_cfg_scale"], + stg_scale=request["audio_stg_scale"], + rescale_scale=request["audio_rescale_scale"], + modality_scale=request["audio_modality_scale"], + skip_step=0, + stg_blocks=request["audio_stg_blocks"], + ), + images=images, + max_batch_size=4, + sigmas=None if request_sigmas is None else torch.tensor(request_sigmas, dtype=torch.float32), + ) + pipeline_name = "TI2VidOneStagePipeline" + checkpoint_metadata = { + "transformer_path": str(args.transformer_path), + "video_vae_path": str(args.video_vae_path), + "connector_model": None if args.connector_model is None else str(args.connector_model), + "pipeline": pipeline_name, + } + else: + if args.checkpoint is None or args.gemma_root is None: + raise ValueError("Official monolith backend requires --checkpoint and --gemma-root") + + from ltx_core.components.guiders import MultiModalGuiderParams + from ltx_pipelines.ti2vid_one_stage import TI2VidOneStagePipeline + + pipeline = TI2VidOneStagePipeline( + model_paths=ModelPaths.from_monolith(str(args.checkpoint), str(args.gemma_root)), + loras=(), + offload_mode=offload_mode, + ) + _configure_official_sdpa(pipeline) + video, audio, _ = pipeline( + prompt=request["prompt"], + negative_prompt=request["negative_prompt"], + seed=request["seed"], + height=request["height"], + width=request["width"], + num_frames=request["num_frames"], + frame_rate=request["fps"], + num_inference_steps=request["num_inference_steps"], + video_guider_params=MultiModalGuiderParams( + cfg_scale=request["video_cfg_scale"], + stg_scale=request["video_stg_scale"], + rescale_scale=request["video_rescale_scale"], + modality_scale=request["video_modality_scale"], + skip_step=0, + stg_blocks=request["video_stg_blocks"], + ), + audio_guider_params=MultiModalGuiderParams( + cfg_scale=request["audio_cfg_scale"], + stg_scale=request["audio_stg_scale"], + rescale_scale=request["audio_rescale_scale"], + modality_scale=request["audio_modality_scale"], + skip_step=0, + stg_blocks=request["audio_stg_blocks"], + ), + images=images, + max_batch_size=4, + ) + checkpoint_metadata = { + "checkpoint": str(args.checkpoint), + "pipeline": "TI2VidOneStagePipeline", + } + + video_tensor = torch.cat([chunk.detach().cpu() for chunk in video], dim=0) + _save_outputs( + args.output_dir, + video=_canonical_video(video_tensor), + audio=audio.waveform, + audio_sample_rate=audio.sampling_rate, + fps=float(request["fps"]), + metadata={ + "backend": "official", + "attention_backend": "torch_sdpa_cudnn_bf16", + "official_revision": os.environ.get("VLLM_TEST_LTX_OFFICIAL_REVISION"), + "seed": request["seed"], + "sigmas": request.get("sigmas"), + "stage_1_sigmas": request.get("stage_1_sigmas"), + "stage_2_sigmas": request.get("stage_2_sigmas"), + **_accelerator_peak_memory_stats(), + **checkpoint_metadata, + }, + ) + + +def _omni_worker_peak_memory_mb(output: Any) -> float: + result = output[0] if isinstance(output, list) and output else output + return float(getattr(result, "peak_memory_mb", 0.0) or 0.0) + + +def _unwrap_omni_output(output: Any) -> tuple[Any, Any, int]: + from vllm_omni.outputs import OmniRequestOutput + + audio = None + audio_sample_rate = None + frames = output[0] if isinstance(output, list) and output else output + if isinstance(frames, OmniRequestOutput): + multimodal_output = frames.multimodal_output or {} + audio = multimodal_output.get("audio") + audio_sample_rate = multimodal_output.get("audio_sample_rate") + nested_output = getattr(frames, "request_output", None) + if frames.is_pipeline_output and isinstance(nested_output, OmniRequestOutput): + frames = nested_output + multimodal_output = frames.multimodal_output or {} + audio = multimodal_output.get("audio", audio) + audio_sample_rate = multimodal_output.get("audio_sample_rate", audio_sample_rate) + if isinstance(frames, OmniRequestOutput): + if not frames.images: + raise ValueError("No video frames found in OmniRequestOutput") + frames = frames.images + + if isinstance(frames, list) and len(frames) == 1: + frames = frames[0] + if isinstance(frames, tuple) and len(frames) == 2: + frames, audio = frames + elif isinstance(frames, dict): + audio = frames.get("audio", audio) + audio_sample_rate = frames.get("audio_sample_rate", audio_sample_rate) + frames = frames.get("frames", frames.get("video")) + + if frames is None or audio is None or audio_sample_rate is None: + raise ValueError("Omni output did not contain video, audio, and audio_sample_rate") + return frames, audio, int(audio_sample_rate) + + +def _canonical_video(video: Any) -> torch.Tensor: + if isinstance(video, list): + if len(video) == 1: + return _canonical_video(video[0]) + tensors = [torch.as_tensor(np.asarray(item) if not isinstance(item, torch.Tensor) else item) for item in video] + if all(item.ndim == 3 for item in tensors): + video = torch.stack(tensors) + elif all(item.ndim == 4 for item in tensors): + video = torch.cat(tensors) + else: + raise ValueError(f"Cannot combine video items with dimensions {[item.ndim for item in tensors]}") + tensor = torch.as_tensor(np.asarray(video) if not isinstance(video, torch.Tensor) else video).detach().cpu() + if tensor.ndim == 5: + tensor = tensor[0] + if tensor.ndim != 4: + raise ValueError(f"Expected a 4D video tensor, got {tuple(tensor.shape)}") + if tensor.shape[-1] in (3, 4): + tensor = tensor[..., :3] + elif tensor.shape[1] in (3, 4): + tensor = tensor[:, :3].permute(0, 2, 3, 1) + elif tensor.shape[0] in (3, 4): + tensor = tensor[:3].permute(1, 2, 3, 0) + else: + raise ValueError(f"Cannot infer video channel dimension from {tuple(tensor.shape)}") + tensor = tensor.float() + if tensor.numel() and tensor.max() > 1: + tensor = tensor / 255.0 + elif tensor.numel() and tensor.min() < 0: + tensor = tensor.clamp(-1.0, 1.0).add(1.0).mul(0.5) + return tensor.clamp(0.0, 1.0) + + +@torch.inference_mode() +def _run_omni(args: argparse.Namespace, request: dict[str, Any]) -> None: + if args.model is None or args.model_class_name is None: + raise ValueError("Omni backend requires --model and --model-class-name") + + # vLLM imports can otherwise change the process-wide Gemma dispatch. + torch.backends.cuda.enable_cudnn_sdp(False) + _reset_accelerator_peak_memory_stats() + + attention_config = {"default": {"backend": "CUDNN_ATTN"}} + + from vllm_omni.diffusion.data import DiffusionParallelConfig + from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args + from vllm_omni.entrypoints.omni import Omni + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + from vllm_omni.model_extras import get_extra_body_params + from vllm_omni.platforms import current_omni_platform + + generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(request["seed"]) + omni = Omni( + model=args.model, + model_class_name=args.model_class_name, + enforce_eager=True, + enable_layerwise_offload=args.enable_layerwise_offload, + diffusion_attention_config=attention_config, + parallel_config=DiffusionParallelConfig(), + ) + try: + # The explicit public class argument selects the worker pipeline. The + # parent Omni proxy can only expose the checkpoint's base class, which + # is insufficient to distinguish LTX execution modes. + model_class_name = args.model_class_name + sampling_params = OmniDiffusionSamplingParams( + height=request["height"], + width=request["width"], + generator=generator, + guidance_scale=None, + num_inference_steps=request["num_inference_steps"], + num_frames=request["num_frames"], + fps=request["fps"], + frame_rate=float(request["fps"]), + output_type="np", + ) + extra_args = { + key: value for key, value in request.items() if key.startswith("video_") or key.startswith("audio_") + } + for key in ("sigmas", "stage_1_sigmas", "stage_2_sigmas", "image_crf"): + if key in request: + extra_args[key] = request[key] + apply_declared_extra_args(sampling_params, get_extra_body_params(model_class_name), extra_args) + prompt: dict[str, Any] = {"prompt": request["prompt"]} + if "negative_prompt" in request: + prompt["negative_prompt"] = request["negative_prompt"] + image_path = request.get("image") + if image_path is not None: + with Image.open(str(image_path)) as source_image: + image = source_image.convert("RGB") + image.load() + prompt["multi_modal_data"] = {"image": image} + output = omni.generate(prompt, sampling_params) + video, audio, audio_sample_rate = _unwrap_omni_output(output) + audio_tensor = torch.as_tensor(np.asarray(audio) if not isinstance(audio, torch.Tensor) else audio) + _save_outputs( + args.output_dir, + video=_canonical_video(video), + audio=audio_tensor, + audio_sample_rate=audio_sample_rate, + fps=float(request["fps"]), + metadata={ + "backend": "omni", + "attention_backend": "torch_sdpa_cudnn_bf16", + "seed": request["seed"], + "sigmas": request.get("sigmas"), + "model": args.model, + "stage_1_sigmas": request.get("stage_1_sigmas"), + "stage_2_sigmas": request.get("stage_2_sigmas"), + "model_class_name": model_class_name, + "worker_peak_memory_mb": _omni_worker_peak_memory_mb(output), + **_accelerator_peak_memory_stats(), + }, + ) + finally: + omni.shutdown() + + +def main() -> None: + args = _parse_args() + request = json.loads(args.request.read_text()) + if args.backend == "official": + _run_official(args, request) + else: + _run_omni(args, request) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py b/tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py new file mode 100644 index 000000000..9f1623c8a --- /dev/null +++ b/tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py @@ -0,0 +1,753 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""E2E accuracy guards against pinned official Lightricks LTX references. + +The LTX-2/2.3 comparison runs both runtimes through PyTorch SDPA and uses +``max_batch_size=4`` in the official reference to match Omni's fused guidance +batch. Video and audio guidance use the official non-HQ one-stage defaults; +only the generation shape and step count are reduced for CI runtime. +LTX-2.5 runs the official split-artifact pipelines with connector weights from +the same Diffusers checkpoint under test. The distilled case covers the fixed +two-stage schedule; the Full/SFT case covers raw dev weights and an explicit +shared one-stage schedule. +""" + +from __future__ import annotations + +import json +import math +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + +import numpy as np +import pytest +import torch +from huggingface_hub import hf_hub_download, snapshot_download +from torchmetrics.image import PeakSignalNoiseRatio, StructuralSimilarityIndexMeasure + +from tests.e2e.accuracy.helpers import reset_artifact_dir +from tests.helpers import skip_if_gated_repo_inaccessible +from tests.helpers.mark import hardware_test + +OFFICIAL_REPOSITORY = "https://github.com/Lightricks/LTX-2.git" +LTX25_OFFICIAL_REVISION = "7954dcb0d986bdc36ef272564a9789ade07fcc65" +LTX25_OMNI_MODEL_ID = "Lightricks/LTX-2.5-Diffusers" +LTX25_OMNI_MODEL_REVISION = "a6de4b5354f078db24d9cf4778c14846788aea3d" +LTX25_OFFICIAL_MODEL_ID = "Lightricks/LTX-2.5" +LTX25_OFFICIAL_MODEL_REVISION = "8a4ff96f581e72bedc1b44367581c49d544a05f1" +LTX25_OFFICIAL_COMMON_FILES = { + "text_encoder": "text_encoders/gemma4-12b-with-proj-ltx-2.5-bf16.safetensors", + "video_vae": "vae/ltx-2.5-video-vae-conv-bf16.safetensors", + "audio_vae": "vae/ltx-2.5-audio-vae-bf16.safetensors", +} +LTX25_OFFICIAL_VARIANT_FILES = { + "distilled": { + "transformer": "diffusion_models/ltx-2.5-22b-distilled-transformer-bf16.safetensors", + "spatial_upsampler": "latent_upscale_models/ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors", + }, + "full": { + "transformer": "diffusion_models/ltx-2.5-22b-dev-transformer-bf16.safetensors", + }, + "full_two_stage": { + "transformer": "diffusion_models/ltx-2.5-22b-dev-transformer-bf16.safetensors", + "spatial_upsampler": "latent_upscale_models/ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors", + "distilled_lora": "loras/ltx-2.5-22b-distilled-lora-450-bf16.safetensors", + }, +} +# Version selected by the pinned official source's uv.lock. Keep it isolated +# from Omni's runtime and development dependencies. +OFFICIAL_OPENIMAGEIO_VERSION = "3.1.11.0" +LTX25_PROMPT = "A cinematic shot of a red fox walking through a snowy forest at dawn, the camera tracking alongside." +LTX25_I2V_IMAGE_REPO = "huggingface/documentation-images" +LTX25_I2V_IMAGE_FILENAME = "diffusers/svd/rocket.png" +LTX25_I2V_IMAGE_REVISION = "645d8364f0c7a101180b364811b5a11a362e4010" +NEGATIVE_PROMPT = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, " + "background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting " + "direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, " + "incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, " + "silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect " + "dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural " + "transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, " + "stylized filters, or AI artifacts." +) + +# Release parity exercises the accelerated BF16 cuDNN path in both runtimes. +# Degenerate one-token attention falls back to PyTorch SDPA MATH. +ATTENTION_BACKEND = "torch_sdpa_cudnn_bf16" +VIDEO_SSIM_MEAN_THRESHOLD = 0.95 +VIDEO_SSIM_MIN_THRESHOLD = 0.90 +VIDEO_PSNR_MEAN_THRESHOLD = 30.0 +AUDIO_RELATIVE_L2_THRESHOLD = 0.2 +AUDIO_COSINE_THRESHOLD = 0.95 + +# Decoded-output gates for the pinned official/Omni BF16 cuDNN comparison. +# The metrics artifact preserves the exact observed values. +LTX25_VIDEO_SSIM_MEAN_THRESHOLD = 0.99 +LTX25_I2V_VIDEO_SSIM_MEAN_THRESHOLD = 0.99 +LTX25_VIDEO_SSIM_MIN_THRESHOLD = 0.99 +LTX25_VIDEO_PSNR_MEAN_THRESHOLD = 40.0 +LTX25_VIDEO_LPIPS_MEAN_THRESHOLD = 0.01 +LTX25_AUDIO_RELATIVE_L2_THRESHOLD = 0.32 +LTX25_AUDIO_COSINE_THRESHOLD = 0.95 + +# Full/SFT uses independently converted dev weights, but the release contract +# still requires decoded-video parity at the same 0.99 SSIM floor as the +# distilled paths when both runtimes use BF16 cuDNN attention. +LTX25_FULL_VIDEO_SSIM_MEAN_THRESHOLD = 0.99 +LTX25_FULL_VIDEO_SSIM_MIN_THRESHOLD = 0.99 +LTX25_FULL_VIDEO_PSNR_MEAN_THRESHOLD = 40.0 +LTX25_FULL_AUDIO_RELATIVE_L2_THRESHOLD = 0.3 +LTX25_FULL_VIDEO_LPIPS_MEAN_THRESHOLD = 0.01 +LTX25_FULL_I2V_VIDEO_LPIPS_MEAN_THRESHOLD = 0.02 +LTX25_FULL_AUDIO_COSINE_THRESHOLD = AUDIO_COSINE_THRESHOLD +LTX25_STAGE_2_SIGMAS = [0.909375, 0.725, 0.421875, 0.0] + + +def test_ltx_reference_runner_unwraps_flattened_pipeline_output() -> None: + from vllm_omni.outputs import OmniRequestOutput + + from .run_ltx25_reference import _omni_worker_peak_memory_mb, _unwrap_omni_output + + frame = object() + audio = object() + output = OmniRequestOutput( + stage_id=0, + images=[frame], + _multimodal_output={"audio": audio, "audio_sample_rate": 48_000}, + ) + + frames, actual_audio, sample_rate = _unwrap_omni_output(output) + + assert frames is frame + assert actual_audio is audio + assert sample_rate == 48_000 + output.peak_memory_mb = 12_345.0 + assert _omni_worker_peak_memory_mb([output]) == 12_345.0 + + +def _run(command: list[str], *, env: dict[str, str], timeout: int = 1800) -> None: + start = time.perf_counter() + subprocess.run(command, env=env, timeout=timeout, check=True) + print(f"{' '.join(command[:3])} finished in {time.perf_counter() - start:.1f}s") + + +def _clone_pinned_source(root: Path, repository: str, revision: str) -> None: + root.mkdir(parents=True) + subprocess.run(["git", "init", "-q", str(root)], check=True) + subprocess.run(["git", "-C", str(root), "remote", "add", "origin", repository], check=True) + last_error: subprocess.CalledProcessError | None = None + for attempt in range(3): + try: + subprocess.run( + ["git", "-C", str(root), "fetch", "--depth", "1", "origin", revision], + check=True, + ) + last_error = None + break + except subprocess.CalledProcessError as error: + last_error = error + if attempt < 2: + time.sleep(5 * (attempt + 1)) + if last_error is not None: + raise last_error + subprocess.run(["git", "-C", str(root), "checkout", "-q", "--detach", "FETCH_HEAD"], check=True) + + +def _git_revision(root: Path) -> str | None: + try: + return subprocess.check_output( + ["git", "-C", str(root), "rev-parse", "HEAD"], + stderr=subprocess.DEVNULL, + text=True, + ).strip() + except subprocess.CalledProcessError: + return None + + +def _ltx25_official_source(artifact_root: Path) -> tuple[Path, str]: + repository = os.environ.get("VLLM_TEST_LTX25_OFFICIAL_REPOSITORY", OFFICIAL_REPOSITORY) + revision = os.environ.get("VLLM_TEST_LTX25_OFFICIAL_REVISION", LTX25_OFFICIAL_REVISION) + configured_root = os.environ.get("VLLM_TEST_LTX25_OFFICIAL_ROOT") + root = Path(configured_root) if configured_root else artifact_root / f"official-source-{revision[:12]}" + actual_revision = _git_revision(root) if root.exists() else None + if actual_revision != revision and configured_root: + raise AssertionError(f"Official LTX-2.5 source revision mismatch: {actual_revision} != {revision}") + if actual_revision != revision: + if root.exists(): + shutil.rmtree(root) + _clone_pinned_source(root, repository, revision) + actual_revision = _git_revision(root) + assert actual_revision == revision, f"Official LTX-2.5 source revision mismatch: {actual_revision} != {revision}" + return root, revision + + +def _official_runner_prefix() -> list[str]: + """Run the reference with its missing binary dependency isolated from CI.""" + uv = shutil.which("uv") + assert uv is not None, "uv is required to run the pinned official LTX reference" + return [ + uv, + "run", + "--no-project", + "--with", + f"openimageio=={OFFICIAL_OPENIMAGEIO_VERSION}", + "--python", + sys.executable, + "python", + ] + + +def _resolve_ltx25_omni_model(*, transformer_subfolder: str = "transformer") -> tuple[Path, str, str | None]: + if transformer_subfolder not in {"transformer", "transformer_full"}: + raise ValueError(f"Unsupported LTX-2.5 transformer subfolder: {transformer_subfolder!r}") + configured_model = os.environ.get("VLLM_TEST_LTX25_MODEL") + configured_revision = os.environ.get("VLLM_TEST_LTX25_MODEL_REVISION") + if configured_model and Path(configured_model).exists(): + model = Path(configured_model).resolve() + snapshot_revision = model.name if model.parent.name == "snapshots" else None + assert (model / transformer_subfolder).is_dir(), f"LTX-2.5 component not found: {model / transformer_subfolder}" + return model, configured_model, configured_revision or snapshot_revision + model_id = configured_model or LTX25_OMNI_MODEL_ID + revision = configured_revision + if revision is None and model_id == LTX25_OMNI_MODEL_ID: + revision = LTX25_OMNI_MODEL_REVISION + model = Path( + snapshot_download( + repo_id=model_id, + revision=revision, + allow_patterns=[ + "model_index.json", + "audio_vae/*", + "connectors/config.json", + "connectors/diffusion_pytorch_model.safetensors.index.json", + "connectors/diffusion_pytorch_model-*.safetensors", + "scheduler/*", + "text_encoder/config.json", + "text_encoder/generation_config.json", + "text_encoder/model*", + "tokenizer/*", + f"{transformer_subfolder}/*", + "vae/*", + "vocoder/*", + ], + ) + ).resolve() + snapshot_revision = model.name if model.parent.name == "snapshots" else None + return model, model_id, snapshot_revision or revision + + +def _resolve_ltx25_official_artifacts( + *, checkpoint_variant: str = "distilled" +) -> tuple[dict[str, Path], str, str | None]: + try: + variant_files = LTX25_OFFICIAL_VARIANT_FILES[checkpoint_variant] + except KeyError as exc: + raise ValueError(f"Unsupported official LTX-2.5 checkpoint variant: {checkpoint_variant!r}") from exc + files = {**LTX25_OFFICIAL_COMMON_FILES, **variant_files} + configured_model = os.environ.get("VLLM_TEST_LTX25_OFFICIAL_MODEL") + configured_revision = os.environ.get("VLLM_TEST_LTX25_OFFICIAL_MODEL_REVISION") + if configured_model and Path(configured_model).exists(): + root = Path(configured_model).resolve() + model_source = configured_model + snapshot_revision = root.name if root.parent.name == "snapshots" else None + resolved_revision = configured_revision or snapshot_revision + else: + model_source = configured_model or LTX25_OFFICIAL_MODEL_ID + revision = configured_revision + if revision is None and model_source == LTX25_OFFICIAL_MODEL_ID: + revision = LTX25_OFFICIAL_MODEL_REVISION + root = Path( + snapshot_download( + repo_id=model_source, + revision=revision, + allow_patterns=list(files.values()), + ) + ).resolve() + snapshot_revision = root.name if root.parent.name == "snapshots" else None + resolved_revision = snapshot_revision or revision + + artifacts = {name: root / relative_path for name, relative_path in files.items()} + missing = [f"{name}: {path}" for name, path in artifacts.items() if not path.is_file()] + assert not missing, f"Official LTX-2.5 artifacts are missing: {', '.join(missing)}" + return artifacts, model_source, resolved_revision + + +def _resolve_gemma_root(model: Path) -> Path: + configured_root = os.environ.get("VLLM_TEST_LTX_GEMMA_ROOT") + if configured_root: + root = Path(configured_root) + assert root.is_dir(), f"Gemma root not found: {root}" + return root + return model + + +def _resolve_ltx25_image() -> Path: + configured_image = os.environ.get("VLLM_TEST_LTX25_I2V_IMAGE") + if configured_image: + image = Path(configured_image) + assert image.is_file(), f"LTX-2.5 I2V conditioning image not found: {image}" + return image + return Path( + hf_hub_download( + repo_id=LTX25_I2V_IMAGE_REPO, + repo_type="dataset", + filename=LTX25_I2V_IMAGE_FILENAME, + revision=LTX25_I2V_IMAGE_REVISION, + ) + ) + + +def _require_ltx25_model_access() -> None: + configured_omni_model = os.environ.get("VLLM_TEST_LTX25_MODEL") + if not (configured_omni_model and Path(configured_omni_model).exists()): + omni_model = configured_omni_model or LTX25_OMNI_MODEL_ID + omni_revision = os.environ.get("VLLM_TEST_LTX25_MODEL_REVISION") + if omni_revision is None and omni_model == LTX25_OMNI_MODEL_ID: + omni_revision = LTX25_OMNI_MODEL_REVISION + skip_if_gated_repo_inaccessible( + omni_model, + revision=omni_revision, + filename="model_index.json", + ) + + configured_official_model = os.environ.get("VLLM_TEST_LTX25_OFFICIAL_MODEL") + if not (configured_official_model and Path(configured_official_model).exists()): + official_model = configured_official_model or LTX25_OFFICIAL_MODEL_ID + official_revision = os.environ.get("VLLM_TEST_LTX25_OFFICIAL_MODEL_REVISION") + if official_revision is None and official_model == LTX25_OFFICIAL_MODEL_ID: + official_revision = LTX25_OFFICIAL_MODEL_REVISION + skip_if_gated_repo_inaccessible( + official_model, + revision=official_revision, + filename=LTX25_OFFICIAL_COMMON_FILES["text_encoder"], + ) + + +def _ltx25_request(image: Path | None = None) -> dict[str, object]: + request: dict[str, object] = { + "prompt": LTX25_PROMPT, + # Keep the nightly parity test small enough to run the pinned official + # pipeline and Omni sequentially on one H100. The public recipe covers + # the release-resolution 1920x1088x121 configuration. + "width": 1024, + "height": 576, + "num_frames": 25, + "fps": 24, + "num_inference_steps": 8, + "seed": 42, + } + if image is not None: + request.update(image=str(image.resolve()), image_crf=18) + return request + + +def _ltx25_full_sigmas(num_inference_steps: int) -> list[float]: + """Materialize the official LTX-2.5 one-stage schedule in FP32.""" + sigmas = torch.linspace(1.0, 0.0, num_inference_steps + 1) + base_anchor = 1024 + max_anchor = 4096 + base_shift = 0.95 + max_shift = 2.05 + slope = (max_shift - base_shift) / (max_anchor - base_anchor) + sigma_shift = max_anchor * slope + (base_shift - slope * base_anchor) + exp_shift = math.exp(sigma_shift) + sigmas = torch.where(sigmas != 0, exp_shift / (exp_shift + (1 / sigmas - 1)), 0) + + non_zero = sigmas != 0 + one_minus_sigmas = 1.0 - sigmas[non_zero] + scale = one_minus_sigmas[-1] / (1.0 - 0.1) + sigmas[non_zero] = 1.0 - one_minus_sigmas / scale + return [float(sigma) for sigma in sigmas] + + +def _ltx25_full_request(image: Path | None = None) -> dict[str, object]: + num_inference_steps = 30 + request: dict[str, object] = { + "prompt": LTX25_PROMPT, + "negative_prompt": NEGATIVE_PROMPT, + # Full guidance expands to a four-way batch. Keep the decoded output + # small enough for the pinned official and Omni runs to share one H100. + "width": 512, + "height": 384, + "num_frames": 25, + "fps": 24, + "num_inference_steps": num_inference_steps, + "seed": 42, + "video_cfg_scale": 3.0, + "audio_cfg_scale": 7.0, + "video_stg_scale": 1.0, + "audio_stg_scale": 1.0, + "video_modality_scale": 3.0, + "audio_modality_scale": 3.0, + "video_rescale_scale": 0.7, + "audio_rescale_scale": 0.7, + "video_stg_blocks": [28], + "audio_stg_blocks": [28], + "sigmas": _ltx25_full_sigmas(num_inference_steps), + } + if image is not None: + request.update(image=str(image.resolve()), image_crf=18) + return request + + +def test_ltx25_full_request_pins_official_schedule() -> None: + request = _ltx25_full_request() + sigmas = request["sigmas"] + + assert isinstance(sigmas, list) + assert isinstance(request["num_inference_steps"], int) + assert len(sigmas) == request["num_inference_steps"] + 1 + assert math.isclose(sigmas[0], 1.0, rel_tol=0.0, abs_tol=1e-6) + assert sigmas[-1] == 0.0 + assert all(sigmas[index] > sigmas[index + 1] for index in range(len(sigmas) - 1)) + + +def test_ltx25_i2v_requests_pin_official_crf() -> None: + image = Path("conditioning.png") + for request in (_ltx25_request(image), _ltx25_full_request(image)): + assert request["image"] == str(image.resolve()) + assert request["image_crf"] == 18 + + +def _video_metrics(reference: np.ndarray, prediction: np.ndarray) -> dict[str, float]: + from benchmarks.diffusion.quantization_quality import compute_lpips_video + + assert reference.shape == prediction.shape + assert reference.ndim == 4 and reference.shape[-1] == 3 + ssim_scores: list[float] = [] + psnr_scores: list[float] = [] + for reference_frame, prediction_frame in zip(reference, prediction, strict=True): + reference_tensor = torch.from_numpy(reference_frame).permute(2, 0, 1).unsqueeze(0) + prediction_tensor = torch.from_numpy(prediction_frame).permute(2, 0, 1).unsqueeze(0) + ssim_scores.append(float(StructuralSimilarityIndexMeasure(data_range=1.0)(prediction_tensor, reference_tensor))) + psnr_scores.append(float(PeakSignalNoiseRatio(data_range=1.0)(prediction_tensor, reference_tensor))) + difference = np.abs(reference.astype(np.float64) - prediction.astype(np.float64)) + return { + "ssim_mean": float(np.mean(ssim_scores)), + "ssim_min": float(np.min(ssim_scores)), + "psnr_mean_db": float(np.mean(psnr_scores)), + "lpips_alex_mean": compute_lpips_video(reference, prediction), + "max_abs": float(difference.max()), + "mean_abs": float(difference.mean()), + } + + +def _canonical_audio(audio: np.ndarray) -> np.ndarray: + while audio.ndim > 2 and audio.shape[0] == 1: + audio = audio[0] + return audio.astype(np.float64) + + +def _audio_metrics(reference: np.ndarray, prediction: np.ndarray) -> dict[str, float | bool]: + reference = _canonical_audio(reference) + prediction = _canonical_audio(prediction) + assert reference.shape == prediction.shape + difference = reference - prediction + reference_norm = max(float(np.linalg.norm(reference)), 1e-12) + prediction_norm = max(float(np.linalg.norm(prediction)), 1e-12) + return { + "bitwise_equal": bool(np.array_equal(reference, prediction)), + "max_abs": float(np.abs(difference).max()), + "mean_abs": float(np.abs(difference).mean()), + "relative_l2": float(np.linalg.norm(difference) / reference_norm), + "cosine_similarity": float(np.vdot(reference.ravel(), prediction.ravel()) / (reference_norm * prediction_norm)), + } + + +@pytest.mark.full_model +@pytest.mark.benchmark +@pytest.mark.diffusion +@hardware_test(res={"cuda": "H100"}, num_cards=1) +@pytest.mark.parametrize("task", ["t2v", "i2v"]) +def test_ltx25_distilled_two_stage_matches_official(accuracy_artifact_root: Path, task: str) -> None: + """Compare Omni with the official runtime using the same checkpoint connector weights.""" + _require_ltx25_model_access() + artifact_parent = accuracy_artifact_root / "ltx_official" + output_root = reset_artifact_dir(artifact_parent / f"ltx2_5_distilled_{task}") + official_root, official_revision = _ltx25_official_source(artifact_parent) + official_artifacts, official_model_source, official_model_revision = _resolve_ltx25_official_artifacts() + omni_model, omni_model_source, omni_model_revision = _resolve_ltx25_omni_model() + image = _resolve_ltx25_image() if task == "i2v" else None + request_path = output_root / "request.json" + request_path.write_text(json.dumps(_ltx25_request(image), indent=2) + "\n") + + runner = Path(__file__).with_name("run_ltx25_reference.py") + runner_args = [ + str(runner), + "--request", + str(request_path), + "--enable-layerwise-offload", + ] + env = os.environ.copy() + env["VLLM_TEST_LTX_OFFICIAL_REVISION"] = official_revision + env["PYTHONUNBUFFERED"] = "1" + repository_root = Path(__file__).resolve().parents[4] + existing_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = ( + str(repository_root) if not existing_pythonpath else f"{repository_root}{os.pathsep}{existing_pythonpath}" + ) + + official_output = output_root / "official" + _run( + _official_runner_prefix() + + runner_args + + [ + "--backend", + "official", + "--output-dir", + str(official_output), + "--official-root", + str(official_root), + "--transformer-path", + str(official_artifacts["transformer"]), + "--text-encoder-path", + str(official_artifacts["text_encoder"]), + "--video-vae-path", + str(official_artifacts["video_vae"]), + "--audio-vae-path", + str(official_artifacts["audio_vae"]), + "--spatial-upsampler-path", + str(official_artifacts["spatial_upsampler"]), + "--connector-model", + str(omni_model), + ], + env=env, + ) + + omni_output = output_root / "omni" + _run( + [sys.executable] + + runner_args + + [ + "--backend", + "omni", + "--output-dir", + str(omni_output), + "--model", + str(omni_model), + "--model-class-name", + "LTX2DistilledTwoStagePipeline", + ], + env=env, + ) + + official_metadata = json.loads((official_output / "metadata.json").read_text()) + omni_metadata = json.loads((omni_output / "metadata.json").read_text()) + assert official_metadata["official_revision"] == official_revision + assert official_metadata["pipeline"] == "DistilledPipeline" + assert official_metadata["attention_backend"] == ATTENTION_BACKEND + assert official_metadata["connector_model"] == str(omni_model) + assert omni_metadata["model_class_name"] == "LTX2DistilledTwoStagePipeline" + assert omni_metadata["attention_backend"] == ATTENTION_BACKEND + assert official_metadata["audio_sample_rate"] == omni_metadata["audio_sample_rate"] + video_metrics = _video_metrics( + np.load(official_output / "video.npy"), + np.load(omni_output / "video.npy"), + ) + audio_metrics = _audio_metrics( + np.load(official_output / "audio.npy"), + np.load(omni_output / "audio.npy"), + ) + result = { + "case": "ltx2_5_distilled", + "task": task, + "attention_backend": ATTENTION_BACKEND, + "official_revision": official_revision, + "official_model": official_model_source, + "official_model_revision": official_model_revision, + "official_connector_model": omni_model_source, + "official_connector_revision": omni_model_revision, + "omni_model": omni_model_source, + "omni_model_revision": omni_model_revision, + "resolved_omni_model_path": str(omni_model), + "peak_memory_mb": { + "official_allocated": official_metadata["peak_memory_allocated_mb"], + "official_reserved": official_metadata["peak_memory_reserved_mb"], + "omni_worker_reported": omni_metadata["worker_peak_memory_mb"], + "omni_parent_allocated": omni_metadata["peak_memory_allocated_mb"], + "omni_parent_reserved": omni_metadata["peak_memory_reserved_mb"], + }, + "video": video_metrics, + "audio": audio_metrics, + } + (output_root / "metrics.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result, indent=2)) + + ssim_mean_threshold = LTX25_VIDEO_SSIM_MEAN_THRESHOLD if task == "t2v" else LTX25_I2V_VIDEO_SSIM_MEAN_THRESHOLD + assert video_metrics["ssim_mean"] >= ssim_mean_threshold + assert video_metrics["ssim_min"] >= LTX25_VIDEO_SSIM_MIN_THRESHOLD + assert video_metrics["psnr_mean_db"] >= LTX25_VIDEO_PSNR_MEAN_THRESHOLD + assert video_metrics["lpips_alex_mean"] <= LTX25_VIDEO_LPIPS_MEAN_THRESHOLD + assert audio_metrics["cosine_similarity"] >= LTX25_AUDIO_COSINE_THRESHOLD + assert audio_metrics["relative_l2"] <= LTX25_AUDIO_RELATIVE_L2_THRESHOLD + + +@pytest.mark.full_model +@pytest.mark.benchmark +@pytest.mark.diffusion +@hardware_test(res={"cuda": "H100"}, num_cards=1) +@pytest.mark.parametrize("task", ["t2v", "i2v"]) +@pytest.mark.parametrize("pipeline_mode", ["full_one_stage", "full_two_stage"]) +def test_ltx25_full_matches_official(accuracy_artifact_root: Path, task: str, pipeline_mode: str) -> None: + """Compare raw official Full/SFT weights with Omni's converted Full pipeline.""" + _require_ltx25_model_access() + artifact_parent = accuracy_artifact_root / "ltx_official" + output_root = reset_artifact_dir(artifact_parent / f"ltx2_5_{pipeline_mode}_{task}") + model_class_name = { + "full_one_stage": "LTX2Pipeline", + "full_two_stage": "LTX2TwoStagePipeline", + }[pipeline_mode] + official_root, official_revision = _ltx25_official_source(artifact_parent) + official_artifacts, official_model_source, official_model_revision = _resolve_ltx25_official_artifacts( + checkpoint_variant="full" if pipeline_mode == "full_one_stage" else "full_two_stage" + ) + omni_model, omni_model_source, omni_model_revision = _resolve_ltx25_omni_model( + transformer_subfolder="transformer_full" + ) + image = _resolve_ltx25_image() if task == "i2v" else None + request = _ltx25_full_request(image) + if pipeline_mode == "full_two_stage": + request["stage_1_sigmas"] = request.pop("sigmas") + request["stage_2_sigmas"] = LTX25_STAGE_2_SIGMAS + request_path = output_root / "request.json" + request_path.write_text(json.dumps(request, indent=2) + "\n") + + runner = Path(__file__).with_name("run_ltx25_reference.py") + runner_args = [ + str(runner), + "--request", + str(request_path), + "--enable-layerwise-offload", + ] + env = os.environ.copy() + env["VLLM_TEST_LTX_OFFICIAL_REVISION"] = official_revision + env["PYTHONUNBUFFERED"] = "1" + repository_root = Path(__file__).resolve().parents[4] + existing_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = ( + str(repository_root) if not existing_pythonpath else f"{repository_root}{os.pathsep}{existing_pythonpath}" + ) + + official_output = output_root / "official" + official_sidecar_args: list[str] = [] + if pipeline_mode == "full_two_stage": + official_sidecar_args = [ + "--spatial-upsampler-path", + str(official_artifacts["spatial_upsampler"]), + "--distilled-lora-path", + str(official_artifacts["distilled_lora"]), + ] + + _run( + _official_runner_prefix() + + runner_args + + [ + "--backend", + "official", + "--output-dir", + str(official_output), + "--official-root", + str(official_root), + "--official-pipeline", + pipeline_mode, + "--transformer-path", + str(official_artifacts["transformer"]), + "--text-encoder-path", + str(official_artifacts["text_encoder"]), + "--video-vae-path", + str(official_artifacts["video_vae"]), + "--audio-vae-path", + str(official_artifacts["audio_vae"]), + "--connector-model", + str(omni_model), + ] + + official_sidecar_args, + env=env, + ) + + omni_output = output_root / "omni" + _run( + [sys.executable] + + runner_args + + [ + "--backend", + "omni", + "--output-dir", + str(omni_output), + "--model", + str(omni_model), + "--model-class-name", + model_class_name, + ], + env=env, + ) + + official_metadata = json.loads((official_output / "metadata.json").read_text()) + omni_metadata = json.loads((omni_output / "metadata.json").read_text()) + assert official_metadata["official_revision"] == official_revision + assert ( + official_metadata["pipeline"] + == { + "full_one_stage": "TI2VidOneStagePipeline", + "full_two_stage": "TI2VidTwoStagesPipeline", + }[pipeline_mode] + ) + assert official_metadata["attention_backend"] == ATTENTION_BACKEND + assert official_metadata["connector_model"] == str(omni_model) + assert omni_metadata["model_class_name"] == model_class_name + assert omni_metadata["attention_backend"] == ATTENTION_BACKEND + assert official_metadata["seed"] == omni_metadata["seed"] + for schedule_name in ("sigmas", "stage_1_sigmas", "stage_2_sigmas"): + assert official_metadata[schedule_name] == omni_metadata[schedule_name] + assert official_metadata["audio_sample_rate"] == omni_metadata["audio_sample_rate"] + video_metrics = _video_metrics( + np.load(official_output / "video.npy"), + np.load(omni_output / "video.npy"), + ) + audio_metrics = _audio_metrics( + np.load(official_output / "audio.npy"), + np.load(omni_output / "audio.npy"), + ) + result = { + "case": f"ltx2_5_{pipeline_mode}", + "task": task, + "attention_backend": ATTENTION_BACKEND, + "official_revision": official_revision, + "official_model": official_model_source, + "official_model_revision": official_model_revision, + "official_connector_model": omni_model_source, + "official_connector_revision": omni_model_revision, + "omni_model": omni_model_source, + "omni_model_revision": omni_model_revision, + "resolved_omni_model_path": str(omni_model), + "peak_memory_mb": { + "official_allocated": official_metadata["peak_memory_allocated_mb"], + "official_reserved": official_metadata["peak_memory_reserved_mb"], + "omni_worker_reported": omni_metadata["worker_peak_memory_mb"], + "omni_parent_allocated": omni_metadata["peak_memory_allocated_mb"], + "omni_parent_reserved": omni_metadata["peak_memory_reserved_mb"], + }, + "video": video_metrics, + "audio": audio_metrics, + } + (output_root / "metrics.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result, indent=2)) + + assert video_metrics["ssim_mean"] >= LTX25_FULL_VIDEO_SSIM_MEAN_THRESHOLD + assert video_metrics["ssim_min"] >= LTX25_FULL_VIDEO_SSIM_MIN_THRESHOLD + assert video_metrics["psnr_mean_db"] >= LTX25_FULL_VIDEO_PSNR_MEAN_THRESHOLD + lpips_threshold = ( + LTX25_FULL_VIDEO_LPIPS_MEAN_THRESHOLD if task == "t2v" else LTX25_FULL_I2V_VIDEO_LPIPS_MEAN_THRESHOLD + ) + assert audio_metrics["relative_l2"] <= LTX25_FULL_AUDIO_RELATIVE_L2_THRESHOLD + assert video_metrics["lpips_alex_mean"] <= lpips_threshold + assert audio_metrics["cosine_similarity"] >= LTX25_FULL_AUDIO_COSINE_THRESHOLD diff --git a/tests/e2e/online_serving/test_ltx25.py b/tests/e2e/online_serving/test_ltx25.py new file mode 100644 index 000000000..2fad9f16a --- /dev/null +++ b/tests/e2e/online_serving/test_ltx25.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Online serving smokes for all four canonical LTX-2.5 pipelines.""" + +import os + +import pytest + +from tests.helpers import skip_if_gated_repo_inaccessible +from tests.helpers.mark import hardware_marks +from tests.helpers.media import generate_synthetic_image +from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +DEFAULT_MODEL = "Lightricks/LTX-2.5-Diffusers" +DEFAULT_REVISION = "a6de4b5354f078db24d9cf4778c14846788aea3d" +MODEL = os.getenv("VLLM_TEST_LTX25_MODEL", DEFAULT_MODEL) +MODEL_REVISION = os.getenv("VLLM_TEST_LTX25_MODEL_REVISION", DEFAULT_REVISION if MODEL == DEFAULT_MODEL else "") +PROMPT = "A red fox walks through a snowy forest while the camera remains fixed." + +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"}) + + +@pytest.fixture(scope="module", autouse=True) +def require_ltx25_model_access() -> None: + if not os.path.isdir(MODEL): + skip_if_gated_repo_inaccessible( + MODEL, + revision=MODEL_REVISION or None, + filename="model_index.json", + ) + + +def _server(model_class_name: str) -> OmniServerParams: + return OmniServerParams( + model=MODEL, + server_args=[ + *(["--revision", MODEL_REVISION] if MODEL_REVISION else []), + "--model-class-name", + model_class_name, + "--enforce-eager", + "--enable-layerwise-offload", + "--diffusion-attention-backend", + "CUDNN_ATTN", + ], + ) + + +def _cases(): + return [ + pytest.param(_server("LTX2Pipeline"), 30, id="full_one_stage", marks=SINGLE_CARD_MARKS), + pytest.param(_server("LTX2TwoStagePipeline"), 30, id="full_two_stage", marks=SINGLE_CARD_MARKS), + pytest.param(_server("LTX2DistilledOneStagePipeline"), 8, id="distilled_one_stage", marks=SINGLE_CARD_MARKS), + pytest.param(_server("LTX2DistilledTwoStagePipeline"), 8, id="distilled_two_stage", marks=SINGLE_CARD_MARKS), + ] + + +@pytest.mark.parametrize(("omni_server", "num_inference_steps"), _cases(), indirect=["omni_server"]) +def test_ltx25_pipeline_entries( + omni_server: OmniServer, + num_inference_steps: int, + openai_client: OpenAIClientHandler, + subtests, +) -> None: + """Generate T2V and first-frame I2V through the synchronous video API.""" + for task in ("t2v", "i2v"): + with subtests.test(task=task): + request_config = { + "model": omni_server.model, + "form_data": { + "model": omni_server.model, + "prompt": PROMPT, + "height": 128, + "width": 128, + "num_frames": 9, + "fps": 24, + "num_inference_steps": num_inference_steps, + "seed": 42, + }, + } + if task == "i2v": + request_config["image_reference"] = ( + f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" + ) + + openai_client.send_video_diffusion_request(request_config) diff --git a/tests/entrypoints/openai_api/test_video_server.py b/tests/entrypoints/openai_api/test_video_server.py index bcda12e2d..b7527e233 100644 --- a/tests/entrypoints/openai_api/test_video_server.py +++ b/tests/entrypoints/openai_api/test_video_server.py @@ -32,7 +32,7 @@ from vllm_omni.entrypoints.openai.protocol.videos import ( VideoParams, VideoResponse, ) -from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo +from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo, ReferenceImage from vllm_omni.entrypoints.openai.storage import LocalStorageManager from vllm_omni.entrypoints.openai.stores import AsyncDictStore, TaskRegistry from vllm_omni.errors import GuardrailViolationError @@ -461,6 +461,51 @@ def test_i2v_video_generation_resizes_input_to_requested_dimensions(test_client, assert input_image.size == (96, 64) +def test_i2v_resize_policy_can_defer_to_pipeline(monkeypatch): + engine = FakeAsyncOmni() + engine.get_diffusion_od_config = lambda: SimpleNamespace( + model="org/model", + model_class_name="ExamplePipeline", + revision="pinned-revision", + ) + captured = {} + + def fake_policy(model_class_name, *, model, revision=None): + captured.update( + model_class_name=model_class_name, + model=model, + revision=revision, + ) + return True + + monkeypatch.setattr( + "vllm_omni.entrypoints.openai.serving_video.should_preserve_reference_image_size", + fake_policy, + ) + handler = OmniOpenAIServingVideo.for_diffusion( + diffusion_engine=engine, + model_name="fallback/model", + ) + image = Image.new("RGB", (48, 32)) + + asyncio.run( + handler._run_and_extract( + VideoGenerationRequest(prompt="A bear playing with yarn.", width=96, height=64), + "pipeline-owned-resize", + reference_image=ReferenceImage(image), + ) + ) + + input_image = engine.captured_prompt["multi_modal_data"]["image"] + assert isinstance(input_image, Image.Image) + assert input_image.size == (48, 32) + assert captured == { + "model_class_name": "ExamplePipeline", + "model": "org/model", + "revision": "pinned-revision", + } + + def test_i2v_extra_params_dimensions_preserve_input_image_geometry(test_client, mocker: MockerFixture): image_bytes = _make_test_image_bytes((48, 48)) mocker.patch( diff --git a/tests/entrypoints/test_resolve_dreamzero_config.py b/tests/entrypoints/test_resolve_dreamzero_config.py index cc459c269..660203710 100644 --- a/tests/entrypoints/test_resolve_dreamzero_config.py +++ b/tests/entrypoints/test_resolve_dreamzero_config.py @@ -49,7 +49,9 @@ def test_dreamzero_config_sets_model_class_and_policy_config(monkeypatch): def test_dreamzero_enrich_config_preserves_explicit_model_class_name(monkeypatch): monkeypatch.setattr( "vllm.transformers_utils.config.get_hf_file_to_dict", - lambda path, _model: None if path == "model_index.json" else {"model_type": "vla", "architectures": ["VLA"]}, + lambda path, _model, **_kwargs: ( + None if path == "model_index.json" else {"model_type": "vla", "architectures": ["VLA"]} + ), ) od_config = OmniDiffusionConfig( diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py index 9929beab8..29cd9567e 100644 --- a/tests/helpers/__init__.py +++ b/tests/helpers/__init__.py @@ -11,11 +11,16 @@ affect in-process Omni (``OmniRunner`` / offline e2e) vs subprocess-based import pytest -def skip_if_gated_repo_inaccessible(repo_id: str) -> None: +def skip_if_gated_repo_inaccessible( + repo_id: str, + *, + revision: str | None = None, + filename: str = "config.json", +) -> None: """Skip the test if a gated HuggingFace repo is not accessible. - Tries to download the model's config.json via ``hf_hub_download``, - which performs an actual file-access check (unlike ``HfApi().model_info()`` + Tries to download ``filename`` via ``hf_hub_download``, which performs an + actual file-access check (unlike ``HfApi().model_info()`` that only checks metadata). If the token has metadata access but not file-download access, ``hf_hub_download`` will raise ``GatedRepoError`` and we skip cleanly. @@ -26,7 +31,7 @@ def skip_if_gated_repo_inaccessible(repo_id: str) -> None: except Exception: return try: - hf_hub_download(repo_id=repo_id, filename="config.json") + hf_hub_download(repo_id=repo_id, filename=filename, revision=revision) except GatedRepoError as exc: pytest.skip( f"Skipping: gated HF repo {repo_id!r} inaccessible to the current " diff --git a/tests/model_extras/test_model_extras.py b/tests/model_extras/test_model_extras.py index 557fb5f6c..573f0bd79 100644 --- a/tests/model_extras/test_model_extras.py +++ b/tests/model_extras/test_model_extras.py @@ -20,6 +20,7 @@ from vllm_omni.model_extras import ( get_output_tensor_range, get_x_to_text_model_family, should_init_extra_args_for_non_diffusion_stages, + should_preserve_reference_image_size, ) @@ -230,18 +231,62 @@ def test_ltx_extra_registry_declares_official_guidance_params() -> None: "audio_rescale_scale", "video_stg_blocks", "audio_stg_blocks", + "sigmas", + "stage_1_sigmas", + "stage_2_sigmas", + "image_crf", } ) for pipeline_name in ( "LTX2Pipeline", "LTX2TwoStagePipeline", + "LTX2DistilledOneStagePipeline", + "LTX2DistilledTwoStagePipeline", "LTX2DistilledPipeline", ): assert get_extra_body_params(pipeline_name) == expected assert get_extra_output_params(pipeline_name) == frozenset() +@pytest.mark.parametrize( + ("model_version", "expected"), + [("2", False), ("2.3", False), ("2.5", True)], +) +def test_ltx_reference_image_size_policy(tmp_path, model_version: str, expected: bool) -> None: + (tmp_path / "model_index.json").write_text( + '{"_class_name": "LTX2Pipeline", "model_version": "' + model_version + '"}' + ) + + assert ( + should_preserve_reference_image_size( + None, + model=str(tmp_path), + ) + is expected + ) + + +def test_reference_image_size_policy_threads_revision(monkeypatch: pytest.MonkeyPatch) -> None: + captured = {} + + def fake_policy(*, model, revision=None): + captured.update(model=model, revision=revision) + return True + + monkeypatch.setattr( + "vllm_omni.diffusion.models.ltx2.ltx2_components.preserves_reference_image_size", + fake_policy, + ) + + assert should_preserve_reference_image_size( + "LTX2Pipeline", + model="org/model", + revision="pinned-revision", + ) + assert captured == {"model": "org/model", "revision": "pinned-revision"} + + @pytest.mark.core_model @pytest.mark.cpu def test_ming_flash_omni_extra_registry_declares_request_and_response_params() -> None: @@ -424,6 +469,13 @@ def test_vace_extra_registry_has_no_pipeline_params() -> None: def test_unknown_pipeline_has_empty_extra_registry() -> None: assert get_extra_body_params("UnknownPipeline") == frozenset() assert get_extra_output_params("UnknownPipeline") == frozenset() + assert ( + should_preserve_reference_image_size( + "UnknownPipeline", + model="org/model", + ) + is False + ) assert should_init_extra_args_for_non_diffusion_stages("UnknownPipeline") is False diff --git a/tests/model_tests/diffusion/test_alignment.py b/tests/model_tests/diffusion/test_alignment.py index f5d6c18c4..fd13fd482 100644 --- a/tests/model_tests/diffusion/test_alignment.py +++ b/tests/model_tests/diffusion/test_alignment.py @@ -28,6 +28,8 @@ EXCLUDED_MODELS = [ "WanPipeline", "WanVACEPipeline", "LTX2TwoStagePipeline", + "LTX2DistilledOneStagePipeline", + "LTX2DistilledTwoStagePipeline", "LTX2DistilledPipeline", "LTX2T2VDMD2Pipeline", "LTX2I2VDMD2Pipeline", diff --git a/vllm_omni/diffusion/attention/backends/cudnn_attn.py b/vllm_omni/diffusion/attention/backends/cudnn_attn.py index ffcc2fb44..ab814f713 100644 --- a/vllm_omni/diffusion/attention/backends/cudnn_attn.py +++ b/vllm_omni/diffusion/attention/backends/cudnn_attn.py @@ -85,6 +85,24 @@ class CuDNNAttentionImpl(AttentionImpl): enable_gqa = query.shape[2] != key.shape[2] query, key, value = (x.permute(0, 2, 1, 3) for x in (query, key, value)) + # cuDNN has no kernel for degenerate attention shapes when either + # sequence has one token. Route those degenerate shapes before entering the + # cuDNN-only context: under torch.compile, FakeTensor evaluation raises + # before the eager RuntimeError fallback below can run. + if query.shape[-2] == 1 or key.shape[-2] == 1: + with sdpa_kernel([SDPBackend.MATH]): + output = torch.nn.functional.scaled_dot_product_attention( + query, + key, + value, + attn_mask=attention_mask, + dropout_p=0.0, + is_causal=self.causal, + scale=self.softmax_scale, + enable_gqa=enable_gqa, + ) + return output.permute(0, 2, 1, 3) + # Pin cuDNN exclusively. A priority list like [CUDNN, FLASH, MATH] hits a # PyTorch SDPA dispatch quirk: when FLASH rejects a non-None attn_mask, # cuDNN gets runtime-disabled in the same call and the dispatcher falls @@ -93,7 +111,7 @@ class CuDNNAttentionImpl(AttentionImpl): # # Fall back to the default SDPA dispatcher if cuDNN rejects the shape, # e.g. under torch.compile where Dynamo sees a symbolic head_dim and - # cuDNN's kernel selection fails (observed in LTX-2 audio attention). + # cuDNN's kernel selection fails for some symbolic attention shapes. # The unpinned dispatcher then picks EFFICIENT/MATH instead of raising. try: with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]): diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index 137fb9860..a5b6d06b5 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -538,7 +538,11 @@ class DiffusionCacheConfig: raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{item}'") -def resolve_model_class_name(model: str | None, diffusion_load_format: str = "default") -> str | None: +def resolve_model_class_name( + model: str | None, + diffusion_load_format: str = "default", + revision: str | None = None, +) -> str | None: """Resolve the diffusion pipeline class name from the model config. Read-only counterpart of ``OmniDiffusionConfig.enrich_config``, safe to call @@ -550,11 +554,10 @@ def resolve_model_class_name(model: str | None, diffusion_load_format: str = "de if not model: return None - is_lance_subfolder = os.path.basename(str(model).rstrip("/")) in {"Lance_3B", "Lance_3B_Video"} # Diffusers models: read _class_name from the pipeline index. - model_index = get_diffusion_model_index(model) + model_index = get_diffusion_model_index(model, revision=revision) if model_index is not None: return model_index.get("_class_name") if diffusion_load_format == "diffusers": @@ -562,7 +565,7 @@ def resolve_model_class_name(model: str | None, diffusion_load_format: str = "de # Other models: map model_type / architecture from config.json. try: - cfg = get_hf_file_to_dict("config.json", model) or {} + cfg = get_hf_file_to_dict("config.json", model, revision=revision) or {} except Exception: cfg = {} model_type = cfg.get("model_type") @@ -1210,9 +1213,20 @@ class OmniDiffusionConfig: exc, ) else: - tf_config_dict = get_hf_file_to_dict("transformer/config.json", self.model) + from vllm_omni.model_extras import get_transformer_config_subfolder + + transformer_subfolder = get_transformer_config_subfolder( + self.model_class_name, + model=self.model, + revision=self.revision, + ) + tf_config_dict = get_hf_file_to_dict( + f"{transformer_subfolder}/config.json", + self.model, + revision=self.revision, + ) if tf_config_dict is None: - tf_config_dict = get_hf_file_to_dict("unet/config.json", self.model) + tf_config_dict = get_hf_file_to_dict("unet/config.json", self.model, revision=self.revision) if tf_config_dict is not None: self.set_tf_model_config(TransformerConfig.from_dict(tf_config_dict)) else: @@ -1232,7 +1246,7 @@ class OmniDiffusionConfig: "that require additional inputs." ) else: - cfg = get_hf_file_to_dict("config.json", self.model) + cfg = get_hf_file_to_dict("config.json", self.model, revision=self.revision) if cfg is None: # Lance ships its top-level config.json one directory above # the per-checkpoint subfolders (``Lance_3B/`` or diff --git a/vllm_omni/diffusion/model_loader/hub_prefetch.py b/vllm_omni/diffusion/model_loader/hub_prefetch.py index 9a9295ab9..3594aa4e1 100644 --- a/vllm_omni/diffusion/model_loader/hub_prefetch.py +++ b/vllm_omni/diffusion/model_loader/hub_prefetch.py @@ -265,6 +265,7 @@ def prefetch_subfolders( subfolders: Iterable[str], *, local_files_only: bool | None = None, + revision: str | None = None, include_root_metadata: bool = True, ) -> None: """Materialise ``model``'s ``subfolders`` in the HF cache before loading. @@ -278,6 +279,7 @@ def prefetch_subfolders( local_files_only: When ``True``, skip the prefetch entirely. When ``None`` (default), auto-detect: skip if *model* is a local directory, run otherwise. + revision: Optional Hub revision shared with component loaders. include_root_metadata: When True, also pull ``*.json`` at the repo root so ``model_index.json`` / ``config.json`` resolution during ``from_pretrained`` also hits a warm cache. @@ -334,6 +336,7 @@ def prefetch_subfolders( with _repo_prefetch_lock(model): snapshot_download( repo_id=model, + revision=revision, allow_patterns=allow_patterns, ) logger.info("Prefetch complete for %s", model) @@ -450,6 +453,7 @@ def from_pretrained_with_prefetch( on the first failure exactly as before. """ prefetch_list = list(prefetch_list) + revision = from_pretrained_kwargs.get("revision") can_heal = not local_files_only and bool(model) and not os.path.isdir(model) last_exc: BaseException | None = None @@ -481,7 +485,7 @@ def from_pretrained_with_prefetch( # Force a fresh, verified snapshot of every component this pipeline # needs - not just ``subfolder`` - so a sibling component that was # also half-written gets repaired in the same pass. - prefetch_subfolders(model, prefetch_list, local_files_only=False) + prefetch_subfolders(model, prefetch_list, local_files_only=False, revision=revision) assert last_exc is not None # loop only exits via return or a caught exc raise last_exc diff --git a/vllm_omni/diffusion/models/ltx2/__init__.py b/vllm_omni/diffusion/models/ltx2/__init__.py index 20ada6adc..a493f1c0b 100644 --- a/vllm_omni/diffusion/models/ltx2/__init__.py +++ b/vllm_omni/diffusion/models/ltx2/__init__.py @@ -8,21 +8,25 @@ from vllm_omni.diffusion.models.ltx2.ltx2_components import ( ) from vllm_omni.diffusion.models.ltx2.ltx2_transformer import LTX2VideoTransformer3DModel from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import ( + LTX2DistilledOneStagePipeline, LTX2I2VDMD2Pipeline, LTX2Pipeline, LTX2T2VDMD2Pipeline, ) from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_two_stage import ( LTX2DistilledPipeline, + LTX2DistilledTwoStagePipeline, LTX2TwoStagePipeline, ) __all__ = [ "LTX2Pipeline", + "LTX2DistilledOneStagePipeline", "LTX2T2VDMD2Pipeline", "LTX2I2VDMD2Pipeline", "LTX2TwoStagePipeline", "LTX2DistilledPipeline", + "LTX2DistilledTwoStagePipeline", "get_ltx2_post_process_func", "load_transformer_config", "create_transformer_from_config", diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_components.py b/vllm_omni/diffusion/models/ltx2/ltx2_components.py index 7f2f31a60..37c3f05dc 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_components.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_components.py @@ -22,7 +22,7 @@ from diffusers.video_processor import VideoProcessor from huggingface_hub import hf_hub_download from safetensors import safe_open from safetensors.torch import load_file -from transformers import AutoTokenizer, Gemma3ForConditionalGeneration +from transformers import AutoModelForImageTextToText, AutoTokenizer, Gemma3ForConditionalGeneration from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata from vllm_omni.diffusion.attention.layer import Attention as OmniAttention @@ -48,6 +48,14 @@ try: except ImportError: LTX2VocoderWithBWE = None +try: + from transformers import Gemma4UnifiedForConditionalGeneration as _Gemma4UnifiedForConditionalGeneration +except ImportError: + _Gemma4UnifiedForConditionalGeneration = None + + +_LTX25_TEXT_ENCODER_CLS = AutoModelForImageTextToText if _Gemma4UnifiedForConditionalGeneration is not None else None + _LTX_COMPONENT_SUBFOLDERS = ( "tokenizer", @@ -73,10 +81,15 @@ class LTXComponentProfile: resident_modules: tuple[str, ...] = () video_vae_cls: type = AutoencoderKLLTX2Video vocoder_cls: type = LTX2Vocoder + text_encoder_cls: type | None = Gemma3ForConditionalGeneration vocoder_fallback_cls: type | None = None artifact_repo_id: str | None = None latent_upsampler_filename: str | None = None distilled_lora_filename: str | None = None + transformer_subfolder: str = "transformer" + scheduler_use_dynamic_shifting: bool = False + scheduler_shift_terminal: float | None = None + preserve_connector_attention_mask: bool = False LTX2_COMPONENT_PROFILE = LTXComponentProfile( @@ -99,6 +112,23 @@ LTX23_COMPONENT_PROFILE = LTXComponentProfile( vocoder_fallback_cls=LTX2Vocoder, ) +LTX25_FULL_COMPONENT_PROFILE = LTXComponentProfile( + name="ltx2_5_full", + dit_modules=("transformer",), + encoder_modules=("text_encoder", "connectors"), + vae_modules=("vae", "audio_vae"), + resident_modules=("vocoder",), + video_vae_cls=DistributedAutoencoderKLLTX2Video, + vocoder_cls=LTX2VocoderWithBWE or LTX2Vocoder, + vocoder_fallback_cls=LTX2Vocoder, + text_encoder_cls=_LTX25_TEXT_ENCODER_CLS, + transformer_subfolder="transformer_full", + scheduler_use_dynamic_shifting=True, + scheduler_shift_terminal=0.1, + preserve_connector_attention_mask=True, +) + + LTX2_DISTILLED_COMPONENT_PROFILE = LTXComponentProfile( name="ltx2_distilled", dit_modules=("transformer",), @@ -108,6 +138,32 @@ LTX2_DISTILLED_COMPONENT_PROFILE = LTXComponentProfile( video_vae_cls=DistributedAutoencoderKLLTX2Video, ) +LTX25_DISTILLED_COMPONENT_PROFILE = LTXComponentProfile( + name="ltx2_5_distilled", + dit_modules=("transformer",), + encoder_modules=("text_encoder", "connectors"), + vae_modules=("vae", "audio_vae"), + resident_modules=("vocoder", "latent_upsampler"), + video_vae_cls=DistributedAutoencoderKLLTX2Video, + vocoder_cls=LTX2VocoderWithBWE or LTX2Vocoder, + vocoder_fallback_cls=LTX2Vocoder, + text_encoder_cls=_LTX25_TEXT_ENCODER_CLS, + preserve_connector_attention_mask=True, +) + +LTX2_DISTILLED_ONE_STAGE_COMPONENT_PROFILE = replace( + LTX2_DISTILLED_COMPONENT_PROFILE, + name="ltx2_distilled_one_stage", + resident_modules=LTX2_COMPONENT_PROFILE.resident_modules, +) + +LTX25_DISTILLED_ONE_STAGE_COMPONENT_PROFILE = replace( + LTX25_DISTILLED_COMPONENT_PROFILE, + name="ltx2_5_distilled_one_stage", + resident_modules=LTX25_FULL_COMPONENT_PROFILE.resident_modules, +) + + LTX23_DISTILLED_COMPONENT_PROFILE = replace( LTX23_COMPONENT_PROFILE, name="ltx2_3_distilled", @@ -116,6 +172,12 @@ LTX23_DISTILLED_COMPONENT_PROFILE = replace( latent_upsampler_filename="ltx-2.3-spatial-upscaler-x2-1.1.safetensors", ) +LTX23_DISTILLED_ONE_STAGE_COMPONENT_PROFILE = replace( + LTX23_DISTILLED_COMPONENT_PROFILE, + name="ltx2_3_distilled_one_stage", + resident_modules=LTX23_COMPONENT_PROFILE.resident_modules, +) + LTX2_TWO_STAGE_COMPONENT_PROFILE = replace( LTX2_COMPONENT_PROFILE, name="ltx2_two_stage", @@ -135,18 +197,44 @@ LTX23_TWO_STAGE_COMPONENT_PROFILE = replace( ) +LTX25_TWO_STAGE_COMPONENT_PROFILE = replace( + LTX25_FULL_COMPONENT_PROFILE, + name="ltx2_5_two_stage", + resident_modules=(*LTX25_FULL_COMPONENT_PROFILE.resident_modules, "latent_upsampler"), + artifact_repo_id="Lightricks/LTX-2.5", + latent_upsampler_filename=("latent_upscale_models/ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors"), + distilled_lora_filename="loras/ltx-2.5-22b-distilled-lora-450-bf16.safetensors", +) + _COMPONENT_PROFILES: dict[tuple[str, str], LTXComponentProfile] = { ("one_stage", "2"): LTX2_COMPONENT_PROFILE, ("one_stage", "2.3"): LTX23_COMPONENT_PROFILE, + ("one_stage", "2.5"): LTX25_FULL_COMPONENT_PROFILE, ("two_stage", "2"): LTX2_TWO_STAGE_COMPONENT_PROFILE, ("two_stage", "2.3"): LTX23_TWO_STAGE_COMPONENT_PROFILE, + ("two_stage", "2.5"): LTX25_TWO_STAGE_COMPONENT_PROFILE, + ("distilled_one_stage", "2"): LTX2_DISTILLED_ONE_STAGE_COMPONENT_PROFILE, + ("distilled_one_stage", "2.3"): LTX23_DISTILLED_ONE_STAGE_COMPONENT_PROFILE, + ("distilled_one_stage", "2.5"): LTX25_DISTILLED_ONE_STAGE_COMPONENT_PROFILE, ("distilled_two_stage", "2"): LTX2_DISTILLED_COMPONENT_PROFILE, ("distilled_two_stage", "2.3"): LTX23_DISTILLED_COMPONENT_PROFILE, + ("distilled_two_stage", "2.5"): LTX25_DISTILLED_COMPONENT_PROFILE, ("dmd2", "2"): LTX2_COMPONENT_PROFILE, ("dmd2", "2.3"): LTX23_COMPONENT_PROFILE, } +def resolve_ltx_checkpoint_kind(pipeline_kind: str) -> LTXCheckpointKind | None: + """Derive checkpoint requirements from the execution contract.""" + if pipeline_kind in {"one_stage", "two_stage"}: + return "regular" + if pipeline_kind in {"distilled_one_stage", "distilled_two_stage"}: + return "distilled" + if pipeline_kind == "dmd2": + return None + raise ValueError(f"Unsupported LTX pipeline kind: {pipeline_kind!r}.") + + def resolve_ltx_artifact( model: str, repo_id: str, @@ -206,7 +294,7 @@ def resolve_ltx_component_profile(pipeline_kind: str, model_version: str) -> LTX raise ValueError(f"Unsupported LTX component kind/version: {pipeline_kind!r}/{model_version!r}.") from exc -def _load_ltx_metadata_json(model: str, filename: str) -> dict[str, Any]: +def _load_ltx_metadata_json(model: str, filename: str, revision: str | None = None) -> dict[str, Any]: """Load small checkpoint metadata without relying on repository names.""" if os.path.isdir(model): path = os.path.join(model, filename) @@ -214,7 +302,7 @@ def _load_ltx_metadata_json(model: str, filename: str) -> dict[str, Any]: return {} else: try: - path = hf_hub_download(repo_id=model, filename=filename) + path = hf_hub_download(repo_id=model, filename=filename, revision=revision) except Exception: return {} try: @@ -225,17 +313,34 @@ def _load_ltx_metadata_json(model: str, filename: str) -> dict[str, Any]: return value if isinstance(value, dict) else {} -def detect_ltx_model_version(model: str) -> str: - """Detect LTX-2 versus LTX-2.3 from checkpoint component metadata. +def detect_ltx_model_version(model: str, revision: str | None = None) -> str: + """Detect the LTX model version from checkpoint component metadata.""" + model_index = _load_ltx_metadata_json(model, "model_index.json", revision) + model_version = str(model_index.get("model_version", "")) + if model_version.startswith("2.5"): + return "2.5" - Official checkpoints use ``model_version`` metadata. Diffusers repositories - expose the equivalent distinction through the BWE vocoder introduced by - LTX-2.3. Unknown conversions retain the official LTX-2 fallback. - """ - model_index = _load_ltx_metadata_json(model, "model_index.json") - if str(model_index.get("model_version", "")).startswith("2.3"): + text_encoder_entry = model_index.get("text_encoder") + if isinstance(text_encoder_entry, (list, tuple)) and text_encoder_entry: + text_encoder_class = str(text_encoder_entry[-1]) + elif isinstance(text_encoder_entry, dict): + text_encoder_class = str(text_encoder_entry.get("_class_name", "")) + else: + text_encoder_class = "" + if text_encoder_class == "Gemma4UnifiedForConditionalGeneration": + return "2.5" + + text_encoder_config = _load_ltx_metadata_json(model, "text_encoder/config.json", revision) + if text_encoder_config.get("model_type") in ("gemma4_unified", "gemma4"): + return "2.5" + + if model_version.startswith("2.3"): return "2.3" + transformer_config = _load_ltx_metadata_json(model, "transformer/config.json", revision) + if transformer_config.get("ff_bias") is False: + return "2.5" + vocoder_entry = model_index.get("vocoder") if isinstance(vocoder_entry, (list, tuple)) and vocoder_entry: vocoder_class = str(vocoder_entry[-1]) @@ -246,21 +351,32 @@ def detect_ltx_model_version(model: str) -> str: if vocoder_class == "LTX2VocoderWithBWE": return "2.3" - vocoder_config = _load_ltx_metadata_json(model, "vocoder/config.json") + vocoder_config = _load_ltx_metadata_json(model, "vocoder/config.json", revision) if str(vocoder_config.get("model_version", "")).startswith("2.3"): return "2.3" if vocoder_config.get("_class_name") == "LTX2VocoderWithBWE": return "2.3" - - logger.info("Using LTX-2 defaults for checkpoint %s", model) + if "bwe" in vocoder_config or "bwe_config" in vocoder_config: + return "2.3" return "2" +def preserves_reference_image_size(*, model: str | None, revision: str | None = None) -> bool: + """Preserve source geometry only for the LTX-2.5 CRF-18 I2V path.""" + return model is not None and detect_ltx_model_version(model, revision=revision) == "2.5" + + class _LTXConnectorAttnProcessor: """Preserve official connector math around Omni attention dispatch.""" - def __init__(self, *, has_learned_registers: bool) -> None: + def __init__( + self, + *, + has_learned_registers: bool = False, + preserve_learned_register_mask: bool = False, + ) -> None: self.has_learned_registers = has_learned_registers + self.preserve_learned_register_mask = preserve_learned_register_mask def __call__( self, @@ -274,12 +390,20 @@ class _LTXConnectorAttnProcessor: encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states gate_logits = attn.to_gate_logits(hidden_states) if attn.to_gate_logits is not None else None - query = attn.norm_q(attn.to_q(hidden_states)) - key = attn.norm_k(attn.to_k(encoder_hidden_states)) + query = attn.to_q(hidden_states) + key = attn.to_k(encoder_hidden_states) value = attn.to_v(encoder_hidden_states) + # Offload hooks may execute affine-free Q/K norms in FP32. Restore the + # projection dtype before attention, matching the fully resident path. + query = attn.norm_q(query).to(dtype=value.dtype) + key = attn.norm_k(key).to(dtype=value.dtype) if query_rotary_emb is not None: + # Diffusers builds connector RoPE in FP32, while the official + # connector materializes it in the hidden-state dtype. + query_rotary_emb = tuple(component.to(value.dtype) for component in query_rotary_emb) key_rotary_emb = key_rotary_emb if key_rotary_emb is not None else query_rotary_emb + key_rotary_emb = tuple(component.to(value.dtype) for component in key_rotary_emb) if attn.rope_type == "interleaved": query = apply_interleaved_rotary_emb(query, query_rotary_emb) key = apply_interleaved_rotary_emb(key, key_rotary_emb) @@ -289,6 +413,10 @@ class _LTXConnectorAttnProcessor: else: raise ValueError(f"Unsupported LTX connector RoPE type: {attn.rope_type}") + # Keep Q/K in the projection dtype expected by the attention backend. + query = query.to(dtype=value.dtype) + key = key.to(dtype=value.dtype) + batch_size, _, inner_dim = query.shape head_dim = inner_dim // attn.heads kv_heads = attn.inner_kv_dim // attn.head_dim @@ -296,10 +424,10 @@ class _LTXConnectorAttnProcessor: key = key.view(batch_size, -1, kv_heads, head_dim) value = value.view(batch_size, -1, kv_heads, head_dim) - # The connector replaces padding tokens with learned registers before - # entering its blocks, so every key is valid and the old padding mask - # becomes an all-keep no-op. - if self.has_learned_registers: + # LTX-2/2.3 replace padding tokens with learned registers, so their + # old padding mask becomes a no-op. LTX-2.5 preserves that all-zero + # additive mask to match the official masked-SDPA dispatch. + if self.has_learned_registers and not self.preserve_learned_register_mask: attention_mask = None elif attention_mask is not None and attn.omni_attention.attn_backend.get_name().upper() == "FLASH_ATTN": attention_mask = to_ltx_padding_mask(attention_mask) @@ -316,7 +444,11 @@ class _LTXConnectorAttnProcessor: return attn.to_out[1](hidden_states) -def _install_connector_attention(connectors: LTX2TextConnectors) -> None: +def _install_connector_attention( + connectors: LTX2TextConnectors, + *, + preserve_learned_register_mask: bool = False, +) -> None: for connector_name in ("video_connector", "audio_connector"): connector = getattr(connectors, connector_name, None) has_learned_registers = getattr(connector, "learnable_registers", None) is not None @@ -335,15 +467,20 @@ def _install_connector_attention(connectors: LTX2TextConnectors) -> None: skip_sequence_parallel=True, disable_kv_quant=True, ) - attention.set_processor(_LTXConnectorAttnProcessor(has_learned_registers=has_learned_registers)) + attention.set_processor( + _LTXConnectorAttnProcessor( + has_learned_registers=has_learned_registers, + preserve_learned_register_mask=preserve_learned_register_mask, + ) + ) -def _detect_vocoder_output_sample_rate(model: str) -> int | None: +def _detect_vocoder_output_sample_rate(model: str, revision: str | None = None) -> int | None: """Read the generated waveform sample rate from the vocoder config.""" vocoder_config_path = os.path.join(model, "vocoder", "config.json") if not os.path.exists(vocoder_config_path): try: - vocoder_config_path = hf_hub_download(model, "vocoder/config.json") + vocoder_config_path = hf_hub_download(model, "vocoder/config.json", revision=revision) except Exception: return None try: @@ -355,7 +492,10 @@ def _detect_vocoder_output_sample_rate(model: str) -> int | None: def get_ltx2_post_process_func(od_config: Any): """Build the common LTX engine-output adapter.""" - output_sample_rate = _detect_vocoder_output_sample_rate(od_config.model) + output_sample_rate = _detect_vocoder_output_sample_rate( + od_config.model, + revision=getattr(od_config, "revision", None), + ) def post_process_func(output: tuple[torch.Tensor, torch.Tensor] | torch.Tensor): if not (isinstance(output, tuple) and len(output) == 2): @@ -378,6 +518,7 @@ def _load_component( *, local_files_only: bool, dtype: torch.dtype, + revision: str | None, ) -> Any: return from_pretrained_with_prefetch( component_cls.from_pretrained, @@ -385,6 +526,7 @@ def _load_component( subfolder=subfolder, prefetch_list=_LTX_COMPONENT_SUBFOLDERS, local_files_only=local_files_only, + revision=revision, torch_dtype=dtype, ) @@ -411,31 +553,36 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None: pipeline.device = get_local_device() dtype = getattr(od_config, "dtype", torch.bfloat16) model = od_config.model + revision = getattr(od_config, "revision", None) local_files_only = os.path.exists(model) pipeline.weights_sources = [ DiffusersPipelineLoader.ComponentSource( model_or_path=model, - subfolder="transformer", - revision=None, + subfolder=profile.transformer_subfolder, + revision=revision, prefix="transformer.", fall_back_to_pt=True, ), ] - prefetch_subfolders(model, _LTX_COMPONENT_SUBFOLDERS, local_files_only=local_files_only) + prefetch_subfolders(model, _LTX_COMPONENT_SUBFOLDERS, local_files_only=local_files_only, revision=revision) pipeline.tokenizer = AutoTokenizer.from_pretrained( model, subfolder="tokenizer", local_files_only=local_files_only, + revision=revision, ) + if profile.text_encoder_cls is None: + raise ImportError("LTX-2.5 requires Gemma4UnifiedForConditionalGeneration; install transformers>=5.10.1,<5.15.") with torch.device("cpu"): pipeline.text_encoder = _load_component( - Gemma3ForConditionalGeneration, + profile.text_encoder_cls, model, "text_encoder", local_files_only=local_files_only, dtype=dtype, + revision=revision, ) pipeline.connectors = _load_component( LTX2TextConnectors, @@ -443,14 +590,19 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None: "connectors", local_files_only=local_files_only, dtype=dtype, + revision=revision, + ) + _install_connector_attention( + pipeline.connectors, + preserve_learned_register_mask=profile.preserve_connector_attention_mask, ) - _install_connector_attention(pipeline.connectors) pipeline.vae = _load_component( profile.video_vae_cls, model, "vae", local_files_only=local_files_only, dtype=dtype, + revision=revision, ) pipeline.audio_vae = _load_component( AutoencoderKLLTX2Audio, @@ -458,6 +610,7 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None: "audio_vae", local_files_only=local_files_only, dtype=dtype, + revision=revision, ) try: pipeline.vocoder = _load_component( @@ -466,6 +619,7 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None: "vocoder", local_files_only=local_files_only, dtype=dtype, + revision=revision, ) except (TypeError, OSError, ValueError): if profile.vocoder_fallback_cls is None or profile.vocoder_fallback_cls is profile.vocoder_cls: @@ -476,6 +630,7 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None: "vocoder", local_files_only=local_files_only, dtype=dtype, + revision=revision, ) if "latent_upsampler" in profile.resident_modules: @@ -488,6 +643,7 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None: "latent_upsampler", local_files_only=local_files_only, dtype=dtype, + revision=revision, ) except (OSError, ValueError): if profile.latent_upsampler_filename is None or profile.artifact_repo_id is None: @@ -508,7 +664,9 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None: ) pipeline.latent_upsampler = _load_ltx_latent_upsampler_single_file(upsampler_path, dtype) - transformer_config = load_transformer_config(model, "transformer", local_files_only) + transformer_config = load_transformer_config( + model, profile.transformer_subfolder, local_files_only, revision=revision + ) quant_config = getattr(od_config, "quantization_config", None) pipeline.transformer = create_transformer_from_config(transformer_config, quant_config=quant_config) _place_aux_components(pipeline) @@ -516,15 +674,17 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None: model, subfolder="scheduler", local_files_only=local_files_only, + revision=revision, ) - expected_checkpoint_kind: LTXCheckpointKind | None = None - if pipeline.pipeline_kind == "two_stage": - expected_checkpoint_kind = "regular" - elif pipeline.pipeline_kind == "distilled_two_stage": - expected_checkpoint_kind = "distilled" + if profile.scheduler_use_dynamic_shifting: + pipeline.scheduler = FlowMatchEulerDiscreteScheduler.from_config( + pipeline.scheduler.config, + use_dynamic_shifting=True, + shift_terminal=profile.scheduler_shift_terminal, + ) validate_ltx_checkpoint( pipeline.scheduler.config, - expected_kind=expected_checkpoint_kind, + expected_kind=resolve_ltx_checkpoint_kind(pipeline.pipeline_kind), pipeline_name=type(pipeline).__name__, ) @@ -553,6 +713,8 @@ def load_transformer_config( model_path: str, subfolder: str = "transformer", local_files_only: bool = True, + *, + revision: str | None = None, ) -> dict: """Load an LTX transformer config from a local model or the HF Hub.""" if local_files_only: @@ -563,6 +725,7 @@ def load_transformer_config( config_path = hf_hub_download( repo_id=model_path, filename=f"{subfolder}/config.json", + revision=revision, ) with open(config_path) as config_file: return json.load(config_file) diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py b/vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py index f4d2f26f6..71beab643 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py @@ -7,6 +7,7 @@ from __future__ import annotations import math from dataclasses import dataclass +from io import BytesIO from typing import TYPE_CHECKING, Any import numpy as np @@ -52,19 +53,18 @@ def _preprocess_i2v_pil_images( *, height: int, width: int, + crf: int = 0, device: torch.device | None = None, dtype: torch.dtype | None = None, ) -> torch.Tensor: - """Match official LTX aspect-preserving resize and center crop.""" + """Match the version-specific official LTX PIL preprocessing.""" image_list = images if isinstance(images, list) else [images] processed = [] for image in image_list: - pixels = ( - torch.from_numpy(np.array(image.convert("RGB"), copy=True)) - .permute(2, 0, 1) - .unsqueeze(0) - .to(device=device, dtype=torch.float32) - ) + image_array = np.array(image.convert("RGB"), copy=True) + if crf: + image_array = _apply_image_conditioning_crf(image_array, crf) + pixels = torch.from_numpy(image_array).permute(2, 0, 1).unsqueeze(0).to(device=device, dtype=torch.float32) source_height, source_width = pixels.shape[-2:] scale = max(height / source_height, width / source_width) resized_height = math.ceil(source_height * scale) @@ -82,6 +82,47 @@ def _preprocess_i2v_pil_images( return torch.cat(processed, dim=0) +def _apply_image_conditioning_crf(image: np.ndarray, crf: int) -> np.ndarray: + """Round-trip one RGB frame through H.264 to match LTX-2.5 training.""" + if image.dtype != np.uint8 or image.ndim != 3 or image.shape[-1] != 3: + raise ValueError("LTX image-conditioning CRF expects a uint8 RGB image.") + if min(image.shape[:2]) < 2: + return image + try: + import av + except ImportError as exc: + raise ImportError("PyAV with a libx264 encoder is required for LTX-2.5 image conditioning (CRF 18).") from exc + + with BytesIO() as output_file: + container = av.open(output_file, "w", format="mp4") + try: + stream = container.add_stream("libx264", rate=1, options={"crf": str(crf), "preset": "veryfast"}) + height = image.shape[0] // 2 * 2 + width = image.shape[1] // 2 * 2 + stream.height = height + stream.width = width + frame = av.VideoFrame.from_ndarray(image[:height, :width], format="rgb24").reformat(format="yuv420p") + container.mux(stream.encode(frame)) + container.mux(stream.encode()) + except Exception as exc: + raise RuntimeError( + "The installed PyAV/FFmpeg build cannot encode libx264 video required " + "for LTX-2.5 image conditioning (CRF 18)." + ) from exc + finally: + container.close() + video_bytes = output_file.getvalue() + + with BytesIO(video_bytes) as video_file: + container = av.open(video_file) + try: + stream = next(stream for stream in container.streams if stream.type == "video") + frame = next(container.decode(stream)) + finally: + container.close() + return frame.to_ndarray(format="rgb24") + + class LTXTextConditioningMixin: """Shared Gemma encoding and text connector orchestration.""" @@ -106,13 +147,30 @@ class LTXTextConditioningMixin: if self.tokenizer.pad_token is None: self.tokenizer.pad_token = self.tokenizer.eos_token - text_inputs = self.tokenizer( - [text.strip() for text in prompt], + bos_token_id = self.tokenizer.bos_token_id + if bos_token_id is None: + raise ValueError("Tokenizer is missing bos_token_id; LTX prompt encoding requires a leading BOS.") + + input_ids = [] + for text in prompt: + encoded = self.tokenizer( + text.strip(), + padding=False, + truncation=True, + max_length=max_sequence_length, + return_tensors=None, + ) + ids = encoded.input_ids + if not ids or ids[0] != bos_token_id: + ids = [bos_token_id, *ids][:max_sequence_length] + input_ids.append(ids) + + text_inputs = self.tokenizer.pad( + {"input_ids": input_ids}, padding="max_length", max_length=max_sequence_length, - truncation=True, - add_special_tokens=True, return_tensors="pt", + return_attention_mask=True, ) text_input_ids = text_inputs.input_ids.to(device) prompt_attention_mask = text_inputs.attention_mask.to(device) @@ -121,7 +179,6 @@ class LTXTextConditioningMixin: attention_mask=prompt_attention_mask, output_hidden_states=True, ).hidden_states - prompt_embeds = torch.stack(hidden_states, dim=-1).flatten(2, 3).to(dtype=dtype) prompt_embeds = _repeat_prompt_tensor_for_outputs(prompt_embeds, num_videos_per_prompt) prompt_attention_mask = prompt_attention_mask.view(batch_size, -1) @@ -501,6 +558,7 @@ class LTXI2VConditioningMixin: vae_temporal_compression_ratio=self.vae_temporal_compression_ratio, ) mask_shape = (batch_size, 1, num_frames, height, width) + is_ltx25 = getattr(self, "model_version", None) == "2.5" if latents is not None: unpacked_latents = latents.ndim == 5 @@ -518,6 +576,7 @@ class LTXI2VConditioningMixin: self.vae.latents_std, self.vae.config.scaling_factor, ) + clean_latents = latents if image is not None: image_latents = self._encode_i2v_image_latents( image, @@ -525,14 +584,19 @@ class LTXI2VConditioningMixin: generator=generator, dtype=dtype, ) - # Official distilled Stage 2 starts from the upsampled Stage 1 - # state, then reapplies the source image at final resolution. - latents = self._replace_first_latent_frame(latents, image_latents) + # Official Stage 2 starts from the upsampled Stage 1 state, + # then reapplies the source image at final resolution. + clean_latents = self._replace_first_latent_frame(clean_latents, image_latents) latents = latent_ops.pack_latents( latents, self.transformer_spatial_patch_size, self.transformer_temporal_patch_size, ) + clean_latents = latent_ops.pack_latents( + clean_latents, + self.transformer_spatial_patch_size, + self.transformer_temporal_patch_size, + ) else: conditioning_mask = latents.new_zeros(mask_shape) conditioning_mask[:, :, 0] = 1.0 @@ -548,14 +612,14 @@ class LTXI2VConditioningMixin: f" {latents.shape}, but the expected shape is {conditioning_mask.shape + (num_channels_latents,)}." ) if unpacked_latents: - # Official LTX patchifies the latent state before drawing noise. - # Sampling the packed shape preserves its seed-to-output contract. - latents = latent_ops.create_noised_state( + latents = latent_ops.create_conditioned_noised_state( latents, - noise_scale * (1 - conditioning_mask).unsqueeze(-1), + clean_latents, + (1 - conditioning_mask.float()).unsqueeze(-1), + noise_scale, generator, ) - elif image is not None: + elif not unpacked_latents and image is not None: image_latents = self._encode_i2v_image_latents( image, batch_size=batch_size, @@ -595,13 +659,27 @@ class LTXI2VConditioningMixin: self.transformer_spatial_patch_size, self.transformer_temporal_patch_size, ) - conditioning_mask = latent_ops.pack_latents( + packed_conditioning_mask = latent_ops.pack_latents( conditioning_mask, self.transformer_spatial_patch_size, self.transformer_temporal_patch_size, ).squeeze(-1) - noise = randn_tensor(init_latents.shape, generator=generator, device=device, dtype=dtype) - latents = init_latents * conditioning_mask.unsqueeze(-1) + noise * (1 - conditioning_mask).unsqueeze(-1) + if is_ltx25: + clean_latents = init_latents + init_latents = torch.zeros_like(init_latents) + latents = latent_ops.create_conditioned_noised_state( + init_latents, + clean_latents, + (1 - packed_conditioning_mask.float()).unsqueeze(-1), + noise_scale, + generator=generator, + ) + else: + noise = randn_tensor(init_latents.shape, generator=generator, device=device, dtype=dtype) + latents = init_latents * packed_conditioning_mask.unsqueeze(-1) + noise * ( + 1 - packed_conditioning_mask + ).unsqueeze(-1) + conditioning_mask = packed_conditioning_mask return latents, conditioning_mask def _step_video_latents_i2v( @@ -665,6 +743,17 @@ class LTXI2VConditioningMixin: ) if image is not None: + default_crf = 18 if getattr(self, "model_version", "2") == "2.5" else 0 + image_crf = getattr(request_inputs, "image_crf", None) + image_crf = default_crf if image_crf is None else int(image_crf) + is_pil_input = isinstance(image, PIL.Image.Image) or ( + isinstance(image, list) and image and all(isinstance(item, PIL.Image.Image) for item in image) + ) + if image_crf != 0 and not is_pil_input: + raise ValueError( + "`image_crf` re-compression requires PIL images. Pass PIL images, or set `image_crf=0` " + "to skip re-compression." + ) if isinstance(image, torch.Tensor): if image.ndim == 3: image = image.unsqueeze(0) @@ -677,6 +766,7 @@ class LTXI2VConditioningMixin: image, height=request_inputs.height, width=request_inputs.width, + crf=image_crf, device=device, dtype=prompt_context.positive_connector_prompt_embeds.dtype, ) diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py b/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py index dd55f3953..7ebd94143 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py @@ -42,6 +42,7 @@ class LTXForwardContext: original_audio_num_frames: int padded_audio_num_frames: int timesteps: torch.Tensor + sampler: str audio_scheduler: Any video_audio_step_adapter: Any @@ -88,6 +89,11 @@ class LTXDenoisePipeline(Protocol): LTXDenoiseStep = Callable[[int, torch.Tensor, LTXAVState], LTXAVState] +LTX_ANCESTRAL_ETA = 1.0 +LTX_ANCESTRAL_S_NOISE = 1.0 +LTX_ANCESTRAL_NOISE_SEED_OFFSET = 10000 +LTX_OFFICIAL_DEFAULT_SEED = 10 + class LTXDenoiseExecutor: """Run the one shared LTX denoise loop. @@ -138,6 +144,9 @@ class LTXVideoAudioStepAdapter: latent_width: int, *, image_conditioned: bool, + sampler: str = "euler", + generator: torch.Generator | list[torch.Generator] | None = None, + conditioning_mask: torch.Tensor | None = None, ) -> None: self._pipeline = pipeline self._audio_scheduler = audio_scheduler @@ -145,11 +154,27 @@ class LTXVideoAudioStepAdapter: self._latent_height = latent_height self._latent_width = latent_width self._image_conditioned = image_conditioned + self._sampler = sampler + self._conditioning_mask = conditioning_mask + self._ancestral_generators = ( + _make_ancestral_generators(generator, pipeline.device) if sampler == "euler_ancestral" else None + ) self._step_index = 0 def step(self, noise_pred, t, latents, return_dict=False, generator=None): del t, return_dict, generator - if self._image_conditioned: + if self._sampler == "euler_ancestral": + video_out = _ancestral_euler_step_from_velocity( + latents[0], + noise_pred[0], + self._pipeline.scheduler.sigmas, + self._step_index, + self._ancestral_generators, + ) + if self._conditioning_mask is not None: + mask = self._conditioning_mask.unsqueeze(-1).to(video_out.dtype) + video_out = torch.lerp(video_out, latents[0], mask) + elif self._image_conditioned: video_out = self._pipeline._step_video_latents_i2v( noise_pred[0], latents[0], @@ -165,17 +190,96 @@ class LTXVideoAudioStepAdapter: self._pipeline.scheduler.sigmas, self._step_index, ) - audio_out = euler_step_from_velocity( - latents[1], - noise_pred[1], - self._audio_scheduler.sigmas, - self._step_index, - ) + if self._sampler == "euler_ancestral": + audio_out = _ancestral_euler_step_from_velocity( + latents[1], + noise_pred[1], + self._audio_scheduler.sigmas, + self._step_index, + self._ancestral_generators, + ) + else: + audio_out = euler_step_from_velocity( + latents[1], + noise_pred[1], + self._audio_scheduler.sigmas, + self._step_index, + ) self._step_index += 1 return ((video_out, audio_out),) -def _official_ltx_sigmas(scheduler: Any, steps: int, device: torch.device) -> torch.Tensor: +def _make_ancestral_generators( + generator: torch.Generator | list[torch.Generator] | None, + device: torch.device, +) -> torch.Generator | list[torch.Generator]: + """Create the official independent ``seed + 10000`` noise stream.""" + + def _offset(source: torch.Generator | None) -> torch.Generator: + seed = LTX_OFFICIAL_DEFAULT_SEED if source is None else source.initial_seed() + return torch.Generator(device=device).manual_seed(seed + LTX_ANCESTRAL_NOISE_SEED_OFFSET) + + if isinstance(generator, list): + return [_offset(item) for item in generator] + return _offset(generator) + + +def _randn_like_with_generators( + sample: torch.Tensor, + generators: torch.Generator | list[torch.Generator], +) -> torch.Tensor: + if not isinstance(generators, list): + return torch.randn(sample.shape, generator=generators, dtype=sample.dtype, device=sample.device) + if len(generators) != sample.shape[0]: + raise ValueError( + f"LTX ancestral sampling received {len(generators)} generators for batch size {sample.shape[0]}." + ) + return torch.cat( + [ + torch.randn((1, *sample.shape[1:]), generator=item, dtype=sample.dtype, device=sample.device) + for item in generators + ] + ) + + +def _ancestral_euler_step_from_velocity( + sample: torch.Tensor, + velocity: torch.Tensor, + sigmas: torch.Tensor, + step_index: int, + generators: torch.Generator | list[torch.Generator] | None, +) -> torch.Tensor: + """Apply the official LTX-2.5 rectified-flow ancestral Euler step.""" + sigma = sigmas[step_index].to(torch.float32) + sigma_next = sigmas[step_index + 1].to(torch.float32) + # Official X0Model materializes the denoised prediction in the model + # dtype before the ancestral step promotes it back to float32. + denoised_model_dtype = (sample.float() - velocity.float() * sigma).to(sample.dtype) + denoised = denoised_model_dtype.float() + if sigma_next == 0: + return denoised.to(sample.dtype) + if generators is None: + raise ValueError("LTX ancestral Euler sampling requires a noise generator.") + + downstep_ratio = 1.0 + (sigma_next / sigma - 1.0) * LTX_ANCESTRAL_ETA + sigma_down = sigma_next * downstep_ratio + sigma_down_ratio = sigma_down / sigma + x_next = sigma_down_ratio * sample.float() + (1.0 - sigma_down_ratio) * denoised + + alpha_next = 1.0 - sigma_next + alpha_down = 1.0 - sigma_down + renoise_coeff = (sigma_next**2 - sigma_down**2 * alpha_next**2 / alpha_down**2).clamp(min=0).sqrt() + noise = _randn_like_with_generators(sample, generators) + x_next = (alpha_next / alpha_down) * x_next + noise.float() * LTX_ANCESTRAL_S_NOISE * renoise_coeff + result = x_next.to(sample.dtype) + return result + + +def _official_ltx_sigmas( + scheduler: Any, + steps: int, + device: torch.device, +) -> torch.Tensor: """Build the official LTX one-stage sigma schedule in torch fp32.""" config = scheduler.config base_anchor = config.get("base_image_seq_len", 1024) @@ -185,7 +289,8 @@ def _official_ltx_sigmas(scheduler: Any, steps: int, device: torch.device) -> to sigmas = torch.linspace(1.0, 0.0, steps + 1) slope = (max_shift - base_shift) / (max_anchor - base_anchor) - # Official LTX2 one-stage intentionally uses the max sequence anchor, so the shift stays at max_shift. + # Official LTX one-stage pipelines omit the latent when constructing + # this schedule, so the shift stays at the max sequence anchor. sigma_shift = max_anchor * slope + (base_shift - slope * base_anchor) exp_shift = math.exp(sigma_shift) sigmas = torch.where(sigmas != 0, exp_shift / (exp_shift + (1 / sigmas - 1)), 0) @@ -224,6 +329,9 @@ def prepare_scheduler_stage( latent_width: int, use_official_sigma_schedule: bool, image_conditioned: bool = False, + sampler: str = "euler", + generator: torch.Generator | list[torch.Generator] | None = None, + conditioning_mask: torch.Tensor | None = None, ) -> tuple[Any, Any, torch.Tensor]: if sigmas is not None and timesteps is not None: raise ValueError("Only one of `sigmas` or `timesteps` may be provided.") @@ -236,6 +344,9 @@ def prepare_scheduler_stage( latent_height, latent_width, image_conditioned=image_conditioned, + sampler=sampler, + generator=generator, + conditioning_mask=conditioning_mask, ) if sigmas is not None: scheduler_sigmas = torch.as_tensor(sigmas, dtype=torch.float32, device=device) @@ -302,6 +413,20 @@ def prepare_rope_coords_stage( return video_coords, audio_coords +def _first_frame_keyframes_mask(reference: torch.Tensor, latent_num_frames: int) -> torch.Tensor: + """Mark the first causal latent frame, matching official LTX-2.5.""" + if latent_num_frames <= 0: + raise ValueError(f"LTX latent frame count must be positive, got {latent_num_frames}.") + tokens_per_frame, remainder = divmod(reference.shape[1], latent_num_frames) + if remainder: + raise ValueError( + f"LTX video token count {reference.shape[1]} is not divisible by {latent_num_frames} latent frames." + ) + mask = reference.new_zeros((reference.shape[0], reference.shape[1], 1)) + mask[:, :tokens_per_frame] = 1 + return mask + + def build_transformer_kwargs( pipeline: Any, forward_ctx: LTXForwardContext, @@ -317,9 +442,13 @@ def build_transformer_kwargs( attention_kwargs: dict[str, Any] | None = None, ) -> dict[str, Any]: del encoder_attention_mask, audio_encoder_attention_mask + keyframes_mask = None + if getattr(pipeline.transformer.config, "use_keyframes_abs_pos_embedding", False): + keyframes_mask = _first_frame_keyframes_mask(hidden_states, forward_ctx.latent_num_frames) return { "hidden_states": hidden_states, "audio_hidden_states": audio_hidden_states, + "keyframes_mask": keyframes_mask, "encoder_hidden_states": encoder_hidden_states, "audio_encoder_hidden_states": audio_encoder_hidden_states, **pipeline._denoise_timestep_kwargs( @@ -426,6 +555,9 @@ class LTXPhaseExecutor: latent_width=latent_width, use_official_sigma_schedule=phase_recipe.use_official_sigma_schedule, image_conditioned=conditioning_mask is not None, + sampler=phase_recipe.sampler, + generator=request_inputs.generator, + conditioning_mask=conditioning_mask, ) forward_ctx = LTXForwardContext( req=req, @@ -441,6 +573,7 @@ class LTXPhaseExecutor: original_audio_num_frames=original_audio_num_frames, padded_audio_num_frames=padded_audio_num_frames, timesteps=timesteps_tensor, + sampler=phase_recipe.sampler, audio_scheduler=audio_scheduler, video_audio_step_adapter=video_audio_step_adapter, ) diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py b/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py index bc84e7c32..9bb8ded75 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py @@ -126,9 +126,9 @@ def x0_from_velocity(sample: torch.Tensor, velocity: torch.Tensor, sigma: torch. def velocity_from_x0(sample: torch.Tensor, x0: torch.Tensor, sigma: torch.Tensor) -> torch.Tensor: """Convert x0 back to velocity using the official fp32 arithmetic order.""" - # Official LTX converts the scalar scheduler sigma to a host value before - # division. A CUDA scalar tensor selects different kernel arithmetic and - # can cross bf16 rounding boundaries late in the denoise trajectory. + # Official LTX materializes the scalar schedule value on the host before + # dividing. CUDA tensor-scalar division can cross bfloat16 rounding bounds + # even when the scalar has the same float32 value. sigma = sigma.to(torch.float32).item() return ((sample.to(torch.float32) - x0.to(torch.float32)) / sigma).to(sample.dtype) @@ -305,10 +305,17 @@ class LTXGuidanceExecutor: ts: torch.Tensor, video_token_count: int, audio_token_count: int, + *, + expand_for_sequence_parallel: bool = False, ) -> dict[str, torch.Tensor]: + timestep = ts + audio_timestep = ts + if expand_for_sequence_parallel: + timestep = ts.reshape(-1, 1).expand(-1, video_token_count) + audio_timestep = ts.reshape(-1, 1).expand(-1, audio_token_count) return { - "timestep": ts.reshape(-1, 1).expand(-1, video_token_count), - "audio_timestep": ts.reshape(-1, 1).expand(-1, audio_token_count), + "timestep": timestep, + "audio_timestep": audio_timestep, "sigma": ts, "audio_sigma": ts, } @@ -342,8 +349,11 @@ class LTXGuidanceExecutor: guidance: LTXModalityGuidance, *, model_sigma: torch.Tensor | None = None, + preserve_positive_velocity: bool = False, rescale_token_count: int | None = None, ) -> torch.Tensor: + if preserve_positive_velocity and tuple(splits) == ("cond",): + return splits["cond"] model_sigma = sigma if model_sigma is None else model_sigma # Official guidance reduces contiguous BSC tensors. The fused # transformer output may be channel-major after splitting, which @@ -368,6 +378,7 @@ class LTXGuidanceExecutor: state: LTXAVState, forward_ctx: LTXForwardContext, denoise_ctx: LTXDenoiseContext, + preserve_positive_velocity: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: guidance_world_size = get_guidance_parallel_world_size() self.validate_guidance_world_size(plan, guidance_world_size) @@ -450,12 +461,14 @@ class LTXGuidanceExecutor: video_sigma, plan.spec.video, model_sigma=pipeline._video_guidance_model_sigma(video_sigma, denoise_ctx), + preserve_positive_velocity=preserve_positive_velocity, ), self._guide_modality( state.audio, audio_splits, forward_ctx.audio_scheduler.sigmas[index], plan.spec.audio, + preserve_positive_velocity=preserve_positive_velocity, rescale_token_count=forward_ctx.original_audio_num_frames, ), ) @@ -469,9 +482,19 @@ class LTXGuidanceExecutor: state: LTXAVState, forward_ctx: LTXForwardContext, denoise_ctx: LTXDenoiseContext, + preserve_positive_velocity: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: if forward_ctx.guidance_parallel_ready: - return self.predict_parallel_guidance(pipeline, plan, index, timestep, state, forward_ctx, denoise_ctx) + return self.predict_parallel_guidance( + pipeline, + plan, + index, + timestep, + state, + forward_ctx, + denoise_ctx, + preserve_positive_velocity=preserve_positive_velocity, + ) prompt = forward_ctx.prompt_context video_contexts: list[torch.Tensor] = [] @@ -526,12 +549,14 @@ class LTXGuidanceExecutor: pipeline.scheduler.sigmas[index], denoise_ctx, ), + preserve_positive_velocity=preserve_positive_velocity, ), self._guide_modality( state.audio, audio_splits, forward_ctx.audio_scheduler.sigmas[index], plan.spec.audio, + preserve_positive_velocity=preserve_positive_velocity, rescale_token_count=forward_ctx.original_audio_num_frames, ), ) diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_latents.py b/vllm_omni/diffusion/models/ltx2/ltx2_latents.py index 11929d331..1b25a8e01 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_latents.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_latents.py @@ -20,6 +20,11 @@ class LTXAVState: audio: torch.Tensor +def official_video_token_layout(latents: torch.Tensor) -> torch.Tensor: + """Match the token-major view produced by the official LTX patchifier.""" + return latents.transpose(1, 2).contiguous().transpose(1, 2) + + def pack_latents( latents: torch.Tensor, patch_size: int = 1, @@ -117,9 +122,23 @@ def create_noised_state( device=latents.device, dtype=latents.dtype, ) - if isinstance(noise_scale, torch.Tensor): - noise_scale = noise_scale.float() - return torch.lerp(latents.float(), noise.float(), noise_scale).to(latents.dtype) + # Official GaussianNoisier blends in fp32, then materializes the next + # latent state in the model dtype. Batched request weights may arrive in + # the model dtype, so promote tensor weights alongside both endpoints. + lerp_weight = noise_scale.float() if isinstance(noise_scale, torch.Tensor) else noise_scale + return torch.lerp(latents.float(), noise.float(), lerp_weight).to(latents.dtype) + + +def create_conditioned_noised_state( + latents: torch.Tensor, + clean_latents: torch.Tensor, + denoise_mask: torch.Tensor, + noise_scale: float | torch.Tensor, + generator: torch.Generator | list[torch.Generator] | None = None, +) -> torch.Tensor: + """Match the official conditioned Gaussian noisier operation order.""" + noised_latents = create_noised_state(latents, noise_scale, generator) + return torch.lerp(clean_latents.float(), noised_latents.float(), denoise_mask.float()).to(latents.dtype) def pack_audio_latents(latents: torch.Tensor) -> torch.Tensor: @@ -138,7 +157,7 @@ def unpad_audio_latents(latents: torch.Tensor, num_frames: int) -> torch.Tensor: def clear_audio_padding(latents: torch.Tensor, num_frames: int) -> torch.Tensor: - """Keep Omni's SP-only audio padding outside the sampler state.""" + """Keep SP-only audio padding outside the logical sampler state.""" if not 0 < num_frames <= latents.shape[1]: raise ValueError(f"Audio frame count must be in [1, {latents.shape[1]}], got {num_frames}.") if num_frames == latents.shape[1]: @@ -196,7 +215,8 @@ def prepare_video_latents( ) if latents.ndim != 3: raise ValueError(f"Provided `latents` has shape {latents.shape}, expected [batch, seq, features].") - return create_noised_state(latents, noise_scale, generator).to(device=device, dtype=dtype) + latents = create_noised_state(latents, noise_scale, generator).to(device=device, dtype=dtype) + return official_video_token_layout(latents) num_frames, height, width = resolve_video_latent_shape( height, @@ -218,11 +238,7 @@ def prepare_video_latents( f" size of {batch_size}. Make sure the batch size matches the length of the generators." ) latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) - # Official LTX samples the 5D video latent first and then patchifies it, - # leaving token-major storage behind the [B, tokens, channels] view. The - # logical values are unchanged, but preserving that layout is important: - # CUDA selects a different bf16 GEMM path for a contiguous token tensor. - return latents.transpose(1, 2).contiguous().transpose(1, 2) + return official_video_token_layout(latents) def prepare_audio_latents( diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_recipes.py b/vllm_omni/diffusion/models/ltx2/ltx2_recipes.py index bb42bc0e9..0caf93f64 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_recipes.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_recipes.py @@ -3,7 +3,7 @@ """Declarative execution recipes for the LTX model family.""" -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Literal from .ltx2_guidance import LTXGuidanceSpec, LTXModalityGuidance @@ -22,6 +22,11 @@ LTX_DEFAULT_NEGATIVE_PROMPT = ( "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." ) +LTX25_DEFAULT_NEGATIVE_PROMPT = ( + "has_subtitles, has_blurbox, transition from black, transition to black, speech_ending_short, " + + LTX_DEFAULT_NEGATIVE_PROMPT +) + LTX_DISTILLED_SIGMAS = (1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875, 0.0) LTX_STAGE_2_DISTILLED_SIGMAS = (0.909375, 0.725, 0.421875, 0.0) LTX_DISTILLED_ADAPTER_SLOT = "ltx_distilled" @@ -38,6 +43,7 @@ class LTXPhaseRecipe: noise_scale: float = 0.0 input_transform: Literal["initial", "spatial_upsample"] = "initial" adapter_slot: str | None = None + sampler: Literal["euler", "euler_ancestral"] = "euler" allow_guidance_override: bool = True use_official_sigma_schedule: bool = True @@ -46,6 +52,8 @@ class LTXPhaseRecipe: raise ValueError("LTX phase spatial_downscale must be positive.") if self.sigmas is not None and len(self.sigmas) < 2: raise ValueError("An explicit LTX sigma schedule must contain at least two denoise sigmas.") + if self.sampler not in ("euler", "euler_ancestral"): + raise ValueError(f"Unsupported LTX sampler: {self.sampler!r}.") @property def num_inference_steps(self) -> int | None: @@ -66,9 +74,11 @@ class LTXPipelineRecipe: video_output_phase: int = -1 audio_output_phase: int = -1 allow_request_sigmas: bool = True + allow_request_phase_sigmas: bool = False allow_request_latents: bool = True allow_negative_prompt: bool = True fixed_num_inference_steps: bool = False + supports_cache_dit: bool = False def __post_init__(self) -> None: if not self.phases: @@ -109,13 +119,29 @@ def _official_guidance(stg_block: int) -> LTXGuidanceSpec: LTX2_ONE_STAGE_RECIPE = LTXPipelineRecipe( + supports_cache_dit=True, phases=(LTXPhaseRecipe(name="generate", guidance=_official_guidance(29)),), ) LTX23_ONE_STAGE_RECIPE = LTXPipelineRecipe( + supports_cache_dit=True, num_inference_steps=30, phases=(LTXPhaseRecipe(name="generate", guidance=_official_guidance(28)),), ) +LTX25_FULL_RECIPE = LTXPipelineRecipe( + height=544, + width=960, + num_inference_steps=30, + negative_prompt=LTX25_DEFAULT_NEGATIVE_PROMPT, + phases=( + LTXPhaseRecipe( + name="generate", + guidance=_official_guidance(28), + noise_scale=1.0, + ), + ), +) LTX_POSITIVE_ONLY_RECIPE = LTXPipelineRecipe( + supports_cache_dit=True, phases=( LTXPhaseRecipe( name="generate", @@ -159,6 +185,41 @@ LTX2_DISTILLED_TWO_STAGE_RECIPE = LTXPipelineRecipe( fixed_num_inference_steps=True, ) +LTX25_DISTILLED_TWO_STAGE_RECIPE = LTXPipelineRecipe( + height=1088, + width=1920, + num_inference_steps=len(LTX_DISTILLED_SIGMAS) - 1, + negative_prompt="", + phases=( + LTXPhaseRecipe( + name="generate_lowres", + guidance=LTXGuidanceSpec.positive_only(), + spatial_downscale=2, + sigmas=LTX_DISTILLED_SIGMAS, + noise_scale=1.0, + allow_guidance_override=False, + use_official_sigma_schedule=False, + sampler="euler_ancestral", + ), + LTXPhaseRecipe( + name="refine", + guidance=LTXGuidanceSpec.positive_only(), + sigmas=LTX_STAGE_2_DISTILLED_SIGMAS, + noise_scale=LTX_STAGE_2_DISTILLED_SIGMAS[0], + input_transform="spatial_upsample", + allow_guidance_override=False, + use_official_sigma_schedule=False, + ), + ), + video_output_phase=1, + audio_output_phase=1, + allow_request_sigmas=False, + allow_request_phase_sigmas=True, + allow_request_latents=False, + allow_negative_prompt=False, + fixed_num_inference_steps=True, +) + # LTX-2.3 full-distilled uses the same official fixed 8 + 3 sigma schedules # and request contract. Its distinct component profile selects the 22B merged # distilled Transformer, BWE vocoder, and matching spatial upsampler; no @@ -166,6 +227,37 @@ LTX2_DISTILLED_TWO_STAGE_RECIPE = LTXPipelineRecipe( LTX23_DISTILLED_TWO_STAGE_RECIPE = LTX2_DISTILLED_TWO_STAGE_RECIPE +def _distilled_one_stage_recipe( + two_stage_recipe: LTXPipelineRecipe, + *, + supports_cache_dit: bool = True, +) -> LTXPipelineRecipe: + """Run the merged-distilled checkpoint at its native Stage-1 resolution.""" + generate_phase = replace( + two_stage_recipe.phases[0], + name="generate", + spatial_downscale=1, + ) + return replace( + two_stage_recipe, + height=two_stage_recipe.height // 2, + width=two_stage_recipe.width // 2, + phases=(generate_phase,), + video_output_phase=0, + audio_output_phase=0, + allow_request_phase_sigmas=False, + supports_cache_dit=supports_cache_dit, + ) + + +LTX2_DISTILLED_ONE_STAGE_RECIPE = _distilled_one_stage_recipe(LTX2_DISTILLED_TWO_STAGE_RECIPE) +LTX23_DISTILLED_ONE_STAGE_RECIPE = _distilled_one_stage_recipe(LTX23_DISTILLED_TWO_STAGE_RECIPE) +LTX25_DISTILLED_ONE_STAGE_RECIPE = _distilled_one_stage_recipe( + LTX25_DISTILLED_TWO_STAGE_RECIPE, + supports_cache_dit=False, +) + + def _official_two_stage_recipe(one_stage_recipe: LTXPipelineRecipe) -> LTXPipelineRecipe: return LTXPipelineRecipe( height=one_stage_recipe.height * 2, @@ -197,21 +289,29 @@ def _official_two_stage_recipe(one_stage_recipe: LTXPipelineRecipe) -> LTXPipeli # discards its audio result. Decode the full-context Stage-1 audio. audio_output_phase=0, allow_request_sigmas=False, + allow_request_phase_sigmas=True, allow_request_latents=False, ) LTX2_TWO_STAGE_RECIPE = _official_two_stage_recipe(LTX2_ONE_STAGE_RECIPE) LTX23_TWO_STAGE_RECIPE = _official_two_stage_recipe(LTX23_ONE_STAGE_RECIPE) +LTX25_TWO_STAGE_RECIPE = _official_two_stage_recipe(LTX25_FULL_RECIPE) _PIPELINE_RECIPES: dict[tuple[str, str], LTXPipelineRecipe] = { ("one_stage", "2"): LTX2_ONE_STAGE_RECIPE, ("one_stage", "2.3"): LTX23_ONE_STAGE_RECIPE, + ("one_stage", "2.5"): LTX25_FULL_RECIPE, ("two_stage", "2"): LTX2_TWO_STAGE_RECIPE, ("two_stage", "2.3"): LTX23_TWO_STAGE_RECIPE, + ("two_stage", "2.5"): LTX25_TWO_STAGE_RECIPE, + ("distilled_one_stage", "2"): LTX2_DISTILLED_ONE_STAGE_RECIPE, + ("distilled_one_stage", "2.3"): LTX23_DISTILLED_ONE_STAGE_RECIPE, + ("distilled_one_stage", "2.5"): LTX25_DISTILLED_ONE_STAGE_RECIPE, ("distilled_two_stage", "2"): LTX2_DISTILLED_TWO_STAGE_RECIPE, ("distilled_two_stage", "2.3"): LTX23_DISTILLED_TWO_STAGE_RECIPE, + ("distilled_two_stage", "2.5"): LTX25_DISTILLED_TWO_STAGE_RECIPE, ("dmd2", "2"): LTX_POSITIVE_ONLY_RECIPE, ("dmd2", "2.3"): LTX_POSITIVE_ONLY_RECIPE, } diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_request.py b/vllm_omni/diffusion/models/ltx2/ltx2_request.py index 91d62943a..35e0cef0b 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_request.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_request.py @@ -43,6 +43,7 @@ class LTXRequestInputs: output_type: str max_sequence_length: int audio_latents_normalized: bool = False + image_crf: int | None = None @property def guidance_scale(self) -> float: @@ -86,6 +87,7 @@ def validate_pipeline_request( vae_temporal_compression_ratio: int, pipeline_name: str, request_sigmas: list[float] | None = None, + request_phase_sigmas: tuple[list[float] | None, ...] | None = None, ) -> None: """Validate the request capabilities declared by an execution recipe.""" alignment = vae_spatial_compression_ratio * pipeline_recipe.max_spatial_downscale @@ -108,9 +110,28 @@ def validate_pipeline_request( if not pipeline_recipe.allow_request_sigmas and request_sigmas is not None: raise ValueError(f"{pipeline_name} uses fixed phase sigma schedules.") + if request_sigmas is not None and len(request_sigmas) < 2: + raise ValueError("A custom LTX sigma schedule must contain at least two values.") - if pipeline_recipe.fixed_num_inference_steps and ( - request_inputs.num_inference_steps != pipeline_recipe.num_inference_steps + has_phase_sigmas = request_phase_sigmas is not None and any(item is not None for item in request_phase_sigmas) + if request_sigmas is not None and has_phase_sigmas: + raise ValueError("Use either `sigmas` or per-stage sigma schedules, not both.") + if has_phase_sigmas and not pipeline_recipe.allow_request_phase_sigmas: + raise ValueError(f"{pipeline_name} does not accept per-stage sigma schedules.") + if request_phase_sigmas is not None and len(request_phase_sigmas) != len(pipeline_recipe.phases): + raise ValueError( + f"{pipeline_name} has {len(pipeline_recipe.phases)} phases but received " + f"{len(request_phase_sigmas)} per-stage sigma schedules." + ) + for index, phase_sigmas in enumerate(request_phase_sigmas or ()): + if phase_sigmas is not None and len(phase_sigmas) < 2: + raise ValueError(f"`stage_{index + 1}_sigmas` must contain at least two values.") + + if ( + request_sigmas is None + and (request_phase_sigmas is None or request_phase_sigmas[0] is None) + and pipeline_recipe.fixed_num_inference_steps + and (request_inputs.num_inference_steps != pipeline_recipe.num_inference_steps) ): raise ValueError( f"{pipeline_name} uses {pipeline_recipe.num_inference_steps} fixed Stage 1 denoise steps; " @@ -165,6 +186,16 @@ def _get_extra_arg(sampling: Any, *names: str) -> Any: return None +def _normalize_image_crf(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError("`image_crf` must be an integer from 0 through 51.") + if not 0 <= value <= 51: + raise ValueError("`image_crf` must be an integer from 0 through 51.") + return value + + def _normalize_stg_blocks(value: Any, default: tuple[int, ...]) -> tuple[int, ...]: if value is None: return default @@ -274,6 +305,7 @@ class LTXRequestMixin: req: DiffusionRequestBatch, *, image: Any | None = None, + image_crf: int | None = None, prompt: str | list[str] | None = None, negative_prompt: str | list[str] | None = None, height: int | None = None, @@ -282,6 +314,8 @@ class LTXRequestMixin: frame_rate: float | None = None, num_inference_steps: int | None = None, sigmas: list[float] | None = None, + stage_1_sigmas: list[float] | None = None, + stage_2_sigmas: list[float] | None = None, timesteps: list[int] | None = None, guidance_scale: float | None = None, guidance_rescale: float | None = None, @@ -315,6 +349,7 @@ class LTXRequestMixin: return self._forward_request( req, image=image, + image_crf=image_crf, prompt=prompt, negative_prompt=negative_prompt, height=height, @@ -323,6 +358,8 @@ class LTXRequestMixin: frame_rate=frame_rate, num_inference_steps=num_inference_steps, sigmas=sigmas, + stage_1_sigmas=stage_1_sigmas, + stage_2_sigmas=stage_2_sigmas, guidance_scale=guidance_scale, num_videos_per_prompt=num_videos_per_prompt, generator=generator, @@ -343,12 +380,30 @@ class LTXRequestMixin: req: DiffusionRequestBatch, fallback: list[float] | None, ) -> list[float] | None: - request_sigmas = [sampling.sigmas for sampling in req.sampling_params_list] + request_sigmas = [ + sampling.sigmas if sampling.sigmas is not None else _get_extra_arg(sampling, "sigmas") + for sampling in req.sampling_params_list + ] first = request_sigmas[0] if request_sigmas else None if any(item != first for item in request_sigmas[1:]): raise ValueError("Batched LTX requests must use identical custom sigmas.") return first if first is not None else fallback + @staticmethod + def _resolve_request_phase_sigmas( + req: DiffusionRequestBatch, + stage_1_fallback: list[float] | None, + stage_2_fallback: list[float] | None, + ) -> tuple[list[float] | None, list[float] | None] | None: + resolved: list[list[float] | None] = [] + for name, fallback in (("stage_1_sigmas", stage_1_fallback), ("stage_2_sigmas", stage_2_fallback)): + values = [_get_extra_arg(sampling, name) for sampling in req.sampling_params_list] + first = values[0] if values else None + if any(item != first for item in values[1:]): + raise ValueError(f"Batched LTX requests must use identical `{name}` values.") + resolved.append(first if first is not None else fallback) + return tuple(resolved) if any(item is not None for item in resolved) else None + def check_inputs( self, prompt: str | list[str] | None, @@ -408,6 +463,7 @@ class LTXRequestMixin: decode_noise_scale: float | list[float] | None, output_type: str, max_sequence_length: int | None, + image_crf: int | None = None, ) -> LTXRequestInputs: sampling_params_list = req.sampling_params_list for sampling in sampling_params_list: @@ -421,6 +477,12 @@ class LTXRequestMixin: if _get_extra_arg(sampling, "flow_shift") is not None: raise ValueError("LTX does not support `flow_shift`; use `sigmas` for a custom schedule.") + request_image_crfs = [_get_extra_arg(item, "image_crf") for item in sampling_params_list] + first_image_crf = request_image_crfs[0] if request_image_crfs else None + if any(item != first_image_crf for item in request_image_crfs[1:]): + raise ValueError("Batched LTX requests must use identical `image_crf` values.") + image_crf = _normalize_image_crf(first_image_crf if first_image_crf is not None else image_crf) + sampling = sampling_params_list[0] is_dummy_run = req.is_dummy_run() prompt = [item if isinstance(item, str) else (item.get("prompt") or "") for item in req.prompts] or prompt @@ -551,4 +613,5 @@ class LTXRequestMixin: decode_noise_scale=decode_noise_scale, output_type=output_type, max_sequence_length=int(max_sequence_length), + image_crf=image_crf, ) diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_runtime.py b/vllm_omni/diffusion/models/ltx2/ltx2_runtime.py index cb4130259..f560f678b 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_runtime.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_runtime.py @@ -144,20 +144,19 @@ class LTXRuntime( f"{self.__class__.__name__} does not support ulysses_mode='advanced_uaa'. " "Use the default ulysses_mode='strict' for LTX sequence parallelism." ) - self.model_version = detect_ltx_model_version(od_config.model) + self.model_version = detect_ltx_model_version(od_config.model, revision=getattr(od_config, "revision", None)) self.component_profile = resolve_ltx_component_profile(self.pipeline_kind, self.model_version) self.pipeline_recipe = resolve_ltx_pipeline_recipe(self.pipeline_kind, self.model_version) - if getattr(od_config, "cache_backend", "none") == "cache_dit" and len(self.pipeline_recipe.phases) > 1: + if getattr(od_config, "cache_backend", "none") == "cache_dit" and not self.pipeline_recipe.supports_cache_dit: raise ValueError( f"{self.__class__.__name__} does not support cache_backend='cache_dit'. " - "Cache-DiT currently supports only one-stage LTX recipes; multi-stage recipes require " - "phase-aware cache enable and refresh." + "Cache-DiT is not qualified for this LTX recipe." ) self._dit_modules = list(self.component_profile.dit_modules) self._encoder_modules = list(self.component_profile.encoder_modules) self._vae_modules = list(self.component_profile.vae_modules) self._resident_modules = list(self.component_profile.resident_modules) - if self.model_version == "2.3": + if self.model_version in ("2.3", "2.5"): self.preserve_sp_padded_audio_duration = True self.reports_stage_durations = True super().__init__() @@ -173,6 +172,7 @@ class LTXRuntime( req: DiffusionRequestBatch, *, image: Any | None = None, + image_crf: int | None = None, prompt: str | list[str] | None = None, negative_prompt: str | list[str] | None = None, height: int | None = None, @@ -181,6 +181,8 @@ class LTXRuntime( frame_rate: float | None = None, num_inference_steps: int | None = None, sigmas: list[float] | None = None, + stage_1_sigmas: list[float] | None = None, + stage_2_sigmas: list[float] | None = None, guidance_scale: float | None = None, num_videos_per_prompt: int | None = 1, generator: torch.Generator | list[torch.Generator] | None = None, @@ -217,11 +219,13 @@ class LTXRuntime( decode_noise_scale=decode_noise_scale, output_type=output_type, max_sequence_length=max_sequence_length, + image_crf=image_crf, ) image = self._resolve_request_image(req, image, request_inputs) if image is not None and not self.support_image_input: raise ValueError(f"{self.__class__.__name__} does not support `image` input.") request_sigmas = self._resolve_request_sigmas(req, sigmas) + request_phase_sigmas = self._resolve_request_phase_sigmas(req, stage_1_sigmas, stage_2_sigmas) validate_pipeline_request( request_inputs, pipeline_recipe=self.pipeline_recipe, @@ -229,6 +233,7 @@ class LTXRuntime( vae_temporal_compression_ratio=self.vae_temporal_compression_ratio, pipeline_name=self.__class__.__name__, request_sigmas=request_sigmas, + request_phase_sigmas=request_phase_sigmas, ) phase_adapter = getattr(self, "_phase_adapter", None) if phase_adapter is not None and any( @@ -237,7 +242,13 @@ class LTXRuntime( raise ValueError( f"{self.__class__.__name__} cannot compose a request LoRA with its internal phase adapter." ) - return self._run_recipe(req, request_inputs, request_sigmas=request_sigmas, image=image) + return self._run_recipe( + req, + request_inputs, + request_sigmas=request_sigmas, + request_phase_sigmas=request_phase_sigmas, + image=image, + ) def _run_recipe( self, @@ -245,24 +256,39 @@ class LTXRuntime( request_inputs: LTXRequestInputs, *, request_sigmas: list[float] | None, + request_phase_sigmas: tuple[list[float] | None, ...] | None = None, image: Any | None = None, ) -> DiffusionOutput | list[DiffusionOutput]: """Execute one- and multi-phase recipes through the same control flow.""" phase_results: list[LTXPhaseResult] = [] prompt_context = None - - for phase_recipe in self.pipeline_recipe.phases: + for phase_index, phase_recipe in enumerate(self.pipeline_recipe.phases): + override_sigmas = None if request_phase_sigmas is None else request_phase_sigmas[phase_index] + phase_sigmas = ( + override_sigmas + if override_sigmas is not None + else ( + request_sigmas + if request_sigmas is not None + else (list(phase_recipe.sigmas) if phase_recipe.sigmas is not None else None) + ) + ) self._enter_phase(phase_recipe) phase_inputs = self._build_phase_inputs( request_inputs, phase_recipe, phase_results[-1] if phase_results else None, ) + if phase_sigmas is not None: + phase_inputs = replace(phase_inputs, num_inference_steps=len(phase_sigmas) - 1) + noise_scale = phase_recipe.noise_scale + if override_sigmas is not None and phase_recipe.input_transform == "spatial_upsample": + noise_scale = float(override_sigmas[0]) phase_result = self.run_phase( req, phase_inputs, - noise_scale=phase_recipe.noise_scale, - sigmas=list(phase_recipe.sigmas) if phase_recipe.sigmas is not None else request_sigmas, + noise_scale=noise_scale, + sigmas=phase_sigmas, timesteps=None, attention_kwargs=None, phase_recipe=phase_recipe, @@ -282,6 +308,7 @@ class LTXRuntime( def _enter_phase(self, phase: LTXPhaseRecipe) -> None: self._active_phase_name = phase.name + self._guidance_plan = LTXGuidancePlan.build(phase.guidance) phase_adapter = getattr(self, "_phase_adapter", None) if phase_adapter is None: if phase.adapter_slot is not None: @@ -645,7 +672,12 @@ class LTXRuntime( audio_token_count: int, ) -> dict[str, torch.Tensor]: del forward_ctx, denoise_ctx - return self.guidance_executor.timestep_kwargs(ts, video_token_count, audio_token_count) + return self.guidance_executor.timestep_kwargs( + ts, + video_token_count, + audio_token_count, + expand_for_sequence_parallel=True, + ) def _video_guidance_model_sigma( self, @@ -700,6 +732,7 @@ class LTXRuntime( state, forward_ctx, denoise_ctx, + preserve_positive_velocity=forward_ctx.sampler == "euler_ancestral", ) def _denoise_step( diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py b/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py index a59655cd3..0b0e7fc08 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py @@ -55,6 +55,18 @@ logger = init_logger(__name__) _RMSNORM_INIT_PARAMS = inspect.signature(RMSNorm.__init__).parameters +def apply_keyframes_absolute_embedding( + hidden_states: torch.Tensor, + keyframes_mask: torch.Tensor | None, + embedding: torch.Tensor | None, +) -> torch.Tensor: + """Add the official learned marker to first-frame/keyframe tokens.""" + if keyframes_mask is None or embedding is None: + return hidden_states + mask = (keyframes_mask > 0).to(dtype=hidden_states.dtype) + return hidden_states + mask * embedding.to(device=hidden_states.device, dtype=hidden_states.dtype) + + def _make_rms_norm(hidden_size: int, *, eps: float, elementwise_affine: bool) -> nn.Module: """Bridge diffusers' RMSNorm API onto vLLM's `has_weight` variant.""" if not elementwise_affine: @@ -82,15 +94,10 @@ def _make_rms_norm(hidden_size: int, *, eps: float, elementwise_affine: bool) -> def apply_interleaved_rotary_emb(x: torch.Tensor, freqs: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor: cos, sin = freqs - if cos.dtype != x.dtype: - cos = cos.to(dtype=x.dtype) - if sin.dtype != x.dtype: - sin = sin.to(dtype=x.dtype) # Concrete pair count instead of -1 keeps SDPA shape static under torch.compile. x_real, x_imag = x.unflatten(2, (x.shape[2] // 2, 2)).unbind(-1) # [B, S, C // 2] x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(2) - out = x * cos + x_rotated * sin - return out + return x * cos + x_rotated * sin def apply_split_rotary_emb( @@ -101,12 +108,6 @@ def apply_split_rotary_emb( ) -> torch.Tensor: # `head_dim` is plumbed in (not inferred via `-1`) so SDPA shape stays static under torch.compile. cos, sin = freqs - if cos.dtype != x.dtype: - cos = cos.to(dtype=x.dtype) - if sin.dtype != x.dtype: - sin = sin.to(dtype=x.dtype) - - x_dtype = x.dtype needs_reshape = False if x.ndim != 4 and cos.ndim == 4: # cos is (#b, h, t, r) -> reshape x to (b, h, t, dim_per_head) @@ -142,7 +143,6 @@ def apply_split_rotary_emb( if needs_reshape: out = out.swapaxes(1, 2).reshape(b, t, h * head_dim) - out = out.to(dtype=x_dtype) return out @@ -269,6 +269,7 @@ class LTX2FeedForward(nn.Module): RowParallelLinear( inner_dim, dim_out, + bias=bias, input_is_parallel=True, return_bias=False, quant_config=quant_config, @@ -517,8 +518,8 @@ class LTX2AudioVideoAttnProcessor: is_self_attention=is_self_attention, ) - query = attn.norm_q(query) - key = attn.norm_k(key) + query = attn.norm_q(query).to(dtype=value.dtype) + key = attn.norm_k(key).to(dtype=value.dtype) if query_rotary_emb is not None: query_rotary_emb = self._slice_rope_for_tp(query_rotary_emb, attn) @@ -537,6 +538,10 @@ class LTX2AudioVideoAttnProcessor: head_dim=attn.head_dim, ) + # Keep FP32 RoPE computation while satisfying attention dtype contracts. + query = query.to(dtype=value.dtype) + key = key.to(dtype=value.dtype) + query = query.unflatten(2, (attn.heads, attn.head_dim)) key = key.unflatten(2, (attn.heads, attn.head_dim)) value_heads = value.unflatten(2, (attn.heads, attn.head_dim)) @@ -846,6 +851,8 @@ class LTX2VideoTransformerBlock(nn.Module): elementwise_affine: bool = False, rope_type: str = "interleaved", perturbed_attn: bool = False, + ff_bias: bool = True, + audio_ff_bias: bool = True, quant_config: "QuantizationConfig | None" = None, prefix: str = "", ): @@ -965,6 +972,7 @@ class LTX2VideoTransformerBlock(nn.Module): self.ff = LTX2FeedForward( dim, activation_fn=activation_fn, + bias=ff_bias, quant_config=quant_config, prefix=f"{prefix}.ff" if prefix else "ff", ) @@ -973,6 +981,7 @@ class LTX2VideoTransformerBlock(nn.Module): self.audio_ff = LTX2FeedForward( audio_dim, activation_fn=activation_fn, + bias=audio_ff_bias, quant_config=quant_config, prefix=f"{prefix}.audio_ff" if prefix else "audio_ff", ) @@ -1078,21 +1087,34 @@ class LTX2VideoTransformerBlock(nn.Module): audio_hidden_states = audio_hidden_states + attn_audio_hidden_states * audio_gate_msa # 2. Video and Audio Cross-Attention with text embeddings (Q: Video/Audio; K,V: Text) - # LTX-2.3: compute prompt modulation params for K/V - if self.cross_attn_adaln and temb_prompt is not None: - video_prompt_ada_params = self.get_mod_params(self.prompt_scale_shift_table, temb_prompt, batch_size) - shift_text_kv, scale_text_kv = video_prompt_ada_params + if self.cross_attn_adaln: + if temb_prompt is not None: + shift_text_kv, scale_text_kv = self.get_mod_params( + self.prompt_scale_shift_table, temb_prompt, batch_size + ) + else: + shift_text_kv, scale_text_kv = ( + self.prompt_scale_shift_table[None, None] + .to(device=hidden_states.device, dtype=hidden_states.dtype) + .unbind(dim=2) + ) - audio_prompt_ada_params = self.get_mod_params( - self.audio_prompt_scale_shift_table, temb_prompt_audio, batch_size - ) - audio_shift_text_kv, audio_scale_text_kv = audio_prompt_ada_params + if temb_prompt_audio is not None: + audio_shift_text_kv, audio_scale_text_kv = self.get_mod_params( + self.audio_prompt_scale_shift_table, temb_prompt_audio, batch_size + ) + else: + audio_shift_text_kv, audio_scale_text_kv = ( + self.audio_prompt_scale_shift_table[None, None] + .to(device=audio_hidden_states.device, dtype=audio_hidden_states.dtype) + .unbind(dim=2) + ) # 2.1. Video-Text Cross-Attention norm_hidden_states = self.norm2(hidden_states) if self.video_cross_attn_adaln: norm_hidden_states = norm_hidden_states * (1 + scale_text_q) + shift_text_q - if self.cross_attn_adaln and temb_prompt is not None: + if self.cross_attn_adaln: encoder_hidden_states = encoder_hidden_states * (1 + scale_text_kv) + shift_text_kv attn_hidden_states = self.attn2( @@ -1109,7 +1131,7 @@ class LTX2VideoTransformerBlock(nn.Module): norm_audio_hidden_states = self.audio_norm2(audio_hidden_states) if self.audio_cross_attn_adaln: norm_audio_hidden_states = norm_audio_hidden_states * (1 + audio_scale_text_q) + audio_shift_text_q - if self.cross_attn_adaln and temb_prompt is not None: + if self.cross_attn_adaln: audio_encoder_hidden_states = audio_encoder_hidden_states * (1 + audio_scale_text_kv) + audio_shift_text_kv attn_audio_hidden_states = self.audio_attn2( @@ -1508,7 +1530,10 @@ class LTX2VideoTransformer3DModel(nn.Module): block_forward_patterns={ "transformer_blocks": ForwardPattern.Pattern_0, }, - has_separate_cfg=True, + # LTX fuses all guidance passes into the batch dimension and invokes + # the transformer once per denoise step. Cache-DiT must therefore + # advance its step counter for every transformer forward. + has_separate_cfg=False, check_forward_pattern=False, ) @@ -1546,6 +1571,7 @@ class LTX2VideoTransformer3DModel(nn.Module): # Shard video/audio latents across sequence "hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False), "audio_hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False), + "keyframes_mask": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False), # Shard prompt embeds across sequence "encoder_hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False), "audio_encoder_hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False), @@ -1620,6 +1646,10 @@ class LTX2VideoTransformer3DModel(nn.Module): cross_attn_mod: bool = False, audio_gated_attn: bool = False, audio_cross_attn_mod: bool = False, + ff_bias: bool = True, + audio_ff_bias: bool = True, + use_prompt_adaln_single: bool = True, + use_keyframes_abs_pos_embedding: bool = False, quant_config: "QuantizationConfig | None" = None, ) -> None: super().__init__() @@ -1666,12 +1696,19 @@ class LTX2VideoTransformer3DModel(nn.Module): timestep_scale_multiplier=timestep_scale_multiplier, cross_attn_timestep_scale_multiplier=cross_attn_timestep_scale_multiplier, rope_type=rope_type, + ff_bias=ff_bias, + audio_ff_bias=audio_ff_bias, + use_prompt_adaln_single=use_prompt_adaln_single, + use_keyframes_abs_pos_embedding=use_keyframes_abs_pos_embedding, ) # 1. Patchification input projections self.proj_in = nn.Linear(in_channels, inner_dim) self.audio_proj_in = nn.Linear(audio_in_channels, audio_inner_dim) + if use_keyframes_abs_pos_embedding: + self.keyframes_abs_pos_embedding = nn.Parameter(torch.zeros(1, inner_dim)) + # 2. Prompt embeddings # LTX-2 (use_prompt_embeddings=True): caption projection in the transformer # LTX-2.3 (use_prompt_embeddings=False): caption projection in the connectors @@ -1695,7 +1732,7 @@ class LTX2VideoTransformer3DModel(nn.Module): # 3.3. LTX-2.3: Prompt modulation from sigma self.prompt_modulation = cross_attn_mod or audio_cross_attn_mod - if self.prompt_modulation: + if self.prompt_modulation and use_prompt_adaln_single: self.prompt_adaln = LTX2AdaLayerNormSingle(inner_dim, num_mod_params=2, use_additional_conditions=False) self.audio_prompt_adaln = LTX2AdaLayerNormSingle( audio_inner_dim, num_mod_params=2, use_additional_conditions=False @@ -1815,6 +1852,8 @@ class LTX2VideoTransformer3DModel(nn.Module): elementwise_affine=norm_elementwise_affine, rope_type=rope_type, perturbed_attn=perturbed_attn, + ff_bias=ff_bias, + audio_ff_bias=audio_ff_bias, quant_config=quant_config, prefix=f"transformer_blocks.{layer_idx}", ) @@ -1852,6 +1891,7 @@ class LTX2VideoTransformer3DModel(nn.Module): audio_encoder_hidden_states: torch.Tensor, timestep: torch.LongTensor, audio_timestep: torch.LongTensor | None = None, + keyframes_mask: torch.Tensor | None = None, sigma: torch.Tensor | None = None, audio_sigma: torch.Tensor | None = None, encoder_attention_mask: torch.Tensor | None = None, @@ -1952,9 +1992,7 @@ class LTX2VideoTransformer3DModel(nn.Module): video_rotary_emb = self.rope(video_coords, device=hidden_states.device, out_dtype=hidden_states.dtype) audio_rotary_emb = self.audio_rope( - audio_coords, - device=audio_hidden_states.device, - out_dtype=audio_hidden_states.dtype, + audio_coords, device=audio_hidden_states.device, out_dtype=audio_hidden_states.dtype ) video_cross_attn_rotary_emb = self.cross_attn_rope( @@ -1970,6 +2008,11 @@ class LTX2VideoTransformer3DModel(nn.Module): # 2. Patchify input projections hidden_states = self.proj_in(hidden_states) + hidden_states = apply_keyframes_absolute_embedding( + hidden_states, + keyframes_mask, + getattr(self, "keyframes_abs_pos_embedding", None), + ) audio_hidden_states = self.audio_proj_in(audio_hidden_states) # 3. Prepare timestep embeddings and modulation parameters @@ -2028,7 +2071,7 @@ class LTX2VideoTransformer3DModel(nn.Module): audio_cross_attn_v2a_gate = audio_cross_attn_v2a_gate.view(batch_size, -1, audio_cross_attn_v2a_gate.shape[-1]) # 3.3. LTX-2.3: Compute prompt modulation from sigma - if self.prompt_modulation and sigma is not None: + if self.prompt_modulation and self.config.use_prompt_adaln_single and sigma is not None: temb_prompt, _ = self.prompt_adaln(sigma.flatten(), batch_size=batch_size, hidden_dtype=hidden_states.dtype) temb_prompt_audio, _ = self.audio_prompt_adaln( audio_sigma.flatten(), batch_size=batch_size, hidden_dtype=audio_hidden_states.dtype diff --git a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py index ff66af5e1..ff2ab0457 100644 --- a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py +++ b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py @@ -10,15 +10,12 @@ from typing import ClassVar from vllm_omni.diffusion.data import OmniDiffusionConfig from vllm_omni.diffusion.models.dmd2 import DMD2PipelineMixin -from .ltx2_components import LTX2_COMPONENT_PROFILE +from .ltx2_components import LTX2_COMPONENT_PROFILE, LTX2_DISTILLED_ONE_STAGE_COMPONENT_PROFILE from .ltx2_components import ( get_ltx2_post_process_func as get_ltx2_post_process_func, # noqa: F401 ) from .ltx2_conditioning import LTXI2VConditioningMixin -from .ltx2_recipes import ( - LTX2_ONE_STAGE_RECIPE, - LTX_POSITIVE_ONLY_RECIPE, -) +from .ltx2_recipes import LTX2_DISTILLED_ONE_STAGE_RECIPE, LTX2_ONE_STAGE_RECIPE, LTX_POSITIVE_ONLY_RECIPE from .ltx2_runtime import LTXRuntime @@ -36,6 +33,14 @@ class LTX2Pipeline(LTXI2VConditioningMixin, LTXRuntime): supports_request_batch = True +class LTX2DistilledOneStagePipeline(LTX2Pipeline): + """Merged-distilled checkpoint at its native one-stage resolution.""" + + pipeline_kind = "distilled_one_stage" + component_profile = LTX2_DISTILLED_ONE_STAGE_COMPONENT_PROFILE + pipeline_recipe = LTX2_DISTILLED_ONE_STAGE_RECIPE + + class LTX2T2VDMD2Pipeline(DMD2PipelineMixin, LTX2Pipeline): """LTX2 T2V entry for FastGen DMD2-distilled models.""" diff --git a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py index 3cb6b84f1..5d50baf42 100644 --- a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py +++ b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py @@ -40,9 +40,13 @@ class LTX2TwoStagePipeline(_LTX2TwoStageBase): pipeline_recipe = LTX2_TWO_STAGE_RECIPE -class LTX2DistilledPipeline(_LTX2TwoStageBase): - """Unified LTX-2/LTX-2.3 full-distilled two-stage T2V/I2V entry.""" +class LTX2DistilledTwoStagePipeline(_LTX2TwoStageBase): + """Merged-distilled checkpoint with latent upsampling and refinement.""" pipeline_kind = "distilled_two_stage" component_profile = LTX2_DISTILLED_COMPONENT_PROFILE pipeline_recipe = LTX2_DISTILLED_TWO_STAGE_RECIPE + + +# Backwards-compatible name retained for existing LTX-2/LTX-2.3 recipes. +LTX2DistilledPipeline = LTX2DistilledTwoStagePipeline diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index 0e814886b..9bac50323 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -76,6 +76,16 @@ _DIFFUSION_MODELS = { "pipeline_ltx2_two_stage", "LTX2TwoStagePipeline", ), + "LTX2DistilledOneStagePipeline": ( + "ltx2", + "pipeline_ltx2", + "LTX2DistilledOneStagePipeline", + ), + "LTX2DistilledTwoStagePipeline": ( + "ltx2", + "pipeline_ltx2_two_stage", + "LTX2DistilledTwoStagePipeline", + ), "LTX2DistilledPipeline": ( "ltx2", "pipeline_ltx2_two_stage", @@ -522,6 +532,8 @@ _DIFFUSION_POST_PROCESS_FUNCS = { "WanVACEPipeline": "get_wan22_vace_post_process_func", "LTX2Pipeline": "get_ltx2_post_process_func", "LTX2TwoStagePipeline": "get_ltx2_post_process_func", + "LTX2DistilledOneStagePipeline": "get_ltx2_post_process_func", + "LTX2DistilledTwoStagePipeline": "get_ltx2_post_process_func", "LTX2DistilledPipeline": "get_ltx2_post_process_func", "LTX2T2VDMD2Pipeline": "get_ltx2_post_process_func", "LTX2I2VDMD2Pipeline": "get_ltx2_post_process_func", diff --git a/vllm_omni/entrypoints/openai/serving_video.py b/vllm_omni/entrypoints/openai/serving_video.py index 9d511ba6d..2628c14f8 100644 --- a/vllm_omni/entrypoints/openai/serving_video.py +++ b/vllm_omni/entrypoints/openai/serving_video.py @@ -7,6 +7,7 @@ import copy import time from collections.abc import Mapping from dataclasses import dataclass +from functools import cached_property from http import HTTPStatus from typing import Any, cast @@ -30,6 +31,7 @@ from vllm_omni.entrypoints.openai.stage_params import ( from vllm_omni.entrypoints.openai.utils import get_stage_type, parse_lora_request from vllm_omni.entrypoints.openai.video_api_utils import _encode_video_bytes, encode_video_base64 from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt +from vllm_omni.model_extras import should_preserve_reference_image_size from vllm_omni.outputs.output_metadata import ( DiffusionMetadataMapping, DiffusionMultimodalOutput, @@ -100,6 +102,21 @@ class OmniOpenAIServingVideo: if self._stage_configs is None and stage_configs is not None: self._stage_configs = stage_configs + @cached_property + def preserves_reference_image_size(self) -> bool: + """Return whether the active pipeline owns reference-image resizing.""" + get_od_config = getattr(self._engine_client, "get_diffusion_od_config", None) + od_config = get_od_config() if callable(get_od_config) else getattr(self._engine_client, "od_config", None) + model_class_name = None if od_config is None else getattr(od_config, "model_class_name", None) + model = getattr(od_config, "model", None) if od_config is not None else None + model = model or self.model_name + revision = getattr(od_config, "revision", None) if od_config is not None else None + return should_preserve_reference_image_size( + model_class_name, + model=None if model is None else str(model), + revision=revision, + ) + @property def supports_mixed_reference_inputs(self) -> bool: """Return whether the configured diffusion model accepts mixed refs.""" @@ -154,7 +171,12 @@ class OmniOpenAIServingVideo: provided_fields = request.model_fields_set fps_provided = self._request_fps_provided(request) vp = request.resolve_video_params() - if input_image is not None and vp.width is not None and vp.height is not None: + if ( + input_image is not None + and vp.width is not None + and vp.height is not None + and not self.preserves_reference_image_size + ): target_size = (vp.width, vp.height) image_items = input_image if isinstance(input_image, list) else [input_image] resized_images = [ diff --git a/vllm_omni/model_extras/__init__.py b/vllm_omni/model_extras/__init__.py index b11ceadec..f10ca9806 100644 --- a/vllm_omni/model_extras/__init__.py +++ b/vllm_omni/model_extras/__init__.py @@ -10,8 +10,10 @@ from vllm_omni.model_extras.registry import ( get_extra_output_params, get_model_class_name, get_output_tensor_range, + get_transformer_config_subfolder, get_x_to_text_model_family, should_init_extra_args_for_non_diffusion_stages, + should_preserve_reference_image_size, ) __all__ = [ @@ -23,6 +25,8 @@ __all__ = [ "get_extra_output_params", "get_model_class_name", "get_output_tensor_range", + "get_transformer_config_subfolder", "get_x_to_text_model_family", "should_init_extra_args_for_non_diffusion_stages", + "should_preserve_reference_image_size", ] diff --git a/vllm_omni/model_extras/ltx2.py b/vllm_omni/model_extras/ltx2.py index 5bac58469..7516a55f2 100644 --- a/vllm_omni/model_extras/ltx2.py +++ b/vllm_omni/model_extras/ltx2.py @@ -21,7 +21,35 @@ LTX_EXTRA_BODY_PARAMS = frozenset( "audio_rescale_scale", "video_stg_blocks", "audio_stg_blocks", + "sigmas", + "stage_1_sigmas", + "stage_2_sigmas", + "image_crf", } ) LTX_EXTRA_OUTPUT_PARAMS = frozenset() + + +def ltx_preserves_reference_image_size(*, model: str | None, revision: str | None = None) -> bool: + """Keep source geometry until LTX-2.5 applies conditioning compression.""" + if model is None: + return False + from vllm_omni.diffusion.models.ltx2.ltx2_components import ( + preserves_reference_image_size as ltx_preserves_reference_image_size, + ) + + return ltx_preserves_reference_image_size(model=model, revision=revision) + + +def ltx_transformer_config_subfolder( + *, + model: str | None, + revision: str | None = None, +) -> str: + """Select the transformer config used by the requested LTX checkpoint.""" + if model is None: + return "transformer" + from vllm_omni.diffusion.models.ltx2.ltx2_components import detect_ltx_model_version + + return "transformer_full" if detect_ltx_model_version(model, revision=revision) == "2.5" else "transformer" diff --git a/vllm_omni/model_extras/registry.py b/vllm_omni/model_extras/registry.py index bc923ea86..c394b8999 100644 --- a/vllm_omni/model_extras/registry.py +++ b/vllm_omni/model_extras/registry.py @@ -4,7 +4,7 @@ from __future__ import annotations from collections.abc import Callable, Mapping -from typing import Any, Literal +from typing import Any, Literal, Protocol from PIL import Image @@ -34,7 +34,12 @@ from vllm_omni.model_extras.helios import ( ) from vllm_omni.model_extras.hunyuan_image3 import build_x_to_text_prompt as build_hunyuan_x_to_text_prompt from vllm_omni.model_extras.lingbot_video import LINGBOT_VIDEO_EXTRA_BODY_PARAMS -from vllm_omni.model_extras.ltx2 import LTX_EXTRA_BODY_PARAMS, LTX_EXTRA_OUTPUT_PARAMS +from vllm_omni.model_extras.ltx2 import ( + LTX_EXTRA_BODY_PARAMS, + LTX_EXTRA_OUTPUT_PARAMS, + ltx_preserves_reference_image_size, + ltx_transformer_config_subfolder, +) from vllm_omni.model_extras.magi_human import ( MAGI_HUMAN_EXTRA_BODY_PARAMS, MAGI_HUMAN_EXTRA_OUTPUT_PARAMS, @@ -96,6 +101,24 @@ XToTextPromptBuilder = Callable[[str, str, bool], tuple[dict[str, Any], list[int OutputTensorRange = Literal["negative_one_to_one", "zero_to_one"] +class ReferenceImageSizeResolver(Protocol): + def __call__( + self, + *, + model: str | None, + revision: str | None = None, + ) -> bool: ... + + +class TransformerConfigSubfolderResolver(Protocol): + def __call__( + self, + *, + model: str | None, + revision: str | None = None, + ) -> str: ... + + def default_x_to_text_prompt( model: str, prompt: str, @@ -202,11 +225,14 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = { model_class_name: { "extra_body_params": LTX_EXTRA_BODY_PARAMS, "extra_output_params": LTX_EXTRA_OUTPUT_PARAMS, + "reference_image_size_resolver": ltx_preserves_reference_image_size, } for model_class_name in ( "LTX2Pipeline", "LTX2TwoStagePipeline", + "LTX2DistilledOneStagePipeline", "LTX2DistilledPipeline", + "LTX2DistilledTwoStagePipeline", ) }, "WanVACEPipeline": { @@ -229,6 +255,10 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = { }, } +for model_class_name in ("LTX2Pipeline", "LTX2TwoStagePipeline"): + _EXTRA_SPECS[model_class_name]["transformer_config_subfolder_resolver"] = ltx_transformer_config_subfolder + + # Multi-stage discovery reports the top-level wrapper rather than its DiT # submodule, so both names must resolve to the same request builders. _EXTRA_SPECS["MammothModa2ForConditionalGeneration"] = _EXTRA_SPECS["MammothModa2DiTPipeline"] @@ -277,6 +307,36 @@ def get_output_tensor_range(model_class_name: str | None) -> OutputTensorRange: return spec.get("output_tensor_range", "negative_one_to_one") +def get_transformer_config_subfolder( + model_class_name: str | None, + *, + model: str | None, + revision: str | None = None, +) -> str: + """Return the model-declared DiT config subfolder, or the standard default.""" + spec = _get_spec(model_class_name) + resolver: TransformerConfigSubfolderResolver | None = ( + spec.get("transformer_config_subfolder_resolver") if spec else None + ) + return resolver(model=model, revision=revision) if resolver else "transformer" + + +def should_preserve_reference_image_size( + model_class_name: str | None, + *, + model: str | None, + revision: str | None = None, +) -> bool: + """Return whether the selected pipeline owns reference-image resizing.""" + if model_class_name is None and model is not None: + from vllm_omni.diffusion.data import resolve_model_class_name + + model_class_name = resolve_model_class_name(model, revision=revision) + spec = _get_spec(model_class_name) + resolver: ReferenceImageSizeResolver | None = spec.get("reference_image_size_resolver") if spec else None + return bool(resolver and resolver(model=model, revision=revision)) + + def should_init_extra_args_for_non_diffusion_stages(model_class_name: str | None) -> bool: spec = _get_spec(model_class_name) return bool(spec and spec.get("init_extra_args_for_non_diffusion_stages", False)) From 136cb27b05385cf2ce7246e6815154ee86525d0f Mon Sep 17 00:00:00 2001 From: Samit <285365963@qq.com> Date: Sat, 15 Aug 2026 18:27:51 +0800 Subject: [PATCH 028/267] [Bugfix] Fix lost async diffusion outputs in request-level batching (#6023) Signed-off-by: samithuang <285365963@qq.com> --- docs/design/feature/async_diffusion_output.md | 18 ++- tests/diffusion/test_async_output_worker.py | 133 ++++++++++++++++++ tests/diffusion/test_diffusion_ipc.py | 43 ++++++ tests/diffusion/test_result_pump.py | 114 +++++++++++++++ tools/pre_commit/check_pickle_imports.py | 4 + vllm_omni/diffusion/diffusion_engine.py | 11 +- .../diffusion/executor/multiproc_executor.py | 129 +++++++++++------ vllm_omni/diffusion/ipc.py | 15 +- .../diffusion/worker/diffusion_worker.py | 63 ++++++++- 9 files changed, 475 insertions(+), 55 deletions(-) diff --git a/docs/design/feature/async_diffusion_output.md b/docs/design/feature/async_diffusion_output.md index a81ed6811..760e81adf 100644 --- a/docs/design/feature/async_diffusion_output.md +++ b/docs/design/feature/async_diffusion_output.md @@ -117,7 +117,22 @@ After (async D2H): ### Batch Split -When `execute_model_batch` returns a `COMPUTE_DONE` with a single `async_output_id` for the entire batch, the executor splits it into per-request `async_output_id`s (format `{batch_id}/{request_id}`) via `_batch_split_map`. When `OUTPUT_READY` arrives, the pump extracts per-request results from the batch output and resolves each request's output future independently. +When `execute_model_batch` returns a `COMPUTE_DONE` with a single `async_output_id` for the entire batch, the executor splits it into per-request `async_output_id`s (format `{batch_id}/{request_id}`) via `_batch_split_map`. When `OUTPUT_READY` arrives, the pump extracts per-request results from the batch output and resolves each request's output future independently. If `OUTPUT_READY` arrives before `execute_batch` registers the split map, `execute_batch` directly adopts the early output. + +### Reliability, Lifecycle & Timeout Behavior + +1. **Drain Before Memory Release (`drain_async_outputs` / `_ASYNC_OUTPUT_DRAIN_TIMEOUT_S`)**: + When workers transition into sleep states (`handle_sleep_task`) or invoke memory-releasing methods, `WorkerProc.drain_async_outputs()` blocks up to `_ASYNC_OUTPUT_DRAIN_TIMEOUT_S` (10.0s) until all in-flight background D2H/SHM packing tasks finish. This acts as a synchronization barrier preventing worker sleep cycles from releasing device tensors while the side CUDA stream is still actively reading them. Draining typically completes within 10–50 ms under normal operation. + +2. **Timeout Diagnostics (`_ASYNC_OUTPUT_TIMEOUT`)**: + The engine waits for `wait_output_ready()` with a timeout of `_ASYNC_OUTPUT_TIMEOUT` (30.0s). If a timeout occurs, the executor logs detailed diagnostic bookkeeping (pending futures, cached outputs, and batch split maps) to identify whether the stall occurred in the worker, pump, or waiter. + +3. **Queue Serialization & Atomic Future Resolution**: + - `result_mq` writes are serialized with `_result_mq_lock` to protect the single-writer `MessageQueue` against concurrent writes from the worker main loop (`COMPUTE_DONE`) and background thread (`OUTPUT_READY`). + - `ResultPumpThread` unpacks shared-memory payloads first, and then atomically resolves waiting futures or populates `_completed_outputs` under `_futures_lock`, avoiding race conditions with concurrent `wait_output_ready()` invocations. + +4. **Storage-Aware IPC Packing**: + `pack_diffusion_output_shm()` evaluates `max(view_bytes, storage_bytes)` against `_SHM_TENSOR_THRESHOLD` (1 MB) to ensure that per-request slices sharing a large batch storage are moved to POSIX shared memory rather than serialized through pickle over the MessageQueue inline buffer. ## Model Coverage @@ -131,6 +146,7 @@ All models running in request-mode (`step_execution=False`, the default) automat |---|---|---| | **HunyuanImage-3.0** | Image | `False` | | **Qwen-Image** | Image | `True` | +| **LTX-2.3** | Video / Audio | `False` | Other models with `step_execution=False` are also supported but not yet verified. diff --git a/tests/diffusion/test_async_output_worker.py b/tests/diffusion/test_async_output_worker.py index da684a4f5..a4bff4625 100644 --- a/tests/diffusion/test_async_output_worker.py +++ b/tests/diffusion/test_async_output_worker.py @@ -3,6 +3,8 @@ """Unit tests for WorkerProc async output logic.""" import queue +import threading +import time from unittest.mock import MagicMock import pytest @@ -29,9 +31,59 @@ def _make_worker_proc(step_execution=False): proc._running = True proc._async_output_queue = queue.Queue() if not step_execution else None proc._async_output_thread = None + proc._async_output_done = threading.Condition() + proc._async_output_pending = 0 + proc._result_mq_lock = threading.Lock() return proc +class _OverlapDetectingQueue: + """Fails if two threads are inside enqueue() at the same time. + + Mirrors vLLM's MessageQueue, whose acquire_write() reads current_idx, + writes that block, then advances it. Overlapping writers can target the + same ring block and silently drop a message. + """ + + def __init__(self): + self._active = 0 + self._guard = threading.Lock() + self.overlaps = 0 + self.messages = [] + + def enqueue(self, msg, timeout=None): + with self._guard: + self._active += 1 + if self._active > 1: + self.overlaps += 1 + time.sleep(0.001) # widen the window a real ring-buffer write would have + with self._guard: + self.messages.append(msg) + self._active -= 1 + + +class TestResultQueueSingleWriter: + """Worker must serialize writes to the single-writer result queue.""" + + def test_concurrent_enqueues_do_not_overlap(self): + proc = _make_worker_proc(step_execution=False) + detector = _OverlapDetectingQueue() + proc.result_mq = detector + + def writer(tag): + for i in range(20): + proc._enqueue_result(f"{tag}-{i}") + + threads = [threading.Thread(target=writer, args=(tag,)) for tag in ("compute", "output")] + for t in threads: + t.start() + for t in threads: + t.join() + + assert detector.overlaps == 0, f"{detector.overlaps} overlapping writes to result_mq" + assert len(detector.messages) == 40 + + class TestGenerateAsyncOutputId: """Test _generate_async_output_id uniqueness.""" @@ -192,6 +244,87 @@ class TestAsyncOutputLoopLogic: assert output_ready_enqueued, "Expected OUTPUT_READY with async_output_id='abc123' in enqueue calls" +class TestDrainAsyncOutputs: + """Async outputs must finish packing before device memory is released.""" + + def test_drain_returns_immediately_when_nothing_in_flight(self): + proc = _make_worker_proc(step_execution=False) + + assert proc.drain_async_outputs(timeout=0.01) is True + + def test_return_result_keeps_output_pending_until_loop_completes(self, mocker): + """_return_result marks work pending; the bg loop clears it.""" + proc = _make_worker_proc(step_execution=False) + + mock_platform = mocker.MagicMock() + mock_platform.record_device_event.return_value = MagicMock() + mocker.patch( + "vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform", + mock_platform, + ) + mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.pack_diffusion_output_shm") + mock_torch = mocker.MagicMock() + mock_torch.accelerator.current_accelerator.return_value.type = "cpu" + mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.torch", mock_torch) + + proc._return_result(DiffusionOutput(output="data"), rpc_id="1") + + # Packing has not run yet, so a sleep here would free tensors in flight. + assert proc._async_output_pending == 1 + assert proc.drain_async_outputs(timeout=0.05) is False + + thread = threading.Thread(target=proc._async_output_loop, daemon=True) + thread.start() + try: + assert proc.drain_async_outputs(timeout=5.0) is True + finally: + proc._async_output_queue.put(None) + thread.join(timeout=2.0) + + def test_sleep_rpc_drains_before_releasing_memory(self, mocker): + proc = _make_worker_proc(step_execution=False) + proc.worker = MagicMock() + + call_order: list[str] = [] + mocker.patch.object( + proc, + "drain_async_outputs", + side_effect=lambda *a, **kw: call_order.append("drain") or True, + ) + proc.worker.execute_method.side_effect = lambda *a, **kw: call_order.append("sleep") + + proc._execute_rpc( + { + "method": "handle_sleep_task", + "args": (), + "kwargs": {}, + "output_rank": 0, + "exec_all_ranks": False, + "collect_rank_status": False, + } + ) + + assert call_order == ["drain", "sleep"] + + def test_non_sleep_rpc_does_not_drain(self, mocker): + proc = _make_worker_proc(step_execution=False) + proc.worker = MagicMock() + drain = mocker.patch.object(proc, "drain_async_outputs", return_value=True) + + proc._execute_rpc( + { + "method": "execute_model", + "args": (), + "kwargs": {}, + "output_rank": 0, + "exec_all_ranks": False, + "collect_rank_status": False, + } + ) + + drain.assert_not_called() + + class TestWorkerProcShutdown: def test_shutdown_stops_async_thread_and_releases_queues(self): import threading diff --git a/tests/diffusion/test_diffusion_ipc.py b/tests/diffusion/test_diffusion_ipc.py index 3e4c587f1..546aa11e7 100644 --- a/tests/diffusion/test_diffusion_ipc.py +++ b/tests/diffusion/test_diffusion_ipc.py @@ -305,6 +305,49 @@ def test_batch_runner_output_round_trips_nested_results_through_shm() -> None: assert output["req-error"].result.error == "boom" +def test_per_request_views_of_a_batch_tensor_do_not_ship_whole_storage() -> None: + """Regression: per-request slices used to pickle the whole batch storage. + + Each view is under the packing threshold, but pickle writes the entire + shared storage per view, so a batch's message grew by the full batch + tensor once per request and overflowed the ring buffer. + """ + import pickle + + num_requests = 16 + # Each row stays under the threshold; the shared storage is well over it. + row_numel = _SHM_TENSOR_THRESHOLD // (2 * torch.empty((), dtype=torch.float32).element_size()) + batch = torch.arange(num_requests * row_numel, dtype=torch.float32).reshape(num_requests, row_numel) + per_request = batch[0] + assert per_request.nelement() * per_request.element_size() <= _SHM_TENSOR_THRESHOLD + assert per_request.untyped_storage().nbytes() > _SHM_TENSOR_THRESHOLD + + output = BatchRunnerOutput.from_list( + [ + RunnerOutput( + request_id=f"req-{i}", + finished=True, + result=DiffusionOutput(trajectory_latents=batch[i]), + ) + for i in range(num_requests) + ] + ) + unpacked_size = len(pickle.dumps(output, protocol=pickle.HIGHEST_PROTOCOL)) + + pack_diffusion_output_shm(output) + packed_size = len(pickle.dumps(output, protocol=pickle.HIGHEST_PROTOCOL)) + + assert output.runner_outputs[0].result.trajectory_latents["__tensor_shm__"] is True + # Whole storage per request before; handles only after. + storage_bytes = per_request.untyped_storage().nbytes() + assert unpacked_size > num_requests * storage_bytes / 2 + assert packed_size < _SHM_TENSOR_THRESHOLD + + unpack_diffusion_output_shm(output) + for i in range(num_requests): + torch.testing.assert_close(output[f"req-{i}"].result.trajectory_latents, batch[i]) + + def test_pack_value_keeps_tensor_at_threshold_inline() -> None: tensor = torch.arange( _SHM_TENSOR_THRESHOLD // torch.empty((), dtype=torch.float32).element_size(), diff --git a/tests/diffusion/test_result_pump.py b/tests/diffusion/test_result_pump.py index cf00cb6cc..2f4a89ca1 100644 --- a/tests/diffusion/test_result_pump.py +++ b/tests/diffusion/test_result_pump.py @@ -6,6 +6,7 @@ import concurrent.futures import queue import threading import time +from types import SimpleNamespace from unittest.mock import MagicMock import pytest @@ -264,6 +265,119 @@ class TestResultPumpDispatch: fut = executor.wait_output_ready(async_output_id) assert fut.done() assert fut.result(timeout=1.0) is output + assert async_output_id not in executor._output_futures + + def test_output_ready_atomic_resolution_when_future_already_waiting(self): + """When OUTPUT_READY arrives and a future is already waiting, resolve it directly.""" + executor = _make_executor() + async_output_id = "abc123" + output = DiffusionOutput(output="waiting") + fut = executor.wait_output_ready(async_output_id) + assert not fut.done() + + msg = AsyncDiffusionOutput( + kind=AsyncOutputKind.OUTPUT_READY, + async_output_id=async_output_id, + output=output, + ) + _feed_one_msg_to_pump(executor, msg) + + assert fut.done() + assert fut.result(timeout=1.0) is output + assert async_output_id not in executor._output_futures + assert async_output_id not in executor._completed_outputs + + +class _FakeBatchOutput: + """Batch-level output exposing per-request results.""" + + def __init__(self, results): + self._results = results + + def get_request_output(self, req_id): + result = self._results.get(req_id) + if result is None: + return None + return SimpleNamespace(result=result) + + +def _make_scheduler_output(req_ids): + return SimpleNamespace( + scheduled_new_reqs=[SimpleNamespace(request_id=rid, req=SimpleNamespace()) for rid in req_ids] + ) + + +class TestBatchSplitDelivery: + """execute_batch must resolve per-request futures in either arrival order.""" + + @staticmethod + def _run(executor, req_ids, batch_id, deliver_early): + # Keep execute_batch on the fused request-batch path (not DLO DP). + executor.od_config.parallel_config.data_parallel_size = 1 + executor.od_config.enable_distributed_layerwise_offload = False + executor._ensure_open = lambda: None + + outputs = {rid: DiffusionOutput(output=f"img-{rid}") for rid in req_ids} + ready = AsyncDiffusionOutput( + kind=AsyncOutputKind.OUTPUT_READY, + async_output_id=batch_id, + output=_FakeBatchOutput(outputs), + ) + + def fake_collective_rpc(*args, **kwargs): + if deliver_early: + # Worker's background D2H/SHM thread wins the race: OUTPUT_READY + # is pumped before execute_batch registers the split map. + _feed_one_msg_to_pump(executor, ready) + return AsyncDiffusionOutput( + kind=AsyncOutputKind.COMPUTE_DONE, + rpc_id="1", + async_output_id=batch_id, + ) + + executor.collective_rpc = fake_collective_rpc + batch = executor.execute_batch(_make_scheduler_output(req_ids)) + if not deliver_early: + _feed_one_msg_to_pump(executor, ready) + return batch, outputs + + def test_output_ready_after_split_map(self): + executor = _make_executor() + req_ids = ["r0", "r1", "r2"] + _, outputs = self._run(executor, req_ids, "batch-1", deliver_early=False) + + for rid in req_ids: + fut = executor.wait_output_ready(f"batch-1/{rid}") + assert fut.done() + assert fut.result(timeout=1.0) is outputs[rid] + + def test_output_ready_before_split_map(self): + """Regression: the whole batch used to be lost, hanging every request.""" + executor = _make_executor() + req_ids = ["r0", "r1", "r2"] + _, outputs = self._run(executor, req_ids, "batch-1", deliver_early=True) + + for rid in req_ids: + fut = executor.wait_output_ready(f"batch-1/{rid}") + assert fut.done(), f"request {rid} never resolved" + assert fut.result(timeout=1.0) is outputs[rid] + + # No stale batch-level state left behind. + assert executor._batch_split_map == {} + assert executor._completed_outputs == {} + + def test_engine_waiter_before_output_is_resolved(self): + """Consumers already blocked in wait_output_ready must be woken.""" + executor = _make_executor() + req_ids = ["r0", "r1"] + batch_id = "batch-2" + futures = {rid: executor.wait_output_ready(f"{batch_id}/{rid}") for rid in req_ids} + + _, outputs = self._run(executor, req_ids, batch_id, deliver_early=True) + + for rid in req_ids: + assert futures[rid].done(), f"request {rid} never resolved" + assert futures[rid].result(timeout=1.0) is outputs[rid] class TestShutdownCleansUpFutures: diff --git a/tools/pre_commit/check_pickle_imports.py b/tools/pre_commit/check_pickle_imports.py index 819a7c8c3..f8f0ef8f2 100644 --- a/tools/pre_commit/check_pickle_imports.py +++ b/tools/pre_commit/check_pickle_imports.py @@ -19,6 +19,10 @@ ALLOWED_FILES = { "tests/helpers/process.py", "vllm_omni/diffusion/distributed/group_coordinator.py", "tests/diffusion/attention/test_attention_sp.py", + # Measures the wire size of diffusion outputs. The transport pickles them, + # and only pickle reproduces the per-view storage duplication under test + # (torch.save deduplicates shared storages). No untrusted data is loaded. + "tests/diffusion/test_diffusion_ipc.py", } PICKLE_RE = re.compile( diff --git a/vllm_omni/diffusion/diffusion_engine.py b/vllm_omni/diffusion/diffusion_engine.py index 08218122e..fe9ed5c8c 100644 --- a/vllm_omni/diffusion/diffusion_engine.py +++ b/vllm_omni/diffusion/diffusion_engine.py @@ -339,7 +339,16 @@ class DiffusionEngine: # Async mode: wait for background D2H/SHM to complete. if output.async_output_id: fut = self.executor.wait_output_ready(output.async_output_id) - output = await asyncio.wait_for(asyncio.wrap_future(fut), timeout=_ASYNC_OUTPUT_TIMEOUT) + try: + output = await asyncio.wait_for(asyncio.wrap_future(fut), timeout=_ASYNC_OUTPUT_TIMEOUT) + except (TimeoutError, asyncio.TimeoutError): + describe = getattr(self.executor, "describe_pending_state", None) + logger.error( + "Timed out after %.0fs waiting for async output; executor state: %s", + _ASYNC_OUTPUT_TIMEOUT, + describe(output.async_output_id) if describe else "unavailable", + ) + raise postprocess_start_time = time.perf_counter() formatted_outputs = self.postprocess_output(request, output) postprocess_time = time.perf_counter() - postprocess_start_time diff --git a/vllm_omni/diffusion/executor/multiproc_executor.py b/vllm_omni/diffusion/executor/multiproc_executor.py index 7cc6488c1..74286dc94 100644 --- a/vllm_omni/diffusion/executor/multiproc_executor.py +++ b/vllm_omni/diffusion/executor/multiproc_executor.py @@ -641,8 +641,23 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): async_output_id=per_req_id, ) ) + # The worker's background D2H/SHM thread can publish OUTPUT_READY + # before this call returns. In that case the pump found no split + # map and cached the whole batch output under batch_id, so adopt it + # here instead of registering a map nothing will ever consume. with self._futures_lock: - self._batch_split_map[batch_id] = per_req_map + early = self._completed_outputs.pop(batch_id, None) + if early is None: + self._batch_split_map[batch_id] = per_req_map + if early is not None: + logger.debug("Batch %s output arrived before split map; splitting now", batch_id) + try: + batch_output = early.result() + error = None + except Exception as exc: + batch_output = None + error = str(exc) + self._deliver_batch_split(per_req_map, batch_output, error) return BatchRunnerOutput.from_list(runner_outputs) if not isinstance(result, BatchRunnerOutput): raise RuntimeError(f"Unexpected response type for execute_batch: {type(result)!r}") @@ -868,52 +883,76 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): unpack_diffusion_output_shm(msg.output) except Exception: logger.exception("SHM unpack failed for batch %s", batch_id) - batch_output = msg.output - for per_req_id, req_id in per_req_map.items(): - req_output = batch_output.get_request_output(req_id) - per_req_result: DiffusionOutput - if req_output is not None and req_output.result is not None: - per_req_result = req_output.result - elif msg.error: - per_req_result = DiffusionOutput(error=msg.error) - else: - per_req_result = DiffusionOutput(error="No output result for batch request") - fut: concurrent.futures.Future = concurrent.futures.Future() - fut.set_result(per_req_result) - with self._futures_lock: - pending = self._output_futures.pop(per_req_id, None) - if pending is not None and not pending.done(): - pending.set_result(per_req_result) - else: - self._completed_outputs[per_req_id] = fut + self._deliver_batch_split(per_req_map, msg.output, msg.error) else: - with self._futures_lock: - fut = self._output_futures.pop(batch_id, None) if batch_id else None - if fut is not None and not fut.done(): - if msg.error: - fut.set_exception(RuntimeError(msg.error)) - else: - try: - unpack_diffusion_output_shm(msg.output) - except Exception as e: - logger.exception("SHM unpack failed in result pump") - fut.set_exception(e) - continue - fut.set_result(msg.output) - elif batch_id: - fut = concurrent.futures.Future() - if msg.error: - fut.set_exception(RuntimeError(msg.error)) - else: - try: - unpack_diffusion_output_shm(msg.output) - except Exception as e: - logger.exception("SHM unpack failed in result pump (cached)") - fut.set_exception(e) - else: - fut.set_result(msg.output) + # Single-request result: unpack SHM first, then resolve or cache atomically. + output_result: DiffusionOutput | None = None + exc: Exception | None = None + if msg.error: + exc = RuntimeError(msg.error) + else: + try: + unpack_diffusion_output_shm(msg.output) + output_result = msg.output + except Exception as e: + logger.exception("SHM unpack failed in result pump") + exc = e + + if batch_id: with self._futures_lock: - self._completed_outputs[batch_id] = fut + pending = self._output_futures.pop(batch_id, None) + if pending is not None and not pending.done(): + if exc is not None: + pending.set_exception(exc) + else: + pending.set_result(output_result) + else: + fut = concurrent.futures.Future() + if exc is not None: + fut.set_exception(exc) + else: + fut.set_result(output_result) + self._completed_outputs[batch_id] = fut + + def _deliver_batch_split( + self, + per_req_map: dict[str, str], + batch_output: Any, + error: str | None = None, + ) -> None: + """Resolve per-request futures from one batch-level output.""" + for per_req_id, req_id in per_req_map.items(): + req_output = batch_output.get_request_output(req_id) if batch_output is not None else None + per_req_result: DiffusionOutput + if req_output is not None and req_output.result is not None: + per_req_result = req_output.result + elif error: + per_req_result = DiffusionOutput(error=error) + else: + per_req_result = DiffusionOutput(error="No output result for batch request") + with self._futures_lock: + pending = self._output_futures.pop(per_req_id, None) + if pending is not None and not pending.done(): + pending.set_result(per_req_result) + else: + fut: concurrent.futures.Future = concurrent.futures.Future() + fut.set_result(per_req_result) + self._completed_outputs[per_req_id] = fut + + def describe_pending_state(self, async_output_id: str | None = None) -> str: + """Summarize async-output bookkeeping for diagnosing stuck waits.""" + with self._futures_lock: + waiting = list(self._output_futures) + cached = list(self._completed_outputs) + batches = list(self._batch_split_map) + rpcs = list(self._rpc_futures) + return ( + f"async_output_id={async_output_id} " + f"waiting={len(waiting)}{waiting[:5]} " + f"cached={len(cached)}{cached[:5]} " + f"unsplit_batches={len(batches)}{batches[:5]} " + f"pending_rpcs={len(rpcs)}{rpcs[:5]}" + ) def _next_rpc_id(self) -> str: with self._rpc_id_lock: diff --git a/vllm_omni/diffusion/ipc.py b/vllm_omni/diffusion/ipc.py index 061f89838..251e33502 100644 --- a/vllm_omni/diffusion/ipc.py +++ b/vllm_omni/diffusion/ipc.py @@ -121,8 +121,19 @@ def _pack_tensor_if_large( val: torch.Tensor, d2h_stream: torch.Stream | None = None, ) -> torch.Tensor | dict: - """Replace a tensor with an SHM handle if it exceeds the threshold.""" - if val.nelement() * val.element_size() > _SHM_TENSOR_THRESHOLD: + """Replace a tensor with an SHM handle if it exceeds the threshold. + + Batch outputs are split into per-request views that share one storage. + Pickle serializes a view's whole storage, so a batch of small views costs + the full batch tensor once per request. Size the decision on the storage + to keep those views off the wire; packing copies just the view. + """ + view_bytes = val.nelement() * val.element_size() + try: + storage_bytes = val.untyped_storage().nbytes() + except Exception: + storage_bytes = view_bytes + if max(view_bytes, storage_bytes) > _SHM_TENSOR_THRESHOLD: return _tensor_to_shm(val, d2h_stream=d2h_stream) return val diff --git a/vllm_omni/diffusion/worker/diffusion_worker.py b/vllm_omni/diffusion/worker/diffusion_worker.py index 17fbb5342..fea78bcf9 100644 --- a/vllm_omni/diffusion/worker/diffusion_worker.py +++ b/vllm_omni/diffusion/worker/diffusion_worker.py @@ -65,6 +65,15 @@ from vllm_omni.worker.gpu_memory_utils import get_process_gpu_memory logger = init_logger(__name__) _ASYNC_OUTPUT_THREAD_JOIN_TIMEOUT_S = 10.0 +# Maximum time (in seconds) to wait for pending background D2H / SHM packing +# to drain before the worker executes memory-releasing lifecycle tasks +# (e.g. during sleep/wake transitions). This barrier prevents device tensors +# from being freed while the side CUDA stream is still actively reading them. +_ASYNC_OUTPUT_DRAIN_TIMEOUT_S = 10.0 + +# Worker entry points that release device memory. Background D2H/SHM packing +# still reads model output tensors, so it must finish before these run. +_MEMORY_RELEASING_METHODS = frozenset({"sleep", "handle_sleep_task"}) @dataclass @@ -850,6 +859,13 @@ class WorkerProc: self._async_output_queue: queue.Queue | None = None self._async_output_thread: threading.Thread | None = None + self._async_output_done = threading.Condition() + self._async_output_pending = 0 + # MessageQueue.acquire_write() is single-writer: it reads current_idx, + # writes that block, then advances the index. The async output thread + # enqueues OUTPUT_READY while the main loop enqueues COMPUTE_DONE, so + # unsynchronized writers can target the same block and drop a message. + self._result_mq_lock = threading.Lock() if not self.od_config.step_execution: self._async_output_queue = queue.Queue() self._async_output_thread = threading.Thread( @@ -882,25 +898,32 @@ class WorkerProc: ) return wrapper + def _enqueue_result(self, msg: Any) -> None: + """Serialize writes to the single-writer result queue.""" + with self._result_mq_lock: + self.result_mq.enqueue(msg) + def _return_result(self, output: Any, rpc_id: str | None = None) -> None: """Reply to client, only on rank 0.""" if self.result_mq is None: return if isinstance(output, OmniACK): - self.result_mq.enqueue(output) + self._enqueue_result(output) return # Async path: enqueue compute_done immediately, bg thread does D2H+SHM. if not self.od_config.step_execution and isinstance(output, (DiffusionOutput, BatchRunnerOutput)): async_output_id = WorkerProc._generate_async_output_id() gpu_event = current_omni_platform.record_device_event() + with self._async_output_done: + self._async_output_pending += 1 self._async_output_queue.put((output, async_output_id, gpu_event)) msg = AsyncDiffusionOutput( kind=AsyncOutputKind.COMPUTE_DONE, rpc_id=rpc_id, async_output_id=async_output_id, ) - self.result_mq.enqueue(msg) + self._enqueue_result(msg) return # Sync path (original, or async fallback). @@ -909,7 +932,7 @@ class WorkerProc: except Exception as e: if hasattr(output, "output"): logger.warning("SHM pack failed for model output: %s", e) - self.result_mq.enqueue(output) + self._enqueue_result(output) def _async_output_loop(self): """Background thread: D2H + SHM packing for async diffusion output. @@ -932,7 +955,7 @@ class WorkerProc: pack_diffusion_output_shm(output, d2h_stream=d2h_stream) d2h_stream.synchronize() - self.result_mq.enqueue( + self._enqueue_result( AsyncDiffusionOutput( kind=AsyncOutputKind.OUTPUT_READY, async_output_id=async_output_id, @@ -944,13 +967,38 @@ class WorkerProc: "Async output packing failed for id '%s'; sending error", async_output_id, ) - self.result_mq.enqueue( + self._enqueue_result( AsyncDiffusionOutput( kind=AsyncOutputKind.OUTPUT_READY, async_output_id=async_output_id, error="Background D2H/SHM packing failed", ) ) + finally: + with self._async_output_done: + # Clamped: only items enqueued by _return_result are counted. + self._async_output_pending = max(0, self._async_output_pending - 1) + self._async_output_done.notify_all() + + def drain_async_outputs(self, timeout: float = _ASYNC_OUTPUT_DRAIN_TIMEOUT_S) -> bool: + """Block until background D2H/SHM packing has no work left. + + Returns False if outputs are still in flight when ``timeout`` expires. + """ + with self._async_output_done: + if self._async_output_pending == 0: + return True + drained = self._async_output_done.wait_for(lambda: self._async_output_pending == 0, timeout=timeout) + pending = self._async_output_pending + if not drained: + logger.warning( + "Worker %d: %d async output(s) still in flight after %.1fs; " + "releasing device memory now may drop OUTPUT_READY messages", + self.gpu_id, + pending, + timeout, + ) + return drained def shutdown(self) -> None: """Stop background work and release worker-owned IPC resources.""" @@ -1049,6 +1097,8 @@ class WorkerProc: } try: + if method in _MEMORY_RELEASING_METHODS: + self.drain_async_outputs() # Use execute_method from WorkerWrapperBase for consistent method resolution result = self.worker.execute_method(method, *args, **kwargs) except Exception as e: @@ -1114,6 +1164,7 @@ class WorkerProc: continue if isinstance(msg, dict) and msg.get("type") == "sleep": + self.drain_async_outputs() task = OmniSleepTask(level=msg.get("level", 2), task_id=msg.get("task_id", "local")) ack = self.worker.handle_sleep_task(task) self._return_result(ack) @@ -1140,7 +1191,7 @@ class WorkerProc: if rpc_id is not None: # Async RPC: must complete the executor's pending # future so collective_rpc() doesn't hang. - self.result_mq.enqueue( + self._enqueue_result( AsyncDiffusionOutput( kind=AsyncOutputKind.RPC_RESULT, rpc_id=rpc_id, From 555336e887da035aa473754d3f87ca7d8eb878a4 Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Sat, 15 Aug 2026 04:51:56 -0700 Subject: [PATCH 029/267] [CI] Refresh Voxtral-4B-TTS perf baselines from a clean window (#6032) Signed-off-by: Yueqian Lin --- tests/dfx/perf/tests/test_tts.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index f398e3271..46aa5e83f 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -424,10 +424,10 @@ "baseline": { "H100": { "median_audio_ttfp_ms": [ - 82.1886 + 82.6344 ], "median_audio_rtf": [ - 0.1241 + 0.1277 ] } } @@ -453,13 +453,13 @@ "baseline": { "H100": { "median_audio_ttfp_ms": [ - 160.9168 + 170.3737 ], "median_audio_rtf": [ - 0.1907 + 0.2076 ], "audio_throughput": [ - 39.2332 + 37.0834 ] } } @@ -485,13 +485,13 @@ "baseline": { "H100": { "median_audio_ttfp_ms": [ - 100.1265 + 101.0293 ], "median_audio_rtf": [ - 0.1437 + 0.1498 ], "audio_throughput": [ - 11.7605 + 10.1186 ] } } From 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 Mon Sep 17 00:00:00 2001 From: m0g3r <87276771+m0g3r@users.noreply.github.com> Date: Sat, 15 Aug 2026 15:00:27 +0200 Subject: [PATCH 030/267] [Bugfix] Restore per-stage runtime env during launch (#6214) Signed-off-by: m0g3r <87276771+m0g3r@users.noreply.github.com> --- .../test_async_omni_engine_stage_init.py | 88 +++++++++++++++++++ vllm_omni/engine/stage_runtime.py | 8 +- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/tests/engine/test_async_omni_engine_stage_init.py b/tests/engine/test_async_omni_engine_stage_init.py index 27aee8706..9ee59f153 100644 --- a/tests/engine/test_async_omni_engine_stage_init.py +++ b/tests/engine/test_async_omni_engine_stage_init.py @@ -161,6 +161,17 @@ def _make_diffusion_plan( ) +def _make_stage_runtime() -> StageRuntime: + return StageRuntime( + stage_configs=[], + model="dummy-model", + config_path="dummy-config", + stage_init_timeout=1, + diffusion_batch_size=1, + async_chunk=False, + ) + + def test_stage_engine_core_client_module_reload_keeps_forward_refs_deferred(): """Regression test for forward references in make_async_mp_client.""" import vllm_omni.engine.stage_engine_core_client as client_mod @@ -233,6 +244,48 @@ def test_collect_initialized_clients_for_cleanup_deduplicates_clients(): assert cleanup_clients == [shared, extra] +def test_initialize_local_diffusion_replica_scopes_runtime_env(monkeypatch): + import vllm_omni.engine.stage_runtime as runtime_mod + from vllm_omni.platforms import current_omni_platform + + runtime = _make_stage_runtime() + plan = _make_diffusion_plan(0, stage_id=0).replicas[0] + + runtime_env_var = "VLLM_OMNI_TEST_STAGE_RUNTIME_ENV" + device_env_var = current_omni_platform.device_control_env_var + runtime._init_visible_devices_baseline = "0,1" + plan.metadata.runtime_cfg = { + "devices": "0", + "env": {runtime_env_var: "stage-value"}, + } + monkeypatch.delenv(runtime_env_var, raising=False) + monkeypatch.setenv(device_env_var, "0,1") + + captured: dict[str, str | None] = {} + monkeypatch.setattr(runtime_mod, "inject_kv_stage_info", lambda *_: None) + + def _capture_launch_diffusion_stage_replica(**_kwargs): + captured["runtime_env"] = os.environ.get(runtime_env_var) + captured["device_env"] = os.environ.get(device_env_var) + raise RuntimeError("stop after capturing launch environment") + + monkeypatch.setattr( + runtime_mod, + "launch_diffusion_stage_replica", + _capture_launch_diffusion_stage_replica, + ) + + with pytest.raises(RuntimeError, match="stop after capturing launch environment"): + runtime._initialize_local_diffusion_replica(plan, stage_init_timeout=1) + + assert captured == { + "runtime_env": "stage-value", + "device_env": "0", + } + assert runtime_env_var not in os.environ + assert os.environ[device_env_var] == "0,1" + + def test_initialize_local_diffusion_replica_restores_device_visibility_after_local_init(monkeypatch): import vllm_omni.engine.stage_runtime as runtime_mod from vllm_omni.engine.stage_engine_startup import StageReplicaResources @@ -311,6 +364,41 @@ def test_initialize_local_diffusion_replica_passes_stage_init_timeout_and_inline } +def test_initialize_local_llm_replica_scopes_runtime_env(monkeypatch): + import vllm_omni.engine.stage_runtime as runtime_mod + + runtime = _make_stage_runtime() + plan = _make_llm_plan(0, stage_id=0, vllm_config=object()).replicas[0] + plan.engine_args_dict = {} + + runtime_env_var = "VLLM_OMNI_TEST_STAGE_RUNTIME_ENV" + plan.metadata.runtime_cfg = { + "devices": "0", + "env": {runtime_env_var: "stage-value"}, + } + monkeypatch.setenv(runtime_env_var, "parent-value") + + captured: dict[str, str | None] = {} + + @contextlib.contextmanager + def _capture_launch_stage_replica(*, stage_visible_devices, **_kwargs): + captured["runtime_env"] = os.environ.get(runtime_env_var) + captured["stage_visible_devices"] = stage_visible_devices + yield None + + monkeypatch.setattr(runtime_mod, "acquire_device_locks", lambda *_args, **_kwargs: []) + monkeypatch.setattr(runtime_mod, "launch_stage_replica", _capture_launch_stage_replica) + + with pytest.raises(RuntimeError, match="launcher returned no resources"): + runtime._initialize_local_llm_replica(plan, stage_init_timeout=1) + + assert captured == { + "runtime_env": "stage-value", + "stage_visible_devices": "0", + } + assert os.environ[runtime_env_var] == "parent-value" + + def test_stage_runtime_initializes_stage_pools(monkeypatch): import vllm_omni.engine.stage_runtime as runtime_mod diff --git a/vllm_omni/engine/stage_runtime.py b/vllm_omni/engine/stage_runtime.py index bf84f7793..c0aafc86a 100644 --- a/vllm_omni/engine/stage_runtime.py +++ b/vllm_omni/engine/stage_runtime.py @@ -58,6 +58,7 @@ from vllm_omni.engine.stage_init_utils import ( load_omni_transfer_config_for_model, prepare_engine_environment, release_device_locks, + stage_runtime_env, ) from vllm_omni.engine.stage_pool import StagePool from vllm_omni.entrypoints.stage_utils import resolve_stage_physical_devices @@ -576,7 +577,7 @@ class StageRuntime: ) # Serialize engine-core spawning across all LLM replicas to avoid # ZMQ port-allocation races and simultaneous CUDA context init. - with self._replica_launch_lock: + with self._replica_launch_lock, stage_runtime_env(plan.metadata.stage_id, plan.metadata.runtime_cfg): with launch_stage_replica( vllm_config=vllm_config, executor_class=executor_class, @@ -645,7 +646,10 @@ class StageRuntime: client = None resources = None try: - with self._stage_device_scope(plan.metadata.stage_id, plan.metadata.runtime_cfg): + with ( + stage_runtime_env(plan.metadata.stage_id, plan.metadata.runtime_cfg), + self._stage_device_scope(plan.metadata.stage_id, plan.metadata.runtime_cfg), + ): omni_conn_cfg, omni_from, omni_to = plan.omni_kv_connector if omni_conn_cfg: inject_omni_kv_config(plan.stage_cfg, omni_conn_cfg, omni_from, omni_to) From d1e230c95ba12aec7664ee6fd18c0b2b2d0d6187 Mon Sep 17 00:00:00 2001 From: SYLAR <125541396+lishunyang12@users.noreply.github.com> Date: Sat, 15 Aug 2026 23:01:26 +0800 Subject: [PATCH 031/267] [Diffusion][Quantization] Enable MiniMax-H3 global FP8 with DLO (#5910) Signed-off-by: lishunyang12 Co-authored-by: Hongsheng Liu --- docs/design/feature/quantization.md | 14 +- docs/user_guide/quantization/fp8.md | 35 ++- recipes/MiniMaxAI/MiniMax-H3.md | 59 +++- .../model_loader/test_diffusers_loader.py | 79 ++++++ .../flux2/test_pipeline_flux2_quantization.py | 10 +- .../minimax_h3/test_minimax_h3_contract.py | 129 +++++++-- .../test_minimax_h3_quantization.py | 15 + .../test_distributed_layerwise_backend.py | 53 ++++ .../offloader/test_layerwise_backend.py | 37 ++- .../quantization/test_component_routing.py | 21 +- .../model_loader/diffusers_loader.py | 79 +++++- .../diffusion/models/flux2/pipeline_flux2.py | 7 +- .../diffusion/models/minimax_h3/encoder.py | 266 +++++++++++++----- .../models/minimax_h3/pipeline_minimax_h3.py | 55 ++-- .../distributed_layerwise_backend.py | 73 ++++- .../diffusion/offloader/layerwise_backend.py | 48 +++- vllm_omni/quantization/__init__.py | 3 +- vllm_omni/quantization/component_config.py | 15 + 18 files changed, 833 insertions(+), 165 deletions(-) diff --git a/docs/design/feature/quantization.md b/docs/design/feature/quantization.md index 7ad612d12..8d6cd09c3 100644 --- a/docs/design/feature/quantization.md +++ b/docs/design/feature/quantization.md @@ -76,6 +76,12 @@ equivalent. `auto-round` and `auto_round` use the Omni INC/AutoRound adapter. ModelOpt checkpoint metadata is detected from its method, producer metadata, or `quant_algo`, then mapped to the corresponding vLLM ModelOpt configuration. +Unless the input is a per-component dictionary, the resolved configuration is +passed unchanged to every quantization-aware component constructed by the +pipeline. It affects only layers supported by that component's quantization +method; it does not rewrite arbitrary `torch.nn` modules. A per-component +dictionary narrows this global scope through runtime layer-prefix routing. + The factory uses Omni-specific overrides for methods whose diffusion behavior or platform dispatch is not provided by the vLLM registry. The current override set includes INT8, BitsAndBytes, MXFP8, MXFP4, MXFP4 dual-scale, and INC/ @@ -154,9 +160,11 @@ quantizable vLLM layers. Each method configuration is responsible for: Model code must keep precision-sensitive or unsupported modules unquantized. The shared `safe_quant_config()` helper is used by model integrations for normalization and modulation layers where a generic FP8 configuration would be -unsafe. Text encoders, audio/vision encoders, VAEs, schedulers, and tokenizers -are not implicitly quantized just because the main transformer is quantized; -they require explicit component routing and method support. +unsafe. Encoders exposed through the quantization factory receive a global +configuration just like the main transformer, but only their supported vLLM +quantizable layers are affected. Ordinary `torch.nn` encoder layers, VAEs, +schedulers, and tokenizers are not rewritten automatically. A component map can +select a narrower scope, but it cannot add quantization support to a layer. Weight quantization and quantized KV cache are separate paths. The former is resolved through `quantization_config`; runtime attention/KV-cache quantization diff --git a/docs/user_guide/quantization/fp8.md b/docs/user_guide/quantization/fp8.md index 05473b15e..397a412e8 100644 --- a/docs/user_guide/quantization/fp8.md +++ b/docs/user_guide/quantization/fp8.md @@ -95,7 +95,7 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)]) | HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | All layers; use the Hunyuan stage config for multi-stage runs | None | | | HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None | | | Cosmos3 | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | Yes | Not validated | All layers | None | | -| MiniMax H3 | `MiniMaxAI/MiniMax-H3` | Yes | Not validated | DiT linears except patch, timestep, and final projections; not compatible with layerwise offload | None | | +| MiniMax-H3 | `MiniMaxAI/MiniMax-H3` (`FL2VA` / `Ref2VA`) | Yes | Not validated | `quantization="fp8"` quantizes eligible DiT and text-encoder linears; mixed-precision input/output heads stay FP32 | None | ✅︎ | ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS) @@ -138,6 +138,39 @@ outputs = omni.generate( ) ``` +### Global and per-component scope + +A plain, global quantization configuration is passed unchanged to every +quantization-aware component constructed by a pipeline. This includes an +eligible encoder when that encoder is implemented with vLLM quantizable +layers; it does not rewrite arbitrary `torch.nn` modules. A structured +component map is the only way to narrow that scope. Pipeline integrations that +do not yet expose an encoder through the quantization factory remain DiT-only. + +For a pipeline that exposes both a transformer and a quantization-aware text +encoder, the scope is: + +| Configuration | Transformer | Text encoder | Components without supported quantizable layers | +|---------------|-------------|--------------|-------------------------------------------------| +| `quantization="fp8"` | FP8 | FP8 | checkpoint precision | +| `{"transformer": {"method": "fp8"}}` | FP8 | checkpoint precision | checkpoint precision | +| `{"text_encoder": {"method": "fp8"}}` | checkpoint precision | FP8 | checkpoint precision | + +Use `quantization_config` for component-selective Python configuration. For +example, quantize only the text decoder: + +```python +omni = Omni( + model="", + quantization_config={"text_encoder": {"method": "fp8"}}, +) +``` + +Component keys are runtime prefixes exposed by the pipeline integration; common +keys include `transformer` and `text_encoder`. Entries may be combined, and +`ignored_layers` can keep named eligible layers in checkpoint precision. Check +the model recipe for supported components and their runtime prefixes. + CLI: ```bash diff --git a/recipes/MiniMaxAI/MiniMax-H3.md b/recipes/MiniMaxAI/MiniMax-H3.md index 114352343..f6e5dd43b 100644 --- a/recipes/MiniMaxAI/MiniMax-H3.md +++ b/recipes/MiniMaxAI/MiniMax-H3.md @@ -357,12 +357,19 @@ No restart is needed: `task=fl2va` routes to `FL2VA/transformer`, while ### Online FP8 quantization -MiniMax H3 supports load-time FP8 quantization of the DiT. The checkpoint -remains BF16 on disk; vLLM-Omni quantizes eligible weights while loading and -uses dynamic activation scaling during inference. By default, attention and -MLP linears in the token refiner and main DiT blocks, the condition -projection, and all AdaLN projections use FP8. Patch, timestep, and final -projections remain FP32; the text encoder and VAEs are unchanged. +MiniMax H3 supports online FP8 quantization of both the DiT and the Qwen3-VL +text decoder. The checkpoint remains BF16 on disk; vLLM-Omni creates FP8 +weights at runtime and uses dynamic activation scaling during inference. By +default, `--quantization fp8` quantizes eligible attention and MLP linears in +the text decoder, token refiner, and main DiT blocks, as well as the DiT +condition and AdaLN projections. The Qwen vision tower, embeddings, norms, +RoPE, both VAEs, and the model's FP32 patch, timestep, and output projections +keep checkpoint precision. + +To select a component, use `--diffusion-quantization-config` with +`{"transformer":{"method":"fp8"}}` for DiT-only FP8 or +`{"text_encoder":{"method":"fp8"}}` for text-decoder-only FP8. The two +entries can be combined. The shorthand below enables both components. Add this option to an existing H3 server command: @@ -370,6 +377,28 @@ Add this option to an existing H3 server command: --quantization fp8 ``` +#### Single 96 GB GPU, no-offload capacity check + +Use the FL2VA-only partition for this capacity test. Loading the combined +service would also load the Ref2VA DiT and would test a different memory +budget. A no-offload capacity check should contain none of +`--enable-cpu-offload`, `--enable-layerwise-offload`, or +`--enable-distributed-layerwise-offload`. VAE tiling changes decode placement +but does not offload model weights to the CPU. + +The run passes the capacity check when the server initializes, the request +finishes without CUDA OOM or Xid errors, `peak_used_mib` remains below the +card's reported `memory_total_mib`, and `ffprobe` reports H.264 video plus +32 kHz stereo AAC audio. Report the measured headroom rather than assuming +that every nominal 96 GB SKU exposes the same MiB total. + +As a capacity proxy only, the same five-second case on one B300 measured a +92,946 MiB whole-device first-request peak and a 92,146 MiB worker peak. Its +encode, diffuse, decode, and client wall times were 8.664 s, 134.504 s, +6.016 s, and 151.583 s. These numbers suggest that a 96 GB card may fit, but +they are not an RTX PRO 6000 validation: kernels, allocator behavior, and +reported device capacity differ across GPUs. + Use `ignored_layers` to keep any otherwise eligible linear in BF16. H3 resolves the `transformer` component before constructing the DiT, so names do not start with `transformer.`. Entries are exact runtime linear prefixes; a @@ -386,10 +415,10 @@ For example, keep the first main block's attention projections in BF16 with: '{"transformer":{"method":"fp8","ignored_layers":["blocks.0.attn.qkv_proj","blocks.0.attn.out_proj"]}}' ``` -The structured option replaces `--quantization fp8`. Online FP8 is currently -incompatible with H3 layerwise offload because the offload path produces a -weight stride rejected by the Cutlass FP8 kernel. Use resident FP8 with tensor -parallelism and VAE tiling instead. +The structured option replaces `--quantization fp8`. Online FP8 can be used +with H3 layerwise offload and with distributed layerwise offload's full-weight +per-rank path (`--dlo-no-use-allgather`). The sharded DLO AllGather path is not +supported for runtime-created FP8 weights. ## AMD ROCm (gfx942 / gfx950) @@ -750,6 +779,13 @@ audio spectral cosine 0.9589 (minimum 0.80), and audio RMS ratio 0.9342. The resident per-GPU peak was 68.52 GiB for BF16 and 53.51 GiB for FP8, a 22% reduction. +For direct human inspection, see the external +[BF16 versus global-FP8 comparison](https://lishunyang12.github.io/vllm-omni-rankings/scripts/minimax_h3_global_fp8_vs_bf16/), +which includes matched five-second T2VA, I2VA, and Ref2VA videos. The +[comparison sources](https://github.com/lishunyang12/vllm-omni-rankings/tree/main/scripts/minimax_h3_global_fp8_vs_bf16) +also record per-task fidelity metrics and provenance without storing generated +media in this repository. + ## TeaCache acceleration TeaCache reuses DiT block residuals across denoising steps when consecutive @@ -834,7 +870,8 @@ vllm serve "${MODEL_ROOT}/FL2VA" \ H3 native `tile` mode only. - A U2 x Ring2 hybrid currently fails with an attention-mask length mismatch; use pure Ulysses. -- Online FP8 is not compatible with layerwise offload. +- Runtime-created FP8 weights do not support the sharded DLO AllGather path; + use `--dlo-no-use-allgather` when combining online FP8 with DLO. - TeaCache and Cache-DiT cannot be enabled on the same server. - Image+audio Ref2VA accepts exactly one image and one audio reference. - Video Ref2VA accepts one or more video files, but not an additional standalone diff --git a/tests/diffusion/model_loader/test_diffusers_loader.py b/tests/diffusion/model_loader/test_diffusers_loader.py index f370d5b49..69600d811 100644 --- a/tests/diffusion/model_loader/test_diffusers_loader.py +++ b/tests/diffusion/model_loader/test_diffusers_loader.py @@ -110,6 +110,85 @@ def test_empty_source_prefix_keeps_full_model_strict_check(): loader.load_weights(model) +def test_stream_online_quant_weights_offloads_layers_after_processing(): + from vllm.model_executor.model_loader.reload.layerwise import ( + get_layerwise_info, + ) + + events: list[str] = [] + + class _OnlineQuantMethod: + uses_meta_device = True + + class _TrackedLayer(nn.Linear): + def __init__(self, name: str): + super().__init__(2, 2, bias=False) + self.name = name + self.quant_method = _OnlineQuantMethod() + get_layerwise_info(self).load_numel_total = self.weight.numel() + + def to(self, *args, **kwargs): + events.append(self.name) + return super().to(*args, **kwargs) + + class _StreamingModel(nn.Module): + def __init__(self): + super().__init__() + self.first = _TrackedLayer("first") + self.second = _TrackedLayer("second") + + model = _StreamingModel() + weights = iter( + [ + ("first.weight", torch.zeros((2, 2))), + ("second.weight", torch.zeros((2, 2))), + ] + ) + streamed = DiffusersPipelineLoader._stream_online_quant_weights_to_cpu(model, weights) + + assert next(streamed)[0] == "first.weight" + get_layerwise_info(model.first).reset() + assert next(streamed)[0] == "second.weight" + assert events == ["first"] + + get_layerwise_info(model.second).reset() + with pytest.raises(StopIteration): + next(streamed) + assert events == ["first", "second"] + + +def test_process_weights_skips_completed_online_quant_layer(monkeypatch): + from unittest.mock import Mock + + from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase + from vllm.model_executor.model_loader.reload import layerwise + + class _TrackedLayer(nn.Linear): + def __init__(self): + super().__init__(2, 2, bias=False) + self.to_calls: list[object] = [] + + def to(self, *args, **kwargs): + self.to_calls.append(args[0] if args else kwargs.get("device")) + return super().to(*args, **kwargs) + + model = nn.Module() + model.layer = _TrackedLayer() + quant_method = Mock(spec=QuantizeMethodBase) + quant_method.uses_meta_device = True + model.layer.quant_method = quant_method + model.layer._already_called_process_weights_after_loading = True + finalize = Mock() + monkeypatch.setattr(layerwise, "finalize_layerwise_processing", finalize) + + loader = _make_loader_with_weights([]) + loader._process_weights_after_loading(model, torch.device("cuda")) + + finalize.assert_called_once_with(model, model_config=None) + quant_method.process_weights_after_loading.assert_not_called() + assert model.layer.to_calls == [] + + class _ConfigAwareModel(nn.Module): def __init__(self, *, od_config): super().__init__() diff --git a/tests/diffusion/models/flux2/test_pipeline_flux2_quantization.py b/tests/diffusion/models/flux2/test_pipeline_flux2_quantization.py index 3900c0c94..ff9e0e110 100644 --- a/tests/diffusion/models/flux2/test_pipeline_flux2_quantization.py +++ b/tests/diffusion/models/flux2/test_pipeline_flux2_quantization.py @@ -7,17 +7,17 @@ from vllm_omni.diffusion.models.flux2.pipeline_flux2 import ( Flux2Pipeline, _resolve_component_quant_config, ) +from vllm_omni.quantization import ComponentQuantizationConfig pytestmark = [pytest.mark.core_model, pytest.mark.cpu] def test_resolve_component_quant_config_routes_components(): text_encoder_config = object() - component_config = Mock() - component_config.resolve.side_effect = { - "text_encoder": text_encoder_config, - "transformer": None, - }.get + component_config = ComponentQuantizationConfig( + component_configs={"text_encoder": text_encoder_config}, + default_config=None, + ) assert _resolve_component_quant_config(component_config, "text_encoder") is text_encoder_config assert _resolve_component_quant_config(component_config, "transformer") is None diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py index f872e381f..34250abf2 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py @@ -5,7 +5,7 @@ import json import sys from multiprocessing.reduction import ForkingPickler from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import Mock, patch import numpy as np import pytest @@ -363,8 +363,18 @@ def test_pipeline_loads_task_selected_dits_and_shared_components_once( if expected_partition == "ref2va": assert pipeline._transformer_for_task("ref2va") is pipeline.transformer assert [source.model_or_path for source in pipeline.weights_sources] == [ - str(tmp_path / partition_name) for partition_name in source_partitions + *(str(tmp_path / partition_name) for partition_name in source_partitions), + str(component_path), ] + assert [source.subfolder for source in pipeline.weights_sources] == [ + *("transformer" for _ in source_partitions), + "text_encoder", + ] + expected_prefixes = ["transformer."] + if expected_dits == 2: + expected_prefixes.append("transformers_ref.") + expected_prefixes.append("text_encoder.") + assert [source.prefix for source in pipeline.weights_sources] == expected_prefixes expected_patterns = ( pipeline_module.MINIMAX_H3_DOWNLOAD_PATTERNS if expected_partition == "combined" @@ -936,6 +946,66 @@ def test_text_encoder_stub_constructs_without_group_or_weights(): assert list(encoder.named_parameters()) == [] +def test_global_quant_config_is_shared_by_dit_and_encoder(): + from vllm_omni.quantization import build_quant_config, resolve_component_quant_config + + global_config = build_quant_config("fp8") + + assert resolve_component_quant_config(global_config, "transformer") is global_config + assert resolve_component_quant_config(global_config, "text_encoder") is global_config + + +def test_minimax_h3_fp8_component_selection(): + from vllm_omni.quantization import build_quant_config, resolve_component_quant_config + + dit_only = build_quant_config({"transformer": {"method": "fp8"}}) + encoder_only = build_quant_config({"text_encoder": {"method": "fp8"}}) + + assert resolve_component_quant_config(dit_only, "transformer").get_name() == "fp8" + assert resolve_component_quant_config(dit_only, "text_encoder") is None + assert resolve_component_quant_config(encoder_only, "transformer") is None + assert resolve_component_quant_config(encoder_only, "text_encoder").get_name() == "fp8" + + +def test_text_encoder_rejects_serialized_fp8(): + from vllm_omni.diffusion.models.minimax_h3.encoder import _validate_encoder_quant_config + + config = SimpleNamespace( + get_name=lambda: "fp8", + is_checkpoint_fp8_serialized=True, + ) + with pytest.raises(ValueError, match="online FP8 only"): + _validate_encoder_quant_config(config) + + +def test_text_encoder_linear_delegates_quantization_to_vllm_factory(): + from vllm.model_executor.layers.linear import UnquantizedLinearMethod + + import vllm_omni.diffusion.models.minimax_h3.encoder as encoder_module + + group = SimpleNamespace(rank_in_group=0, world_size=1) + method = UnquantizedLinearMethod() + quant_config = Mock() + quant_config.get_quant_method.return_value = method + prefix = "text_encoder.text_model.layers.0.mlp.gate_up_proj" + + with ( + patch("vllm.model_executor.parameter.get_tensor_model_parallel_rank", return_value=0), + patch("vllm.model_executor.parameter.get_tensor_model_parallel_world_size", return_value=1), + ): + linear = encoder_module.MiniMaxH3Qwen3VLMergedColumnParallelLinear( + group, + input_size=4, + intermediate_size=8, + dtype=torch.bfloat16, + quant_config=quant_config, + prefix=prefix, + ) + + assert linear.quant_method is method + quant_config.get_quant_method.assert_called_once_with(linear, prefix=prefix) + + def test_no_offload_keeps_text_encoder_resident(): from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline @@ -978,15 +1048,21 @@ def test_model_offload_uses_hooked_text_encoder_call(): pipeline.text_encoder.load_to_device.assert_not_called() -def test_layerwise_offload_releases_text_encoder(): +@pytest.mark.parametrize( + "offload_flag", + ["enable_layerwise_offload", "enable_distributed_layerwise_offload"], +) +def test_layerwise_offload_releases_text_encoder(offload_flag): from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline pipeline = object.__new__(MiniMaxH3Pipeline) torch.nn.Module.__init__(pipeline) pipeline.od_config = SimpleNamespace( enable_cpu_offload=False, - enable_layerwise_offload=True, + enable_layerwise_offload=False, + enable_distributed_layerwise_offload=False, ) + setattr(pipeline.od_config, offload_flag, True) pipeline.text_encoder = Mock() expected = torch.ones(2, 3) pipeline.text_encoder.encode_ids.return_value = expected @@ -1988,21 +2064,25 @@ def _one_layer_text_encoder(): group = _SingleRankEncoderGroup(0) layer = nn.Module() layer.self_attn = nn.Module() - layer.self_attn.qkv_proj = MiniMaxH3Qwen3VLQKVParallelLinear( - group, - hidden_size=_ENCODER_HIDDEN, - num_heads=_ENCODER_NUM_HEADS, - num_kv_heads=_ENCODER_NUM_KV_HEADS, - head_dim=_ENCODER_HEAD_DIM, - dtype=torch.float32, - ) layer.mlp = nn.Module() - layer.mlp.gate_up_proj = MiniMaxH3Qwen3VLMergedColumnParallelLinear( - group, - input_size=_ENCODER_HIDDEN, - intermediate_size=_ENCODER_INTERMEDIATE, - dtype=torch.float32, - ) + with ( + patch("vllm.model_executor.parameter.get_tensor_model_parallel_rank", return_value=0), + patch("vllm.model_executor.parameter.get_tensor_model_parallel_world_size", return_value=1), + ): + layer.self_attn.qkv_proj = MiniMaxH3Qwen3VLQKVParallelLinear( + group, + hidden_size=_ENCODER_HIDDEN, + num_heads=_ENCODER_NUM_HEADS, + num_kv_heads=_ENCODER_NUM_KV_HEADS, + head_dim=_ENCODER_HEAD_DIM, + dtype=torch.float32, + ) + layer.mlp.gate_up_proj = MiniMaxH3Qwen3VLMergedColumnParallelLinear( + group, + input_size=_ENCODER_HIDDEN, + intermediate_size=_ENCODER_INTERMEDIATE, + dtype=torch.float32, + ) layer.input_layernorm = MiniMaxH3Qwen3VLRMSNorm(_ENCODER_HIDDEN) encoder = object.__new__(MiniMaxH3Qwen3VLEncoder) @@ -2035,6 +2115,7 @@ def _write_text_encoder_checkpoint(path, *, omit=()): safetensors.torch.save_file(tensors, str(path / "model.safetensors")) index = {"weight_map": dict.fromkeys(tensors, "model.safetensors")} (path / "model.safetensors.index.json").write_text(json.dumps(index)) + return tensors @pytest.mark.parametrize( @@ -2056,10 +2137,10 @@ def test_encoder_load_reports_missing_fused_source_shards( tmp_path, omitted_sources, expected_detail, unreported_target ): encoder = _one_layer_text_encoder() - _write_text_encoder_checkpoint(tmp_path, omit=omitted_sources) + weights = _write_text_encoder_checkpoint(tmp_path, omit=omitted_sources) with pytest.raises(RuntimeError) as excinfo: - encoder._load_weights(str(tmp_path)) + encoder.load_weights(weights.items()) message = str(excinfo.value) assert expected_detail in message @@ -2068,17 +2149,17 @@ def test_encoder_load_reports_missing_fused_source_shards( def test_encoder_load_rejects_unloaded_plain_param(tmp_path): encoder = _one_layer_text_encoder() - _write_text_encoder_checkpoint(tmp_path, omit=("input_layernorm.weight",)) + weights = _write_text_encoder_checkpoint(tmp_path, omit=("input_layernorm.weight",)) with pytest.raises(RuntimeError, match=r"weights not loaded.*input_layernorm"): - encoder._load_weights(str(tmp_path)) + encoder.load_weights(weights.items()) def test_encoder_load_places_every_source_in_its_own_rows(tmp_path): encoder = _one_layer_text_encoder() - _write_text_encoder_checkpoint(tmp_path) + weights = _write_text_encoder_checkpoint(tmp_path) - encoder._load_weights(str(tmp_path)) + encoder.load_weights(weights.items()) params = dict(encoder.named_parameters()) q_rows = _ENCODER_NUM_HEADS * _ENCODER_HEAD_DIM diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py index b9ae3335c..5f220d6ab 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py @@ -211,3 +211,18 @@ def test_pipeline_resolves_transformer_component_quant_config(): assert transformer_config.ignored_layers == ignored_layers assert _resolve_component_quant_config(component_config, "transformer") is transformer_config assert _resolve_component_quant_config(transformer_config, "transformer") is transformer_config + + +def test_pipeline_strips_prequantized_text_encoder_config(): + from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import _resolve_minimax_h3_text_encoder_quant_config + from vllm_omni.quantization import ComponentQuantizationConfig + + prequantized = Mock() + prequantized.get_name.return_value = "modelopt" + online_fp8 = Mock() + online_fp8.get_name.return_value = "fp8" + component_config = ComponentQuantizationConfig({"text_encoder": prequantized}) + + assert _resolve_minimax_h3_text_encoder_quant_config(online_fp8) is online_fp8 + assert _resolve_minimax_h3_text_encoder_quant_config(prequantized) is None + assert _resolve_minimax_h3_text_encoder_quant_config(component_config) is None diff --git a/tests/diffusion/offloader/test_distributed_layerwise_backend.py b/tests/diffusion/offloader/test_distributed_layerwise_backend.py index fd004d056..4c0cceb68 100644 --- a/tests/diffusion/offloader/test_distributed_layerwise_backend.py +++ b/tests/diffusion/offloader/test_distributed_layerwise_backend.py @@ -139,6 +139,41 @@ class TestDistributedLayerwiseOffloadHook: assert shard.numel() == 4 assert torch.equal(shard, _make_values(10.0)) + def test_prefetch_preserves_transposed_weight_stride( + self, + dist_group, + patched_offload_runtime, + ): + """Online-FP8 Cutlass weights must retain their transposed layout.""" + + class StridedBlock(nn.Module): + def __init__(self, start: float): + super().__init__() + base = torch.arange(start, start + 12).reshape(3, 4) + self.weight = nn.Parameter(base.t(), requires_grad=False) + + current_block = StridedBlock(1.0) + next_block = StridedBlock(20.0) + expected = next_block.weight.detach().clone() + expected_stride = next_block.weight.stride() + assert expected_stride == (1, 4) + + hook = DistributedLayerwiseOffloadHook( + next_block=next_block, + device=torch.device("cpu"), + dp_group=None, + dp_size=1, + rank=0, + copy_stream=DummyStream(), + comm_stream=DummyStream(), + pin_memory=False, + ) + hook.initialize_hook(current_block) + hook.prefetch_layer(slot=0, non_blocking=False) + + assert next_block.weight.stride() == expected_stride + assert torch.equal(next_block.weight, expected) + def test_dtensor_wrapper_preserved_across_prefetch_and_offload(self, dist_group, patched_offload_runtime): """DTensor wrapper should be preserved through prefetch/offload cycle.""" current_block = TinyBlock(_make_values(1.0)) @@ -386,6 +421,24 @@ class TestPinnedResidentLayerGroup: group.offload() + def test_load_preserves_noncontiguous_weight_stride(self, patched_offload_runtime): + block = nn.Module() + weight = torch.arange(6, dtype=torch.float32).reshape(2, 3).t() + expected = weight.detach().clone() + expected_stride = weight.stride() + block.weight = nn.Parameter(weight) + group = PinnedResidentLayerGroup( + [block], + device=torch.device("cpu"), + copy_stream=DummyStream(), + pin_memory=False, + ) + + group.load() + assert torch.equal(block.weight, expected) + assert block.weight.stride() == expected_stride + group.offload() + def test_mmap_loader_attrs_survive_to_empty_parameter_replacement(self): module = nn.Linear(2, 2, bias=False) diff --git a/tests/diffusion/offloader/test_layerwise_backend.py b/tests/diffusion/offloader/test_layerwise_backend.py index 5fd80e75c..f62bd6472 100644 --- a/tests/diffusion/offloader/test_layerwise_backend.py +++ b/tests/diffusion/offloader/test_layerwise_backend.py @@ -79,11 +79,11 @@ def dist_group(): @pytest.fixture -def patched_offload_runtime(mocker): - mocker.patch.object(layerwise_backend_module.current_omni_platform, "Stream", DummyStream) - mocker.patch.object(layerwise_backend_module.current_omni_platform, "Event", DummyEvent) - mocker.patch.object(layerwise_backend_module.current_omni_platform, "current_stream", lambda: DummyStream()) - mocker.patch.object(layerwise_backend_module.current_omni_platform, "stream", dummy_stream) +def patched_offload_runtime(monkeypatch): + monkeypatch.setattr(layerwise_backend_module.current_omni_platform, "Stream", DummyStream) + monkeypatch.setattr(layerwise_backend_module.current_omni_platform, "Event", DummyEvent) + monkeypatch.setattr(layerwise_backend_module.current_omni_platform, "current_stream", lambda: DummyStream()) + monkeypatch.setattr(layerwise_backend_module.current_omni_platform, "stream", dummy_stream) class TinyBlock(nn.Module): @@ -128,6 +128,33 @@ class TestLayerwiseOffloadHook: assert current_block.weight.to_local().shape == torch.Size([4]) assert not hook.is_materialized + def test_prefetch_preserves_transposed_weight_stride(self, patched_offload_runtime): + """Online-FP8 Cutlass weights must retain their transposed layout.""" + + class StridedBlock(nn.Module): + def __init__(self, start: float): + super().__init__() + base = torch.arange(start, start + 12).reshape(3, 4) + self.weight = nn.Parameter(base.t(), requires_grad=False) + + current_block = StridedBlock(1.0) + next_block = StridedBlock(20.0) + expected = next_block.weight.detach().clone() + expected_stride = next_block.weight.stride() + assert expected_stride == (1, 4) + + hook = LayerwiseOffloadHook( + next_block=next_block, + device=torch.device("cpu"), + stream=DummyStream(), + pin_memory=False, + ) + hook.initialize_hook(current_block) + hook.prefetch_layer(non_blocking=False) + + assert next_block.weight.stride() == expected_stride + assert torch.equal(next_block.weight, expected) + class _DummyBlock(nn.Module): def __init__(self): diff --git a/tests/diffusion/quantization/test_component_routing.py b/tests/diffusion/quantization/test_component_routing.py index c8b3837e2..9a306b6e4 100644 --- a/tests/diffusion/quantization/test_component_routing.py +++ b/tests/diffusion/quantization/test_component_routing.py @@ -16,10 +16,11 @@ from vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker import ( ) from vllm_omni.quantization.component_config import ( ComponentQuantizationConfig, + resolve_component_quant_config, ) from vllm_omni.quantization.inc_config import OmniINCConfig -pytestmark = [pytest.mark.core_model] +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] # --------------------------------------------------------------------------- @@ -347,6 +348,24 @@ class TestTalkerVisualRouting: # =================================================================== +class TestResolveComponentQuantConfig: + def test_global_config_applies_to_each_quantization_aware_component(self): + config = _MockQuantConfig("fp8") + + assert resolve_component_quant_config(config, "transformer") is config + assert resolve_component_quant_config(config, "text_encoder") is config + + def test_component_map_is_the_only_scope_override(self): + transformer = _MockQuantConfig("fp8") + config = ComponentQuantizationConfig( + component_configs={"transformer": transformer}, + default_config=None, + ) + + assert resolve_component_quant_config(config, "transformer") is transformer + assert resolve_component_quant_config(config, "text_encoder") is None + + class TestComponentResolve: def test_longest_prefix_match(self): a = _MockQuantConfig("a") diff --git a/vllm_omni/diffusion/model_loader/diffusers_loader.py b/vllm_omni/diffusion/model_loader/diffusers_loader.py index d1a1ed15e..bde9a1543 100644 --- a/vllm_omni/diffusion/model_loader/diffusers_loader.py +++ b/vllm_omni/diffusion/model_loader/diffusers_loader.py @@ -298,6 +298,60 @@ class DiffusersPipelineLoader: for source in sources: yield from self._get_weights_iterator(source, model=model) + @staticmethod + def _stream_online_quant_weights_to_cpu( + model: nn.Module, + weights: Iterable[tuple[str, torch.Tensor]], + ) -> Generator[tuple[str, torch.Tensor], None, None]: + """Offload each online-quantized layer as soon as it is complete. + + Upstream vLLM's online layerwise loader materializes and quantizes a + layer synchronously while consuming ``weights``. Generator execution + resumes after that weight has been consumed, which gives us a safe + point to move completed layers to CPU before the next layer is loaded. + This bounds accelerator residency during CPU-offloaded model startup + instead of retaining the entire quantized model until loading ends. + """ + from vllm.model_executor.model_loader.reload.layerwise import ( + get_layerwise_info, + ) + + pending = { + module + for module in model.modules() + if getattr(getattr(module, "quant_method", None), "uses_meta_device", False) + and get_layerwise_info(module).can_load() + } + offloaded = 0 + + def offload_completed() -> None: + nonlocal offloaded + for module in tuple(pending): + if get_layerwise_info(module).can_load(): + continue + module.to("cpu") + pending.remove(module) + offloaded += 1 + + for weight in weights: + # This runs after the consumer has handled the previous yield. + offload_completed() + yield weight + offload_completed() + + # Quantization workspaces and the old accelerator-side parameter + # storages are now reusable cache blocks. Release them before the + # remaining (unquantized) model tensors are copied to CPU; otherwise + # the cached quantization footprint and final offload overlap in the + # process-level startup peak. + if offloaded: + torch.accelerator.empty_cache() + + logger.info( + "Stream-offloaded %d online-quantized layers to CPU during weight loading", + offloaded, + ) + def _get_weight_sources(self, model: nn.Module) -> tuple["ComponentSource", ...]: return tuple( cast( @@ -403,7 +457,10 @@ class DiffusersPipelineLoader: if load_format == "diffusers": cast(DiffusersAdapterPipeline, model).load_weights() else: - self.load_weights(model) + if offload_after_quant: + self.load_weights(model, stream_online_quant_to_cpu=True) + else: + self.load_weights(model) self._process_weights_after_loading(model, target_device) if offload_after_quant: @@ -498,6 +555,14 @@ class DiffusersPipelineLoader: continue if has_online_quant: + # finalize_layerwise_processing() and the synchronous online + # loader already processed these layers. Avoid moving their + # quantized CPU weights back to the accelerator merely to call + # an idempotent no-op; doing so rebuilds a large CUDA allocator + # cache and defeats streaming CPU offload's startup-memory bound. + if getattr(module, "_already_called_process_weights_after_loading", False): + continue + # Online quant may leave straggler params on the ``meta`` device. # Move only real (non-meta) params onto the target device for # processing and restore them afterward, mirroring upstream vLLM's @@ -535,9 +600,17 @@ class DiffusersPipelineLoader: if needs_device_move: module.to(module_device) - def load_weights(self, model: nn.Module) -> None: + def load_weights( + self, + model: nn.Module, + *, + stream_online_quant_to_cpu: bool = False, + ) -> None: weights_to_load = self._get_expected_parameter_names(model) - loaded_weights = model.load_weights(self.get_all_weights(model)) + weights = self.get_all_weights(model) + if stream_online_quant_to_cpu: + weights = self._stream_online_quant_weights_to_cpu(model, weights) + loaded_weights = model.load_weights(weights) self.counter_after_loading_weights = time.perf_counter() logger.info_once( diff --git a/vllm_omni/diffusion/models/flux2/pipeline_flux2.py b/vllm_omni/diffusion/models/flux2/pipeline_flux2.py index 0dbfe0f8c..973f8b7ff 100644 --- a/vllm_omni/diffusion/models/flux2/pipeline_flux2.py +++ b/vllm_omni/diffusion/models/flux2/pipeline_flux2.py @@ -39,16 +39,11 @@ from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific from vllm_omni.platforms import current_omni_platform +from vllm_omni.quantization import resolve_component_quant_config as _resolve_component_quant_config logger = logging.getLogger(__name__) -def _resolve_component_quant_config(quant_config, component: str): - if hasattr(quant_config, "resolve"): - return quant_config.resolve(component) - return quant_config - - class Flux2ImageProcessor(VaeImageProcessor): """Image processor to preprocess the reference image for Flux2.""" diff --git a/vllm_omni/diffusion/models/minimax_h3/encoder.py b/vllm_omni/diffusion/models/minimax_h3/encoder.py index 2e07ace96..88f0ab193 100644 --- a/vllm_omni/diffusion/models/minimax_h3/encoder.py +++ b/vllm_omni/diffusion/models/minimax_h3/encoder.py @@ -30,14 +30,16 @@ complete ``[seq, 5120]`` hidden state. from __future__ import annotations import itertools -import json import re +from collections.abc import Iterable from typing import Any import torch import torch.nn as nn import torch.nn.functional as F from vllm.logger import init_logger +from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm_omni.diffusion.layers.norm import RMSNorm as DiffusionRMSNorm @@ -47,6 +49,19 @@ MINIMAX_H3_QWEN3VL_HIDDEN_DIM = 5120 logger = init_logger(__name__) +def _validate_encoder_quant_config(quant_config: QuantizationConfig | None) -> None: + if ( + quant_config is not None + and quant_config.get_name() == "fp8" + and bool(getattr(quant_config, "is_checkpoint_fp8_serialized", False)) + ): + raise ValueError("MiniMax-H3 text encoder supports online FP8 only, not serialized FP8 checkpoints") + + +def _is_quantized_linear(module: LinearBase) -> bool: + return not isinstance(module.quant_method, UnquantizedLinearMethod) + + def _default_weight_loader( param: nn.Parameter, loaded_weight: torch.Tensor, @@ -114,7 +129,7 @@ class MiniMaxH3Qwen3VLVocabParallelEmbedding(nn.Module): param.data.copy_(loaded_weight.narrow(0, start_idx, shard_size)) -class MiniMaxH3Qwen3VLMergedColumnParallelLinear(nn.Module): +class MiniMaxH3Qwen3VLMergedColumnParallelLinear(LinearBase): """Packed gate/up projection sharded along the output dimension.""" def __init__( @@ -123,23 +138,43 @@ class MiniMaxH3Qwen3VLMergedColumnParallelLinear(nn.Module): input_size: int, intermediate_size: int, dtype: torch.dtype, + quant_config: QuantizationConfig | None = None, + prefix: str = "", ) -> None: - super().__init__() self.group = group - self.input_size = input_size self.intermediate_size = intermediate_size tp_rank, tp_size = _tp_range(group) assert intermediate_size % tp_size == 0, ( f"intermediate_size {intermediate_size} must be divisible by text_encoder_tp_size {tp_size}" ) self.intermediate_size_per_partition = intermediate_size // tp_size - self.weight = nn.Parameter(torch.empty(2 * self.intermediate_size_per_partition, input_size, dtype=dtype)) - self.weight.weight_loader = self.weight_loader # type: ignore[attr-defined] + self.output_dtype = dtype + super().__init__( + input_size=input_size, + output_size=2 * intermediate_size, + bias=False, + params_dtype=dtype, + quant_config=quant_config, + prefix=prefix, + disable_tp=True, + ) + self.quant_method.create_weights( + self, + input_size_per_partition=input_size, + output_partition_sizes=[ + self.intermediate_size_per_partition, + self.intermediate_size_per_partition, + ], + input_size=input_size, + output_size=2 * intermediate_size, + params_dtype=dtype, + weight_loader=self.weight_loader, + ) self._tp_rank = tp_rank self._tp_size = tp_size def forward(self, input_: torch.Tensor) -> torch.Tensor: - return F.linear(input_, self.weight) + return self.quant_method.apply(self, input_) def weight_loader( self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: str | None = None @@ -152,7 +187,7 @@ class MiniMaxH3Qwen3VLMergedColumnParallelLinear(nn.Module): param.data[0:shard_size].copy_(loaded_weight.narrow(0, start_idx, shard_size)) -class MiniMaxH3Qwen3VLQKVParallelLinear(nn.Module): +class MiniMaxH3Qwen3VLQKVParallelLinear(LinearBase): """QKV projection with GQA head sharding along the output dimension.""" def __init__( @@ -163,8 +198,9 @@ class MiniMaxH3Qwen3VLQKVParallelLinear(nn.Module): num_kv_heads: int, head_dim: int, dtype: torch.dtype, + quant_config: QuantizationConfig | None = None, + prefix: str = "", ) -> None: - super().__init__() self.group = group self.hidden_size = hidden_size self.num_heads = num_heads @@ -179,15 +215,33 @@ class MiniMaxH3Qwen3VLQKVParallelLinear(nn.Module): ) self.local_num_heads = num_heads // tp_size self.local_num_kv_heads = num_kv_heads // tp_size + self.output_dtype = dtype q_local = self.local_num_heads * head_dim kv_local = self.local_num_kv_heads * head_dim - self.weight = nn.Parameter(torch.empty(q_local + 2 * kv_local, hidden_size, dtype=dtype)) - self.weight.weight_loader = self.weight_loader # type: ignore[attr-defined] + output_size = (num_heads + 2 * num_kv_heads) * head_dim + super().__init__( + input_size=hidden_size, + output_size=output_size, + bias=False, + params_dtype=dtype, + quant_config=quant_config, + prefix=prefix, + disable_tp=True, + ) + self.quant_method.create_weights( + self, + input_size_per_partition=hidden_size, + output_partition_sizes=[q_local, kv_local, kv_local], + input_size=hidden_size, + output_size=output_size, + params_dtype=dtype, + weight_loader=self.weight_loader, + ) self._tp_rank = tp_rank self._tp_size = tp_size def forward(self, input_: torch.Tensor) -> torch.Tensor: - return F.linear(input_, self.weight) + return self.quant_method.apply(self, input_) def weight_loader( self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: str | None = None @@ -240,7 +294,7 @@ def _verify_fused_shards_complete( raise RuntimeError(f"MiniMax H3 text encoder fused weights are missing source shards: {details}") -class MiniMaxH3Qwen3VLRowParallelLinear(nn.Module): +class MiniMaxH3Qwen3VLRowParallelLinear(LinearBase): def __init__( self, group: Any, @@ -248,17 +302,33 @@ class MiniMaxH3Qwen3VLRowParallelLinear(nn.Module): output_size: int, dtype: torch.dtype, input_is_parallel: bool = True, + quant_config: QuantizationConfig | None = None, + prefix: str = "", ) -> None: - super().__init__() self.group = group - self.input_size = input_size - self.output_size = output_size self.input_is_parallel = input_is_parallel + self.output_dtype = dtype tp_rank, tp_size = _tp_range(group) assert input_size % tp_size == 0, f"input_size {input_size} must be divisible by text_encoder_tp_size {tp_size}" self.input_size_per_partition = input_size // tp_size - self.weight = nn.Parameter(torch.empty(output_size, self.input_size_per_partition, dtype=dtype)) - self.weight.weight_loader = self.weight_loader # type: ignore[attr-defined] + super().__init__( + input_size=input_size, + output_size=output_size, + bias=False, + params_dtype=dtype, + quant_config=quant_config, + prefix=prefix, + disable_tp=True, + ) + self.quant_method.create_weights( + self, + input_size_per_partition=self.input_size_per_partition, + output_partition_sizes=[output_size], + input_size=input_size, + output_size=output_size, + params_dtype=dtype, + weight_loader=self.weight_loader, + ) self._tp_rank = tp_rank self._tp_size = tp_size @@ -268,7 +338,7 @@ class MiniMaxH3Qwen3VLRowParallelLinear(nn.Module): else: split_input = input_.split(self.input_size_per_partition, dim=-1) input_parallel = split_input[self._tp_rank].contiguous() - output_parallel = F.linear(input_parallel, self.weight) + output_parallel = self.quant_method.apply(self, input_parallel) if self._tp_size > 1: # Reduce in fp32: the reference path accumulates the full (K=8192) # dot product inside a single cuBLAS GEMM before rounding to bf16. @@ -276,7 +346,7 @@ class MiniMaxH3Qwen3VLRowParallelLinear(nn.Module): # and amplify error on this model's large-magnitude activations. output_parallel = output_parallel.float() self.group.all_reduce(output_parallel) - output_parallel = output_parallel.to(self.weight.dtype) + output_parallel = output_parallel.to(self.output_dtype) return output_parallel def weight_loader( @@ -665,7 +735,14 @@ class MiniMaxH3Qwen3VLVisionModel(nn.Module): class MiniMaxH3Qwen3VLTextAttention(nn.Module): - def __init__(self, group: Any, config: Any, dtype: torch.dtype) -> None: + def __init__( + self, + group: Any, + config: Any, + dtype: torch.dtype, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: super().__init__() self.group = group self.config = config @@ -683,6 +760,8 @@ class MiniMaxH3Qwen3VLTextAttention(nn.Module): num_kv_heads=self.num_kv_heads, head_dim=self.head_dim, dtype=dtype, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", ) self.o_proj = MiniMaxH3Qwen3VLRowParallelLinear( group, @@ -690,6 +769,8 @@ class MiniMaxH3Qwen3VLTextAttention(nn.Module): output_size=self.hidden_size, dtype=dtype, input_is_parallel=True, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", ) self.q_norm = MiniMaxH3Qwen3VLRMSNorm(self.head_dim, eps=config.rms_norm_eps, dtype=dtype) self.k_norm = MiniMaxH3Qwen3VLRMSNorm(self.head_dim, eps=config.rms_norm_eps, dtype=dtype) @@ -703,14 +784,23 @@ class MiniMaxH3Qwen3VLTextAttention(nn.Module): hidden_shape = (*input_shape, -1, self.head_dim) q_local = self.qkv_proj.local_num_heads * self.head_dim kv_local = self.qkv_proj.local_num_kv_heads * self.head_dim - q_weight = self.qkv_proj.weight[0:q_local] - k_weight = self.qkv_proj.weight[q_local : q_local + kv_local] - v_weight = self.qkv_proj.weight[q_local + kv_local : q_local + 2 * kv_local] - # Three separate GEMMs keep the numerics identical to the reference - # path (packed QKV changes cuBLAS tiling and shifts bf16 rounding). - query_states = self.q_norm(F.linear(hidden_states, q_weight).view(hidden_shape)).transpose(1, 2) - key_states = self.k_norm(F.linear(hidden_states, k_weight).view(hidden_shape)).transpose(1, 2) - value_states = F.linear(hidden_states, v_weight).view(hidden_shape).transpose(1, 2) + if _is_quantized_linear(self.qkv_proj): + query, key, value = self.qkv_proj(hidden_states).split( + (q_local, kv_local, kv_local), + dim=-1, + ) + else: + q_weight = self.qkv_proj.weight[0:q_local] + k_weight = self.qkv_proj.weight[q_local : q_local + kv_local] + v_weight = self.qkv_proj.weight[q_local + kv_local : q_local + 2 * kv_local] + # Three separate GEMMs keep the numerics identical to the reference + # path (packed QKV changes cuBLAS tiling and shifts bf16 rounding). + query = F.linear(hidden_states, q_weight) + key = F.linear(hidden_states, k_weight) + value = F.linear(hidden_states, v_weight) + query_states = self.q_norm(query.view(hidden_shape)).transpose(1, 2) + key_states = self.k_norm(key.view(hidden_shape)).transpose(1, 2) + value_states = value.view(hidden_shape).transpose(1, 2) cos, sin = position_embeddings query_states, key_states = _apply_rotary_pos_emb(query_states, key_states, cos, sin) @@ -734,7 +824,14 @@ class MiniMaxH3Qwen3VLTextAttention(nn.Module): class MiniMaxH3Qwen3VLTextMLP(nn.Module): - def __init__(self, group: Any, config: Any, dtype: torch.dtype) -> None: + def __init__( + self, + group: Any, + config: Any, + dtype: torch.dtype, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: super().__init__() self.hidden_size = config.hidden_size self.intermediate_size = config.intermediate_size @@ -743,6 +840,8 @@ class MiniMaxH3Qwen3VLTextMLP(nn.Module): input_size=self.hidden_size, intermediate_size=self.intermediate_size, dtype=dtype, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", ) self.down_proj = MiniMaxH3Qwen3VLRowParallelLinear( group, @@ -750,22 +849,46 @@ class MiniMaxH3Qwen3VLTextMLP(nn.Module): output_size=self.hidden_size, dtype=dtype, input_is_parallel=True, + quant_config=quant_config, + prefix=f"{prefix}.down_proj", ) def forward(self, x: torch.Tensor) -> torch.Tensor: ip_local = self.gate_up_proj.intermediate_size_per_partition - gate = F.linear(x, self.gate_up_proj.weight[0:ip_local]) - up = F.linear(x, self.gate_up_proj.weight[ip_local : 2 * ip_local]) + if _is_quantized_linear(self.gate_up_proj): + gate, up = self.gate_up_proj(x).split(ip_local, dim=-1) + else: + gate = F.linear(x, self.gate_up_proj.weight[0:ip_local]) + up = F.linear(x, self.gate_up_proj.weight[ip_local : 2 * ip_local]) x = F.silu(gate) * up return self.down_proj(x) class MiniMaxH3Qwen3VLTextDecoderLayer(nn.Module): - def __init__(self, group: Any, config: Any, dtype: torch.dtype) -> None: + def __init__( + self, + group: Any, + config: Any, + dtype: torch.dtype, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: super().__init__() self.hidden_size = config.hidden_size - self.self_attn = MiniMaxH3Qwen3VLTextAttention(group, config, dtype) - self.mlp = MiniMaxH3Qwen3VLTextMLP(group, config, dtype) + self.self_attn = MiniMaxH3Qwen3VLTextAttention( + group, + config, + dtype, + quant_config=quant_config, + prefix=f"{prefix}.self_attn", + ) + self.mlp = MiniMaxH3Qwen3VLTextMLP( + group, + config, + dtype, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) self.input_layernorm = MiniMaxH3Qwen3VLRMSNorm(config.hidden_size, eps=config.rms_norm_eps, dtype=dtype) self.post_attention_layernorm = MiniMaxH3Qwen3VLRMSNorm( config.hidden_size, eps=config.rms_norm_eps, dtype=dtype @@ -791,7 +914,15 @@ class MiniMaxH3Qwen3VLTextDecoderLayer(nn.Module): class MiniMaxH3Qwen3VLTextModel(nn.Module): """TP-sharded Qwen3-VL text model returning unnormalized layer-50 states.""" - def __init__(self, group: Any, config: Any, selected_layer: int, dtype: torch.dtype) -> None: + def __init__( + self, + group: Any, + config: Any, + selected_layer: int, + dtype: torch.dtype, + quant_config: QuantizationConfig | None = None, + prefix: str = "text_encoder.text_model", + ) -> None: super().__init__() self.group = group self.config = config @@ -804,7 +935,16 @@ class MiniMaxH3Qwen3VLTextModel(nn.Module): ) self.rotary_emb = MiniMaxH3Qwen3VLTextRotaryEmbedding(config) self.layers = nn.ModuleList( - [MiniMaxH3Qwen3VLTextDecoderLayer(group, config, dtype) for _ in range(self.num_layers)] + [ + MiniMaxH3Qwen3VLTextDecoderLayer( + group, + config, + dtype, + quant_config=quant_config, + prefix=f"{prefix}.layers.{layer_idx}", + ) + for layer_idx in range(self.num_layers) + ] ) def _deepstack_process( @@ -951,10 +1091,13 @@ class MiniMaxH3Qwen3VLEncoder(nn.Module): device: torch.device, load_model: bool, encoder_group: Any | None = None, + quant_config: QuantizationConfig | None = None, ) -> None: super().__init__() self.device_target = device self.encoder_group = encoder_group + self.quant_config = quant_config + _validate_encoder_quant_config(quant_config) self.image_token_id = 151655 self.video_token_id = 151656 self._tp_size = 1 @@ -975,9 +1118,9 @@ class MiniMaxH3Qwen3VLEncoder(nn.Module): config.text_config, MINIMAX_H3_QWEN3VL_SELECTED_LM_LAYER, dtype, + quant_config=quant_config, ) self.text_model.to(dtype=dtype) - self._load_weights(model_path) logger.info( "MiniMax H3 Qwen3-VL encoder: %d retained decoder layers, text_encoder_tp_size=%d, vision replicated", MINIMAX_H3_QWEN3VL_SELECTED_LM_LAYER, @@ -1032,48 +1175,37 @@ class MiniMaxH3Qwen3VLEncoder(nn.Module): return ("text_model." + rest, None) return None - def _load_weights(self, model_path: str) -> None: - import os - - import safetensors.torch - - index_path = os.path.join(model_path, "model.safetensors.index.json") - with open(index_path, encoding="utf-8") as handle: - index = json.load(handle) - weight_map: dict[str, str] = index["weight_map"] - files = sorted(set(weight_map.values())) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: params = dict(self.named_parameters()) loaded: set[str] = set() - # Seeded from the modules that own a multi-shard loader, so a fused weight - # that receives no source shard at all is caught too, not only a partially - # filled one. + # Seeded from modules that own a multi-shard loader, so a fused weight + # receiving no source shard is caught as well as a partially filled one. expected_fused_shards: dict[str, tuple[tuple[str, str | int], ...]] = { f"{module_name}.weight": sources for module_name, module in self.named_modules() if (sources := _fused_source_shards(module)) is not None } loaded_fused_shards: dict[str, set[str | int]] = {name: set() for name in expected_fused_shards} - for shard in files: - tensors = safetensors.torch.load_file(os.path.join(model_path, shard), device="cpu") - for name, tensor in tensors.items(): - mapped = self._map_weight_name(name) - if mapped is None: - continue - param_name, shard_id = mapped - param = params.get(param_name) - if param is None: - logger.warning("MiniMax H3 text encoder weight %s has no target parameter", name) - continue - weight_loader = getattr(param, "weight_loader", _default_weight_loader) - weight_loader(param, tensor, shard_id) - loaded.add(param_name) - if shard_id is not None and param_name in loaded_fused_shards: - loaded_fused_shards[param_name].add(shard_id) + for name, tensor in weights: + mapped = self._map_weight_name(name) + if mapped is None: + continue + param_name, shard_id = mapped + param = params.get(param_name) + if param is None: + logger.warning("MiniMax H3 text encoder weight %s has no target parameter", name) + continue + weight_loader = getattr(param, "weight_loader", _default_weight_loader) + weight_loader(param, tensor, shard_id) + loaded.add(param_name) + if shard_id is not None and param_name in loaded_fused_shards: + loaded_fused_shards[param_name].add(shard_id) _verify_fused_shards_complete(expected_fused_shards, loaded_fused_shards) missing = sorted(set(params) - loaded) if missing: # Listed in full: this aborts startup, so the message is the only diagnostic. raise RuntimeError(f"MiniMax H3 text encoder weights not loaded: {len(missing)} params: {missing}") + return loaded def load_to_device(self) -> None: if not self.is_loaded: diff --git a/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py b/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py index 3b96b8ab7..5de203721 100644 --- a/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py +++ b/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py @@ -20,6 +20,7 @@ import torch.nn as nn from PIL import Image from transformers import Qwen2TokenizerFast, Qwen3VLProcessor from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm_omni.diffusion import envs from vllm_omni.diffusion.cache.cachedit import ( @@ -54,6 +55,12 @@ from vllm_omni.model_executor.model_loader.weight_utils import ( download_weights_from_hf_specific, ) from vllm_omni.platforms import current_omni_platform +from vllm_omni.quantization import ( + resolve_component_quant_config as _resolve_component_quant_config, +) +from vllm_omni.quantization.component_config import ( + resolve_encoder_quant_config as _resolve_encoder_quant_config, +) from .condition_noise import ( minimax_h3_audio_cond_noise_aug_rows, @@ -128,6 +135,13 @@ MINIMAX_H3_TASK_DOWNLOAD_PATTERNS = { } +def _resolve_minimax_h3_text_encoder_quant_config( + quant_config: QuantizationConfig | None, +) -> QuantizationConfig | None: + resolved = _resolve_component_quant_config(quant_config, "text_encoder") + return _resolve_encoder_quant_config(resolved) + + def _minimax_h3_partition_for_task( task_type: str | None, model: str | None = None, @@ -175,12 +189,6 @@ def _read_base_schedule(release: Mapping[str, Any]) -> DMD2SigmaSchedule | None: return DMD2SigmaSchedule.from_metadata(release) -def _resolve_component_quant_config(quant_config, component: str): - if hasattr(quant_config, "resolve"): - return quant_config.resolve(component) - return quant_config - - def _minimax_h3_post_process(output, output_type: str = "np"): """Convert the joint video/audio output without capturing worker state. @@ -688,12 +696,24 @@ class MiniMaxH3Pipeline( ) self.text_encoder_tp_size = text_encoder_tp_size self.text_encoder_group = self._build_text_encoder_group(text_encoder_tp_size) + load_text_encoder = rank < text_encoder_tp_size self.text_encoder = MiniMaxH3Qwen3VLEncoder( os.path.join(model_path, "text_encoder"), device=self.device, - load_model=rank < text_encoder_tp_size, + load_model=load_text_encoder, encoder_group=self.text_encoder_group, + quant_config=_resolve_minimax_h3_text_encoder_quant_config(od_config.quantization_config), ) + if load_text_encoder: + self.weights_sources.append( + DiffusersPipelineLoader.ComponentSource( + model_or_path=str(model_path), + subfolder="text_encoder", + revision=od_config.revision, + prefix="text_encoder.", + fall_back_to_pt=False, + ) + ) stage_components = bool( od_config.enable_layerwise_offload or getattr(od_config, "enable_distributed_layerwise_offload", False) ) @@ -725,7 +745,7 @@ class MiniMaxH3Pipeline( def source_prefix(item: tuple[str, torch.Tensor]) -> str: name, _ = item prefix = name.partition(".")[0] + "." - if prefix in {"transformer.", "transformers_ref."}: + if prefix in {"transformer.", "transformers_ref.", "text_encoder."}: return prefix raise ValueError(f"unexpected MiniMax-H3 weight {name!r}") @@ -735,17 +755,16 @@ class MiniMaxH3Pipeline( if prefix in loaded_prefixes: raise ValueError(f"MiniMax-H3 weight source {prefix!r} is not contiguous") loaded_prefixes.add(prefix) - transformer = getattr(self, prefix.removesuffix(".")) - loaded = transformer.load_weights((name[len(prefix) :], tensor) for name, tensor in grouped_weights) - transformer.post_load_weights() + component = getattr(self, prefix.removesuffix(".")) + loaded = component.load_weights((name[len(prefix) :], tensor) for name, tensor in grouped_weights) + if prefix != "text_encoder.": + component.post_load_weights() loaded_with_prefix.update(prefix + name for name in loaded) - # The text encoder and both VAEs load eagerly in ``__init__`` rather - # than through ``weights_sources``. Record them for the runner's strict - # missing-parameter check. For the text encoder that report is exact - # because its loader raises on any unloaded parameter or partially filled - # fused parameter; weaken that and gaps here become invisible. The two - # VAEs carry no equivalent guarantee. - for component_name in ("text_encoder", "video_vae", "audio_vae"): + # Both VAEs load eagerly in ``__init__`` rather than through + # ``weights_sources``. The text encoder uses the shared component + # loader so online quantization and offload processing follow the same + # path as the DiT. + for component_name in ("video_vae", "audio_vae"): component = getattr(self, component_name) loaded_with_prefix.update(f"{component_name}.{name}" for name, _ in component.named_parameters()) return loaded_with_prefix diff --git a/vllm_omni/diffusion/offloader/distributed_layerwise_backend.py b/vllm_omni/diffusion/offloader/distributed_layerwise_backend.py index 7da7c8d37..2d6a34978 100644 --- a/vllm_omni/diffusion/offloader/distributed_layerwise_backend.py +++ b/vllm_omni/diffusion/offloader/distributed_layerwise_backend.py @@ -178,7 +178,16 @@ class DistributedLayerwiseOffloadHook(ModelHook): if m["name"] in self.next_block_parameters else self.next_block_buffers[m["name"]] ) - repoint.append((target, dtype, m["offset"], m["numel"], m["shape"])) + repoint.append( + ( + target, + dtype, + m["offset"], + m["numel"], + m["shape"], + m["stride"], + ) + ) self._cached_repoint.append(repoint) # Pre-compute AG output sizes (avoid sum() per layer). @@ -228,9 +237,15 @@ class DistributedLayerwiseOffloadHook(ModelHook): # parameter as an mmap view avoids a private full-model # copy in every worker. local_t = mmap_transform(local_t) - weights_with_local.append((name, t, local_t)) + stride = local_t.stride() + storage_numel = ( + 0 + if local_t.numel() == 0 + else 1 + sum((size - 1) * axis_stride for size, axis_stride in zip(local_t.shape, stride)) + ) + weights_with_local.append((name, t, local_t, storage_numel, stride)) - total_numel = sum(local.numel() for _, _, local in weights_with_local) + total_numel = sum(storage_numel for _, _, _, storage_numel, _ in weights_with_local) # Equal-sized shards (ceil division) for all_gather_into_tensor shard_size = (total_numel + dp_size - 1) // dp_size # ceil @@ -242,8 +257,13 @@ class DistributedLayerwiseOffloadHook(ModelHook): shard = torch.zeros(shard_size, dtype=dtype, device="cpu") current_offset = 0 - for name, original_tensor, local_tensor in weights_with_local: - numel = local_tensor.numel() + for ( + name, + original_tensor, + local_tensor, + storage_numel, + stride, + ) in weights_with_local: if dtype not in dtype_metadata: dtype_metadata[dtype] = [] # Offsets remain relative to the FULL flattened buffer @@ -252,27 +272,48 @@ class DistributedLayerwiseOffloadHook(ModelHook): { "name": name, "offset": current_offset, - "numel": numel, + "numel": storage_numel, "shape": local_tensor.shape, + "stride": stride, } ) # Copy ONLY the portion within [shard_start, shard_end) overlap_start = max(current_offset, shard_start) - overlap_end = min(current_offset + numel, shard_end) + overlap_end = min(current_offset + storage_numel, shard_end) if overlap_start < overlap_end: + if local_tensor.is_contiguous(): + flat_storage = local_tensor.flatten() + else: + # Online FP8 stores Cutlass weights as transposed views + # (e.g. stride=(1, K)). Flattening such a tensor in + # logical order and later rebuilding it with .view() + # changes its layout and makes scaled_mm reject it. + # Pack the physical storage order and preserve the + # original stride for zero-copy reconstruction. + flat_storage = torch.zeros( + storage_numel, + dtype=dtype, + device=local_tensor.device, + ) + physical_view = torch.as_strided( + flat_storage, + size=local_tensor.shape, + stride=stride, + ) + physical_view.copy_(local_tensor) src_start = overlap_start - current_offset src_end = overlap_end - current_offset dst_start = overlap_start - shard_start dst_end = overlap_end - shard_start - shard[dst_start:dst_end].copy_(local_tensor.flatten()[src_start:src_end]) + shard[dst_start:dst_end].copy_(flat_storage[src_start:src_end]) # Replace original tensor with placeholder (frees CPU storage) set_tensor_storage( original_tensor, make_offload_placeholder(original_tensor), ) - current_offset += numel + current_offset += storage_numel if pin_memory: shard = shard.pin_memory() @@ -365,10 +406,14 @@ class DistributedLayerwiseOffloadHook(ModelHook): self._shared_slot_group[slot] = self._group_id # Re-point using cached metadata (avoids per-layer dict lookups). - for target, dtype, offset, numel, shape in self._cached_repoint[slot]: + for target, dtype, offset, numel, shape, stride in self._cached_repoint[slot]: set_tensor_storage( target, - gpu_weights[dtype][offset : offset + numel].view(shape), + torch.as_strided( + gpu_weights[dtype][offset : offset + numel], + size=shape, + stride=stride, + ), ) def get_weights(self, slot: int) -> dict[torch.dtype, torch.Tensor] | None: @@ -577,7 +622,11 @@ class PinnedResidentLayerGroup: for meta in metas: set_tensor_storage( targets[meta["name"]], - gpu_buffer[meta["offset"] : meta["offset"] + meta["numel"]].view(meta["shape"]), + torch.as_strided( + gpu_buffer[meta["offset"] : meta["offset"] + meta["numel"]], + size=meta["shape"], + stride=meta["stride"], + ), ) current_omni_platform.current_stream().wait_event(ready) diff --git a/vllm_omni/diffusion/offloader/layerwise_backend.py b/vllm_omni/diffusion/offloader/layerwise_backend.py index 9979d01b1..662e1304f 100644 --- a/vllm_omni/diffusion/offloader/layerwise_backend.py +++ b/vllm_omni/diffusion/offloader/layerwise_backend.py @@ -139,29 +139,57 @@ class LayerwiseOffloadHook(ModelHook): weights_with_local = [] for name, t in name2weights.items(): local_t = t.to_local() if hasattr(t, "to_local") else t - weights_with_local.append((name, t, local_t)) - total_numel = sum(local.numel() for _, _, local in weights_with_local) + stride = local_t.stride() + storage_numel = ( + 0 + if local_t.numel() == 0 + else 1 + sum((size - 1) * axis_stride for size, axis_stride in zip(local_t.shape, stride)) + ) + weights_with_local.append((name, t, local_t, storage_numel, stride)) + total_numel = sum(storage_numel for _, _, _, storage_numel, _ in weights_with_local) cpu_tensor = torch.empty(total_numel, dtype=dtype, device="cpu", pin_memory=pin_memory) current_offset = 0 - for name, original_tensor, local_tensor in weights_with_local: - numel = local_tensor.numel() - cpu_tensor[current_offset : current_offset + numel].copy_(local_tensor.flatten()) + for ( + name, + original_tensor, + local_tensor, + storage_numel, + stride, + ) in weights_with_local: + if local_tensor.is_contiguous(): + flat_storage = local_tensor.flatten() + else: + # Cutlass FP8 weights use a transposed physical layout. + # Preserve it across the flattened CPU staging buffer. + flat_storage = torch.zeros( + storage_numel, + dtype=dtype, + device=local_tensor.device, + ) + physical_view = torch.as_strided( + flat_storage, + size=local_tensor.shape, + stride=stride, + ) + physical_view.copy_(local_tensor) + cpu_tensor[current_offset : current_offset + storage_numel].copy_(flat_storage) if dtype not in dtype_metadata: dtype_metadata[dtype] = [] dtype_metadata[dtype].append( { "name": name, "offset": current_offset, - "numel": numel, + "numel": storage_numel, "shape": local_tensor.shape, + "stride": stride, } ) LayerwiseOffloadHook._set_tensor_storage( original_tensor, LayerwiseOffloadHook._make_offload_placeholder(original_tensor) ) - current_offset += numel + current_offset += storage_numel dtype_cpu_flattened_weights[dtype] = cpu_tensor @@ -210,7 +238,11 @@ class LayerwiseOffloadHook(ModelHook): LayerwiseOffloadHook._set_tensor_storage( target_param_or_buf, - gpu_weight[metadata["offset"] : metadata["offset"] + metadata["numel"]].view(metadata["shape"]), + torch.as_strided( + gpu_weight[metadata["offset"] : metadata["offset"] + metadata["numel"]], + size=metadata["shape"], + stride=metadata["stride"], + ), ) self._prefetch_done = evt diff --git a/vllm_omni/quantization/__init__.py b/vllm_omni/quantization/__init__.py index 8896903e8..89d445d61 100644 --- a/vllm_omni/quantization/__init__.py +++ b/vllm_omni/quantization/__init__.py @@ -11,7 +11,7 @@ Adds per-component quantization for multi-stage models. config = build_quant_config({"transformer": {"method": "fp8"}, "vae": None}) """ -from .component_config import ComponentQuantizationConfig +from .component_config import ComponentQuantizationConfig, resolve_component_quant_config from .factory import SUPPORTED_QUANTIZATION_METHODS, build_quant_config, register_quantization_override from .inc_config import OmniINCConfig @@ -23,6 +23,7 @@ from .inc_config import OmniINCConfig __all__ = [ "build_quant_config", "ComponentQuantizationConfig", + "resolve_component_quant_config", "OmniINCConfig", "SUPPORTED_QUANTIZATION_METHODS", "register_quantization_override", diff --git a/vllm_omni/quantization/component_config.py b/vllm_omni/quantization/component_config.py index 28c33d637..638419689 100644 --- a/vllm_omni/quantization/component_config.py +++ b/vllm_omni/quantization/component_config.py @@ -29,6 +29,21 @@ if TYPE_CHECKING: PRE_QUANTIZED_METHODS: frozenset[str] = frozenset({"modelopt", "modelopt_fp4", "modelopt_mxfp8", "modelopt_mixed"}) +def resolve_component_quant_config( + quant_config: QuantizationConfig | None, + component: str, +) -> QuantizationConfig | None: + """Resolve one pipeline component from a global or component config. + + A plain config is global and therefore applies unchanged to every + quantization-aware component. Only ``ComponentQuantizationConfig`` narrows + the scope through its explicit prefix map. + """ + if isinstance(quant_config, ComponentQuantizationConfig): + return quant_config.resolve(component) + return quant_config + + def resolve_encoder_quant_config( quant_config: QuantizationConfig | None, ) -> QuantizationConfig | None: From 2a315e1a2d3ef87fca690f7c58e00a1d35a047d0 Mon Sep 17 00:00:00 2001 From: hurukawa <61525444+nagisa-kunhah@users.noreply.github.com> Date: Sat, 15 Aug 2026 23:58:36 +0800 Subject: [PATCH 032/267] [Perf] Support request-level batching for Wan2.2 pipelines (#5676) Signed-off-by: nagisa-kun <1434936049@qq.com> Signed-off-by: nagisa <1434936049@qq.com> --- .../models/wan2_2/test_wan22_i2v_pipeline.py | 286 ++++++++++++++- .../wan2_2/test_wan22_pipeline_diffuse.py | 156 +++++++- .../wan2_2/test_wan22_pipeline_helpers.py | 12 + .../models/wan2_2/test_wan22_s2v_pipeline.py | 345 ++++++++++++++++-- .../models/wan2_2/test_wan22_vace_pipeline.py | 210 ++++++++++- .../test_diffusion_engine_rpc_routing.py | 1 + tests/diffusion/test_diffusion_scheduler.py | 134 ++++++- .../test_async_omni_engine_stage_init.py | 95 +++++ tests/engine/test_single_stage_mode.py | 9 +- .../models/wan2_2/pipeline_wan2_2.py | 171 +++++---- .../models/wan2_2/pipeline_wan2_2_i2v.py | 257 ++++++++----- .../models/wan2_2/pipeline_wan2_2_s2v.py | 317 ++++++++++------ .../models/wan2_2/pipeline_wan2_2_vace.py | 208 ++++++----- vllm_omni/diffusion/request.py | 4 + vllm_omni/diffusion/sched/interface.py | 15 +- .../diffusion/sched/request_scheduler.py | 23 +- vllm_omni/engine/stage_engine_startup.py | 1 + vllm_omni/engine/stage_init_utils.py | 1 + 18 files changed, 1871 insertions(+), 374 deletions(-) diff --git a/tests/diffusion/models/wan2_2/test_wan22_i2v_pipeline.py b/tests/diffusion/models/wan2_2/test_wan22_i2v_pipeline.py index 9d5b11d3b..12c0ebf45 100644 --- a/tests/diffusion/models/wan2_2/test_wan22_i2v_pipeline.py +++ b/tests/diffusion/models/wan2_2/test_wan22_i2v_pipeline.py @@ -8,15 +8,28 @@ import torch from PIL import Image from torch import nn -from tests.diffusion.models.wan2_2.conftest import StubTransformer, StubVAE, noop_progress_bar +from tests.diffusion.models.wan2_2.conftest import StubScheduler, StubTransformer, StubVAE, noop_progress_bar from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_i2v import ( Wan22I2VPipeline, + get_wan22_i2v_post_process_func, get_wan22_i2v_pre_process_func, ) +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] +def test_wan22_i2v_postprocess_honors_request_output_type() -> None: + video = torch.zeros(1, 4, 1, 2, 2) + + output = get_wan22_i2v_post_process_func(SimpleNamespace())( + video, + sampling_params=SimpleNamespace(output_type="latent"), + ) + + assert output is video + + def _make_i2v_pipeline(*, expand_timesteps: bool) -> Wan22I2VPipeline: pipeline = object.__new__(Wan22I2VPipeline) nn.Module.__init__(pipeline) @@ -31,6 +44,29 @@ def _make_i2v_pipeline(*, expand_timesteps: bool) -> Wan22I2VPipeline: return pipeline +def _make_i2v_sampling(**overrides): + values = { + "height": 16, + "width": 16, + "num_frames": 5, + "num_inference_steps": 1, + "guidance_scale_provided": True, + "guidance_scale": 1.0, + "guidance_scale_2": None, + "guidance_scale_2_provided": False, + "boundary_ratio": None, + "generator": None, + "seed": None, + "num_outputs_per_prompt": 1, + "max_sequence_length": 8, + "latents": None, + "output_type": "latent", + "extra_args": {}, + } + values.update(overrides) + return SimpleNamespace(**values) + + def test_i2v_preprocess_requires_image_and_resizes_to_480p_aspect() -> None: preprocess = get_wan22_i2v_pre_process_func(SimpleNamespace()) request = SimpleNamespace( @@ -53,6 +89,88 @@ def test_i2v_preprocess_requires_image_and_resizes_to_480p_aspect() -> None: preprocess(missing_image) +def _make_i2v_preprocess_request(last_image): + return SimpleNamespace( + prompt={ + "prompt": "p", + "multi_modal_data": { + "image": Image.new("RGB", (320, 160), "red"), + "last_image": last_image, + }, + }, + sampling_params=SimpleNamespace(height=16, width=16), + ) + + +def test_i2v_preprocess_treats_empty_last_image_list_as_absent() -> None: + preprocess = get_wan22_i2v_pre_process_func(SimpleNamespace()) + + result = preprocess(_make_i2v_preprocess_request([])) + + assert result.prompt["multi_modal_data"]["last_image"] is None + assert result.batch_compatibility_key == ("wan22_i2v_last_image", False) + + +def test_i2v_preprocess_unwraps_single_last_image_list() -> None: + preprocess = get_wan22_i2v_pre_process_func(SimpleNamespace()) + last_image = Image.new("RGB", (16, 16), "blue") + + result = preprocess(_make_i2v_preprocess_request([last_image])) + + assert result.prompt["multi_modal_data"]["last_image"] is last_image + assert result.batch_compatibility_key == ("wan22_i2v_last_image", True) + + +@pytest.mark.parametrize( + "last_image", + [ + Image.new("RGB", (16, 16), "blue"), + torch.zeros(3, 16, 16), + ], +) +def test_i2v_preprocess_preserves_supported_last_image(last_image) -> None: + preprocess = get_wan22_i2v_pre_process_func(SimpleNamespace()) + + result = preprocess(_make_i2v_preprocess_request(last_image)) + + assert result.prompt["multi_modal_data"]["last_image"] is last_image + assert result.batch_compatibility_key == ("wan22_i2v_last_image", True) + + +def test_i2v_preprocess_loads_last_image_path(tmp_path) -> None: + preprocess = get_wan22_i2v_pre_process_func(SimpleNamespace()) + path = tmp_path / "last.png" + Image.new("RGB", (16, 16), "blue").save(path) + + result = preprocess(_make_i2v_preprocess_request(str(path))) + + last_image = result.prompt["multi_modal_data"]["last_image"] + assert isinstance(last_image, Image.Image) + assert last_image.mode == "RGB" + assert result.batch_compatibility_key == ("wan22_i2v_last_image", True) + + +def test_i2v_preprocess_rejects_multiple_last_images() -> None: + preprocess = get_wan22_i2v_pre_process_func(SimpleNamespace()) + + with pytest.raises(ValueError, match="at most one last_image"): + preprocess( + _make_i2v_preprocess_request( + [ + Image.new("RGB", (16, 16), "blue"), + Image.new("RGB", (16, 16), "green"), + ] + ) + ) + + +def test_i2v_preprocess_rejects_unsupported_last_image_type() -> None: + preprocess = get_wan22_i2v_pre_process_func(SimpleNamespace()) + + with pytest.raises(TypeError, match="Unsupported last_image format"): + preprocess(_make_i2v_preprocess_request({"not": "an image"})) + + def test_i2v_diffuse_selects_stage_guidance_and_expands_timesteps() -> None: pipeline = _make_i2v_pipeline(expand_timesteps=True) latents = torch.zeros(1, 4, 2, 4, 4) @@ -126,3 +244,169 @@ def test_i2v_prepare_latents_builds_expand_condition_and_first_frame_mask() -> N assert first_frame_mask.shape == (1, 1, 2, 2, 2) assert first_frame_mask[:, :, 0].sum() == 0 assert first_frame_mask[:, :, 1].sum() == 4 + + +def test_i2v_prepare_latents_preserves_batched_image_conditions() -> None: + pipeline = _make_i2v_pipeline(expand_timesteps=True) + generators = [ + torch.Generator(device="cpu").manual_seed(1), + torch.Generator(device="cpu").manual_seed(2), + ] + + latents, condition, first_frame_mask = pipeline.prepare_latents( + image=torch.zeros(2, 3, 16, 16), + batch_size=2, + num_channels_latents=4, + height=16, + width=16, + num_frames=5, + dtype=torch.float32, + device=torch.device("cpu"), + generator=generators, + ) + + assert latents.shape == (2, 4, 2, 2, 2) + assert condition.shape == (2, 4, 1, 2, 2) + assert first_frame_mask.shape == (2, 1, 2, 2, 2) + + +def test_i2v_forward_batches_conditions_random_inputs_and_outputs(monkeypatch) -> None: + monkeypatch.setattr( + "vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_i2v.current_omni_platform", + SimpleNamespace(is_available=lambda: False), + ) + pipeline = _make_i2v_pipeline(expand_timesteps=True) + pipeline.scheduler = StubScheduler([9]) + pipeline.od_config = SimpleNamespace(flow_shift=5.0) + pipeline._sample_solver = "unipc" + pipeline._flow_shift = 5.0 + pipeline.boundary_ratio = 0.875 + pipeline.has_image_encoder = False + pipeline._guidance_scale = None + pipeline._guidance_scale_2 = None + pipeline._num_timesteps = None + pipeline._current_timestep = None + pipeline.check_inputs = lambda **kwargs: None + prepare_call = {} + + def _fake_encode_prompt(**kwargs): + batch_size = len(kwargs["prompt"]) + n = batch_size * kwargs["num_videos_per_prompt"] + return torch.zeros(n, 2, 3), None + + def _fake_prepare_latents(**kwargs): + prepare_call.update(kwargs) + batch_size = kwargs["batch_size"] + return ( + kwargs["latents"], + torch.zeros(batch_size, 4, 1, 2, 2), + torch.ones(batch_size, 1, 2, 2, 2), + ) + + pipeline.encode_prompt = _fake_encode_prompt # type: ignore[method-assign] + pipeline.prepare_latents = _fake_prepare_latents # type: ignore[method-assign] + pipeline.diffuse = lambda **kwargs: kwargs["latents"] # type: ignore[method-assign] + + gen_a = torch.Generator(device="cpu").manual_seed(1) + gen_b = torch.Generator(device="cpu").manual_seed(2) + latents_a = torch.zeros(2, 4, 2, 2, 2) + latents_b = torch.ones(2, 4, 2, 2, 2) + image_a = torch.zeros(1, 3, 16, 16) + image_b = torch.ones(1, 3, 16, 16) + batch = DiffusionRequestBatch( + requests=[ + SimpleNamespace( + request_id="a", + prompt={"prompt": "first", "multi_modal_data": {"image": image_a}}, + sampling_params=_make_i2v_sampling( + generator=gen_a, + latents=latents_a, + num_outputs_per_prompt=2, + ), + ), + SimpleNamespace( + request_id="b", + prompt={"prompt": "second", "multi_modal_data": {"image": image_b}}, + sampling_params=_make_i2v_sampling( + generator=gen_b, + latents=latents_b, + num_outputs_per_prompt=2, + ), + ), + ] + ) + + outputs = pipeline.forward(batch) + + assert prepare_call["batch_size"] == 4 + assert prepare_call["generator"] == [gen_a, gen_a, gen_b, gen_b] + torch.testing.assert_close(prepare_call["latents"], torch.cat([latents_a, latents_b])) + torch.testing.assert_close( + prepare_call["image"], + torch.cat([image_a, image_a, image_b, image_b]), + ) + assert len(outputs) == 2 + torch.testing.assert_close(outputs[0].output, latents_a) + torch.testing.assert_close(outputs[1].output, latents_b) + + +def test_i2v_forward_rejects_mismatched_tensor_condition_shapes(monkeypatch) -> None: + monkeypatch.setattr( + "vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_i2v.current_omni_platform", + SimpleNamespace(is_available=lambda: False), + ) + pipeline = _make_i2v_pipeline(expand_timesteps=True) + pipeline.scheduler = StubScheduler([9]) + pipeline.od_config = SimpleNamespace(flow_shift=5.0) + pipeline._sample_solver = "unipc" + pipeline._flow_shift = 5.0 + pipeline.boundary_ratio = 0.875 + pipeline.has_image_encoder = False + pipeline._guidance_scale = None + pipeline._guidance_scale_2 = None + pipeline._num_timesteps = None + pipeline._current_timestep = None + pipeline.check_inputs = lambda **kwargs: None + pipeline.encode_prompt = lambda **kwargs: (torch.zeros(2, 2, 3), None) # type: ignore[method-assign] + batch = DiffusionRequestBatch( + requests=[ + SimpleNamespace( + request_id="a", + prompt={"prompt": "first", "multi_modal_data": {"image": torch.zeros(1, 3, 16, 16)}}, + sampling_params=_make_i2v_sampling(), + ), + SimpleNamespace( + request_id="b", + prompt={"prompt": "second", "multi_modal_data": {"image": torch.zeros(1, 3, 16, 8)}}, + sampling_params=_make_i2v_sampling(), + ), + ] + ) + + with pytest.raises(ValueError, match="image condition"): + pipeline.forward(batch) + + +def test_i2v_forward_rejects_mixed_last_image_presence() -> None: + pipeline = _make_i2v_pipeline(expand_timesteps=True) + image = torch.zeros(1, 3, 16, 16) + batch = DiffusionRequestBatch( + requests=[ + SimpleNamespace( + request_id="a", + prompt={ + "prompt": "first", + "multi_modal_data": {"image": image, "last_image": image}, + }, + sampling_params=_make_i2v_sampling(), + ), + SimpleNamespace( + request_id="b", + prompt={"prompt": "second", "multi_modal_data": {"image": image}}, + sampling_params=_make_i2v_sampling(), + ), + ] + ) + + with pytest.raises(ValueError, match="mix of provided and missing last_image"): + pipeline.forward(batch) diff --git a/tests/diffusion/models/wan2_2/test_wan22_pipeline_diffuse.py b/tests/diffusion/models/wan2_2/test_wan22_pipeline_diffuse.py index 8c06df034..325bd59da 100644 --- a/tests/diffusion/models/wan2_2/test_wan22_pipeline_diffuse.py +++ b/tests/diffusion/models/wan2_2/test_wan22_pipeline_diffuse.py @@ -93,6 +93,29 @@ def _make_pipeline() -> Wan22Pipeline: return pipeline +def _make_sampling(**overrides): + values = { + "height": None, + "width": None, + "num_frames": 1, + "num_inference_steps": 2, + "guidance_scale_provided": True, + "guidance_scale": 1.0, + "guidance_scale_2": None, + "guidance_scale_2_provided": False, + "boundary_ratio": None, + "generator": None, + "seed": None, + "num_outputs_per_prompt": 1, + "max_sequence_length": 32, + "latents": None, + "output_type": "latent", + "extra_args": {}, + } + values.update(overrides) + return SimpleNamespace(**values) + + @pytest.mark.parametrize( ("sampling_params_kwargs", "expected_low", "expected_high"), [ @@ -129,9 +152,10 @@ def test_forward_delegates_denoising_to_diffuse( ) batch = DiffusionRequestBatch(requests=[mock_req]) - output = pipeline.forward(batch) + outputs = pipeline.forward(batch) - assert torch.equal(output.output, torch.ones((1, 4, 1, 8, 8))) + assert len(outputs) == 1 + assert torch.equal(outputs[0].output, torch.ones((1, 4, 1, 8, 8))) assert torch.equal(captured["prompt_embeds"], torch.zeros(1, 32, 8)) assert torch.equal(captured["timesteps"], pipeline.scheduler.timesteps) assert captured["guidance_low"] == expected_low @@ -142,6 +166,134 @@ def test_forward_delegates_denoising_to_diffuse( assert pipeline.scheduler.set_timesteps_calls == [(2, torch.device("cpu"))] +def test_forward_batches_text_generators_latents_and_splits_outputs() -> None: + pipeline = _make_pipeline() + encode_call = {} + prepare_call = {} + + def _fake_encode_prompt(**kwargs): + encode_call.update(kwargs) + batch_size = len(kwargs["prompt"]) + n = batch_size * kwargs["num_videos_per_prompt"] + return torch.arange(n, dtype=torch.float32).view(n, 1, 1), torch.zeros(n, 1, 1) + + def _fake_prepare_latents(**kwargs): + prepare_call.update(kwargs) + return kwargs["latents"] + + pipeline.encode_prompt = _fake_encode_prompt # type: ignore[method-assign] + pipeline.prepare_latents = _fake_prepare_latents # type: ignore[method-assign] + pipeline.diffuse = lambda **kwargs: kwargs["latents"] # type: ignore[method-assign] + + gen_a = torch.Generator(device="cpu").manual_seed(1) + gen_b = torch.Generator(device="cpu").manual_seed(2) + latents_a = torch.zeros(2, 4, 1, 2, 2) + latents_b = torch.ones(2, 4, 1, 2, 2) + batch = DiffusionRequestBatch( + requests=[ + SimpleNamespace( + request_id="a", + prompt={"prompt": "first", "negative_prompt": "bad first"}, + sampling_params=_make_sampling( + generator=gen_a, + latents=latents_a, + num_outputs_per_prompt=2, + ), + ), + SimpleNamespace( + request_id="b", + prompt={"prompt": "second", "negative_prompt": "bad second"}, + sampling_params=_make_sampling( + generator=gen_b, + latents=latents_b, + num_outputs_per_prompt=2, + ), + ), + ] + ) + + outputs = pipeline.forward(batch) + + assert encode_call["prompt"] == ["first", "second"] + assert encode_call["negative_prompt"] == ["bad first", "bad second"] + assert prepare_call["batch_size"] == 4 + assert prepare_call["generator"] == [gen_a, gen_a, gen_b, gen_b] + torch.testing.assert_close(prepare_call["latents"], torch.cat([latents_a, latents_b])) + assert len(outputs) == 2 + torch.testing.assert_close(outputs[0].output, latents_a) + torch.testing.assert_close(outputs[1].output, latents_b) + + +def test_forward_batches_precomputed_prompt_embeddings() -> None: + pipeline = _make_pipeline() + diffuse_call = {} + pipeline.encode_prompt = lambda **kwargs: pytest.fail("text encoder must not run") # type: ignore[method-assign] + pipeline.prepare_latents = lambda **kwargs: torch.zeros(kwargs["batch_size"], 4, 1, 2, 2) # type: ignore[method-assign] + + def _fake_diffuse(**kwargs): + diffuse_call.update(kwargs) + return kwargs["latents"] + + pipeline.diffuse = _fake_diffuse # type: ignore[method-assign] + embeds_a = torch.zeros(3, 4) + embeds_b = torch.ones(3, 4) + negative_a = torch.full((3, 4), 2.0) + negative_b = torch.full((3, 4), 3.0) + batch = DiffusionRequestBatch( + requests=[ + SimpleNamespace( + request_id="a", + prompt={"prompt_embeds": embeds_a, "negative_prompt_embeds": negative_a}, + sampling_params=_make_sampling(guidance_scale=4.0), + ), + SimpleNamespace( + request_id="b", + prompt={"prompt_embeds": embeds_b, "negative_prompt_embeds": negative_b}, + sampling_params=_make_sampling(guidance_scale=4.0), + ), + ] + ) + + outputs = pipeline.forward(batch) + + torch.testing.assert_close(diffuse_call["prompt_embeds"], torch.stack([embeds_a, embeds_b])) + torch.testing.assert_close(diffuse_call["negative_prompt_embeds"], torch.stack([negative_a, negative_b])) + assert len(outputs) == 2 + + +def test_prepare_latents_with_request_generators_matches_single_generation() -> None: + pipeline = _make_pipeline() + kwargs = { + "num_channels_latents": 4, + "height": 16, + "width": 16, + "num_frames": 5, + "dtype": torch.float32, + "device": torch.device("cpu"), + } + + batched = Wan22Pipeline.prepare_latents( + pipeline, + batch_size=2, + generator=[torch.Generator().manual_seed(1), torch.Generator().manual_seed(2)], + **kwargs, + ) + singles = torch.cat( + [ + Wan22Pipeline.prepare_latents( + pipeline, + batch_size=1, + generator=torch.Generator().manual_seed(seed), + **kwargs, + ) + for seed in (1, 2) + ] + ) + + torch.testing.assert_close(batched, singles) + assert not torch.equal(batched[0], batched[1]) + + def test_diffuse_runs_prediction_and_scheduler_for_each_timestep() -> None: pipeline = _make_pipeline() latents = torch.zeros((1, 1, 1, 2, 2), dtype=torch.float32) diff --git a/tests/diffusion/models/wan2_2/test_wan22_pipeline_helpers.py b/tests/diffusion/models/wan2_2/test_wan22_pipeline_helpers.py index 314717869..5a37e14f2 100644 --- a/tests/diffusion/models/wan2_2/test_wan22_pipeline_helpers.py +++ b/tests/diffusion/models/wan2_2/test_wan22_pipeline_helpers.py @@ -10,6 +10,7 @@ import torch import vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 as wan22_module from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import ( create_transformer_from_config, + get_wan22_post_process_func, load_transformer_config, retrieve_latents, ) @@ -26,6 +27,17 @@ class _LatentDist: return torch.tensor([2.0]) +def test_wan22_postprocess_honors_request_output_type() -> None: + video = torch.zeros(1, 4, 1, 2, 2) + + output = get_wan22_post_process_func(SimpleNamespace())( + video, + sampling_params=SimpleNamespace(output_type="latent"), + ) + + assert output is video + + def test_retrieve_latents_supports_sample_mode_argmax_and_direct_latents() -> None: generator = torch.Generator(device="cpu") diff --git a/tests/diffusion/models/wan2_2/test_wan22_s2v_pipeline.py b/tests/diffusion/models/wan2_2/test_wan22_s2v_pipeline.py index 4440a8f35..e5bae65df 100644 --- a/tests/diffusion/models/wan2_2/test_wan22_s2v_pipeline.py +++ b/tests/diffusion/models/wan2_2/test_wan22_s2v_pipeline.py @@ -1,14 +1,307 @@ +from types import SimpleNamespace from unittest.mock import MagicMock, patch +import numpy as np +import PIL.Image import pytest import torch import torch.nn as nn +from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_s2v import ( + Wan22S2VPipeline, + _make_clip_generators, +) from vllm_omni.diffusion.models.wan2_2.wan2_2_s2v_transformer import WanS2VTransformer3DModel +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +def _make_s2v_sampling(**overrides): + values = { + "height": 16, + "width": 16, + "num_frames": 8, + "num_inference_steps": 1, + "guidance_scale_provided": True, + "guidance_scale": 1.0, + "generator": None, + "seed": None, + "num_outputs_per_prompt": 1, + "max_sequence_length": 8, + "latents": None, + "output_type": "latent", + "extra_args": {}, + } + values.update(overrides) + return SimpleNamespace(**values) + + +def _make_s2v_validation_pipeline() -> Wan22S2VPipeline: + pipeline = object.__new__(Wan22S2VPipeline) + nn.Module.__init__(pipeline) + pipeline.device = torch.device("cpu") + pipeline.transformer = SimpleNamespace(dtype=torch.float32) + pipeline.vae_scale_factor_spatial = 8 + pipeline.resolution_divisor = 16 + pipeline.motion_frames = 7 + pipeline.drop_first_motion = True + pipeline._DEFAULT_INFER_FRAMES = 8 + pipeline._guidance_scale = None + pipeline.check_inputs = lambda *args, **kwargs: None + pipeline.encode_prompt = lambda **kwargs: (torch.zeros(2, 2, 3), None) # type: ignore[method-assign] + return pipeline + + +def test_s2v_predict_noise_keeps_batch_dimension() -> None: + pipeline = object.__new__(Wan22S2VPipeline) + nn.Module.__init__(pipeline) + pipeline.device = torch.device("cpu") + transformer = MagicMock() + transformer.parameters.return_value = iter([torch.zeros(1, dtype=torch.bfloat16)]) + transformer.return_value = (torch.zeros(1, 16, 2, 2, 2),) + pipeline.transformer = transformer + + result = pipeline.predict_noise(hidden_states=torch.zeros(1, 16, 2, 2, 2)) + + assert result.shape == (1, 16, 2, 2, 2) + + +def test_s2v_clip_generators_preserve_main_seed_per_clip_behavior() -> None: + explicit_generator = torch.Generator(device="cpu").manual_seed(999) + + generators = _make_clip_generators( + seeds=[1234, 5678, None, None], + request_generators=[ + torch.Generator(device="cpu").manual_seed(1234), + torch.Generator(device="cpu").manual_seed(5678), + explicit_generator, + None, + ], + clip_index=2, + device=torch.device("cpu"), + ) + + assert [generator.initial_seed() for generator in generators if generator is not None] == [1236, 5680, 999, 2] + assert generators[2] is explicit_generator + + +def test_s2v_forward_rejects_different_num_repeat() -> None: + pipeline = _make_s2v_validation_pipeline() + pipeline.encode_audio = lambda *args, **kwargs: (torch.zeros(1, 1, 2, 16), 2, 16) # type: ignore[method-assign] + image = PIL.Image.new("RGB", (16, 16)) + audio = np.zeros(16000, dtype=np.float32) + batch = DiffusionRequestBatch( + requests=[ + SimpleNamespace( + request_id="a", + prompt={ + "prompt": "first", + "multi_modal_data": {"image": image, "audio": audio}, + "additional_information": {"num_repeat": 1}, + }, + sampling_params=_make_s2v_sampling(), + ), + SimpleNamespace( + request_id="b", + prompt={ + "prompt": "second", + "multi_modal_data": {"image": image, "audio": audio}, + "additional_information": {"num_repeat": 2}, + }, + sampling_params=_make_s2v_sampling(), + ), + ] + ) + + with pytest.raises(ValueError, match="num_repeat"): + pipeline.forward(batch) + + +def test_s2v_forward_rejects_different_audio_time_lengths() -> None: + pipeline = _make_s2v_validation_pipeline() + pipeline.encode_audio = lambda audio, **kwargs: ( # type: ignore[method-assign] + torch.zeros(1, 1, 2, 16 if audio[0] == 0 else 24), + 1, + 16 if audio[0] == 0 else 24, + ) + image = PIL.Image.new("RGB", (16, 16)) + batch = DiffusionRequestBatch( + requests=[ + SimpleNamespace( + request_id="a", + prompt={ + "prompt": "first", + "multi_modal_data": {"image": image, "audio": np.zeros(16000, dtype=np.float32)}, + }, + sampling_params=_make_s2v_sampling(), + ), + SimpleNamespace( + request_id="b", + prompt={ + "prompt": "second", + "multi_modal_data": {"image": image, "audio": np.ones(16000, dtype=np.float32)}, + }, + sampling_params=_make_s2v_sampling(), + ), + ] + ) + + with pytest.raises(ValueError, match="audio embedding time lengths"): + pipeline.forward(batch) + + +def test_s2v_forward_rejects_different_init_first_frame() -> None: + pipeline = _make_s2v_validation_pipeline() + image = PIL.Image.new("RGB", (16, 16)) + audio = np.zeros(16000, dtype=np.float32) + batch = DiffusionRequestBatch( + requests=[ + SimpleNamespace( + request_id="a", + prompt={ + "prompt": "first", + "multi_modal_data": {"image": image, "audio": audio}, + "additional_information": {"init_first_frame": True}, + }, + sampling_params=_make_s2v_sampling(), + ), + SimpleNamespace( + request_id="b", + prompt={ + "prompt": "second", + "multi_modal_data": {"image": image, "audio": audio}, + "additional_information": {"init_first_frame": False}, + }, + sampling_params=_make_s2v_sampling(), + ), + ] + ) + + with pytest.raises(ValueError, match="init_first_frame"): + pipeline.forward(batch) + + +def test_s2v_forward_rejects_different_raw_audio_shapes() -> None: + pipeline = _make_s2v_validation_pipeline() + pipeline.encode_audio = lambda *args, **kwargs: (torch.zeros(1, 1, 2, 16), 1, 16) # type: ignore[method-assign] + image = PIL.Image.new("RGB", (16, 16)) + batch = DiffusionRequestBatch( + requests=[ + SimpleNamespace( + request_id="a", + prompt={ + "prompt": "first", + "multi_modal_data": {"image": image, "audio": np.zeros(16000, dtype=np.float32)}, + }, + sampling_params=_make_s2v_sampling(), + ), + SimpleNamespace( + request_id="b", + prompt={ + "prompt": "second", + "multi_modal_data": {"image": image, "audio": np.zeros(8000, dtype=np.float32)}, + }, + sampling_params=_make_s2v_sampling(), + ), + ] + ) + + with pytest.raises(ValueError, match="raw audio shapes and sample rates"): + pipeline.forward(batch) + + +def test_s2v_forward_batches_request_local_inputs_and_splits_outputs() -> None: + pipeline = object.__new__(Wan22S2VPipeline) + nn.Module.__init__(pipeline) + pipeline.device = torch.device("cpu") + pipeline.transformer = MagicMock() + pipeline.transformer.dtype = torch.float32 + pipeline.transformer.parameters.return_value = iter([torch.zeros(1, dtype=torch.float32)]) + pipeline.transformer.casual_audio_encoder = None + pipeline.transformer.encode_audio.side_effect = lambda audio, _motion: {"audio_emb": audio} + pipeline.vae = MagicMock() + pipeline.vae.dtype = torch.float32 + pipeline.vae.decode.return_value = (torch.zeros(4, 3, 8, 16, 16),) + pipeline.od_config = SimpleNamespace( + enable_cpu_offload=False, + parallel_config=SimpleNamespace(use_hsdp=False), + ) + pipeline.scheduler = MagicMock(timesteps=torch.tensor([1.0])) + pipeline.vae_scale_factor_spatial = 8 + pipeline.resolution_divisor = 16 + pipeline.motion_frames = 7 + pipeline.drop_first_motion = True + pipeline._DEFAULT_INFER_FRAMES = 8 + pipeline._guidance_scale = None + pipeline._num_timesteps = None + pipeline.check_inputs = lambda *args, **kwargs: None + pipeline.encode_prompt = MagicMock(return_value=(torch.zeros(4, 2, 3), torch.ones(4, 2, 3))) + pipeline.encode_audio = MagicMock(return_value=(torch.zeros(1, 1, 2, 8), 1, 8)) + pipeline.encode_ref_image = MagicMock(return_value=torch.zeros(1, 16, 1, 2, 2)) + pipeline.prepare_motion_latents = MagicMock( + side_effect=lambda pixels, **_: torch.zeros(pixels.shape[0], 16, 2, 2, 2) + ) + pipeline._denormalize_latents = lambda latents: latents + pipeline.diffuse = MagicMock(side_effect=lambda **kwargs: kwargs["latents"]) + + image = PIL.Image.new("RGB", (16, 16)) + audio_a = np.zeros(16000, dtype=np.float32) + audio_b = np.ones(16000, dtype=np.float32) + latents_a = torch.zeros(2, 16, 2, 2, 2) + latents_b = torch.ones(2, 16, 2, 2, 2) + generators_a = [torch.Generator().manual_seed(1), torch.Generator().manual_seed(2)] + generators_b = [torch.Generator().manual_seed(3), torch.Generator().manual_seed(4)] + batch = DiffusionRequestBatch( + requests=[ + SimpleNamespace( + request_id="a", + prompt={ + "prompt": "first", + "negative_prompt": "negative first", + "multi_modal_data": {"image": image, "audio": audio_a}, + }, + sampling_params=_make_s2v_sampling( + num_outputs_per_prompt=2, + generator=generators_a, + latents=latents_a, + ), + ), + SimpleNamespace( + request_id="b", + prompt={ + "prompt": "second", + "negative_prompt": "negative second", + "multi_modal_data": {"image": image, "audio": audio_b}, + }, + sampling_params=_make_s2v_sampling( + num_outputs_per_prompt=2, + generator=generators_b, + latents=latents_b, + ), + ), + ] + ) + + with patch("vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_s2v.current_omni_platform") as platform: + platform.is_available.return_value = False + outputs = pipeline.forward(batch) + + assert len(outputs) == 2 + assert outputs[0].output[0].shape[0] == 2 + assert outputs[1].output[0].shape[0] == 2 + assert outputs[0].output[1].shape == (16000,) + assert outputs[1].output[1].shape == (16000,) + np.testing.assert_array_equal(outputs[0].output[1], audio_a) + np.testing.assert_array_equal(outputs[1].output[1], audio_b) + assert np.shares_memory(outputs[0].output[1], audio_a) + assert np.shares_memory(outputs[1].output[1], audio_b) + torch.testing.assert_close(pipeline.diffuse.call_args.kwargs["latents"], torch.cat([latents_a, latents_b])) + assert pipeline.diffuse.call_args.kwargs["clip_generator"] == generators_a + generators_b + assert pipeline.encode_prompt.call_args.kwargs["prompt"] == ["first", "second"] + assert pipeline.encode_prompt.call_args.kwargs["negative_prompt"] == ["negative first", "negative second"] + + def test_s2v_exposes_hsdp_shard_conditions_for_transformer_blocks(): model = object.__new__(WanS2VTransformer3DModel) nn.Module.__init__(model) @@ -260,31 +553,37 @@ def test_s2v_pipeline_hsdp_forward_complete_process(): pipeline._prompt_clean = Wan22S2VPipeline._prompt_clean # -- Build request -- - from vllm_omni.diffusion.request import OmniDiffusionRequest - - sampling_params = MagicMock() - sampling_params.height = 704 - sampling_params.width = 1024 - sampling_params.num_frames = 80 - sampling_params.num_inference_steps = 5 - sampling_params.guidance_scale = 4.5 - sampling_params.guidance_scale_provided = True - sampling_params.generator = None - sampling_params.seed = 42 + sampling_params = _make_s2v_sampling( + height=704, + width=1024, + num_frames=80, + num_inference_steps=5, + guidance_scale=4.5, + generator=torch.Generator(device="cpu").manual_seed(42), + max_sequence_length=512, + ) ref_image = PIL.Image.new("RGB", (1024, 704)) audio_data = np.zeros(16000, dtype=np.float32) - req = MagicMock(spec=OmniDiffusionRequest) - req.prompts = [ - { - "prompt": "test prompt", - "negative_prompt": "bad quality", - "multi_modal_data": {"image": ref_image, "audio": audio_data}, - "additional_information": {"audio_path": audio_data, "pose_video": None, "init_first_frame": False}, - } - ] - req.sampling_params = sampling_params + req = DiffusionRequestBatch( + requests=[ + SimpleNamespace( + request_id="test", + prompt={ + "prompt": "test prompt", + "negative_prompt": "bad quality", + "multi_modal_data": {"image": ref_image, "audio": audio_data}, + "additional_information": { + "audio_path": audio_data, + "pose_video": None, + "init_first_frame": False, + }, + }, + sampling_params=sampling_params, + ) + ] + ) # -- Mock methods that use complex internal state -- pipeline.encode_audio = MagicMock(return_value=(torch.zeros(1, 25, 64, 80), 1, 80)) @@ -295,7 +594,7 @@ def test_s2v_pipeline_hsdp_forward_complete_process(): pipeline.progress_bar.return_value.__exit__ = MagicMock(return_value=False) # Mock predict_noise_maybe_with_cfg from CFGParallelMixin - pipeline.predict_noise_maybe_with_cfg = MagicMock(return_value=torch.zeros(16, 20, 88, 128)) + pipeline.predict_noise_maybe_with_cfg = MagicMock(return_value=torch.zeros(1, 16, 20, 88, 128)) pipeline.predict_noise = Wan22S2VPipeline.predict_noise.__get__(pipeline) # Mock platform methods @@ -306,7 +605,7 @@ def test_s2v_pipeline_hsdp_forward_complete_process(): with patch( "vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_s2v.load_audio", return_value=(audio_data, 16000) ): - result = Wan22S2VPipeline.forward(pipeline, req=req) + result = Wan22S2VPipeline.forward(pipeline, req=req)[0] # -- Assertions -- # Text encoder was called diff --git a/tests/diffusion/models/wan2_2/test_wan22_vace_pipeline.py b/tests/diffusion/models/wan2_2/test_wan22_vace_pipeline.py index 714b79d50..6534b7cff 100644 --- a/tests/diffusion/models/wan2_2/test_wan22_vace_pipeline.py +++ b/tests/diffusion/models/wan2_2/test_wan22_vace_pipeline.py @@ -8,13 +8,14 @@ import torch from PIL import Image from torch import nn -from tests.diffusion.models.wan2_2.conftest import StubTransformer, StubVAE, noop_progress_bar +from tests.diffusion.models.wan2_2.conftest import StubScheduler, StubTransformer, StubVAE, noop_progress_bar from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_vace import ( Wan22VACEPipeline, create_vace_transformer_from_config, get_wan22_vace_pre_process_func, ) from vllm_omni.diffusion.models.wan2_2.wan2_2_vace_transformer import WanVACETransformer3DModel +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] @@ -32,6 +33,27 @@ def _make_vace_pipeline() -> Wan22VACEPipeline: return pipeline +def _make_vace_sampling(**overrides): + values = { + "height": 16, + "width": 16, + "num_frames": 5, + "num_inference_steps": 1, + "guidance_scale_provided": True, + "guidance_scale": 1.0, + "boundary_ratio": None, + "generator": None, + "seed": None, + "num_outputs_per_prompt": 1, + "max_sequence_length": 8, + "latents": None, + "output_type": "latent", + "extra_args": {}, + } + values.update(overrides) + return SimpleNamespace(**values) + + def test_vace_preprocess_collects_reference_video_and_mask_inputs() -> None: preprocess = get_wan22_vace_pre_process_func(SimpleNamespace()) ref = Image.new("RGB", (320, 160), "green") @@ -138,3 +160,189 @@ def test_vace_diffuse_passes_context_and_scale_to_cfg_branches() -> None: assert calls[0]["positive_kwargs"]["vace_context"] is vace_context assert calls[0]["negative_kwargs"]["vace_context_scale"] == 0.75 torch.testing.assert_close(result, torch.ones_like(latents)) + + +def test_vace_forward_batches_random_inputs_and_splits_outputs(monkeypatch) -> None: + monkeypatch.setattr( + "vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_vace.current_omni_platform", + SimpleNamespace(is_available=lambda: False), + ) + pipeline = _make_vace_pipeline() + pipeline.transformer.vace_patch_embedding = None + pipeline.transformer_2 = None + pipeline.scheduler = StubScheduler([9]) + pipeline.od_config = SimpleNamespace(flow_shift=3.0) + pipeline._sample_solver = "unipc" + pipeline._flow_shift = 3.0 + pipeline.boundary_ratio = None + pipeline._guidance_scale = None + pipeline._num_timesteps = None + pipeline._current_timestep = None + pipeline.check_inputs = lambda **kwargs: None + prepare_call = {} + + def _fake_encode_prompt(**kwargs): + batch_size = len(kwargs["prompt"]) + n = batch_size * kwargs["num_videos_per_prompt"] + return torch.zeros(n, 2, 3), None + + def _fake_prepare_latents(**kwargs): + prepare_call.update(kwargs) + return kwargs["latents"] + + pipeline.encode_prompt = _fake_encode_prompt # type: ignore[method-assign] + pipeline.prepare_latents = _fake_prepare_latents # type: ignore[method-assign] + pipeline.diffuse = lambda **kwargs: kwargs["latents"] # type: ignore[method-assign] + gen_a = torch.Generator(device="cpu").manual_seed(1) + gen_b = torch.Generator(device="cpu").manual_seed(2) + latents_a = torch.zeros(2, 4, 2, 2, 2) + latents_b = torch.ones(2, 4, 2, 2, 2) + batch = DiffusionRequestBatch( + requests=[ + SimpleNamespace( + request_id="a", + prompt={"prompt": "first"}, + sampling_params=_make_vace_sampling( + generator=gen_a, + latents=latents_a, + num_outputs_per_prompt=2, + ), + ), + SimpleNamespace( + request_id="b", + prompt={"prompt": "second"}, + sampling_params=_make_vace_sampling( + generator=gen_b, + latents=latents_b, + num_outputs_per_prompt=2, + ), + ), + ] + ) + + outputs = pipeline.forward(batch) + + assert prepare_call["batch_size"] == 4 + assert prepare_call["generator"] == [gen_a, gen_a, gen_b, gen_b] + torch.testing.assert_close(prepare_call["latents"], torch.cat([latents_a, latents_b])) + assert len(outputs) == 2 + torch.testing.assert_close(outputs[0].output, latents_a) + torch.testing.assert_close(outputs[1].output, latents_b) + + +def test_vace_forward_rejects_mismatched_condition_shapes(monkeypatch) -> None: + monkeypatch.setattr( + "vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_vace.current_omni_platform", + SimpleNamespace(is_available=lambda: False), + ) + pipeline = _make_vace_pipeline() + pipeline.transformer.vace_patch_embedding = object() + pipeline.transformer_2 = None + pipeline.scheduler = StubScheduler([9]) + pipeline.od_config = SimpleNamespace(flow_shift=3.0) + pipeline._sample_solver = "unipc" + pipeline._flow_shift = 3.0 + pipeline.boundary_ratio = None + pipeline._guidance_scale = None + pipeline._num_timesteps = None + pipeline._current_timestep = None + pipeline.check_inputs = lambda **kwargs: None + pipeline.encode_prompt = lambda **kwargs: (torch.zeros(2, 2, 3), None) # type: ignore[method-assign] + + def _fake_preprocess(video, **kwargs): + del kwargs + mask = torch.ones_like(video) + return video, mask, [[]] + + pipeline.preprocess_conditions = _fake_preprocess # type: ignore[method-assign] + pipeline.prepare_video_latents = ( # type: ignore[method-assign] + lambda video, mask, refs, generator, device: torch.zeros(1, 8, video.shape[2], 2, 2) + ) + pipeline.prepare_masks = lambda mask, refs: torch.zeros(1, 4, mask.shape[2], 2, 2) # type: ignore[method-assign] + batch = DiffusionRequestBatch( + requests=[ + SimpleNamespace( + request_id="a", + prompt={ + "prompt": "first", + "additional_information": {"source_video": torch.zeros(1, 3, 5, 16, 16)}, + }, + sampling_params=_make_vace_sampling(), + ), + SimpleNamespace( + request_id="b", + prompt={ + "prompt": "second", + "additional_information": {"source_video": torch.zeros(1, 3, 3, 16, 16)}, + }, + sampling_params=_make_vace_sampling(), + ), + ] + ) + + with pytest.raises(ValueError, match="VACE condition"): + pipeline.forward(batch) + + +def test_vace_forward_rejects_mismatched_reference_image_counts(monkeypatch) -> None: + monkeypatch.setattr( + "vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_vace.current_omni_platform", + SimpleNamespace(is_available=lambda: False), + ) + pipeline = _make_vace_pipeline() + pipeline.transformer.vace_patch_embedding = object() + pipeline.transformer_2 = None + pipeline.scheduler = StubScheduler([9]) + pipeline.od_config = SimpleNamespace(flow_shift=3.0) + pipeline._sample_solver = "unipc" + pipeline._flow_shift = 3.0 + pipeline.boundary_ratio = None + pipeline._guidance_scale = None + pipeline._num_timesteps = None + pipeline._current_timestep = None + pipeline.check_inputs = lambda **kwargs: None + pipeline.encode_prompt = lambda **kwargs: (torch.zeros(2, 2, 3), None) # type: ignore[method-assign] + + def _fake_preprocess(video, mask, reference_images, **kwargs): + del kwargs + return video, mask, [reference_images] + + pipeline.preprocess_conditions = _fake_preprocess # type: ignore[method-assign] + pipeline.prepare_video_latents = ( # type: ignore[method-assign] + lambda video, mask, refs, generator, device: torch.zeros(1, 8, 5, 2, 2) + ) + pipeline.prepare_masks = lambda mask, refs: torch.zeros(1, 4, 5, 2, 2) # type: ignore[method-assign] + video = torch.zeros(1, 3, 5, 16, 16) + mask = torch.ones_like(video) + reference = torch.zeros(3, 16, 16) + batch = DiffusionRequestBatch( + requests=[ + SimpleNamespace( + request_id="a", + prompt={ + "prompt": "first", + "additional_information": { + "source_video": video, + "mask": mask, + "reference_images": [reference], + }, + }, + sampling_params=_make_vace_sampling(), + ), + SimpleNamespace( + request_id="b", + prompt={ + "prompt": "second", + "additional_information": { + "source_video": video, + "mask": mask, + "reference_images": [reference, reference], + }, + }, + sampling_params=_make_vace_sampling(), + ), + ] + ) + + with pytest.raises(ValueError, match="same number of reference images"): + pipeline.forward(batch) diff --git a/tests/diffusion/test_diffusion_engine_rpc_routing.py b/tests/diffusion/test_diffusion_engine_rpc_routing.py index 1d4c2d17e..5aef831aa 100644 --- a/tests/diffusion/test_diffusion_engine_rpc_routing.py +++ b/tests/diffusion/test_diffusion_engine_rpc_routing.py @@ -123,6 +123,7 @@ class _ConcurrencyTrackingExecutor: def _make_request(tag: str): sampling_params = dict(_SAMPLING_KEY_DEFAULTS) sampling_params["num_inference_steps"] = 1 + sampling_params["extra_args"] = {} return SimpleNamespace( request_id=tag, prompt=f"prompt_{tag}", diff --git a/tests/diffusion/test_diffusion_scheduler.py b/tests/diffusion/test_diffusion_scheduler.py index ce9f0ebe1..46ec0cbe6 100644 --- a/tests/diffusion/test_diffusion_scheduler.py +++ b/tests/diffusion/test_diffusion_scheduler.py @@ -217,13 +217,21 @@ class TestGetRequestBatchSamplingParamsKey: num_inference_steps: int = 2, seed: int | None = 123, generator: torch.Generator | None = None, + extra_args: dict | None = None, + condition_key: tuple | None = None, ) -> OmniDiffusionRequest: sp = OmniDiffusionSamplingParams( num_inference_steps=num_inference_steps, seed=seed, generator=generator, + extra_args=extra_args or {}, + ) + return OmniDiffusionRequest( + prompt="prompt", + sampling_params=sp, + request_id=f"req-{num_inference_steps}", + batch_compatibility_key=condition_key, ) - return OmniDiffusionRequest(prompt="prompt", sampling_params=sp, request_id=f"req-{num_inference_steps}") def test_distinguishes_num_inference_steps(self) -> None: scheduler = RequestScheduler() @@ -240,6 +248,57 @@ class TestGetRequestBatchSamplingParamsKey: self._make(seed=1, generator=gen_a) ) == scheduler._build_sampling_params_key(self._make(seed=2, generator=gen_b)) + @pytest.mark.parametrize( + ("first_extra_args", "second_extra_args"), + [ + ({"sample_solver": "unipc"}, {"sample_solver": "euler"}), + ({"flow_shift": 3.0}, {"flow_shift": 5.0}), + ], + ) + def test_distinguishes_wan_scheduler_structure( + self, + first_extra_args: dict, + second_extra_args: dict, + ) -> None: + scheduler = RequestScheduler() + + assert scheduler._build_sampling_params_key( + self._make(extra_args=first_extra_args) + ) != scheduler._build_sampling_params_key(self._make(extra_args=second_extra_args)) + + @pytest.mark.parametrize( + ("first_extra_args", "second_extra_args"), + [ + ({"sample_solver": " Euler "}, {"sample_solver": "euler"}), + ({"flow_shift": "5.0"}, {"flow_shift": 5.0}), + ], + ) + def test_normalizes_equivalent_wan_scheduler_structure( + self, + first_extra_args: dict, + second_extra_args: dict, + ) -> None: + scheduler = RequestScheduler() + + assert scheduler._build_sampling_params_key( + self._make(extra_args=first_extra_args) + ) == scheduler._build_sampling_params_key(self._make(extra_args=second_extra_args)) + + def test_uses_none_for_unspecified_wan_scheduler_structure(self) -> None: + scheduler = RequestScheduler() + + key = scheduler._build_sampling_params_key(self._make()) + + assert key.sample_solver is None + assert key.flow_shift is None + + def test_distinguishes_pipeline_condition_structure(self) -> None: + scheduler = RequestScheduler() + + assert scheduler._build_sampling_params_key( + self._make(condition_key=("wan22_s2v_condition", True)) + ) != scheduler._build_sampling_params_key(self._make(condition_key=("wan22_s2v_condition", False))) + class TestRequestScheduler: def setup_method(self) -> None: @@ -602,6 +661,38 @@ class TestRequestScheduler: assert first.num_running_reqs == 1 assert first.num_waiting_reqs == 1 + def test_batches_incompatible_pipeline_conditions_separately(self) -> None: + scheduler = RequestScheduler() + scheduler.initialize(SimpleNamespace(max_num_seqs=2)) + + request_a = OmniDiffusionRequest( + prompt="a", + sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, seed=123), + request_id="a", + batch_compatibility_key=("wan22_s2v_condition", True), + ) + request_b = OmniDiffusionRequest( + prompt="b", + sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, seed=456), + request_id="b", + batch_compatibility_key=("wan22_s2v_condition", False), + ) + scheduler.add_request(request_a) + scheduler.add_request(request_b) + + first = scheduler.schedule() + + assert _new_ids(first) == ["a"] + assert first.num_running_reqs == 1 + assert first.num_waiting_reqs == 1 + + scheduler.update_from_output(first, _make_request_output("a")) + second = scheduler.schedule() + + assert _new_ids(second) == ["b"] + assert second.num_running_reqs == 1 + assert second.num_waiting_reqs == 0 + def test_batches_different_request_local_seed_together(self) -> None: scheduler = RequestScheduler() scheduler.initialize(SimpleNamespace(max_num_seqs=2)) @@ -625,6 +716,47 @@ class TestRequestScheduler: assert first.num_running_reqs == 2 assert first.num_waiting_reqs == 0 + @pytest.mark.parametrize( + ("first_extra_args", "second_extra_args"), + [ + ({"sample_solver": "unipc"}, {"sample_solver": "euler"}), + ({"flow_shift": 3.0}, {"flow_shift": 5.0}), + ({"sample_solver": "unipc"}, {}), + ({"flow_shift": 3.0}, {}), + ], + ) + def test_batches_incompatible_wan_scheduler_structure_separately( + self, + first_extra_args: dict, + second_extra_args: dict, + ) -> None: + scheduler = RequestScheduler() + scheduler.initialize(SimpleNamespace(max_num_seqs=2)) + first_id = scheduler.add_request( + _make_step_request( + "a", + sampling_params=OmniDiffusionSamplingParams( + num_inference_steps=2, + extra_args=first_extra_args, + ), + ) + ) + scheduler.add_request( + _make_step_request( + "b", + sampling_params=OmniDiffusionSamplingParams( + num_inference_steps=2, + extra_args=second_extra_args, + ), + ) + ) + + first = scheduler.schedule() + + assert _new_ids(first) == [first_id] + assert first.num_running_reqs == 1 + assert first.num_waiting_reqs == 1 + def test_incompatible_waiting_head_blocks_later_compatible_request(self) -> None: scheduler = RequestScheduler() scheduler.initialize(SimpleNamespace(max_num_seqs=3)) diff --git a/tests/engine/test_async_omni_engine_stage_init.py b/tests/engine/test_async_omni_engine_stage_init.py index 9ee59f153..5fae5f1b2 100644 --- a/tests/engine/test_async_omni_engine_stage_init.py +++ b/tests/engine/test_async_omni_engine_stage_init.py @@ -399,6 +399,101 @@ def test_initialize_local_llm_replica_scopes_runtime_env(monkeypatch): assert os.environ[runtime_env_var] == "parent-value" +def test_initialize_diffusion_stage_applies_client_batch_size_to_engine(monkeypatch): + import vllm_omni.diffusion.stage_diffusion_client as client_mod + import vllm_omni.engine.stage_init_utils as init_mod + + od_config = types.SimpleNamespace(max_num_seqs=1) + captured: dict[str, object] = {} + monkeypatch.setattr(init_mod, "build_diffusion_config", lambda *args: od_config) + + def _capture_client(model, config, metadata, stage_init_timeout, batch_size, use_inline): + captured.update( + model=model, + config=config, + metadata=metadata, + stage_init_timeout=stage_init_timeout, + batch_size=batch_size, + use_inline=use_inline, + ) + return object() + + monkeypatch.setattr(client_mod, "create_diffusion_client", _capture_client) + metadata = _make_diffusion_metadata(0) + + init_mod.initialize_diffusion_stage( + 0, + "dummy-model", + types.SimpleNamespace(), + metadata, + stage_init_timeout=12, + batch_size=4, + use_inline=True, + ) + + assert od_config.max_num_seqs == 4 + assert captured == { + "model": "dummy-model", + "config": od_config, + "metadata": metadata, + "stage_init_timeout": 12, + "batch_size": 4, + "use_inline": True, + } + + +def test_launch_diffusion_stage_replica_applies_batch_size_to_config(monkeypatch): + import vllm_omni.diffusion.stage_diffusion_client as client_mod + import vllm_omni.diffusion.stage_diffusion_proc as proc_mod + import vllm_omni.engine.stage_engine_startup as startup_mod + + od_config = types.SimpleNamespace(max_num_seqs=1, parallel_config=types.SimpleNamespace(world_size=1)) + monkeypatch.setattr(startup_mod, "build_diffusion_config", lambda *args: od_config) + monkeypatch.setattr(startup_mod, "acquire_device_locks", lambda *args: []) + monkeypatch.setattr( + startup_mod, + "register_stage_with_omni_master", + lambda **kwargs: types.SimpleNamespace( + handshake_address="tcp://127.0.0.1:26001", + input_address="tcp://127.0.0.1:26002", + output_address="tcp://127.0.0.1:26003", + ), + ) + + omni_master_server = types.SimpleNamespace( + address="127.0.0.1", + port=25000, + release_route_port_reservations=lambda *args, **kwargs: None, + ) + proc_manager = types.SimpleNamespace( + addresses=types.SimpleNamespace( + inputs=["tcp://127.0.0.1:26002"], + outputs=["tcp://127.0.0.1:26003"], + ) + ) + monkeypatch.setattr(proc_mod, "StageDiffusionProcManager", lambda **kwargs: proc_manager) + sentinel_client = object() + monkeypatch.setattr( + client_mod.StageDiffusionClient, + "from_addresses", + lambda metadata, **kwargs: sentinel_client, + ) + + result, resources = startup_mod.launch_diffusion_stage_replica( + model="dummy-model", + stage_config=types.SimpleNamespace(), + metadata=types.SimpleNamespace(stage_id=0), + stage_init_timeout=12, + batch_size=4, + use_inline=False, + omni_master_server=omni_master_server, + ) + + assert result is sentinel_client + assert od_config.max_num_seqs == 4 + assert resources.manager is proc_manager + + def test_stage_runtime_initializes_stage_pools(monkeypatch): import vllm_omni.engine.stage_runtime as runtime_mod diff --git a/tests/engine/test_single_stage_mode.py b/tests/engine/test_single_stage_mode.py index 65ec024d5..64ae6627c 100644 --- a/tests/engine/test_single_stage_mode.py +++ b/tests/engine/test_single_stage_mode.py @@ -1138,7 +1138,8 @@ class TestSingleStageReplicaInitialization: runtime._init_visible_devices_baseline = "0" mocker.patch.object(runtime_mod, "inject_kv_stage_info") - mocker.patch("vllm_omni.engine.stage_engine_startup.build_diffusion_config", return_value="diffusion-config") + od_config = SimpleNamespace(max_num_seqs=None, parallel_config=SimpleNamespace(world_size=1)) + mocker.patch("vllm_omni.engine.stage_engine_startup.build_diffusion_config", return_value=od_config) mock_register = mocker.patch( "vllm_omni.engine.stage_engine_startup.register_stage_with_omni_master", return_value=StageRegistrationResponse( @@ -1174,6 +1175,7 @@ class TestSingleStageReplicaInitialization: os.environ[device_env_var] = prev_device_env assert result is sentinel_client + assert od_config.max_num_seqs == 4 mock_register.assert_called_once_with( omni_master_address="127.0.0.1", omni_master_port=25000, @@ -1183,7 +1185,7 @@ class TestSingleStageReplicaInitialization: ) mock_manager.assert_called_once_with( model="fake-model", - od_config="diffusion-config", + od_config=od_config, stage_init_timeout=60, handshake_address="tcp://127.0.0.1:26001", addresses=mocker.ANY, @@ -1230,7 +1232,8 @@ class TestSingleStageReplicaInitialization: runtime._init_visible_devices_baseline = "0" mocker.patch.object(runtime_mod, "inject_kv_stage_info") - mocker.patch("vllm_omni.engine.stage_engine_startup.build_diffusion_config", return_value="diffusion-config") + od_config = SimpleNamespace(max_num_seqs=None, parallel_config=SimpleNamespace(world_size=1)) + mocker.patch("vllm_omni.engine.stage_engine_startup.build_diffusion_config", return_value=od_config) mocker.patch( "vllm_omni.engine.stage_engine_startup.register_stage_with_omni_master", return_value=StageRegistrationResponse( diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py index f9bc9f04f..7ec6c3060 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py @@ -37,7 +37,7 @@ from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3 from vllm_omni.diffusion.postprocess import interpolate_video_tensor from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch, split_diffusion_output_by_request from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt from vllm_omni.platforms import current_omni_platform @@ -204,6 +204,8 @@ def get_wan22_post_process_func( output_type: str = "np", sampling_params=None, ): + if sampling_params is not None and sampling_params.output_type is not None: + output_type = sampling_params.output_type if output_type == "latent": return video video_metadata = {} @@ -233,6 +235,8 @@ def get_wan22_pre_process_func( prompt = request.prompt multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None + has_image = raw_image is not None and (not isinstance(raw_image, list) or bool(raw_image)) + request.batch_compatibility_key = ("wan22_image_condition", has_image) if isinstance(prompt, str): prompt = OmniTextPrompt(prompt=prompt) if "additional_information" not in prompt: @@ -288,6 +292,7 @@ class Wan22Pipeline( SupportsComponentDiscovery, WanLoraLoaderMixin, ): + supports_request_batch = True _dit_modules: ClassVar[list[str]] = ["transformer", "transformer_2"] _encoder_modules: ClassVar[list[str]] = ["text_encoder"] _vae_modules: ClassVar[list[str]] = ["vae"] @@ -561,27 +566,33 @@ class Wan22Pipeline( return latents - def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput: - prompt: str | None = None - negative_prompt: str | None = None - prompt_embeds: torch.Tensor | None = None - negative_prompt_embeds: torch.Tensor | None = None - if len(req.prompts) > 1: - raise ValueError( - """This model only supports a single prompt, not a batched request.""", - """Please pass in a single prompt object or string, or a single-item list.""", - ) - if len(req.prompts) == 1: - first_prompt = req.prompts[0] - prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "") - negative_prompt = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt") + def forward(self, req: DiffusionRequestBatch) -> list[DiffusionOutput]: + sampling_params_list = req.sampling_params_list + common = sampling_params_list[0] + prompt_texts = [prompt if isinstance(prompt, str) else (prompt.get("prompt") or "") for prompt in req.prompts] + negative_prompts = [ + None if isinstance(prompt, str) else prompt.get("negative_prompt") for prompt in req.prompts + ] + prompt_fields = DiffusionRequestBatch.collate_prompt_field_map( + req.prompts, + { + "prompt_embeds": None, + "negative_prompt_embeds": None, + }, + ) + prompt_embeds = prompt_fields["prompt_embeds"] + negative_prompt_embeds = prompt_fields["negative_prompt_embeds"] + prompt: list[str] | None = prompt_texts if prompt_embeds is None else None + negative_prompt: list[str] | None = None + if negative_prompt_embeds is None and any(value is not None for value in negative_prompts): + negative_prompt = [value or "" for value in negative_prompts] - if not prompt: - raise ValueError("Prompt is required for Wan2.2 generation.") + if prompt is not None and not all(prompt): + raise ValueError("Prompt is required for Wan2.2 generation when prompt_embeds are not provided.") - height = req.sampling_params.height or 480 - width = req.sampling_params.width or 832 - num_frames = req.sampling_params.num_frames or 81 + height = common.height or 480 + width = common.width or 832 + num_frames = common.num_frames or 81 # Ensure dimensions are compatible with VAE and patch size # For expand_timesteps mode, we need latent dims to be even (divisible by patch_size) @@ -589,19 +600,20 @@ class Wan22Pipeline( mod_value = self.vae_scale_factor_spatial * patch_size[1] # 16*2=32 for TI2V, 8*2=16 for I2V height = (height // mod_value) * mod_value width = (width // mod_value) * mod_value - num_steps = 40 if req.sampling_params.num_inference_steps is None else req.sampling_params.num_inference_steps + num_steps = 40 if common.num_inference_steps is None else common.num_inference_steps - output_type = req.sampling_params.output_type or "np" + output_type = common.output_type or "np" + num_outputs_per_prompt = common.num_outputs_per_prompt or 1 attention_kwargs: dict | None = None - guidance_low, guidance_high = resolve_wan_guidance_scales(req.sampling_params, default_guidance_scale=4.0) + guidance_low, guidance_high = resolve_wan_guidance_scales(common, default_guidance_scale=4.0) # record guidance for properties self._guidance_scale = guidance_low self._guidance_scale_2 = guidance_high # Prefer engine-configured boundary_ratio, but allow per-request fallback. - boundary_ratio = self.boundary_ratio if self.boundary_ratio is not None else req.sampling_params.boundary_ratio + boundary_ratio = self.boundary_ratio if self.boundary_ratio is not None else common.boundary_ratio if boundary_ratio is None: boundary_ratio = 0.875 @@ -633,32 +645,49 @@ class Wan22Pipeline( # Fallback to text_encoder dtype if no transformer loaded dtype = self.text_encoder.dtype - # Seed / generator - generator = req.sampling_params.generator - if generator is None and req.sampling_params.seed is not None: - generator = torch.Generator(device=device).manual_seed(req.sampling_params.seed) + generator = req.collate_request_generators(num_outputs_per_prompt, None) + request_latents = req.collate_request_tensors("latents", None) if DEBUG_PERF: # Sync GPU before timing to ensure accurate measurements current_omni_platform.synchronize() _t_pipeline_start = time.perf_counter() _t_text_enc_start = _t_pipeline_start - prompt_embeds, negative_prompt_embeds = self.encode_prompt( - prompt=prompt, - negative_prompt=negative_prompt, - do_classifier_free_guidance=guidance_low > 1.0 or guidance_high > 1.0, - num_videos_per_prompt=req.sampling_params.num_outputs_per_prompt or 1, - max_sequence_length=req.sampling_params.max_sequence_length or 512, - device=device, - dtype=dtype, - ) + do_classifier_free_guidance = guidance_low > 1.0 or guidance_high > 1.0 + if prompt_embeds is None: + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt=prompt, + negative_prompt=negative_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + num_videos_per_prompt=num_outputs_per_prompt, + max_sequence_length=common.max_sequence_length or 512, + device=device, + dtype=dtype, + ) + else: + prompt_embeds = prompt_embeds.to(device=device, dtype=dtype) + prompt_embeds = prompt_embeds.repeat_interleave(num_outputs_per_prompt, dim=0) + if negative_prompt_embeds is not None: + negative_prompt_embeds = negative_prompt_embeds.to(device=device, dtype=dtype) + negative_prompt_embeds = negative_prompt_embeds.repeat_interleave(num_outputs_per_prompt, dim=0) + elif do_classifier_free_guidance: + _, negative_prompt_embeds = self.encode_prompt( + prompt=[""] * req.num_reqs, + negative_prompt=negative_prompt, + do_classifier_free_guidance=True, + num_videos_per_prompt=num_outputs_per_prompt, + max_sequence_length=common.max_sequence_length or 512, + device=device, + dtype=dtype, + ) if DEBUG_PERF: current_omni_platform.synchronize() _t_text_enc_ms = (time.perf_counter() - _t_text_enc_start) * 1000 - sample_solver = resolve_wan_sample_solver(req, default=self._sample_solver) - flow_shift = resolve_wan_flow_shift(req, self.od_config) + first_request = req.requests[0] + sample_solver = resolve_wan_sample_solver(first_request, default=self._sample_solver) + flow_shift = resolve_wan_flow_shift(first_request, self.od_config) if sample_solver != self._sample_solver or abs(flow_shift - self._flow_shift) > 1e-6: self.scheduler = build_wan_scheduler(sample_solver, flow_shift) self._sample_solver = sample_solver @@ -674,37 +703,41 @@ class Wan22Pipeline( if DEBUG_PERF: _t_latent_prep_start = time.perf_counter() - multi_modal_data = req.prompts[0].get("multi_modal_data", {}) if not isinstance(req.prompts[0], str) else None - raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None - if isinstance(raw_image, list): - if len(raw_image) > 1: - logger.warning( - """Received a list of image. Only a single image is supported by this model.""" - """Taking only the first image for now.""" - ) - raw_image = raw_image[0] - if raw_image is None: - image = None - elif isinstance(raw_image, str): - image = PIL.Image.open(raw_image) - else: - image = cast(PIL.Image.Image | torch.Tensor, raw_image) + images: list[PIL.Image.Image | torch.Tensor | None] = [] + for request_prompt in req.prompts: + multi_modal_data = request_prompt.get("multi_modal_data", {}) if not isinstance(request_prompt, str) else {} + raw_image = multi_modal_data.get("image") + if isinstance(raw_image, list): + if len(raw_image) > 1: + logger.warning("Received multiple images for one Wan request; using only the first image.") + raw_image = raw_image[0] if raw_image else None + if isinstance(raw_image, str): + raw_image = PIL.Image.open(raw_image) + images.append(cast(PIL.Image.Image | torch.Tensor | None, raw_image)) latent_condition = None first_frame_mask = None - if self.expand_timesteps and image is not None: + if self.expand_timesteps and any(image is not None for image in images): + if not all(image is not None for image in images): + raise ValueError("Cannot batch Wan requests with a mix of provided and missing image conditions.") # I2V mode: encode image and prepare condition from diffusers.video_processor import VideoProcessor video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) - # Preprocess image - if isinstance(image, PIL.Image.Image): - image = image.resize((width, height), PIL.Image.Resampling.LANCZOS) - image_tensor = video_processor.preprocess(image, height=height, width=width) - else: - image_tensor = image + image_tensors = [] + for image in images: + assert image is not None + if isinstance(image, PIL.Image.Image): + image = image.resize((width, height), PIL.Image.Resampling.LANCZOS) + image_tensor = video_processor.preprocess(image, height=height, width=width) + else: + image_tensor = image.unsqueeze(0) if image.ndim == 3 else image + image_tensors.append(image_tensor) + image_tensor = DiffusionRequestBatch.collate_tensors(image_tensors, "image condition", None) + assert image_tensor is not None + image_tensor = image_tensor.repeat_interleave(num_outputs_per_prompt, dim=0) # Use out_channels for noise latents (not in_channels which includes condition) num_channels_latents = self.transformer_config.out_channels @@ -720,7 +753,7 @@ class Wan22Pipeline( dtype=torch.float32, device=device, generator=generator, - latents=req.sampling_params.latents, + latents=request_latents, ) # Encode image condition @@ -731,7 +764,6 @@ class Wan22Pipeline( image_tensor = image_tensor.unsqueeze(2) # [B, C, 1, H, W] image_tensor = image_tensor.to(device=device, dtype=self.vae.dtype) latent_condition = retrieve_latents(self.vae.encode(image_tensor), sample_mode="argmax") - latent_condition = latent_condition.repeat(batch_size, 1, 1, 1, 1) # Normalize condition latents latents_mean = ( @@ -747,7 +779,7 @@ class Wan22Pipeline( # Create mask: 0 for first frame (condition), 1 for rest (to denoise) first_frame_mask = torch.ones( - 1, 1, num_latent_frames, latent_height, latent_width, dtype=torch.float32, device=device + batch_size, 1, num_latent_frames, latent_height, latent_width, dtype=torch.float32, device=device ) first_frame_mask[:, :, 0] = 0 else: @@ -762,7 +794,7 @@ class Wan22Pipeline( dtype=torch.float32, device=device, generator=generator, - latents=req.sampling_params.latents, + latents=request_latents, ) if DEBUG_PERF: current_omni_platform.synchronize() @@ -839,8 +871,13 @@ class Wan22Pipeline( _t_pipeline_wall_ms - _t_stages_sum, ) - return DiffusionOutput( - output=output, stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None + return split_diffusion_output_by_request( + DiffusionOutput( + output=output, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + ), + req, + num_outputs_per_prompt=num_outputs_per_prompt, ) def predict_noise( diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py index a2e444bb0..8e149d389 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py @@ -45,7 +45,7 @@ from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3 from vllm_omni.diffusion.postprocess import interpolate_video_tensor from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch, split_diffusion_output_by_request from vllm_omni.inputs.data import OmniTextPrompt from vllm_omni.platforms import current_omni_platform @@ -65,6 +65,8 @@ def get_wan22_i2v_post_process_func( output_type: str = "np", sampling_params=None, ): + if sampling_params is not None and sampling_params.output_type is not None: + output_type = sampling_params.output_type if output_type == "latent": return video video_metadata = {} @@ -84,6 +86,32 @@ def get_wan22_i2v_post_process_func( return post_process_func +def _normalize_i2v_last_image( + value: str | PIL.Image.Image | torch.Tensor | list[object] | None, +) -> PIL.Image.Image | torch.Tensor | None: + """Normalize the optional I2V last-frame condition for batching.""" + if value is None: + return None + + if isinstance(value, list): + if not value: + return None + if len(value) != 1: + raise ValueError("I2V accepts at most one last_image.") + value = value[0] + + if isinstance(value, str): + value = PIL.Image.open(value).convert("RGB") + + if not isinstance(value, (PIL.Image.Image, torch.Tensor)): + raise TypeError( + f"Unsupported last_image format {value.__class__}. " + "Expected a file path, PIL.Image.Image, torch.Tensor, or None." + ) + + return value + + def get_wan22_i2v_pre_process_func( od_config: OmniDiffusionConfig, ): @@ -109,6 +137,9 @@ def get_wan22_i2v_pre_process_func( """Please correctly set `"multi_modal_data": {"image": , …}`""", ) image = PIL.Image.open(raw_image).convert("RGB") if isinstance(raw_image, str) else raw_image + last_image = _normalize_i2v_last_image(multi_modal_data.get("last_image")) # type: ignore[union-attr] + prompt["multi_modal_data"]["last_image"] = last_image # type: ignore[index] + request.batch_compatibility_key = ("wan22_i2v_last_image", last_image is not None) # Calculate dimensions based on aspect ratio if not provided if request.sampling_params.height is None or request.sampling_params.width is None: @@ -157,6 +188,7 @@ class Wan22I2VPipeline( Wan2.2-style I2V (with expand_timesteps for TI2V-5B). """ + supports_request_batch = True _dit_modules: ClassVar[list[str]] = ["transformer", "transformer_2"] _encoder_modules: ClassVar[list[str]] = ["text_encoder", "image_encoder"] _vae_modules: ClassVar[list[str]] = ["vae"] @@ -433,60 +465,67 @@ class Wan22I2VPipeline( quant_config = getattr(self.od_config, "quantization_config", None) return create_transformer_from_config(config, quant_config=quant_config) - def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput: - prompt: str | None = None - negative_prompt: str | None = None - prompt_embeds: torch.Tensor | None = None - negative_prompt_embeds: torch.Tensor | None = None + def forward(self, req: DiffusionRequestBatch) -> list[DiffusionOutput]: + sampling_params_list = req.sampling_params_list + common = sampling_params_list[0] + prompt_texts = [prompt if isinstance(prompt, str) else (prompt.get("prompt") or "") for prompt in req.prompts] + negative_prompts = [ + None if isinstance(prompt, str) else prompt.get("negative_prompt") for prompt in req.prompts + ] + prompt_fields = DiffusionRequestBatch.collate_prompt_field_map( + req.prompts, + { + "prompt_embeds": None, + "negative_prompt_embeds": None, + }, + ) + prompt_embeds = prompt_fields["prompt_embeds"] + negative_prompt_embeds = prompt_fields["negative_prompt_embeds"] image_embeds: torch.Tensor | None = None - if len(req.prompts) > 1: - raise ValueError( - """This model only supports a single prompt, not a batched request.""", - """Please pass in a single prompt object or string, or a single-item list.""", - ) - if len(req.prompts) == 1: - first_prompt = req.prompts[0] - prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "") - negative_prompt = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt") + prompt: list[str] | None = prompt_texts if prompt_embeds is None else None + negative_prompt: list[str] | None = None + if negative_prompt_embeds is None and any(value is not None for value in negative_prompts): + negative_prompt = [value or "" for value in negative_prompts] + if prompt is not None and not all(prompt): + raise ValueError("Prompt is required for Wan2.2 I2V generation when prompt_embeds are not provided.") - if not prompt: - raise ValueError("Prompt is required for Wan2.2 generation.") + images: list[PIL.Image.Image | torch.Tensor] = [] + last_images: list[PIL.Image.Image | torch.Tensor | None] = [] + for request_prompt in req.prompts: + multi_modal_data = request_prompt.get("multi_modal_data", {}) if not isinstance(request_prompt, str) else {} + raw_image = multi_modal_data.get("image") + if raw_image is None: + raise ValueError("Image is required for I2V generation.") + if isinstance(raw_image, list): + if len(raw_image) > 1: + logger.warning("Received multiple images for one I2V request; using only the first image.") + raw_image = raw_image[0] + if isinstance(raw_image, str): + raw_image = PIL.Image.open(raw_image).convert("RGB") + images.append(cast(PIL.Image.Image | torch.Tensor, raw_image)) - # Get image from request - multi_modal_data = req.prompts[0].get("multi_modal_data", {}) if not isinstance(req.prompts[0], str) else None - raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None - if raw_image is None: - raise ValueError("Image is required for I2V generation.") - if isinstance(raw_image, list): - if len(raw_image) > 1: - logger.warning( - """Received a list of image. Only a single image is supported by this model.""" - """Taking only the first image for now.""" - ) - raw_image = raw_image[0] - if isinstance(raw_image, str): - image = PIL.Image.open(raw_image) - else: - image = cast(PIL.Image.Image | torch.Tensor, raw_image) + last_image = multi_modal_data.get("last_image") + if isinstance(last_image, str): + last_image = PIL.Image.open(last_image).convert("RGB") + last_images.append(cast(PIL.Image.Image | torch.Tensor | None, last_image)) + if any(image is not None for image in last_images) and not all(image is not None for image in last_images): + raise ValueError("Cannot batch I2V requests with a mix of provided and missing last_image conditions.") - last_image: PIL.Image.Image | torch.Tensor | None = None - if multi_modal_data is not None: - last_image = multi_modal_data.get("last_image", None) + height = common.height or 480 + width = common.width or 832 + num_frames = common.num_frames or 81 + num_steps = 40 if common.num_inference_steps is None else common.num_inference_steps - height = req.sampling_params.height or 480 - width = req.sampling_params.width or 832 - num_frames = req.sampling_params.num_frames or 81 - num_steps = 40 if req.sampling_params.num_inference_steps is None else req.sampling_params.num_inference_steps - - output_type = req.sampling_params.output_type or "np" + output_type = common.output_type or "np" + num_outputs_per_prompt = common.num_outputs_per_prompt or 1 attention_kwargs: dict | None = None - guidance_low, guidance_high = resolve_wan_guidance_scales(req.sampling_params, default_guidance_scale=5.0) + guidance_low, guidance_high = resolve_wan_guidance_scales(common, default_guidance_scale=5.0) self._guidance_scale = guidance_low self._guidance_scale_2 = guidance_high - boundary_ratio = self.boundary_ratio if self.boundary_ratio is not None else req.sampling_params.boundary_ratio + boundary_ratio = self.boundary_ratio if self.boundary_ratio is not None else common.boundary_ratio if boundary_ratio is None: boundary_ratio = 0.875 logger.warning("boundary_ratio is required for I2V generation. using default value 0.875") @@ -495,7 +534,7 @@ class Wan22I2VPipeline( self.check_inputs( prompt=prompt, negative_prompt=negative_prompt, - image=image, + image=images, height=height, width=width, prompt_embeds=prompt_embeds, @@ -513,10 +552,8 @@ class Wan22I2VPipeline( device = self.device dtype = self.transformer.dtype - # Generator setup - generator = req.sampling_params.generator - if generator is None and req.sampling_params.seed is not None: - generator = torch.Generator(device=device).manual_seed(req.sampling_params.seed) + generator = req.collate_request_generators(num_outputs_per_prompt, None) + request_latents = req.collate_request_tensors("latents", None) if DEBUG_PERF: # Sync GPU before timing to ensure accurate measurements @@ -524,15 +561,33 @@ class Wan22I2VPipeline( _t_pipeline_start = time.perf_counter() _t_text_enc_start = _t_pipeline_start - prompt_embeds, negative_prompt_embeds = self.encode_prompt( - prompt=prompt, - negative_prompt=negative_prompt, - do_classifier_free_guidance=guidance_low > 1.0 or guidance_high > 1.0, - num_videos_per_prompt=req.sampling_params.num_outputs_per_prompt or 1, - max_sequence_length=req.sampling_params.max_sequence_length or 512, - device=device, - dtype=dtype, - ) + do_classifier_free_guidance = guidance_low > 1.0 or guidance_high > 1.0 + if prompt_embeds is None: + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt=prompt, + negative_prompt=negative_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + num_videos_per_prompt=num_outputs_per_prompt, + max_sequence_length=common.max_sequence_length or 512, + device=device, + dtype=dtype, + ) + else: + prompt_embeds = prompt_embeds.to(device=device, dtype=dtype) + prompt_embeds = prompt_embeds.repeat_interleave(num_outputs_per_prompt, dim=0) + if negative_prompt_embeds is not None: + negative_prompt_embeds = negative_prompt_embeds.to(device=device, dtype=dtype) + negative_prompt_embeds = negative_prompt_embeds.repeat_interleave(num_outputs_per_prompt, dim=0) + elif do_classifier_free_guidance: + _, negative_prompt_embeds = self.encode_prompt( + prompt=[""] * req.num_reqs, + negative_prompt=negative_prompt, + do_classifier_free_guidance=True, + num_videos_per_prompt=num_outputs_per_prompt, + max_sequence_length=common.max_sequence_length or 512, + device=device, + dtype=dtype, + ) if DEBUG_PERF: current_omni_platform.synchronize() @@ -543,12 +598,18 @@ class Wan22I2VPipeline( if DEBUG_PERF: _t_img_enc_start = time.perf_counter() if self.has_image_encoder and self.transformer.config.image_dim is not None: - if image_embeds is None: - if last_image is None: - image_embeds = self.encode_image(image, device) - else: - image_embeds = self.encode_image([image, last_image], device) - image_embeds = image_embeds.repeat(batch_size, 1, 1) + if all(last_image is None for last_image in last_images): + image_embeds = self.encode_image(images, device) + image_embeds = image_embeds.repeat_interleave(num_outputs_per_prompt, dim=0) + else: + image_pairs = [] + for first_image, last_image in zip(images, last_images): + assert last_image is not None + image_pairs.extend([first_image, last_image]) + image_embeds = self.encode_image(image_pairs, device) + image_embeds = image_embeds.view(req.num_reqs, 2, *image_embeds.shape[1:]) + image_embeds = image_embeds.repeat_interleave(num_outputs_per_prompt, dim=0) + image_embeds = image_embeds.flatten(0, 1) image_embeds = image_embeds.to(dtype) else: image_embeds = None @@ -557,8 +618,9 @@ class Wan22I2VPipeline( current_omni_platform.synchronize() _t_img_enc_ms = (time.perf_counter() - _t_img_enc_start) * 1000 - sample_solver = resolve_wan_sample_solver(req, default=self._sample_solver) - flow_shift = resolve_wan_flow_shift(req, self.od_config) + first_request = req.requests[0] + sample_solver = resolve_wan_sample_solver(first_request, default=self._sample_solver) + flow_shift = resolve_wan_flow_shift(first_request, self.od_config) if sample_solver != self._sample_solver or abs(flow_shift - self._flow_shift) > 1e-6: self.scheduler = build_wan_scheduler(sample_solver, flow_shift) self._sample_solver = sample_solver @@ -582,23 +644,36 @@ class Wan22I2VPipeline( video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) - if isinstance(image, PIL.Image.Image): - image = TF.to_tensor(image).to(device) - image_tensor = video_processor.preprocess(image, height=height, width=width) - else: - image_tensor = image - image_tensor = image_tensor.to(device=device, dtype=torch.float32) - - # Handle last_image if provided - if last_image is not None: - if isinstance(last_image, PIL.Image.Image): - image = TF.to_tensor(last_image).to(device) - last_image_tensor = video_processor.preprocess(last_image, height=height, width=width) + image_tensors = [] + for image in images: + if isinstance(image, PIL.Image.Image): + image = TF.to_tensor(image).to(device) + image_tensor = video_processor.preprocess(image, height=height, width=width) else: - last_image_tensor = last_image - last_image_tensor = last_image_tensor.to(device=device, dtype=torch.float32) - else: - last_image_tensor = None + image_tensor = image.unsqueeze(0) if image.ndim == 3 else image + image_tensors.append(image_tensor.to(device=device, dtype=torch.float32)) + image_tensor = DiffusionRequestBatch.collate_tensors(image_tensors, "image condition", None) + assert image_tensor is not None + image_tensor = image_tensor.repeat_interleave(num_outputs_per_prompt, dim=0) + + last_image_tensor = None + if all(last_image is not None for last_image in last_images): + last_image_tensors = [] + for last_image in last_images: + assert last_image is not None + if isinstance(last_image, PIL.Image.Image): + last_image = TF.to_tensor(last_image).to(device) + current_last_image = video_processor.preprocess(last_image, height=height, width=width) + else: + current_last_image = last_image.unsqueeze(0) if last_image.ndim == 3 else last_image + last_image_tensors.append(current_last_image.to(device=device, dtype=torch.float32)) + last_image_tensor = DiffusionRequestBatch.collate_tensors( + last_image_tensors, + "last image condition", + None, + ) + assert last_image_tensor is not None + last_image_tensor = last_image_tensor.repeat_interleave(num_outputs_per_prompt, dim=0) latents, condition, first_frame_mask = self.prepare_latents( image=image_tensor, @@ -610,7 +685,7 @@ class Wan22I2VPipeline( dtype=torch.float32, device=device, generator=generator, - latents=req.sampling_params.latents, + latents=request_latents, last_image=last_image_tensor, ) @@ -694,8 +769,13 @@ class Wan22I2VPipeline( _t_pipeline_wall_ms - _t_stages_sum, ) - return DiffusionOutput( - output=output, stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None + return split_diffusion_output_by_request( + DiffusionOutput( + output=output, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + ), + req, + num_outputs_per_prompt=num_outputs_per_prompt, ) def predict_noise( @@ -849,7 +929,6 @@ class Wan22I2VPipeline( # Encode through VAE latent_condition = retrieve_latents(self.vae.encode(video_condition), sample_mode="argmax") - latent_condition = latent_condition.repeat(batch_size, 1, 1, 1, 1) # Normalize latents latents_mean = ( @@ -866,7 +945,7 @@ class Wan22I2VPipeline( if self.expand_timesteps: # TI2V-5B style: create mask where first frame is 0 (condition), rest is 1 (to denoise) first_frame_mask = torch.ones( - 1, 1, num_latent_frames, latent_height, latent_width, dtype=dtype, device=device + batch_size, 1, num_latent_frames, latent_height, latent_width, dtype=dtype, device=device ) first_frame_mask[:, :, 0] = 0 return latents, latent_condition, first_frame_mask @@ -892,7 +971,9 @@ class Wan22I2VPipeline( condition = torch.concat([mask_lat_size, latent_condition], dim=1) # For non-expand mode, first_frame_mask is not used in the same way - first_frame_mask = torch.ones(1, 1, num_latent_frames, latent_height, latent_width, dtype=dtype, device=device) + first_frame_mask = torch.ones( + batch_size, 1, num_latent_frames, latent_height, latent_width, dtype=dtype, device=device + ) return latents, condition, first_frame_mask diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py index 52a42cb79..bce71633c 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py @@ -43,7 +43,7 @@ from vllm_omni.diffusion.models.wan2_2.wan2_2_s2v_transformer import ( ) from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch, split_diffusion_output_by_request from vllm_omni.inputs.data import OmniTextPrompt from vllm_omni.platforms import current_omni_platform @@ -63,6 +63,30 @@ _S2V_DEFAULT_NEG_PROMPT = ( _AUDIO_VIDEO_RATE = 30 +def _s2v_audio_condition_key(audio: str | np.ndarray | None) -> tuple[Any, ...]: + if isinstance(audio, str): + # File paths are deliberately conservative: two different files may + # have different durations even when their request metadata matches. + return ("path", audio) + if isinstance(audio, np.ndarray): + return ("array", tuple(audio.shape), str(audio.dtype)) + return (type(audio).__name__,) + + +def _make_clip_generators( + seeds: list[int | None], + request_generators: list[torch.Generator | None], + clip_index: int, + device: torch.device, +) -> list[torch.Generator | None]: + return [ + torch.Generator(device=device).manual_seed((seed if seed is not None else 0) + clip_index) + if seed is not None or request_generator is None + else request_generator + for seed, request_generator in zip(seeds, request_generators) + ] + + def _linear_interpolation( features: torch.Tensor, input_fps: float, @@ -337,6 +361,12 @@ def get_wan22_s2v_pre_process_func( prompt["additional_information"]["init_first_frame"] = ( multi_modal_data.get("init_first_frame", False) if multi_modal_data is not None else False ) + request.batch_compatibility_key = ( + "wan22_s2v_condition", + bool(prompt["additional_information"]["init_first_frame"]), + prompt["additional_information"].get("num_repeat"), + _s2v_audio_condition_key(raw_audio), + ) request.prompt = prompt return request @@ -486,6 +516,7 @@ class Wan22S2VPipeline( """ # Default config values from Wan2.2/wan/configs/wan_s2v_14B.py + supports_request_batch = True _DEFAULT_MOTION_FRAMES = 73 _DEFAULT_INFER_FRAMES = 80 _DEFAULT_FPS = 16 @@ -942,7 +973,6 @@ class Wan22S2VPipeline( """Forward pass through the S2V transformer to predict noise. With return_dict=False, the model returns (output,) where output is [B, C, T, H, W]. - We squeeze batch dim since S2V processes one sample at a time. """ if current_model is None: current_model = self.transformer @@ -952,8 +982,8 @@ class Wan22S2VPipeline( if isinstance(result, list): return result[0] if isinstance(result, tuple): - return result[0].squeeze(0) - return result.sample.squeeze(0) + return result[0] + return result.sample def diffuse( self, @@ -962,7 +992,7 @@ class Wan22S2VPipeline( prompt_embeds: torch.Tensor, negative_prompt_embeds: torch.Tensor | None, guidance_scale: float, - clip_generator: torch.Generator, + clip_generator: torch.Generator | list[torch.Generator] | None, dtype: torch.dtype, device: torch.device, max_seq_len: int, @@ -971,13 +1001,13 @@ class Wan22S2VPipeline( ref_latents: torch.Tensor, motion_frames: list[int], drop_first_motion: bool, - positive_audio_emb: torch.Tensor, - negative_audio_emb: torch.Tensor | None, + positive_audio_emb: dict[str, torch.Tensor], + negative_audio_emb: dict[str, torch.Tensor] | None, ) -> torch.Tensor: - """Denoising diffusion loop for one S2V clip. + """Denoising diffusion loop for a batch of compatible S2V clips. Args: - latents: Initial noise tensor [C, T, H, W] + latents: Initial noise tensor [B, C, T, H, W] timesteps: Denoising timesteps prompt_embeds: Text embeddings [1, seq_len, dim] negative_prompt_embeds: Negative text embeddings (optional) @@ -995,7 +1025,7 @@ class Wan22S2VPipeline( negative_audio_emb: Precomputed audio embeddings for negative prompt (optional) Returns: - Denoised latents [C, T, H, W] + Denoised latents [B, C, T, H, W] """ do_true_cfg = self.do_classifier_free_guidance and negative_prompt_embeds is not None @@ -1004,13 +1034,13 @@ class Wan22S2VPipeline( self._current_timestep = t self.record_denoise_step(step_idx, t) - latent_model_input = [latents.to(device)] - timestep = t.unsqueeze(0).to(device) if t.dim() == 0 else t.to(device) + latent_model_input = latents.to(device) + timestep = t.expand(latents.shape[0]).to(device) positive_kwargs = { "hidden_states": latent_model_input, "timestep": timestep, - "encoder_hidden_states": prompt_embeds[0:1], + "encoder_hidden_states": prompt_embeds, "cond_states": cond_latents, "motion_latents": input_motion_latents, "ref_latents": ref_latents, @@ -1023,7 +1053,7 @@ class Wan22S2VPipeline( negative_kwargs = { "hidden_states": latent_model_input, "timestep": timestep, - "encoder_hidden_states": negative_prompt_embeds[0:1], + "encoder_hidden_states": negative_prompt_embeds, "cond_states": cond_latents, "motion_latents": input_motion_latents, "ref_latents": ref_latents, @@ -1044,12 +1074,12 @@ class Wan22S2VPipeline( # Scheduler step latents = self.scheduler.step( - noise_pred.unsqueeze(0), + noise_pred, t, - latents.unsqueeze(0), + latents, return_dict=False, - generator=clip_generator, - )[0].squeeze(0) + generator=clip_generator if isinstance(clip_generator, torch.Generator) else None, + )[0] pbar.update() @@ -1080,111 +1110,178 @@ class Wan22S2VPipeline( prompt_embeds: torch.Tensor | None = None, negative_prompt_embeds: torch.Tensor | None = None, **kwargs, - ) -> DiffusionOutput: + ) -> list[DiffusionOutput]: """Run S2V generation — may produce multiple autoregressive clips. This method mirrors ``WanS2V.generate()``, reorganized into the vLLM-Omni pipeline structure. """ - # ---- Extract params from request ---- - if len(req.prompts) > 1: - raise ValueError("S2V only supports a single prompt per request.") - - if len(req.prompts) == 1: - prompt_data = req.prompts[0] - prompt = prompt_data if isinstance(prompt_data, str) else prompt_data.get("prompt") - negative_prompt = None if isinstance(prompt_data, str) else prompt_data.get("negative_prompt") + sampling_params_list = req.sampling_params_list + common = sampling_params_list[0] + prompt_fields = DiffusionRequestBatch.collate_prompt_field_map( + req.prompts, + { + "prompt_embeds": prompt_embeds, + "negative_prompt_embeds": negative_prompt_embeds, + }, + ) + prompt_embeds = prompt_fields["prompt_embeds"] + negative_prompt_embeds = prompt_fields["negative_prompt_embeds"] + prompt_texts: list[str] = [] + negative_prompts: list[str] = [] + images: list[PIL.Image.Image | None] = [] + audio_paths: list[str | np.ndarray | None] = [] + requested_repeats: list[int | None] = [] + init_first_frames: list[bool] = [] + for prompt_data in req.prompts: multi_modal_data = prompt_data.get("multi_modal_data", {}) if not isinstance(prompt_data, str) else {} additional_info = prompt_data.get("additional_information", {}) if not isinstance(prompt_data, str) else {} - if image is None: - raw_image = multi_modal_data.get("image", None) - if isinstance(raw_image, str): - image = PIL.Image.open(raw_image).convert("RGB") - elif isinstance(raw_image, PIL.Image.Image): - image = raw_image + prompt_texts.append( + prompt_data if isinstance(prompt_data, str) else (prompt_data.get("prompt") or prompt or "") + ) + request_negative_prompt = ( + negative_prompt if isinstance(prompt_data, str) else prompt_data.get("negative_prompt", negative_prompt) + ) + negative_prompts.append(request_negative_prompt or _S2V_DEFAULT_NEG_PROMPT) - if audio_path is None: - audio_path = additional_info.get("audio_path") - if audio_path is None: - audio_path = multi_modal_data.get("audio") + raw_image = multi_modal_data.get("image", image) + if isinstance(raw_image, list): + raw_image = raw_image[0] if raw_image else None + if isinstance(raw_image, str): + raw_image = PIL.Image.open(raw_image).convert("RGB") + images.append(raw_image if isinstance(raw_image, PIL.Image.Image) else None) - if pose_video is None: - pose_video = additional_info.get("pose_video") + request_audio_path = additional_info.get("audio_path", audio_path) + if request_audio_path is None: + request_audio_path = multi_modal_data.get("audio") + audio_paths.append(request_audio_path) + requested_repeats.append(additional_info.get("num_repeat", num_repeat)) + init_first_frames.append(additional_info.get("init_first_frame", init_first_frame)) - init_first_frame = additional_info.get("init_first_frame", init_first_frame) - - if negative_prompt is None: - negative_prompt = _S2V_DEFAULT_NEG_PROMPT - - height = req.sampling_params.height or height - width = req.sampling_params.width or width + height = common.height or height + width = common.width or width # num_frames defaults to 1 (image-model sentinel); S2V needs many frames. - req_num_frames = req.sampling_params.num_frames + req_num_frames = common.num_frames infer_frames = ( infer_frames or (req_num_frames if req_num_frames and req_num_frames > 1 else None) or self._DEFAULT_INFER_FRAMES ) - num_steps = req.sampling_params.num_inference_steps or num_inference_steps + num_steps = common.num_inference_steps or num_inference_steps + num_outputs_per_prompt = common.num_outputs_per_prompt or 1 - if req.sampling_params.guidance_scale_provided: - guidance_scale = req.sampling_params.guidance_scale + if common.guidance_scale_provided: + guidance_scale = common.guidance_scale self._guidance_scale = guidance_scale # ---- Validate ---- - self.check_inputs(prompt, image, audio_path, height, width, prompt_embeds) + if len(set(init_first_frames)) != 1: + raise ValueError("Batched S2V requests must have matching init_first_frame values.") + init_first_frame = init_first_frames[0] + for request_prompt, request_image, request_audio_path in zip(prompt_texts, images, audio_paths): + self.check_inputs( + request_prompt if prompt_embeds is None else None, + request_image, + request_audio_path, + height, + width, + prompt_embeds, + ) device = self.device dtype = self.transformer.dtype if hasattr(self.transformer, "dtype") else torch.bfloat16 - - if generator is None: - generator = req.sampling_params.generator - seed = req.sampling_params.seed - if generator is None and seed is not None: - generator = torch.Generator(device=device).manual_seed(seed) - if seed is None: - seed = 0 + request_generators = req.collate_request_generators(num_outputs_per_prompt, None) + batch_size = req.num_reqs * num_outputs_per_prompt + if isinstance(request_generators, list): + generators = request_generators + else: + generators = [request_generators] * batch_size + seeds = [sampling.seed for sampling in sampling_params_list for _ in range(num_outputs_per_prompt)] + request_latents = req.collate_request_tensors("latents", None) # ---- 1. Text encoding ---- if prompt_embeds is None: prompt_embeds, negative_prompt_embeds = self.encode_prompt( - prompt=prompt, - negative_prompt=negative_prompt, + prompt=prompt_texts, + negative_prompt=negative_prompts, do_classifier_free_guidance=self.do_classifier_free_guidance, - num_videos_per_prompt=1, - max_sequence_length=512, + num_videos_per_prompt=num_outputs_per_prompt, + max_sequence_length=common.max_sequence_length or 512, device=device, dtype=dtype, ) else: prompt_embeds = prompt_embeds.to(device=device, dtype=dtype) + prompt_embeds = prompt_embeds.repeat_interleave(num_outputs_per_prompt, dim=0) if negative_prompt_embeds is not None: negative_prompt_embeds = negative_prompt_embeds.to(device=device, dtype=dtype) + negative_prompt_embeds = negative_prompt_embeds.repeat_interleave(num_outputs_per_prompt, dim=0) # ---- 2. Audio encoding ---- - audio_emb, audio_num_repeat, target_video_frames = self.encode_audio( - audio_path, infer_frames=infer_frames, device=device, dtype=dtype - ) - if num_repeat is None or num_repeat > audio_num_repeat: - num_repeat = audio_num_repeat + audio_embeddings: list[torch.Tensor] = [] + effective_repeats: list[int] = [] + target_video_frames: list[int] = [] + for request_audio_path, requested_repeat in zip(audio_paths, requested_repeats): + assert request_audio_path is not None + audio_emb, audio_num_repeat, request_target_video_frames = self.encode_audio( + request_audio_path, infer_frames=infer_frames, device=device, dtype=dtype + ) + audio_embeddings.append(audio_emb.repeat_interleave(num_outputs_per_prompt, dim=0)) + effective_repeats.append( + audio_num_repeat + if requested_repeat is None or requested_repeat > audio_num_repeat + else requested_repeat + ) + target_video_frames.append(request_target_video_frames) + + if len(set(effective_repeats)) != 1: + raise ValueError(f"Batched S2V requests must have matching num_repeat values, got {effective_repeats}.") + if len(set(target_video_frames)) != 1: + raise ValueError( + "Batched S2V requests must have matching audio embedding time lengths; " + f"got target video frames {target_video_frames}." + ) + num_repeat = effective_repeats[0] + audio_emb = DiffusionRequestBatch.collate_tensors(audio_embeddings, "S2V audio embeddings", None) + assert audio_emb is not None # Load raw audio waveform for muxing into output video - if isinstance(audio_path, np.ndarray): - raw_audio_waveform = audio_path.astype(np.float32) - raw_audio_sr = 16000 - else: - raw_audio_waveform, raw_audio_sr = load_audio(audio_path, sr=None, mono=True) + raw_audio_waveforms: list[np.ndarray] = [] + raw_audio_sample_rates: list[int] = [] + for request_audio_path in audio_paths: + assert request_audio_path is not None + if isinstance(request_audio_path, np.ndarray): + raw_audio_waveforms.append(request_audio_path.astype(np.float32, copy=False)) + raw_audio_sample_rates.append(16000) + else: + raw_audio_waveform, raw_audio_sr = load_audio(request_audio_path, sr=None, mono=True) + raw_audio_waveforms.append(raw_audio_waveform) + raw_audio_sample_rates.append(raw_audio_sr) + if len(set(raw_audio_sample_rates)) != 1 or len({waveform.shape for waveform in raw_audio_waveforms}) != 1: + raise ValueError("Batched S2V requests must have matching raw audio shapes and sample rates.") + raw_audio_sr = raw_audio_sample_rates[0] # ---- 3. Reference image encoding ---- - ref_latents = self.encode_ref_image(image, height, width, device=device).to(dtype=dtype) + ref_latents = DiffusionRequestBatch.collate_tensors( + [ + self.encode_ref_image(request_image, height, width, device=device) + .to(dtype=dtype) + .repeat_interleave(num_outputs_per_prompt, dim=0) + for request_image in images + if request_image is not None + ], + "S2V reference image latents", + None, + ) + assert ref_latents is not None # ---- 4. Initial motion latents (zeros) ---- motion_frames = min(self.motion_frames, infer_frames - 1) motion_pixels = torch.zeros( - [1, 3, motion_frames, height, width], + [batch_size, 3, motion_frames, height, width], dtype=dtype, device=device, ) @@ -1193,8 +1290,11 @@ class Wan22S2VPipeline( drop_first_motion = False # Place reference image in the last 6 frames of motion tensor_trans = transforms.ToTensor() - ref_tensor = tensor_trans(image).to(device=device, dtype=dtype) * 2 - 1.0 # [C, H, W] - motion_pixels[:, :, -6:] = ref_tensor.unsqueeze(0).unsqueeze(2).expand(-1, -1, 6, -1, -1) + ref_tensors = torch.stack( + [tensor_trans(request_image) for request_image in images if request_image is not None] + ).to(device=device, dtype=dtype) + ref_tensors = ref_tensors.repeat_interleave(num_outputs_per_prompt, dim=0) * 2 - 1.0 + motion_pixels[:, :, -6:] = ref_tensors.unsqueeze(2).expand(-1, -1, 6, -1, -1) motion_latents = self.prepare_motion_latents(motion_pixels, device=device).to(dtype=dtype) @@ -1211,24 +1311,29 @@ class Wan22S2VPipeline( clips: list[torch.Tensor] = [] # Keep a pixel-space buffer of the trailing motion_frames for the # autoregressive connection between clips. - videos_last_frames = torch.zeros([1, 3, motion_frames, height, width], dtype=dtype, device=device) + videos_last_frames = torch.zeros([batch_size, 3, motion_frames, height, width], dtype=dtype, device=device) for r in range(num_repeat): - # Per-clip seed - clip_seed = seed + r - clip_generator = torch.Generator(device=device).manual_seed(clip_seed) - + clip_generators = _make_clip_generators(seeds, generators, r, device) # -- Noise -- - latents = self.prepare_latents( - infer_frames=infer_frames, - height=height, - width=width, - motion_frames=motion_frames, - lat_motion_frames=lat_motion_frames, - dtype=dtype, - device=device, - generator=clip_generator, - ) + if r == 0 and request_latents is not None: + latents = request_latents.to(device=device, dtype=dtype) + else: + latents = torch.stack( + [ + self.prepare_latents( + infer_frames=infer_frames, + height=height, + width=width, + motion_frames=motion_frames, + lat_motion_frames=lat_motion_frames, + dtype=dtype, + device=device, + generator=clip_generator, + ) + for clip_generator in clip_generators + ] + ) # -- Scheduler -- self.scheduler.set_timesteps(num_steps, device=device) @@ -1242,11 +1347,11 @@ class Wan22S2VPipeline( # -- Pose condition for this clip -- # Default: zero condition (no pose driving) - # Shape: [B=1, C=16, T, H, W] — passed to transformer which iterates over batch dim + # Shape: [B, C=16, T, H, W] lat_h = height // self.vae_scale_factor_spatial lat_w = width // self.vae_scale_factor_spatial lat_target_frames = (infer_frames + 3 + motion_frames) // 4 - lat_motion_frames - cond_latents = torch.zeros([1, 16, lat_target_frames, lat_h, lat_w], dtype=dtype, device=device) + cond_latents = torch.zeros([batch_size, 16, lat_target_frames, lat_h, lat_w], dtype=dtype, device=device) # -- Clone motion latents for this clip -- input_motion_latents = motion_latents.clone() @@ -1282,7 +1387,7 @@ class Wan22S2VPipeline( prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_prompt_embeds, guidance_scale=guidance_scale, - clip_generator=clip_generator, + clip_generator=clip_generators, dtype=dtype, device=device, max_seq_len=max_seq_len, @@ -1300,11 +1405,10 @@ class Wan22S2VPipeline( self.transformer.to("cpu") current_omni_platform.empty_cache() - latents_for_decode = latents.unsqueeze(0) # [1, C, T, H, W] if not (drop_first_motion and r == 0): - decode_latents = torch.cat([motion_latents, latents_for_decode], dim=2) + decode_latents = torch.cat([motion_latents, latents], dim=2) else: - decode_latents = torch.cat([ref_latents, latents_for_decode], dim=2) + decode_latents = torch.cat([ref_latents, latents], dim=2) decode_latents = self._denormalize_latents(decode_latents) decode_latents = decode_latents.to(self.vae.dtype) @@ -1321,7 +1425,7 @@ class Wan22S2VPipeline( # Create buffer for broadcast clip_video = torch.empty( - (1, 3, total_frames, height, width), + (batch_size, 3, total_frames, height, width), device=decode_latents.device, dtype=decode_latents.dtype, ) @@ -1356,12 +1460,19 @@ class Wan22S2VPipeline( current_omni_platform.empty_cache() # ---- Concatenate all clips ---- - output = torch.cat(clips, dim=2) # [1, C, T_total, H, W] + output = torch.cat(clips, dim=2) # [B, C, T_total, H, W] - return DiffusionOutput( - output=(output, raw_audio_waveform, raw_audio_sr), - stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + outputs = split_diffusion_output_by_request( + DiffusionOutput( + output=output, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + ), + req, + num_outputs_per_prompt=num_outputs_per_prompt, ) + for request_output, raw_audio_waveform in zip(outputs, raw_audio_waveforms): + request_output.output = (request_output.output, raw_audio_waveform, raw_audio_sr) + return outputs def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: """Load weights using AutoWeightsLoader for vLLM integration.""" diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py index d0cdb1250..ae869fd57 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py @@ -35,13 +35,27 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import ( ) from vllm_omni.diffusion.models.wan2_2.wan2_2_vace_transformer import WanVACETransformer3DModel from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch, split_diffusion_output_by_request from vllm_omni.inputs.data import OmniTextPrompt from vllm_omni.platforms import current_omni_platform logger = init_logger(__name__) +def _vace_condition_structure(value: object) -> tuple: + if value is None: + return ("none",) + if isinstance(value, str): + return ("path", value) + if isinstance(value, PIL.Image.Image): + return ("image", value.size) + if isinstance(value, torch.Tensor): + return ("tensor", tuple(value.shape), str(value.dtype)) + if isinstance(value, list): + return ("list", tuple(_vace_condition_structure(item) for item in value)) + return (type(value).__name__,) + + def create_vace_transformer_from_config( config: dict, quant_config: QuantizationConfig | None = None, @@ -104,6 +118,7 @@ def get_wan22_vace_pre_process_func(od_config: OmniDiffusionConfig): prompt["additional_information"] = {} if not multi_modal_data: + request.batch_compatibility_key = ("wan22_vace_condition", ("none",), ("none",), ("none",)) request.prompt = prompt return request @@ -155,6 +170,13 @@ def get_wan22_vace_pre_process_func(od_config: OmniDiffusionConfig): mask = [mask] prompt["additional_information"]["mask"] = mask + request.batch_compatibility_key = ( + "wan22_vace_condition", + _vace_condition_structure(ref_images), + _vace_condition_structure(source_video), + _vace_condition_structure(mask), + ) + request.prompt = prompt return request @@ -482,7 +504,7 @@ class Wan22VACEPipeline(Wan22Pipeline, SupportImageInput): attention_kwargs: dict | None = None, vace_context_scale: float | list[float] = 1.0, **kwargs, - ) -> DiffusionOutput: + ) -> list[DiffusionOutput]: """Generate or edit video using VACE. The mode is determined by which inputs are provided in the request: @@ -507,47 +529,48 @@ class Wan22VACEPipeline(Wan22Pipeline, SupportImageInput): attention_kwargs: Additional kwargs for attention layers. vace_context_scale: VACE conditioning strength. """ - # Get parameters from request or arguments - if len(req.prompts) > 1: - raise ValueError( - "This model only supports a single prompt, not a batched request. " - "Please pass in a single prompt object or string, or a single-item list." - ) - - reference_images = None - source_video = None - source_mask = None - - if len(req.prompts) == 1: - first_prompt = req.prompts[0] - if isinstance(first_prompt, str): - prompt = first_prompt - else: - prompt = first_prompt.get("prompt") - negative_prompt = negative_prompt or first_prompt.get("negative_prompt") - prompt_embeds = prompt_embeds if prompt_embeds is not None else first_prompt.get("prompt_embeds") - negative_prompt_embeds = ( - negative_prompt_embeds - if negative_prompt_embeds is not None - else first_prompt.get("negative_prompt_embeds") - ) - - additional_info = first_prompt.get("additional_information", {}) - reference_images = additional_info.get("reference_images") - source_video = additional_info.get("source_video") - source_mask = additional_info.get("mask") - - if prompt is None and prompt_embeds is None: + sampling_params_list = req.sampling_params_list + common = sampling_params_list[0] + prompt_texts = [item if isinstance(item, str) else (item.get("prompt") or "") for item in req.prompts] + negative_prompts = [None if isinstance(item, str) else item.get("negative_prompt") for item in req.prompts] + prompt_fields = DiffusionRequestBatch.collate_prompt_field_map( + req.prompts, + { + "prompt_embeds": prompt_embeds, + "negative_prompt_embeds": negative_prompt_embeds, + }, + ) + prompt_embeds = prompt_fields["prompt_embeds"] + negative_prompt_embeds = prompt_fields["negative_prompt_embeds"] + prompt = prompt_texts if prompt_embeds is None else None + negative_prompt = None + if negative_prompt_embeds is None and any(value is not None for value in negative_prompts): + negative_prompt = [value or "" for value in negative_prompts] + if prompt is not None and not all(prompt): raise ValueError("Prompt or prompt_embeds is required for VACE generation.") - height = req.sampling_params.height or height - width = req.sampling_params.width or width - num_frames = req.sampling_params.num_frames or frame_num - num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps - generator = req.sampling_params.generator or generator + reference_images_list = [] + source_videos = [] + source_masks = [] + for request_prompt in req.prompts: + additional_info = ( + request_prompt.get("additional_information", {}) if not isinstance(request_prompt, str) else {} + ) + reference_images_list.append(additional_info.get("reference_images")) + source_videos.append(additional_info.get("source_video")) + source_masks.append(additional_info.get("mask")) - if req.sampling_params.guidance_scale_provided: - guidance_scale = req.sampling_params.guidance_scale + height = common.height or height + width = common.width or width + num_frames = common.num_frames or frame_num + num_inference_steps = common.num_inference_steps or num_inference_steps + num_outputs_per_prompt = common.num_outputs_per_prompt or 1 + generator = req.collate_request_generators(num_outputs_per_prompt, None) + request_latents = req.collate_request_tensors("latents", None) + output_type = common.output_type or output_type + + if common.guidance_scale_provided: + guidance_scale = common.guidance_scale # Ensure dimensions are compatible with VAE and patch size mod_value = self.vae_scale_factor_spatial * 2 # 8 * 2 = 16 @@ -558,47 +581,47 @@ class Wan22VACEPipeline(Wan22Pipeline, SupportImageInput): num_frames = num_frames // self.vae_scale_factor_temporal * self.vae_scale_factor_temporal + 1 num_frames = max(num_frames, 1) - self.check_inputs( - prompt=prompt, - negative_prompt=negative_prompt, - height=height, - width=width, - prompt_embeds=prompt_embeds, - negative_prompt_embeds=negative_prompt_embeds, - video=source_video, - mask=source_mask, - reference_images=reference_images, - ) + for index in range(req.num_reqs): + self.check_inputs( + prompt=None if prompt is None else prompt[index], + negative_prompt=None if negative_prompt is None else negative_prompt[index], + height=height, + width=width, + prompt_embeds=None if prompt_embeds is None else prompt_embeds[index], + negative_prompt_embeds=None if negative_prompt_embeds is None else negative_prompt_embeds[index], + video=source_videos[index], + mask=source_masks[index], + reference_images=reference_images_list[index], + ) device = self.device self._guidance_scale = guidance_scale dtype = self.transformer.dtype if self.transformer is not None else torch.bfloat16 - if generator is None and req.sampling_params.seed is not None: - generator = torch.Generator(device=device).manual_seed(req.sampling_params.seed) - # Encode prompts if prompt_embeds is None: - if prompt is None: - raise ValueError("Either prompt or prompt_embeds must be provided.") prompt_embeds, negative_prompt_embeds = self.encode_prompt( prompt=prompt, negative_prompt=negative_prompt, do_classifier_free_guidance=guidance_scale > 1.0, - num_videos_per_prompt=req.sampling_params.num_outputs_per_prompt or 1, - max_sequence_length=req.sampling_params.max_sequence_length or 512, + num_videos_per_prompt=num_outputs_per_prompt, + max_sequence_length=common.max_sequence_length or 512, device=device, dtype=dtype, ) else: prompt_embeds = prompt_embeds.to(device=device, dtype=dtype) + prompt_embeds = prompt_embeds.repeat_interleave(num_outputs_per_prompt, dim=0) if negative_prompt_embeds is not None: negative_prompt_embeds = negative_prompt_embeds.to(device=device, dtype=dtype) + negative_prompt_embeds = negative_prompt_embeds.repeat_interleave(num_outputs_per_prompt, dim=0) elif guidance_scale > 1.0: _, negative_prompt_embeds = self.encode_prompt( - prompt="", - negative_prompt=None, + prompt=[""] * req.num_reqs, + negative_prompt=negative_prompt, do_classifier_free_guidance=True, + num_videos_per_prompt=num_outputs_per_prompt, + max_sequence_length=common.max_sequence_length or 512, device=device, dtype=dtype, ) @@ -608,24 +631,43 @@ class Wan22VACEPipeline(Wan22Pipeline, SupportImageInput): # so read it from whichever transformer exists. active_transformer = self.transformer if self.transformer is not None else self.transformer_2 if active_transformer.vace_patch_embedding is not None: - video, mask, ref_images_processed = self.preprocess_conditions( - video=source_video, - mask=source_mask, - reference_images=reference_images, - height=height, - width=width, - num_frames=num_frames, - dtype=dtype, - device=device, - ) + contexts = [] + reference_counts = [] + for index, (source_video, source_mask, reference_images) in enumerate( + zip(source_videos, source_masks, reference_images_list) + ): + video, mask, ref_images_processed = self.preprocess_conditions( + video=source_video, + mask=source_mask, + reference_images=reference_images, + height=height, + width=width, + num_frames=num_frames, + dtype=dtype, + device=device, + ) + request_generator = sampling_params_list[index].generator + if isinstance(request_generator, list): + request_generator = request_generator[0] + conditioning_latents = self.prepare_video_latents( + video, + mask, + ref_images_processed, + request_generator, + device, + ) + mask_encoded = self.prepare_masks(mask, ref_images_processed) + context = torch.cat([conditioning_latents, mask_encoded], dim=1) + contexts.append(context.repeat_interleave(num_outputs_per_prompt, dim=0)) + reference_counts.append(len(ref_images_processed[0]) if ref_images_processed else 0) - conditioning_latents = self.prepare_video_latents(video, mask, ref_images_processed, generator, device) - mask_encoded = self.prepare_masks(mask, ref_images_processed) - - # Unified VACE context: [video_latents, mask] along channels -> [B, C, T, H, W] - vace_context = torch.cat([conditioning_latents, mask_encoded], dim=1) - - num_reference_images = len(ref_images_processed[0]) if ref_images_processed else 0 + if len(set(reference_counts)) != 1: + raise ValueError( + f"Batched VACE conditions must have the same number of reference images; got {reference_counts}." + ) + vace_context = DiffusionRequestBatch.collate_tensors(contexts, "VACE condition", None) + assert vace_context is not None + num_reference_images = reference_counts[0] else: vace_context = None @@ -641,7 +683,7 @@ class Wan22VACEPipeline(Wan22Pipeline, SupportImageInput): dtype=torch.float32, device=device, generator=generator, - latents=req.sampling_params.latents, + latents=request_latents, ) # Set up scheduler @@ -654,9 +696,7 @@ class Wan22VACEPipeline(Wan22Pipeline, SupportImageInput): # stays None and diffuse() uses the same transformer for every step. boundary_timestep = None if self.transformer_2 is not None: - boundary_ratio = ( - self.boundary_ratio if self.boundary_ratio is not None else req.sampling_params.boundary_ratio - ) + boundary_ratio = self.boundary_ratio if self.boundary_ratio is not None else common.boundary_ratio if boundary_ratio is None: boundary_ratio = 0.875 logger.warning_once("boundary_ratio is required for Wan2.2 VACE generation. using default value 0.875") @@ -700,7 +740,11 @@ class Wan22VACEPipeline(Wan22Pipeline, SupportImageInput): latents = latents / latents_std + latents_mean output = self.vae.decode(latents, return_dict=False)[0] - return DiffusionOutput(output=output) + return split_diffusion_output_by_request( + DiffusionOutput(output=output), + req, + num_outputs_per_prompt=num_outputs_per_prompt, + ) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: """Load weights using AutoWeightsLoader for vLLM integration.""" diff --git a/vllm_omni/diffusion/request.py b/vllm_omni/diffusion/request.py index 8834e408e..ce097a2d9 100644 --- a/vllm_omni/diffusion/request.py +++ b/vllm_omni/diffusion/request.py @@ -43,6 +43,10 @@ class OmniDiffusionRequest: # BaseScheduler takes ownership and clears this field before the request is # sent to a Worker. diffusion_kv_requests: tuple[DiffusionKVRequest, ...] | None = None + # Optional model-specific condition structure used by request-batch admission. + # This is populated by a pipeline preprocessor before the request reaches + # the scheduler; ``None`` keeps the default behavior for other pipelines. + batch_compatibility_key: tuple[Any, ...] | None = None def __post_init__(self): """Initialize dependent fields after dataclass initialization.""" diff --git a/vllm_omni/diffusion/sched/interface.py b/vllm_omni/diffusion/sched/interface.py index 8fb36f405..e543ed251 100644 --- a/vllm_omni/diffusion/sched/interface.py +++ b/vllm_omni/diffusion/sched/interface.py @@ -88,9 +88,10 @@ class RequestBatchSamplingParamsKey: """Request level Batch-compatibility key derived from ``OmniDiffusionSamplingParams``. Only request-batch-wide fields belong here. Request-local values such as - seeds, generators, latent tensors, timesteps, and pipeline-specific + seeds, generators, latent tensors, timesteps, and request-local ``extra_args`` are read per request from - ``DiffusionRequestBatch.sampling_params_list``. + ``DiffusionRequestBatch.sampling_params_list``. Structural ``extra_args`` + that affect batching are listed explicitly below. """ # Spatial / temporal shape. @@ -131,6 +132,16 @@ class RequestBatchSamplingParamsKey: # the model and must remain distinct from explicit ``lossless``. quality: str | None = None + # Wan scheduler structure is carried through extra_args. Requests using + # different solvers or flow shifts must not share a request batch. + sample_solver: str | None = None + flow_shift: float | None = None + + # Pipeline-specific condition structure populated during preprocessing. + # It prevents independently valid requests with incompatible conditions + # from being admitted to the same request batch. + condition_key: tuple[Any, ...] | None = None + # LoRA identity. lora_int_id: int | None = None lora_scale: float = 1.0 diff --git a/vllm_omni/diffusion/sched/request_scheduler.py b/vllm_omni/diffusion/sched/request_scheduler.py index 968360183..4168ae974 100644 --- a/vllm_omni/diffusion/sched/request_scheduler.py +++ b/vllm_omni/diffusion/sched/request_scheduler.py @@ -22,7 +22,21 @@ if TYPE_CHECKING: # on sampling params, so it must be resolved separately from the bulk lookup. _REQUEST_BATCH_SAMPLING_PARAMS_KEY_FIELD_NAMES = frozenset( field.name for field in fields(RequestBatchSamplingParamsKey) -) - {"lora_int_id"} +) - {"condition_key", "flow_shift", "lora_int_id", "sample_solver"} + + +def _normalize_explicit_sample_solver(value: object | None) -> str | None: + """Normalize an explicitly provided solver without selecting a default.""" + if value is None: + return None + return str(value).strip().lower() + + +def _normalize_explicit_flow_shift(value: object | None) -> float | None: + """Normalize an explicitly provided flow shift without selecting a default.""" + if value is None: + return None + return float(value) def build_request_batch_sampling_params_key(request: OmniDiffusionRequest) -> RequestBatchSamplingParamsKey: @@ -31,6 +45,13 @@ def build_request_batch_sampling_params_key(request: OmniDiffusionRequest) -> Re # LoRA identity is optional on sampling params (and on test stubs). lora_request = getattr(sampling, "lora_request", None) key_kwargs = {name: getattr(sampling, name) for name in _REQUEST_BATCH_SAMPLING_PARAMS_KEY_FIELD_NAMES} + extra_args = sampling.extra_args or {} + # Match pipeline resolution for explicit overrides, but preserve None: + # pipeline/engine defaults are configuration-dependent and must not be + # inferred while building the request-batch key. + key_kwargs["sample_solver"] = _normalize_explicit_sample_solver(extra_args.get("sample_solver")) + key_kwargs["flow_shift"] = _normalize_explicit_flow_shift(extra_args.get("flow_shift")) + key_kwargs["condition_key"] = getattr(request, "batch_compatibility_key", None) key_kwargs["lora_int_id"] = lora_request.lora_int_id if lora_request is not None else None return RequestBatchSamplingParamsKey(**key_kwargs) diff --git a/vllm_omni/engine/stage_engine_startup.py b/vllm_omni/engine/stage_engine_startup.py index 5dacc31df..081b14703 100644 --- a/vllm_omni/engine/stage_engine_startup.py +++ b/vllm_omni/engine/stage_engine_startup.py @@ -1558,6 +1558,7 @@ def launch_diffusion_stage_replica( from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient od_config = build_diffusion_config(model, stage_config, metadata) + od_config.max_num_seqs = batch_size parallel_config = getattr(od_config, "parallel_config", None) world_size = getattr(parallel_config, "world_size", 1) try: diff --git a/vllm_omni/engine/stage_init_utils.py b/vllm_omni/engine/stage_init_utils.py index c2e1ab5bc..054a2fb22 100644 --- a/vllm_omni/engine/stage_init_utils.py +++ b/vllm_omni/engine/stage_init_utils.py @@ -1441,6 +1441,7 @@ def initialize_diffusion_stage( from vllm_omni.diffusion.stage_diffusion_client import create_diffusion_client od_config = build_diffusion_config(model, stage_cfg, metadata) + od_config.max_num_seqs = batch_size return create_diffusion_client(model, od_config, metadata, stage_init_timeout, batch_size, use_inline) From 444485650b19b792403b12976f1b0eeb2ac1451c Mon Sep 17 00:00:00 2001 From: yiminghub2024 <482890@qq.com> Date: Sun, 16 Aug 2026 00:04:38 +0800 Subject: [PATCH 033/267] Add MiniMax-H3 recipe for NPU 950PR (#6120) Signed-off-by: yiminghub2024 <482890@qq.com> Co-authored-by: Hongsheng Liu --- recipes/MiniMaxAI/MiniMax-H3-NPU-950PR.md | 195 ++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 recipes/MiniMaxAI/MiniMax-H3-NPU-950PR.md diff --git a/recipes/MiniMaxAI/MiniMax-H3-NPU-950PR.md b/recipes/MiniMaxAI/MiniMax-H3-NPU-950PR.md new file mode 100644 index 000000000..36eb78a5d --- /dev/null +++ b/recipes/MiniMaxAI/MiniMax-H3-NPU-950PR.md @@ -0,0 +1,195 @@ +# MiniMax-H3 on a single NPU 950PR + +This recipe runs MiniMax-H3 with online INT8 quantization on one NPU 950PR +NPU (128 GB HiBL 1.0 HBM). It covers the single-card T2VA configuration at +1024x576. For the eight-card Atlas 800I A3 BF16 route at 768P, see +[MiniMax-H3-NPU.md](MiniMax-H3-NPU.md). + +## Capacity requirements + +| Resource | Requirement | +| --- | ---: | +| NPU | 1x NPU 950PR | +| NPU HBM | 128 GiB (131,072 MiB reported by `npu-smi`) | +| Observed HBM high-water mark | 118,442 MiB (90.4% of capacity) | +| Checkpoint storage | 135 GiB per partition, local disk strongly preferred | +| Container shared memory | 8 GiB, or a writable local filesystem for spill | + +The observed high-water mark leaves about 12.3 GiB of headroom, so this card is +sized for exactly one resident task partition at 1024x576. Do not expect a +larger output shape or concurrency greater than one to fit. + +Because a single 950PR holds the whole partition in HBM, this route does not +need layerwise offload, and the 200 GiB system-RAM floor carried by the 72 GiB +GPU recipes does not apply. The validated run completed inside a container +limited to 32 GiB of RAM. Passing `--enable-layerwise-offload` on this card is +actively harmful: it stages weights in non-reclaimable host memory and the +container's OOM killer terminates the server with exit code -9. + + +## Environment + +- Host architecture: x86_64 +- driver: 25.7.rc1.6 (hal 7.35.23) +- firmware: 9.0.0.105.229 +- CANN toolkit: 9.1.0 (`/usr/local//cann-9.1.0`) +- npu-smi: 25.7.rc1.6 +- Python: 3.12.13 +- PyTorch: 2.10.0+cpu +- torch_npu: 2.10.0.post2 +- vLLM: 0.26.0 +- vLLM-Omni: 0.26.1.dev103+g584d78c67.npu (commit `584d78c6`) + +Install vLLM-Omni from a checkout with MiniMax-H3 support: + +```bash +uv venv +source .venv/bin/activate +uv pip install -e . +``` + +## Start a server + +```bash +export MODEL=/path/to/MiniMax-H3/FL2VA +export PORT=8000 +export VLLM_WORKER_MULTIPROC_METHOD=spawn +export VLLM_OMNI_VIDEO_SYNC_TIMEOUT=14400 + +vllm serve "${MODEL}" \ + --omni \ + --host 0.0.0.0 \ + --port "${PORT}" \ + --trust-remote-code \ + --num-gpus 1 \ + --tensor-parallel-size 1 \ + --usp 1 \ + --ring 1 \ + --text-encoder-tp-size 1 \ + --vae-patch-parallel-size 1 \ + --vae-parallel-mode tile \ + --vae-use-tiling \ + --init-timeout 14400 \ + --stage-init-timeout 14400 \ + --quantization int8 +``` + +Do not pass `--diffusion-attention-backend CUDNN_ATTN`. cuDNN attention has no +NPU implementation and fails at the first denoise step. Leave the backend +unset: it resolves to `TORCH_SDPA`, or to `FLASH_ATTN` when MindIE-SD is +installed (see [MiniMax-H3-NPU.md § Environment](MiniMax-H3-NPU.md#environment)). + +H3 is CFG-distilled, so `--cfg-parallel-size` must remain 1. + +## Validated evidence + +Measured on one NPU 950PR with the server command above plus +`--enable-diffusion-pipeline-profiler`, generating a 5 s 1024x576 T2VA clip from +the request in [§ T2VA request example](#t2va-request-example). MiniMax-H3 +requested 60 denoise steps and executed 59 denoise updates, so per-step latency +is `denoise / 59`. `ffprobe` confirms every returned MP4 is 1024x576, 24 fps, +124 video frames, 5.175 s, with a 32 kHz audio track of 165,600 samples. + +Five requests were issued back to back with no idle time between them. The card +is measurably faster on the first request than in steady state, so both regimes +are reported. + +| Stage | First request, cold card | Thermal steady state | +| --- | ---: | ---: | +| End-to-end request | 462.13 s | 506.38 s | +| Text encode | 0.55 s | 0.05 s | +| Denoise (59 updates) | 429.29 s | 471.74 s | +| Per denoise update | 7,276 ms | 7,996 ms | +| VAE decode | 8.43 s | 8.14 s | +| Worker-to-server handoff | 24 s | 27 s | +| MP4 muxing | 0.57 s | 0.50 s | +| Server-reported `denoise_step_latency_ms` | 7,702 ms | 8,440 ms | +| Peak HBM (`npu-smi`) | 118,442 MiB | 118,442 MiB | +| Average NPU power | 566.9 W | 556.9 W | +| Peak NPU temperature | 104 C | 104 C | + +Sample counts differ per row. The cold column is a single request. In the steady +column, text encode, denoise and VAE decode are means over the four subsequent +requests — denoise spanned 470.37 to 473.21 s, a 0.60% spread — while +end-to-end, handoff and muxing are means over the two of those four that +survived the handoff timeout described above. + +**Report the steady-state column.** The cold card completes denoise 9.9% faster +while drawing only 1.8% more average power, and both regimes reach the same +104 C ceiling, so the first request is buying a short window of higher clocks +before the die saturates. Any benchmark that issues a single request against an +idle card will overstate throughput by about 10%. + +**Do not read `denoise_step_latency_ms` as a per-step time.** The server divides +the entire stage wall time by the 60 *requested* steps, so it absorbs text +encoding, VAE decode and the 837 MiB handoff. All three successful requests +satisfy `denoise_step_latency_ms == e2e_stage_wall_time_ms / 60` exactly. The +per-update figure in the table is `diffuse / 59` from the pipeline profiler. + +Denoise accounts for 93% of the steady-state request and the handoff accounts +for 5.3%, so keeping the transfer in shared memory should bring end-to-end down +to roughly 480 s. + +Peak HBM is sampled externally with `npu-smi` at a 3 s interval. It is a +reserved high-water mark rather than live allocation: it reads 111,272 MiB after +weight loading and before the first request, and does not fall back after +requests complete. Re-measure for longer outputs, a different output shape, or +concurrency greater than one. + +### Startup + +Weight loading took **31 min 50 s** on the validated host (first log line +08:04:59, `Application startup complete` 08:36:49), with the checkpoint on a +shared GlusterFS network mount. This is why the server command sets +`--init-timeout 14400` and `--stage-init-timeout 14400`; the stock 600 s and +1800 s timeouts both abort mid-load with `TimeoutError` and exit code 143. +Reduce both timeouts when the partition is staged on local disk. + +## T2VA request example + +```bash +export API_URL="http://127.0.0.1:${PORT}/v1/videos/sync" + +curl -sS --max-time 1800 -X POST "${API_URL}" \ + -F 'prompt=At night, three cats march into a bedroom playing tiny brass instruments, then abruptly file out, with synchronized room ambience.' \ + -F 'width=1024' \ + -F 'height=576' \ + -F 'aspect_ratio=16:9' \ + -F 'fps=24' \ + -F 'num_inference_steps=60' \ + -F 'flow_shift=12' \ + -F 'seed=1101' \ + -F 'extra_params={"task":"t2va","duration":5,"audio_flow_shift":3.0}' \ + -o t2va.mp4 +``` + +## Known limitations + +- INT8 online quantization is validated for T2VA on this card. Use BF16 for + FL2VA and Ref2VA, or re-measure before relying on it. +- Single-card serving loads one task partition at a time. For Ref2VA, stop the + server and restart it against the `Ref2VA` directory. +- Sustained load drives the die to 104 C and costs about 10% of denoise + throughput relative to a cold card. Warm the server with at least one full + request before measuring, and treat steady state as the reportable number. +- Requests can fail during result handoff on hosts where the IPC spill path is + slow. See [§ Result handoff and shared + memory](#result-handoff-and-shared-memory). +- Loading the checkpoint from a network mount takes over 30 minutes and forces + very large init timeouts. Stage the partition on local disk when possible. +- The configuration measured here is 1024x576 at 60 steps, which differs from + the 1344x768 at 50 steps used by the GPU recipes in this directory. Denoise + cost scales superlinearly with token count, so the numbers are not directly + comparable across recipes. +- The image ships a `triton` package whose backend is not built + (`No module named 'triton._C.libtriton.'`). vLLM logs this as an error + at startup and disables Triton; the diffusion path does not need it and the + run completes normally. + +## Additional resources + +- [MiniMax-H3.md](MiniMax-H3.md) — full GPU guide +- [MiniMax-H3-NPU.md](MiniMax-H3-NPU.md) — eight-card Atlas 800I A3 BF16 guide +- [Int8 quantization](../../docs/user_guide/quantization/int8.md) +- [Supported models](../../docs/models/supported_models.md) +- [Video API](../../docs/serving/videos_api.md) From 68bf8c2c4296fb851c36173e5dcdca55ea56500c Mon Sep 17 00:00:00 2001 From: Anurag <75563673+anurag12-webster@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:49:45 +0530 Subject: [PATCH 034/267] [Bugfix] Prevent DiffusionResultPump crash on cancelled futures (#5793) (#5983) Signed-off-by: ANURAG KANADE --- tests/diffusion/test_result_pump.py | 74 +++++++++++++++++++ .../diffusion/executor/multiproc_executor.py | 15 ++-- vllm_omni/diffusion/utils/future_utils.py | 27 +++++++ 3 files changed, 109 insertions(+), 7 deletions(-) create mode 100644 vllm_omni/diffusion/utils/future_utils.py diff --git a/tests/diffusion/test_result_pump.py b/tests/diffusion/test_result_pump.py index 2f4a89ca1..fa172fd0c 100644 --- a/tests/diffusion/test_result_pump.py +++ b/tests/diffusion/test_result_pump.py @@ -418,3 +418,77 @@ class TestShutdownCleansUpFutures: assert len(executor._rpc_futures) == 0 assert len(executor._output_futures) == 0 + + +class _RacyFuture(concurrent.futures.Future): + # done() always lies and reports False, to deterministically force the + # pump's check-then-act race window without needing real thread timing. + def done(self) -> bool: + return False + + +def _racy_cancelled_future() -> concurrent.futures.Future: + fut = _RacyFuture() + fut.cancel() + assert fut.cancelled() + return fut + + +class TestResultPumpCancelledFutureRace: + """Regression for #5793: a future cancelled concurrently with the pump's + resolve call (e.g. an asyncio.wait_for timeout, or a request abort) must + be dropped, not raise InvalidStateError and kill the pump thread. + """ + + def test_compute_done_racing_cancel_does_not_crash_pump(self): + executor = _make_executor() + fut = _racy_cancelled_future() + with executor._futures_lock: + executor._rpc_futures["1"] = fut + + msg = AsyncDiffusionOutput(kind=AsyncOutputKind.COMPUTE_DONE, rpc_id="1", async_output_id="abc") + _feed_one_msg_to_pump(executor, msg) + + assert fut.cancelled() + + def test_output_ready_racing_cancel_does_not_crash_pump(self): + executor = _make_executor() + fut = _racy_cancelled_future() + with executor._futures_lock: + executor._output_futures["abc123"] = fut + + msg = AsyncDiffusionOutput( + kind=AsyncOutputKind.OUTPUT_READY, + async_output_id="abc123", + output=DiffusionOutput(output="late"), + ) + _feed_one_msg_to_pump(executor, msg) + + assert fut.cancelled() + + def test_batch_split_racing_cancel_does_not_crash_pump(self): + executor = _make_executor() + cancelled_fut = _racy_cancelled_future() + healthy_fut = concurrent.futures.Future() + with executor._futures_lock: + executor._output_futures["batch-1/r-aborted"] = cancelled_fut + executor._output_futures["batch-1/r-healthy"] = healthy_fut + executor._batch_split_map["batch-1"] = { + "batch-1/r-aborted": "r-aborted", + "batch-1/r-healthy": "r-healthy", + } + + outputs = { + "r-aborted": DiffusionOutput(output="late"), + "r-healthy": DiffusionOutput(output="ok"), + } + msg = AsyncDiffusionOutput( + kind=AsyncOutputKind.OUTPUT_READY, + async_output_id="batch-1", + output=_FakeBatchOutput(outputs), + ) + _feed_one_msg_to_pump(executor, msg) + + assert cancelled_fut.cancelled() + assert healthy_fut.done() + assert healthy_fut.result(timeout=1.0) is outputs["r-healthy"] diff --git a/vllm_omni/diffusion/executor/multiproc_executor.py b/vllm_omni/diffusion/executor/multiproc_executor.py index 74286dc94..6f0917e32 100644 --- a/vllm_omni/diffusion/executor/multiproc_executor.py +++ b/vllm_omni/diffusion/executor/multiproc_executor.py @@ -24,6 +24,7 @@ from vllm_omni.diffusion.data import SHUTDOWN_MESSAGE, AsyncDiffusionOutput, Asy from vllm_omni.diffusion.executor.abstract import DiffusionExecutor from vllm_omni.diffusion.ipc import DIFFUSION_RPC_RESULT_ENVELOPE, unpack_diffusion_output_shm from vllm_omni.diffusion.sched.request_scheduler import build_request_batch_sampling_params_key +from vllm_omni.diffusion.utils.future_utils import try_set_exception, try_set_result from vllm_omni.diffusion.worker import WorkerProc if TYPE_CHECKING: @@ -870,9 +871,9 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): fut = self._rpc_futures.pop(msg.rpc_id, None) if msg.rpc_id else None if fut is not None and not fut.done(): if msg.error: - fut.set_exception(RuntimeError(msg.error)) + try_set_exception(fut, RuntimeError(msg.error)) else: - fut.set_result(msg) + try_set_result(fut, msg) elif msg.kind == AsyncOutputKind.OUTPUT_READY: batch_id = msg.async_output_id with self._futures_lock: @@ -903,9 +904,9 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): pending = self._output_futures.pop(batch_id, None) if pending is not None and not pending.done(): if exc is not None: - pending.set_exception(exc) + try_set_exception(pending, exc) else: - pending.set_result(output_result) + try_set_result(pending, output_result) else: fut = concurrent.futures.Future() if exc is not None: @@ -933,7 +934,7 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): with self._futures_lock: pending = self._output_futures.pop(per_req_id, None) if pending is not None and not pending.done(): - pending.set_result(per_req_result) + try_set_result(pending, per_req_result) else: fut: concurrent.futures.Future = concurrent.futures.Future() fut.set_result(per_req_result) @@ -999,10 +1000,10 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): with self._futures_lock: for fut in self._rpc_futures.values(): if not fut.done(): - fut.set_exception(RuntimeError("Executor shut down")) + try_set_exception(fut, RuntimeError("Executor shut down")) for fut in self._output_futures.values(): if not fut.done(): - fut.set_exception(RuntimeError("Executor shut down")) + try_set_exception(fut, RuntimeError("Executor shut down")) self._rpc_futures.clear() self._output_futures.clear() self._batch_split_map.clear() diff --git a/vllm_omni/diffusion/utils/future_utils.py b/vllm_omni/diffusion/utils/future_utils.py new file mode 100644 index 000000000..5492c6a21 --- /dev/null +++ b/vllm_omni/diffusion/utils/future_utils.py @@ -0,0 +1,27 @@ +"""Helpers for resolving ``concurrent.futures.Future`` objects shared across threads.""" + +import concurrent.futures +from typing import Any, TypeVar + +from vllm.logger import init_logger + +logger = init_logger(__name__) + +_ResultT = TypeVar("_ResultT") + + +def try_set_result(fut: concurrent.futures.Future[_ResultT], result: _ResultT) -> None: + # fut may be cancelled concurrently (e.g. asyncio.wait_for timeout) between + # the caller's fut.done() check and this call; drop the late result instead + # of crashing the pump thread. + try: + fut.set_result(result) + except concurrent.futures.InvalidStateError: + logger.debug("Dropping late result for already-resolved/cancelled future") + + +def try_set_exception(fut: concurrent.futures.Future[Any], exc: BaseException) -> None: + try: + fut.set_exception(exc) + except concurrent.futures.InvalidStateError: + logger.debug("Dropping late exception for already-resolved/cancelled future") From 9798bc3a8d5a400b97fba7402f42510577e7302d Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:18:47 -0700 Subject: [PATCH 035/267] [Docs] Unify recipe serve commands on `vllm serve --omni` (#6221) Signed-off-by: Yueqian Lin --- recipes/BosonAI/Higgs-Audio-V3-TTS.md | 2 +- recipes/NVIDIA/Nemotron-Labs-Audex.md | 4 ++-- recipes/StabilityAI/Stable-Audio-Open.md | 2 +- recipes/TEMPLATE.md | 3 +++ recipes/inclusionAI/Ming-omni-tts.md | 2 +- recipes/zai-org/GLM-TTS.md | 6 +++--- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/recipes/BosonAI/Higgs-Audio-V3-TTS.md b/recipes/BosonAI/Higgs-Audio-V3-TTS.md index b924ac472..0391fe03c 100644 --- a/recipes/BosonAI/Higgs-Audio-V3-TTS.md +++ b/recipes/BosonAI/Higgs-Audio-V3-TTS.md @@ -45,7 +45,7 @@ MusicGen-style delay pattern). **Online serving:** ```bash -vllm-omni serve bosonai/higgs-audio-v3-tts-4b \ +vllm serve bosonai/higgs-audio-v3-tts-4b \ --host 0.0.0.0 --port 8095 \ --trust-remote-code --omni ``` diff --git a/recipes/NVIDIA/Nemotron-Labs-Audex.md b/recipes/NVIDIA/Nemotron-Labs-Audex.md index 21ee7e655..315d31804 100644 --- a/recipes/NVIDIA/Nemotron-Labs-Audex.md +++ b/recipes/NVIDIA/Nemotron-Labs-Audex.md @@ -51,7 +51,7 @@ NemotronH (~3B active parameters). Pick the pipeline by task: `model_type` auto-resolves `audex_tts.yaml`): ```bash -vllm-omni serve nvidia/Nemotron-Labs-Audex-2B \ +vllm serve nvidia/Nemotron-Labs-Audex-2B \ --host 0.0.0.0 --port 8097 \ --trust-remote-code --omni # other modes: add --stage-configs-path vllm_omni/deploy/audex_{tta,thinker_only,s2s}.yaml @@ -109,7 +109,7 @@ The 30B REQUIRES its explicit deploy yaml (the shared HF `model_type` auto-resolves to the 2B-tuned config otherwise): ```bash -vllm-omni serve nvidia/Nemotron-Labs-Audex-30B-A3B \ +vllm serve nvidia/Nemotron-Labs-Audex-30B-A3B \ --host 0.0.0.0 --port 8097 \ --trust-remote-code --omni \ --stage-configs-path vllm_omni/deploy/audex_tts_30b.yaml diff --git a/recipes/StabilityAI/Stable-Audio-Open.md b/recipes/StabilityAI/Stable-Audio-Open.md index a7f18128d..97c39f7ee 100644 --- a/recipes/StabilityAI/Stable-Audio-Open.md +++ b/recipes/StabilityAI/Stable-Audio-Open.md @@ -78,7 +78,7 @@ python examples/offline_inference/text_to_audio/text_to_audio.py \ Start the online serving endpoint: ```bash -vllm-omni serve /path/to/stable-audio-open-1.0 \ +vllm serve /path/to/stable-audio-open-1.0 \ --host 0.0.0.0 \ --port 8091 \ --gpu-memory-utilization 0.9 \ diff --git a/recipes/TEMPLATE.md b/recipes/TEMPLATE.md index 6af66f721..f2a145660 100644 --- a/recipes/TEMPLATE.md +++ b/recipes/TEMPLATE.md @@ -39,6 +39,9 @@ platform section, document one or more tested hardware configurations. #### Command +Serve commands use `vllm serve --omni`. The `--omni` flag is what +selects the Omni pipeline, so it is required and cannot be dropped. + ```bash # Add the exact command(s) here ``` diff --git a/recipes/inclusionAI/Ming-omni-tts.md b/recipes/inclusionAI/Ming-omni-tts.md index e42959e8a..0f0034721 100644 --- a/recipes/inclusionAI/Ming-omni-tts.md +++ b/recipes/inclusionAI/Ming-omni-tts.md @@ -39,7 +39,7 @@ Other hardware is welcome as community validation lands. Launch the two-stage talker: ```bash -vllm-omni serve inclusionAI/Ming-omni-tts-0.5B \ +vllm serve inclusionAI/Ming-omni-tts-0.5B \ --deploy-config vllm_omni/deploy/ming_tts.yaml \ --omni \ --port 8091 \ diff --git a/recipes/zai-org/GLM-TTS.md b/recipes/zai-org/GLM-TTS.md index 2bf1a202a..c7f331d96 100644 --- a/recipes/zai-org/GLM-TTS.md +++ b/recipes/zai-org/GLM-TTS.md @@ -44,20 +44,20 @@ conditioned on reference audio and its transcript. Start the server from the repository root: ```bash -vllm-omni serve zai-org/GLM-TTS --omni --trust-remote-code --port 8091 +vllm serve zai-org/GLM-TTS --omni --trust-remote-code --port 8091 ``` Async chunking is enabled by default in the bundled deployment config. For the sync (non-streaming) path: ```bash -vllm-omni serve zai-org/GLM-TTS --omni --trust-remote-code --port 8091 --no-async-chunk +vllm serve zai-org/GLM-TTS --omni --trust-remote-code --port 8091 --no-async-chunk ``` Use a custom deploy config for advanced cases: ```bash -vllm-omni serve zai-org/GLM-TTS --omni --trust-remote-code --port 8091 \ +vllm serve zai-org/GLM-TTS --omni --trust-remote-code --port 8091 \ --deploy-config /path/to/your_glm_tts_overrides.yaml ``` From 3875f8ded41d5015aa98e5c35f12e6d2770c15b1 Mon Sep 17 00:00:00 2001 From: Jack Chen Date: Sun, 16 Aug 2026 09:11:20 +0800 Subject: [PATCH 036/267] [Core] Add realtime AR-Diffusion tick sessions for LingBot World 2.0 (#5491) Signed-off-by: zhangbin.327 Signed-off-by: Jack47 Co-authored-by: zhangbin.327 Co-authored-by: Hongsheng Liu --- docs/.nav.yml | 1 + .../ar_diffusion_pipeline_capability.md | 18 +- docs/design/feature/realtime_ar_diffusion.md | 213 ++ docs/design/index.md | 1 + docs/models/supported_models.md | 5 + .../diffusion/lingbot_world_v2.py | 288 +++ .../diffusion/lingbot_world_v2_realtime.py | 239 +++ recipes/README.md | 1 + recipes/Robbyant/LingBot-World-2.0.md | 122 ++ .../ar_diffusion/test_capability_runner.py | 155 +- tests/diffusion/ar_diffusion/test_consumer.py | 231 +++ tests/diffusion/ar_diffusion/test_kv_cache.py | 159 +- .../ar_diffusion/test_kv_orchestration.py | 8 +- .../ar_diffusion/test_paged_attention.py | 6 +- .../ar_diffusion/test_pipeline_bridge.py | 13 + tests/diffusion/ar_diffusion/test_session.py | 505 +++++ .../ar_diffusion/test_tick_protocol.py | 108 + .../ar_diffusion/test_worker_lifecycle.py | 53 + .../models/dreamzero/test_pipeline_state.py | 36 +- .../models/lingbot_world/__init__.py | 1 + .../lingbot_world_model_index.json.fixture | 13 + ...lingbot_world_official_shapes.json.fixture | 42 + ...ingbot_world_scheduler_config.json.fixture | 30 + ...gbot_world_weight_index_names.json.fixture | 112 ++ .../test_lingbot_world_actions.py | 294 +++ .../test_lingbot_world_attention.py | 543 +++++ .../test_lingbot_world_camera.py | 525 +++++ .../test_lingbot_world_transformer.py | 685 +++++++ .../test_lingbot_world_typing.py | 39 + .../test_pipeline_lingbot_world.py | 1765 +++++++++++++++++ .../test_lingbot_world_v2.py | 84 + tests/entrypoints/test_utils.py | 6 +- tests/model_tests/diffusion/test_alignment.py | 3 + .../models/dreamzero/pipeline_dreamzero.py | 4 + .../models/lingbot_world/__init__.py | 18 + .../diffusion/models/lingbot_world/actions.py | 302 +++ .../diffusion/models/lingbot_world/camera.py | 481 +++++ .../models/lingbot_world/pipeline.py | 1292 ++++++++++++ .../models/lingbot_world/transformer.py | 1201 +++++++++++ vllm_omni/diffusion/registry.py | 8 + .../diffusion/worker/diffusion_worker.py | 19 + vllm_omni/engine/async_omni_engine.py | 3 + .../experimental/ar_diffusion/__init__.py | 20 + .../experimental/ar_diffusion/capability.py | 9 + .../experimental/ar_diffusion/consumer.py | 168 ++ .../ar_diffusion/kv_cache/__init__.py | 2 - .../ar_diffusion/kv_cache/config.py | 3 +- .../ar_diffusion/kv_cache/manager.py | 160 +- .../ar_diffusion/kv_cache/paged_attention.py | 72 +- .../ar_diffusion/kv_cache/state.py | 13 +- vllm_omni/experimental/ar_diffusion/runner.py | 24 +- .../experimental/ar_diffusion/session.py | 578 ++++++ .../ar_diffusion/tick_protocol.py | 202 ++ 53 files changed, 10754 insertions(+), 129 deletions(-) create mode 100644 docs/design/feature/realtime_ar_diffusion.md create mode 100644 examples/offline_inference/diffusion/lingbot_world_v2.py create mode 100644 examples/offline_inference/diffusion/lingbot_world_v2_realtime.py create mode 100644 recipes/Robbyant/LingBot-World-2.0.md create mode 100644 tests/diffusion/ar_diffusion/test_consumer.py create mode 100644 tests/diffusion/ar_diffusion/test_session.py create mode 100644 tests/diffusion/ar_diffusion/test_tick_protocol.py create mode 100644 tests/diffusion/ar_diffusion/test_worker_lifecycle.py create mode 100644 tests/diffusion/models/lingbot_world/__init__.py create mode 100644 tests/diffusion/models/lingbot_world/fixtures/lingbot_world_model_index.json.fixture create mode 100644 tests/diffusion/models/lingbot_world/fixtures/lingbot_world_official_shapes.json.fixture create mode 100644 tests/diffusion/models/lingbot_world/fixtures/lingbot_world_scheduler_config.json.fixture create mode 100644 tests/diffusion/models/lingbot_world/fixtures/lingbot_world_weight_index_names.json.fixture create mode 100644 tests/diffusion/models/lingbot_world/test_lingbot_world_actions.py create mode 100644 tests/diffusion/models/lingbot_world/test_lingbot_world_attention.py create mode 100644 tests/diffusion/models/lingbot_world/test_lingbot_world_camera.py create mode 100644 tests/diffusion/models/lingbot_world/test_lingbot_world_transformer.py create mode 100644 tests/diffusion/models/lingbot_world/test_lingbot_world_typing.py create mode 100644 tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py create mode 100644 tests/e2e/offline_inference/test_lingbot_world_v2.py create mode 100644 vllm_omni/diffusion/models/lingbot_world/__init__.py create mode 100644 vllm_omni/diffusion/models/lingbot_world/actions.py create mode 100644 vllm_omni/diffusion/models/lingbot_world/camera.py create mode 100644 vllm_omni/diffusion/models/lingbot_world/pipeline.py create mode 100644 vllm_omni/diffusion/models/lingbot_world/transformer.py create mode 100644 vllm_omni/experimental/ar_diffusion/consumer.py create mode 100644 vllm_omni/experimental/ar_diffusion/session.py create mode 100644 vllm_omni/experimental/ar_diffusion/tick_protocol.py diff --git a/docs/.nav.yml b/docs/.nav.yml index ffaa46368..d367ded05 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -116,6 +116,7 @@ nav: - Async Diffusion Output: design/feature/async_diffusion_output.md - Async Omni Output Materialization: design/feature/omni_async_output_materialization.md - Automatic Prefix Caching: design/feature/prefix_caching.md + - Realtime AR-Diffusion: design/feature/realtime_ar_diffusion.md - Communication: - OmniConnector Implementations: - Mooncake Store Connector: design/feature/omni_connectors/mooncake_store_connector.md diff --git a/docs/design/ar_diffusion_pipeline_capability.md b/docs/design/ar_diffusion_pipeline_capability.md index c9e05c8e8..3c1b4b9cd 100644 --- a/docs/design/ar_diffusion_pipeline_capability.md +++ b/docs/design/ar_diffusion_pipeline_capability.md @@ -4,6 +4,9 @@ > `experimental/ar_diffusion` contract. It follows the ownership and lifecycle > direction in [#5137](https://github.com/vllm-project/vllm-omni/issues/5137), > but it is not yet part of the stable module-design hierarchy. +> +> The end-to-end request, identity, failure, and memory contracts are documented +> in [Realtime AR-Diffusion sessions](feature/realtime_ar_diffusion.md). `ARDiffusionEngine` is an opt-in diffusion backend for pipelines that keep autoregressive attention KV across requests. The engine only selects @@ -51,14 +54,13 @@ time, and its inherited batch/step entry points also fail explicitly. Supporting those paths requires a batch-aware state-binding contract rather than silently bypassing `bind_ar_diffusion_state()`. -The current implementation retains one session per model instance. A request -for a different `session_id` closes the resident session through the same LRU -lifecycle used by explicit close and failure cleanup before allocating the new -one. Self-attention and cross-attention memory are therefore sized for one -resident session even if the pipeline declares a larger `session_capacity`. -This prioritizes rollout throughput and prevents block-pool exhaustion before a -count-based eviction can run. Multi-session residency can be added later with a -memory-backed capacity calculation or allocation-pressure eviction. +The implementation may retain multiple sessions per model instance. It selects +the largest feasible capacity up to the pipeline's `session_capacity`, based on +self-attention pages, cross-attention allocations, scratch, and declared +model-owned state. One session is admitted whenever it fits actual free device +memory; `gpu_memory_fraction` controls additional residency. Requests beyond +that capacity evict the least-recently-used session through the same lifecycle +hook used by explicit close and failure cleanup. Scratch storage is also capability-driven. `frames_per_block` reserves space for an uncommitted current video block, while `max_scratch_tokens_per_branch` diff --git a/docs/design/feature/realtime_ar_diffusion.md b/docs/design/feature/realtime_ar_diffusion.md new file mode 100644 index 000000000..5963befe7 --- /dev/null +++ b/docs/design/feature/realtime_ar_diffusion.md @@ -0,0 +1,213 @@ +# Realtime AR-Diffusion sessions + +> **Status:** experimental. This document describes the internal execution +> contract implemented under `vllm_omni.experimental.ar_diffusion`. It does not +> define a public HTTP or WebSocket schema. + +## Scope + +Realtime AR-Diffusion serves world models that generate one autoregressive +latent block per request while retaining model state across requests. The +generic runtime owns identity, event ordering, session lifecycle, routing, and +KV allocation. A model adapter owns model-specific controls and conditioning. + +LingBot World v2 is the first single-KV-branch integration. DreamZero uses the +same runner capability with two logical CFG branches. + +## Identities + +Four identifiers have distinct lifetimes: + +| Identifier | Lifetime | Owner | Contract | +| --- | --- | --- | --- | +| `session_id` | One persistent world | Session manager | Selects runner KV and model-owned state. | +| `event_id` | One control/prompt update | Transport adapter | Unique and monotonically increasing within a session, including across reset. | +| `chunk_index` | One committed AR block position | Session | Contiguous from zero; reset starts again at zero. | +| `request_id` | One chunk execution | Session | Correlates the tick with its returned metadata. | + +`AsyncOmni` keeps its normal UUID-suffixed engine routing ID. The tick +`request_id` remains inside the immutable AR-Diffusion snapshot and the +standard output metadata, while caller-visible `OmniRequestOutput.request_id` +continues to use AsyncOmni's existing external-ID contract. Protocol identity +therefore does not require a private entrypoint override. + +## Request lifecycle + +1. `ARDiffusionSession.accept_event()` validates and queues an immutable + `ARDiffusionSessionEvent`. Prompt and opaque control tracks may be updated in + the same event. +2. `ARDiffusionSession.next_chunk()` serializes chunk execution and calls + `_build_tick_locked()`. The resulting `ARDiffusionTickRequest` is a snapshot + of the current prompt, reduced controls, ordered event IDs, and chunk index. +3. `ARDiffusionOmniTickConsumer.execute_tick()` copies the stage sampling + parameters, stores the typed tick in + `sampling_params.extra_args["ar_diffusion_tick"]`, constructs the normal Omni + prompt through its `prompt_provider`, and invokes `AsyncOmni.generate()`. +4. `AsyncOmni` allocates its normal unique engine routing ID and submits one + `OmniDiffusionRequest` through the orchestrator. +5. `ARDiffusionModelRunner._request_session()` parses the typed tick. + `_get_or_create_session()` obtains runner-owned paged KV, and + `bind_ar_diffusion_state()` exposes it to the pipeline only for the duration + of `execute_model()`. +6. The model pipeline validates the tick and generates exactly one latent + block. LingBot interprets camera controls in + `LingBotCameraControlReducer` and + `LingBotWorldCausalDMDPipeline.forward()`; the generic runtime does not parse + camera or action schemas. +7. The pipeline returns one `DiffusionOutput` with the generated payload and an + `ARDiffusionChunkMetadata` mapping under `metadata["ar_diffusion"]`. + `AsyncOmni` exposes it as + `OmniRequestOutput.multimodal_output["metadata"]["ar_diffusion"]`. +8. The consumer parses the standard envelope. The session commits its reducer + state, prompt, controls, event IDs, and next chunk index only when returned + metadata exactly equals the submitted tick snapshot. + +The metadata envelope is: + +```json +{ + "metadata": { + "ar_diffusion": { + "session_id": "world-7", + "request_id": "ar-world-7-3-...", + "chunk_index": 3, + "applied_event_ids": [10, 11] + } + } +} +``` + +## Transaction and failure semantics + +Accepted events are not acknowledged as applied until the model output and +metadata are validated. Reducer `prepare()` is speculative; reducer `commit()` +and the session snapshot form one logical commit. A model, metadata, or reducer +failure leaves the chunk index and queued events uncommitted and transitions the +session to `FAILED`. + +After a tick failure, worker state is closed because a forward may have +partially modified or already released KV. The same chunk is not retried in +place. A caller must explicitly reset the session, starting again at chunk +zero, or close it. + +Explicit close follows this state machine: + +```text +ACTIVE or FAILED + | + v + CLOSING ---- worker cleanup succeeds ----> CLOSED + | + +------- worker cleanup fails --------> CLEANUP_FAILED + | + +-- close retry --> CLOSING +``` + +`CLEANUP_FAILED` is a local tombstone. The session manager retains it, rejects +creation of the same `session_id`, and rejects reset. Only an idempotent close +retry that succeeds removes the session. Transport disconnect uses the same +path. + +## KV ownership and capacity + +`ARDiffusionKVCacheSpec` is the pipeline-owned immutable geometry contract. It +declares: + +- TP-local self-attention geometry, AR block size, sink, and recent window; +- logical KV branches and their worker-local slots; +- fixed-length cross-attention caches; +- scratch tokens required by an uncommitted forward; +- persistent model-owned CUDA bytes per resident session; and +- the requested resident-session capacity. + +The runner computes: + +```text +required(capacity) = + scratch + + managed_self_attention_pages(capacity) + + capacity * cross_attention_bytes_per_session + + capacity * model_owned_state_bytes_per_session +``` + +`gpu_memory_fraction` is a soft expansion budget: + +```text +configured_budget = available_device_bytes * gpu_memory_fraction +effective_budget = max(configured_budget, required(1)) +``` + +If `required(1)` exceeds actual available device memory, initialization fails. +Otherwise the runner always admits one viable session and uses the configured +fraction to select additional resident sessions up to the model-declared cap. +All model-owned reservations are deducted before the paged self-attention pool +is allocated. + +LingBot reports its persistent full-horizon image-condition tensor. DreamZero +reports the measured 603 MiB per-session Wan VAE causal-convolution state. + +## Concurrency and routing limits + +The current runner executes one request at a time: + +- `max_num_seqs` must be one; +- request batching and diffusion step execution are rejected; +- one AR block is generated per engine request; and +- an AR-Diffusion stage must have exactly one replica. + +The single-replica restriction is intentional. Multi-replica support requires +session-affine routing so every tick, reset, and close reaches the worker that +owns the session. Random or round-robin routing is not correct. + +Multiple resident sessions may share one runner. Capacity selection accounts +for all persistent pools, and the runner performs LRU eviction through the same +model lifecycle hook used by explicit close. + +## Model boundary + +The generic runtime transports controls as opaque +`ARDiffusionControlInput(track, schema, data)` values. Model adapters validate +and reduce those values. This keeps target/velocity, key state, pose, +intrinsics, and other model-specific conventions out of the session protocol. + +For LingBot World v2: + +- `lingbot_world/actions.py` parses key-state and trajectory controls; +- `lingbot_world/camera.py` loads or constructs camera trajectories and Plücker embeddings; +- `lingbot_world/transformer.py` implements checkpoint-compatible causal attention; and +- `lingbot_world/pipeline.py` constructs conditioning, owns small non-KV session state, and + produces one block plus the standard metadata envelope. + +## Non-goals + +This contract does not currently provide: + +- a public HTTP or WebSocket request schema (the structured-interaction + frontend is tracked separately in + [#5527](https://github.com/vllm-project/vllm-omni/pull/5527)); +- camera/action semantics shared by every world model; +- session migration or replication across workers; +- retry of an ambiguous partially executed chunk; +- cross-stage KV transfer; or +- stateful streaming VAE decode. + +Those features can be layered on top without adding LingBot-specific identity +or lifecycle fields to the generic runtime. + +## Required regression coverage + +CPU contract tests cover: + +- separation of tick correlation IDs from internal engine routing IDs; +- event ordering, metadata equality, and reducer fail-closed commit; +- close/disconnect failure tombstones and cleanup retry; +- capacity one under the default fraction for shipped LingBot geometry; +- capacity expansion under a tuned fraction; +- DreamZero model-owned state accounting; +- runner session reuse, reset, close, LRU eviction, and forward cleanup; and +- LingBot model registration, imports, camera/action reduction, and one-block + output metadata. + +GPU validation must additionally exercise real model loading, action input, +prompt switching, rolling-window replacement, reset, close, finite output +latents, and exact metadata/request identity. diff --git a/docs/design/index.md b/docs/design/index.md index 0db152335..acac90458 100644 --- a/docs/design/index.md +++ b/docs/design/index.md @@ -22,6 +22,7 @@ implementation contract; it is not, by itself, a general support claim. - [Async Diffusion Output](feature/async_diffusion_output.md) - [Async Omni Output Materialization](feature/omni_async_output_materialization.md) - [Automatic Prefix Caching in Omni Models](feature/prefix_caching.md) +- [Realtime AR-Diffusion Sessions](feature/realtime_ar_diffusion.md) ### Communication diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 5097c83fa..62645aae6 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -37,6 +37,7 @@ th { | `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | | — | | `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | | `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | +| `LingBotWorldCausalDMDPipeline` | LingBot-World 2.0 v1 (experimental) | `robbyant/lingbot-world-v2-14b-causal-fast-diffusers` | Experimental | | | | — | | `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, V2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | | `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | @@ -117,6 +118,10 @@ support metadata until their recipe is audited. ✅︎ indicates the model is supported on that backend. Empty cells mean not listed as supported on that backend. +See the [LingBot-World 2.0 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Robbyant/LingBot-World-2.0.md) +for offline and experimental realtime usage, supported controls, validation, +and current limitations. + H3 MiniMax H3 AMD GPU support is validated on gfx942 (MI300X) and gfx950 (MI350) in BF16, using the AITER `FLASH_ATTN` diffusion attention backend. Other AMD SKUs (e.g. MI325X) are not listed until their diff --git a/examples/offline_inference/diffusion/lingbot_world_v2.py b/examples/offline_inference/diffusion/lingbot_world_v2.py new file mode 100644 index 000000000..145f5f641 --- /dev/null +++ b/examples/offline_inference/diffusion/lingbot_world_v2.py @@ -0,0 +1,288 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Generate a LingBot-World v2 video from an image and camera trajectory. + +The official checkpoint is licensed separately under CC BY-NC-SA and is +restricted to non-commercial use. This vLLM-Omni integration remains +Apache-2.0 licensed. + +Example: + python examples/offline_inference/diffusion/lingbot_world_v2.py \ + --prompt "The camera moves slowly forward through the scene." \ + --image /path/to/first_frame.png \ + --action-dir /path/to/actions/forward \ + --output lingbot_world_v2.mp4 + +``--action-dir`` must contain ``poses.npy`` and ``intrinsics.npy``. The +example configures its parent as the trusted action root and sends the +contained directory through ``extra_args["action_path"]``. +""" + +from __future__ import annotations + +import argparse +import math +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + +_MODEL = "robbyant/lingbot-world-v2-14b-causal-fast-diffusers" +_NUM_INFERENCE_STEPS = 4 +_MAX_SEQUENCE_LENGTH = 512 +_MAX_PIXEL_AREA = 480 * 832 +_MAX_RAW_FRAMES = 117 +_MAX_SOURCE_CAMERA_FRAMES = 4096 +_TEMPORAL_COMPRESSION = 4 +_LATENT_FRAMES_PER_BLOCK = 3 +_SPATIAL_ALIGNMENT = 16 + + +@dataclass(frozen=True) +class LingBotPaths: + image: Path + action_dir: Path + action_root: Path + action_relative: Path + camera_frames: int + output: Path + + +def _validate_camera_arrays(action_dir: Path) -> int: + """Validate the bounded camera contract before starting the engine.""" + + try: + poses = np.load(action_dir / "poses.npy", mmap_mode="r", allow_pickle=False) + intrinsics = np.load(action_dir / "intrinsics.npy", mmap_mode="r", allow_pickle=False) + except (OSError, TypeError, ValueError): + raise ValueError("Camera files must be valid numeric poses.npy and intrinsics.npy arrays.") from None + if poses.ndim != 3 or poses.shape[1:] != (4, 4): + raise ValueError("poses.npy must have shape [frames, 4, 4].") + if intrinsics.ndim != 2 or intrinsics.shape[1:] != (4,): + raise ValueError("intrinsics.npy must have shape [frames, 4].") + if poses.shape[0] != intrinsics.shape[0]: + raise ValueError("poses.npy and intrinsics.npy must contain the same frame count.") + if not 1 <= poses.shape[0] <= _MAX_SOURCE_CAMERA_FRAMES: + raise ValueError("Camera arrays must contain between 1 and 4096 source frames.") + if poses.dtype.kind not in "fiu" or intrinsics.dtype.kind not in "fiu": + raise ValueError("Camera arrays must contain real numeric values.") + if not np.isfinite(poses).all() or not np.isfinite(intrinsics).all(): + raise ValueError("Camera arrays must contain only finite values.") + return int(poses.shape[0]) + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Generate a LingBot-World v2 video from a first-frame image and camera trajectory." + ) + parser.add_argument("--model", default=_MODEL, help="Official Hugging Face model ID or local checkpoint path.") + parser.add_argument("--prompt", required=True, help="Text description of the scene and desired motion.") + parser.add_argument("--image", required=True, help="Path to the first-frame image.") + parser.add_argument( + "--action-dir", + required=True, + help="Camera action directory containing poses.npy and intrinsics.npy.", + ) + parser.add_argument("--height", type=int, default=480, help="Output height; must be divisible by 16.") + parser.add_argument("--width", type=int, default=832, help="Output width; must be divisible by 16.") + parser.add_argument( + "--num-frames", + type=int, + default=9, + help="Raw output frame count. Nine frames are one three-latent-frame causal block.", + ) + parser.add_argument("--seed", type=int, default=42, help="Random seed.") + parser.add_argument( + "--tensor-parallel-size", + type=int, + default=1, + help="Number of GPUs used for tensor parallelism inside the DiT.", + ) + parser.add_argument("--flow-shift", type=float, default=5.0, help="Positive FlowUniPC scheduler shift.") + parser.add_argument("--fps", type=int, default=16, help="Frames per second in the exported MP4.") + parser.add_argument("--output", default="lingbot_world_v2.mp4", help="Output MP4 path.") + parser.add_argument( + "--enforce-eager", + action="store_true", + help="Disable compilation and force eager execution.", + ) + return parser.parse_args(argv) + + +def resolve_cli_paths(args: argparse.Namespace) -> LingBotPaths: + """Canonicalize local inputs and construct a safe trusted-root mapping.""" + + image = Path(args.image).expanduser().resolve() + if not image.is_file(): + raise ValueError("--image must point to an existing file.") + + action_dir = Path(args.action_dir).expanduser().resolve() + if not action_dir.is_dir(): + raise ValueError("--action-dir must point to an existing directory.") + if not all((action_dir / name).is_file() for name in ("poses.npy", "intrinsics.npy")): + raise ValueError("--action-dir must contain poses.npy and intrinsics.npy.") + camera_frames = _validate_camera_arrays(action_dir) + + action_root = action_dir.parent.resolve() + try: + action_relative = action_dir.relative_to(action_root) + except ValueError: + raise ValueError("--action-dir must be contained by its canonical trusted action root.") from None + if not action_relative.parts: + raise ValueError("--action-dir must name a directory contained by a trusted action root.") + + output = Path(args.output).expanduser().resolve() + return LingBotPaths( + image=image, + action_dir=action_dir, + action_root=action_root, + action_relative=action_relative, + camera_frames=camera_frames, + output=output, + ) + + +def _positive_finite(value: float, flag: str) -> float: + result = float(value) + if not math.isfinite(result) or result <= 0: + raise ValueError(f"{flag} must be a positive finite number.") + return result + + +def build_omni_kwargs( + args: argparse.Namespace, + paths: LingBotPaths, + *, + parallel_config: Any, +) -> dict[str, Any]: + """Build Omni configuration without overriding checkpoint class discovery.""" + + if args.tensor_parallel_size <= 0: + raise ValueError("--tensor-parallel-size must be a positive integer.") + flow_shift = _positive_finite(args.flow_shift, "--flow-shift") + model_path = Path(args.model).expanduser() + model = str(model_path.resolve()) if model_path.exists() else args.model + return { + "model": model, + "flow_shift": flow_shift, + "parallel_config": parallel_config, + "enforce_eager": args.enforce_eager, + "model_config": {"lingbot_action_root": str(paths.action_root)}, + } + + +def build_request( + args: argparse.Namespace, + paths: LingBotPaths, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Build the prompt and fixed v1 causal-DMD sampling contract.""" + + prompt_text = args.prompt.strip() + if not prompt_text: + raise ValueError("--prompt must contain non-whitespace text.") + if args.height <= 0 or args.width <= 0: + raise ValueError("--height and --width must be positive integers.") + if args.height % _SPATIAL_ALIGNMENT or args.width % _SPATIAL_ALIGNMENT: + raise ValueError("--height and --width must each be divisible by 16.") + if args.height * args.width > _MAX_PIXEL_AREA: + raise ValueError("--height * --width must not exceed 480 * 832 pixels.") + if args.num_frames <= 0: + raise ValueError("--num-frames must be a positive integer.") + if args.num_frames > _MAX_RAW_FRAMES: + raise ValueError("--num-frames must not exceed 117 raw frames.") + if args.num_frames > paths.camera_frames: + raise ValueError("Camera arrays must contain at least --num-frames frames.") + if (args.num_frames - 1) % _TEMPORAL_COMPRESSION: + raise ValueError("--num-frames must satisfy (num_frames - 1) divisible by 4.") + latent_frames = (args.num_frames - 1) // _TEMPORAL_COMPRESSION + 1 + if latent_frames % _LATENT_FRAMES_PER_BLOCK: + raise ValueError("--num-frames must map to whole three-frame latent blocks.") + if args.fps <= 0: + raise ValueError("--fps must be a positive integer.") + flow_shift = _positive_finite(args.flow_shift, "--flow-shift") + + prompt = { + "prompt": prompt_text, + "multi_modal_data": {"image": str(paths.image)}, + } + sampling_kwargs = { + "height": args.height, + "width": args.width, + "num_frames": args.num_frames, + "num_inference_steps": _NUM_INFERENCE_STEPS, + "max_sequence_length": _MAX_SEQUENCE_LENGTH, + "seed": args.seed, + "fps": args.fps, + "extra_args": { + "action_path": paths.action_relative.as_posix(), + "flow_shift": flow_shift, + }, + } + return prompt, sampling_kwargs + + +def extract_video_array(outputs: Any) -> np.ndarray: + """Unwrap one standard Omni diffusion video into ``[frames, H, W, C]``.""" + + if not isinstance(outputs, list) or len(outputs) != 1: + raise ValueError("Expected exactly one Omni output for the single LingBot request.") + output = outputs[0] + request_output = getattr(output, "request_output", None) + if request_output is not None and getattr(request_output, "images", None): + output = request_output + + images = getattr(output, "images", None) + if not images: + multimodal_output = getattr(output, "multimodal_output", None) or {} + images = multimodal_output.get("video") + if images is None: + raise ValueError("No video frames found in Omni output.") + if isinstance(images, list) and len(images) == 1: + images = images[0] + + video: np.ndarray = np.asarray(images) + if video.ndim == 5: + if video.shape[0] != 1: + raise ValueError(f"Expected one generated video, got batch shape {video.shape}.") + video = video[0] + if video.ndim != 4 or video.shape[-1] not in (3, 4): + raise ValueError(f"Expected video shape [frames, height, width, channels], got {video.shape}.") + return video + + +def main(argv: Sequence[str] | None = None) -> Path: + args = parse_args(argv) + paths = resolve_cli_paths(args) + prompt, sampling_kwargs = build_request(args, paths) + paths.output.parent.mkdir(parents=True, exist_ok=True) + + # Keep vLLM imports below pure CLI/path validation so ``--help`` and the + # helper tests work on machines without the native vLLM runtime. + from diffusers.utils import export_to_video + + from vllm_omni.diffusion.data import DiffusionParallelConfig + from vllm_omni.entrypoints.omni import Omni + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + parallel_config = DiffusionParallelConfig(tensor_parallel_size=args.tensor_parallel_size) + omni_kwargs = build_omni_kwargs(args, paths, parallel_config=parallel_config) + omni = Omni(**omni_kwargs) + try: + outputs = omni.generate( + prompt, + OmniDiffusionSamplingParams(**sampling_kwargs), + use_tqdm=False, + ) + finally: + omni.close() + + frames = extract_video_array(outputs) + export_to_video(frames, str(paths.output), fps=args.fps) + print(f"Saved {frames.shape[0]} frames to {paths.output}") + return paths.output + + +if __name__ == "__main__": + main() diff --git a/examples/offline_inference/diffusion/lingbot_world_v2_realtime.py b/examples/offline_inference/diffusion/lingbot_world_v2_realtime.py new file mode 100644 index 000000000..3d6d7a7e1 --- /dev/null +++ b/examples/offline_inference/diffusion/lingbot_world_v2_realtime.py @@ -0,0 +1,239 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Run LingBot-World 2.0 as an in-process realtime AR-Diffusion session. + +This example exercises the internal session API. It is not a public HTTP or +WebSocket protocol. Each JSONL event advances the world by one three-latent +frame AR block and writes the latent plus its identity metadata. The current +scoped implementation supports at most ten ticks per generation epoch because +its image condition is bounded to 117 pixel frames; reset or create a session +to start a new world. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import math +import time +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +_MODEL = "robbyant/lingbot-world-v2-14b-causal-fast-diffusers" +_CAMERA_ACTION_SCHEMA = "lingbot.camera_actions.v1" +_FRAMES_PER_BLOCK = 3 +# ((117 pixel frames - 1) / VAE temporal factor 4 + 1) / 3 latent frames. +_MAX_REALTIME_TICKS = 10 + + +def _camera_event_data(frames: list[list[str]]) -> dict[str, Any]: + """Build the event-side script consumed by LingBotCameraControlReducer.""" + return {"mode": "script", "frames": frames} + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run an in-process realtime LingBot-World 2.0 session.") + parser.add_argument("--model", default=_MODEL, help="Hugging Face model ID or local checkpoint path.") + parser.add_argument("--image", required=True, help="Initial RGB image.") + parser.add_argument("--prompt", required=True, help="Initial scene prompt.") + parser.add_argument( + "--events", + required=True, + help="JSONL file with one prompt/action event per AR block; at most 10 events.", + ) + parser.add_argument("--output-dir", required=True, help="Directory for chunk latents and metadata.") + parser.add_argument("--session-id", default="lingbot-world", help="Persistent world session identifier.") + parser.add_argument("--height", type=int, default=480) + parser.add_argument("--width", type=int, default=832) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--gpu-memory-fraction", type=float, default=0.1) + parser.add_argument("--tensor-parallel-size", type=int, default=1) + parser.add_argument("--enforce-eager", action="store_true") + return parser.parse_args(argv) + + +def _load_events(path: Path) -> list[dict[str, Any]]: + events: list[dict[str, Any]] = [] + for line_number, raw_line in enumerate(path.read_text().splitlines(), start=1): + line = raw_line.strip() + if not line: + continue + try: + value = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"events line {line_number} is not valid JSON: {exc.msg}.") from None + if not isinstance(value, dict): + raise ValueError(f"events line {line_number} must be a JSON object.") + event_id = value.get("event_id") + if isinstance(event_id, bool) or not isinstance(event_id, int) or event_id < 0: + raise ValueError(f"events line {line_number} requires a non-negative integer event_id.") + prompt = value.get("prompt") + if prompt is not None and (not isinstance(prompt, str) or not prompt.strip()): + raise ValueError(f"events line {line_number} prompt must be a non-empty string.") + frames = value.get("frames") + if frames is not None: + if not isinstance(frames, list) or len(frames) != _FRAMES_PER_BLOCK: + raise ValueError(f"events line {line_number} frames must contain exactly three lists.") + for frame in frames: + if not isinstance(frame, list) or any(not isinstance(action, str) for action in frame): + raise ValueError(f"events line {line_number} frames must contain only action strings.") + if prompt is None and frames is None: + raise ValueError(f"events line {line_number} must update prompt and/or frames.") + events.append({"event_id": event_id, "prompt": prompt, "frames": frames}) + if not events: + raise ValueError("events file must contain at least one event.") + if len(events) > _MAX_REALTIME_TICKS: + raise ValueError( + "events file must contain at most " + f"{_MAX_REALTIME_TICKS} events because the current LingBot realtime " + "image-condition horizon is 117 pixel frames." + ) + event_ids = [event["event_id"] for event in events] + if event_ids != sorted(set(event_ids)): + raise ValueError("event_id values must be unique and strictly increasing.") + return events + + +def _validate_args(args: argparse.Namespace) -> tuple[Path, Path, Path]: + image = Path(args.image).expanduser().resolve() + events = Path(args.events).expanduser().resolve() + output_dir = Path(args.output_dir).expanduser().resolve() + if not image.is_file(): + raise ValueError("--image must point to an existing file.") + if not events.is_file(): + raise ValueError("--events must point to an existing JSONL file.") + if not args.prompt.strip(): + raise ValueError("--prompt must contain non-whitespace text.") + if args.height <= 0 or args.width <= 0 or args.height % 16 or args.width % 16: + raise ValueError("--height and --width must be positive multiples of 16.") + if args.tensor_parallel_size <= 0: + raise ValueError("--tensor-parallel-size must be positive.") + if not math.isfinite(args.gpu_memory_fraction) or not 0 < args.gpu_memory_fraction <= 1: + raise ValueError("--gpu-memory-fraction must be in (0, 1].") + return image, events, output_dir + + +async def run(argv: Sequence[str] | None = None) -> Path: + args = parse_args(argv) + image, events_path, output_dir = _validate_args(args) + events = _load_events(events_path) + output_dir.mkdir(parents=True, exist_ok=True) + + # Imports stay below pure input validation so --help and helper tests do not + # require a CUDA-enabled vLLM installation. + import torch + + from vllm_omni.diffusion.data import DiffusionParallelConfig + from vllm_omni.diffusion.models.lingbot_world.actions import LingBotCameraControlReducer + from vllm_omni.entrypoints.async_omni import AsyncOmni + from vllm_omni.experimental.ar_diffusion.consumer import ARDiffusionOmniTickConsumer + from vllm_omni.experimental.ar_diffusion.session import ( + ARDiffusionSessionEvent, + ARDiffusionSessionManager, + ARDiffusionWorkerLifecycle, + ) + from vllm_omni.experimental.ar_diffusion.tick_protocol import ARDiffusionControlInput + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + engine = AsyncOmni( + model=args.model, + engine_backend="vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine", + enforce_eager=args.enforce_eager, + parallel_config=DiffusionParallelConfig(tensor_parallel_size=args.tensor_parallel_size), + max_num_seqs=1, + model_config={ + "ar_diffusion_height": args.height, + "ar_diffusion_width": args.width, + "ar_diffusion_kv_config": { + "gpu_memory_fraction": args.gpu_memory_fraction, + "warmup_cudagraph": True, + }, + }, + ) + sampling = OmniDiffusionSamplingParams( + height=args.height, + width=args.width, + num_frames=9, + num_inference_steps=4, + max_sequence_length=512, + seed=args.seed, + output_type="latent", + extra_args={"flow_shift": 5.0}, + ) + consumer = ARDiffusionOmniTickConsumer( + engine, + prompt_provider=lambda tick: { + "prompt": tick.prompt, + "multi_modal_data": {"image": str(image)}, + }, + sampling_params_list=[sampling], + diffusion_stage_id=0, + ) + manager = ARDiffusionSessionManager( + tick_consumer=consumer, + lifecycle=ARDiffusionWorkerLifecycle(engine, stage_ids=[0], timeout=180.0), + max_pending_events=32, + control_reducer_factory=LingBotCameraControlReducer, + ) + session = await manager.create_session(args.session_id) + measurements: list[dict[str, Any]] = [] + try: + for chunk_index, event in enumerate(events): + controls = () + if event["frames"] is not None: + controls = ( + ARDiffusionControlInput( + track="camera", + schema=_CAMERA_ACTION_SCHEMA, + data=_camera_event_data(event["frames"]), + ), + ) + await session.accept_event( + ARDiffusionSessionEvent( + event_id=event["event_id"], + prompt=event["prompt"] + if event["prompt"] is not None + else (args.prompt if chunk_index == 0 else None), + controls=controls, + ) + ) + started = time.perf_counter() + output = await session.next_chunk() + elapsed = time.perf_counter() - started + if len(output.images) != 1 or not isinstance(output.images[0], torch.Tensor): + raise RuntimeError("Expected one latent tensor from each realtime LingBot chunk.") + latent = output.images[0].detach().float().cpu() + metadata = consumer.chunk_metadata(output).to_dict() + torch.save(latent, output_dir / f"chunk_{chunk_index:03d}.pt") + (output_dir / f"chunk_{chunk_index:03d}.json").write_text( + json.dumps(metadata, indent=2, sort_keys=True) + "\n" + ) + measurements.append( + { + "chunk_index": chunk_index, + "latency_seconds": elapsed, + "shape": list(latent.shape), + "finite": bool(torch.isfinite(latent).all()), + "metadata": metadata, + } + ) + print(json.dumps(measurements[-1], sort_keys=True), flush=True) + finally: + try: + await manager.close_session(args.session_id) + finally: + engine.shutdown() + + summary_path = output_dir / "summary.json" + summary_path.write_text(json.dumps({"chunks": measurements}, indent=2, sort_keys=True) + "\n") + return summary_path + + +def main(argv: Sequence[str] | None = None) -> Path: + return asyncio.run(run(argv)) + + +if __name__ == "__main__": + main() diff --git a/recipes/README.md b/recipes/README.md index 6a3b23770..5fee4fdae 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -60,6 +60,7 @@ recipes/ | [`meituan-longcat/LongCat-Video-Avatar-1.5.md`](./meituan-longcat/LongCat-Video-Avatar-1.5.md) | Audio-driven avatar video generation (AT2V / AI2V, single- and multi-speaker, AVC continuation) | 1x H100 80GB | | [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB | | [`Robbyant/LingBot-Video.md`](./Robbyant/LingBot-Video.md) | Native dense and MoE text-to-video serving | Dense: 1x L20X; MoE: 1x L20X (~67.7 GiB peak) | +| [`Robbyant/LingBot-World-2.0.md`](./Robbyant/LingBot-World-2.0.md) | Offline and experimental realtime interactive world generation | 1x H200/B200 or 2x B200 for TP=2 | | [`cosmos3/Cosmos3-Nano.md`](./cosmos3/Cosmos3-Nano.md) | Text-to-image, text-to-video, image-to-video, video-to-video generation, text to video with sound, action policy | 1x H200 141GB / B300 | | [`cosmos3/Cosmos3-Edge.md`](./cosmos3/Cosmos3-Edge.md) | T2I / T2V / I2V + action (Physical-AI) generation on the Nemotron-based Edge transformer | 1x GPU (~8 GiB) | | [`cosmos3/Cosmos3-Super.md`](./cosmos3/Cosmos3-Super.md) | 64B T2I / T2V / I2V / V2V generation (+ optional audio) / Action policy | 8x H200/H100/A100 / 2x H200 / B300 | diff --git a/recipes/Robbyant/LingBot-World-2.0.md b/recipes/Robbyant/LingBot-World-2.0.md new file mode 100644 index 000000000..851aeb930 --- /dev/null +++ b/recipes/Robbyant/LingBot-World-2.0.md @@ -0,0 +1,122 @@ +# LingBot-World 2.0 + +> Offline and experimental realtime interactive world generation + +## Summary + +- Vendor: Robbyant +- Model: `robbyant/lingbot-world-v2-14b-causal-fast-diffusers` +- Task: image-conditioned interactive world generation +- Modes: offline trajectory replay and in-process realtime AR-Diffusion ticks +- Hardware validated: NVIDIA H200 and B200 +- Maintainer: Community + +The checkpoint is separately licensed under CC BY-NC-SA and restricted to +non-commercial use. The vLLM-Omni integration code remains Apache-2.0. + +## Offline generation + +The offline path consumes one source image and an action directory containing +`poses.npy` with shape `[frames, 4, 4]` and `intrinsics.npy` with shape +`[frames, 4]`. + +```bash +python examples/offline_inference/diffusion/lingbot_world_v2.py \ + --prompt "The camera moves slowly forward through the scene." \ + --image /path/to/first_frame.png \ + --action-dir /path/to/actions/forward \ + --num-frames 81 \ + --output lingbot_world_v2.mp4 +``` + +The current causal-fast checkpoint uses four DMD sampling steps. Raw frame +counts must be `9 + 12k`, up to 117 frames. Camera files may contain a +longer bounded source trajectory (the official example contains 269 frames); +the runtime consumes only the prefix needed by the request. + +## Realtime in-process generation + +The realtime example keeps AR-Diffusion KV and model-owned state across +requests. Each JSONL line describes the prompt and/or three latent-frame camera +actions applied at the next chunk boundary: + +```json +{"event_id":1,"prompt":"A road through a forest","frames":[["j"],[],[]]} +{"event_id":2,"frames":[["w"],["w"],["w"]]} +{"event_id":3,"prompt":"The road enters a snowy valley","frames":[[],[],[]]} +``` + +Run: + +```bash +python examples/offline_inference/diffusion/lingbot_world_v2_realtime.py \ + --image /path/to/first_frame.png \ + --events /path/to/events.jsonl \ + --output-dir /tmp/lingbot-realtime \ + --gpu-memory-fraction 0.6 +``` + +The example writes one latent tensor and one metadata JSON file per chunk. It +exercises the same `ARDiffusionSessionManager -> ARDiffusionOmniTickConsumer +-> AsyncOmni -> ARDiffusionEngine` path used by a future HTTP or WebSocket +transport. + +This PR intentionally does not define a public realtime HTTP/WebSocket schema. +An online serving client should be added together with the public transport +API rather than exposing LingBot-specific event fields from the generic model +runtime. The structured camera-interaction frontend and transport work is +tracked separately in +[vllm-project/vllm-omni#5527](https://github.com/vllm-project/vllm-omni/pull/5527). + +## Realtime identity and controls + +- `session_id` identifies the persistent world and its worker-owned state. +- `event_id` identifies a prompt/control update and remains monotonic across reset. +- `chunk_index` is contiguous from zero and restarts from zero after reset. +- `request_id` correlates one chunk snapshot with its output metadata. +- AsyncOmni uses a separate UUID-suffixed internal engine routing ID. + +The generic runtime transports controls as opaque snapshots. LingBot's adapter +accepts: + +- `lingbot.camera_actions.v1` for per-latent-frame key states such as `w`, `a`, + `s`, `d`, `i`, `j`, `k`, and `l`; +- `lingbot.camera_trajectory.v1` for explicit pose/intrinsics trajectories. + +## Validation + +Real-checkpoint validation uses 480x832 output, four DMD steps, and seed 42. +The exercised matrix includes: + +- TP=1 and TP=2 execution; +- two interleaved resident sessions; +- action input and prompt switching; +- seven contiguous chunks crossing the sink plus recent rolling window; +- direct versus paged replay; +- CUDA Graph execution; +- reset, close, failure cleanup, and exact metadata matching; and +- nine-frame VAE decode. + +Official `generate.py` accuracy and performance numbers, together with +generation artifacts, are recorded in the PR validation section for the exact +tested commit. + +## Current limitations + +- Only the 14B causal-fast checkpoint is supported. +- The realtime control plane is internal; there is no public server transport yet. +- AR-Diffusion stages currently require one replica because session-affine + routing across replicas is not implemented. +- One AR block is generated per request and `max_num_seqs` must be one. +- Stateful streaming VAE decode is not implemented; the realtime example emits + latent chunks. +- SP/USP, pipeline/CFG parallelism, HSDP, VAE parallelism, quantization, + Cache-DiT, TeaCache, causal-pretrain, and the 1.3B checkpoint are not claimed. +- No AMD GPU, Ascend NPU, or Intel GPU support is claimed. + +## References + +- Checkpoint: +- Official implementation: +- Offline example: [`examples/offline_inference/diffusion/lingbot_world_v2.py`](../../examples/offline_inference/diffusion/lingbot_world_v2.py) +- Realtime design: [`docs/design/feature/realtime_ar_diffusion.md`](../../docs/design/feature/realtime_ar_diffusion.md) diff --git a/tests/diffusion/ar_diffusion/test_capability_runner.py b/tests/diffusion/ar_diffusion/test_capability_runner.py index 7b94fe80f..10451c134 100644 --- a/tests/diffusion/ar_diffusion/test_capability_runner.py +++ b/tests/diffusion/ar_diffusion/test_capability_runner.py @@ -18,6 +18,7 @@ from vllm_omni.experimental.ar_diffusion.capability import ( ) from vllm_omni.experimental.ar_diffusion.kv_cache import ARDiffusionKVConfig from vllm_omni.experimental.ar_diffusion.runner import ARDiffusionModelRunner +from vllm_omni.experimental.ar_diffusion.tick_protocol import ARDiffusionTickRequest BLOCK = 16 POS = "positive" @@ -56,6 +57,21 @@ def dreamzero_like_spec(*, capacity: int = 2) -> ARDiffusionKVCacheSpec: ) +def tiny_spec(*, capacity: int = 2) -> ARDiffusionKVCacheSpec: + return ARDiffusionKVCacheSpec( + num_layers=1, + num_kv_heads=1, + head_size=1, + tokens_per_frame=1, + frames_per_block=3, + window_frames=3, + sink_frames=3, + kv_branches=(ARDiffusionKVBranchSpec("main", 0),), + session_capacity=capacity, + cross_attention=(ARDiffusionCrossAttentionKVSpec("text", 2),), + ) + + class CapablePipeline: def __init__(self, spec: ARDiffusionKVCacheSpec) -> None: self.spec = spec @@ -103,6 +119,7 @@ def make_runner( *, available_bytes: int = 1 << 28, step_execution: bool = False, + gpu_memory_fraction: float = 0.1, ) -> ARDiffusionModelRunner: runner = object.__new__(ARDiffusionModelRunner) runner.od_config = SimpleNamespace( @@ -113,7 +130,10 @@ def make_runner( ) runner.device = torch.device("cpu") runner.pipeline = pipeline - runner.ar_diffusion_kv_config = ARDiffusionKVConfig(enable=True) + runner.ar_diffusion_kv_config = ARDiffusionKVConfig( + enable=True, + gpu_memory_fraction=gpu_memory_fraction, + ) runner.kv_cache = None runner._ar_diffusion_capability = None runner._ar_diffusion_kv_cache_spec = None @@ -132,6 +152,16 @@ def commit_one_frame(runner: ARDiffusionModelRunner, session_id: str, kv_branch: return state +def commit_one_block(runner: ARDiffusionModelRunner, session_id: str, kv_branch: str): + state = runner._get_or_create_session(session_id) + assert runner.kv_cache is not None + seq_len = BLOCK * runner.kv_cache.frames_per_block + ctx = state.get_kv_caches(kv_branch, seq_len=seq_len, commit_current=True)[0].forward_ctx + ctx.ensure_video_slots(torch.device("cpu")) + state.commit_paged_context(kv_branch) + return state + + def test_ar_runner_rejects_pipeline_without_capability(): runner = object.__new__(ARDiffusionModelRunner) runner.od_config = SimpleNamespace(max_num_seqs=1) @@ -140,6 +170,43 @@ def test_ar_runner_rejects_pipeline_without_capability(): runner._preallocate_kv_cache(available_bytes=1 << 20) +def test_runner_uses_typed_tick_as_authoritative_session_contract(): + tick = ARDiffusionTickRequest( + session_id="world-7", + request_id="request-3", + chunk_index=3, + reset=True, + ) + req = SimpleNamespace( + request_id="request-3", + sampling_params=SimpleNamespace(extra_args=tick.to_extra_args()), + ) + + session_id, extra_args, parsed = ARDiffusionModelRunner._request_session(req) + + assert session_id == "world-7" + assert extra_args == tick.to_extra_args() + assert parsed == tick + + +def test_runner_keeps_engine_request_id_separate_from_tick_correlation_id(): + tick = ARDiffusionTickRequest( + session_id="world-7", + request_id="client-request-3", + chunk_index=3, + ) + req = SimpleNamespace( + request_id="engine-request-uuid", + sampling_params=SimpleNamespace(extra_args=tick.to_extra_args()), + ) + + session_id, _, parsed = ARDiffusionModelRunner._request_session(req) + + assert session_id == "world-7" + assert req.request_id == "engine-request-uuid" + assert parsed.request_id == "client-request-3" + + def test_lingbot_like_single_branch_session_reuse_reset_and_close(): pipeline = CapablePipeline(lingbot_like_spec()) runner = make_runner(pipeline) @@ -170,6 +237,31 @@ def test_lingbot_like_single_branch_session_reuse_reset_and_close(): assert pipeline.closes == ["s1"] +def test_lingbot_like_interleaved_sessions_keep_independent_kv_partitions(): + runner = make_runner(CapablePipeline(lingbot_like_spec(capacity=2))) + kv = runner.kv_cache + assert kv is not None + + session_a = commit_one_block(runner, "world-a", "main") + a0_blocks = kv.window_block_ids(session_a.adapter("main")) + session_b = commit_one_block(runner, "world-b", "main") + b0_blocks = kv.window_block_ids(session_b.adapter("main")) + session_a_again = commit_one_block(runner, "world-a", "main") + a1_blocks = kv.window_block_ids(session_a_again.adapter("main")) + + assert session_a_again is session_a + assert session_a.adapter("main").request_id == "ar::world-a::main" + assert session_b.adapter("main").request_id == "ar::world-b::main" + assert session_a.adapter("main").completed_chunks == 6 + assert session_b.adapter("main").completed_chunks == 3 + assert len(a0_blocks) == 3 + assert len(b0_blocks) == 3 + assert len(a1_blocks) == 6 + assert set(a0_blocks) <= set(a1_blocks) + assert set(a1_blocks).isdisjoint(b0_blocks) + assert tuple(runner._sessions) == ("world-b", "world-a") + + def test_lingbot_like_sink_survives_sliding_window_eviction(): runner = make_runner(CapablePipeline(lingbot_like_spec())) kv = runner.kv_cache @@ -198,12 +290,12 @@ def test_dreamzero_like_two_branches_are_independent(): def test_lru_eviction_releases_blocks_and_notifies_pipeline(): - pipeline = CapablePipeline(lingbot_like_spec(capacity=8)) + pipeline = CapablePipeline(lingbot_like_spec(capacity=2)) runner = make_runner(pipeline) kv = runner.kv_cache assert kv is not None - assert runner._session_capacity == 1 - assert kv.session_capacity == 1 + assert runner._session_capacity == 2 + assert kv.session_capacity == 2 free_total = kv.manager.block_pool.get_num_free_blocks() old = commit_one_frame(runner, "old", "main") k = torch.randn(1, 8, 4, 64) @@ -211,13 +303,56 @@ def test_lru_eviction_releases_blocks_and_notifies_pipeline(): assert kv.manager.block_pool.get_num_free_blocks() < free_total runner._get_or_create_session("new") + assert tuple(runner._sessions) == ("old", "new") + runner._get_or_create_session("newest") - assert tuple(runner._sessions) == ("new",) + assert tuple(runner._sessions) == ("new", "newest") assert pipeline.closes == ["old"] assert "old" not in kv._cross_sessions assert kv.manager.block_pool.get_num_free_blocks() == free_total +def test_budget_reduced_capacity_drives_runner_lru(): + pipeline = CapablePipeline(tiny_spec(capacity=2)) + # One tiny LingBot-like session requires 128 bytes; two require 192. + runner = make_runner( + pipeline, + available_bytes=128, + gpu_memory_fraction=1.0, + ) + kv = runner.kv_cache + assert kv is not None + assert kv.requested_session_capacity == 2 + assert kv.session_capacity == 1 + assert runner._session_capacity == 1 + + runner._get_or_create_session("old") + runner._get_or_create_session("new") + + assert tuple(runner._sessions) == ("new",) + assert pipeline.closes == ["old"] + + +def test_dreamzero_like_requested_capacity_is_capped_by_budget(): + spec = dreamzero_like_spec(capacity=64) + # Per all-layer self-KV page: 65,536 bytes. Two resident sessions need: + # managed=(2 * (2 * 6 + 4) + 2)=34 pages, scratch=8 pages, + # cross-attention=1 page/session, for 44 pages total. + page_bytes = 65_536 + pipeline = CapablePipeline(spec) + runner = make_runner( + pipeline, + available_bytes=44 * page_bytes, + gpu_memory_fraction=1.0, + ) + kv = runner.kv_cache + assert kv is not None + assert kv.requested_session_capacity == 64 + assert kv.session_capacity == 2 + assert runner._session_capacity == 2 + assert kv.cross_attention_reserved_bytes == 2 * page_bytes + + def test_forward_exception_releases_pending_allocation_and_model_state(monkeypatch): pipeline = CapablePipeline(lingbot_like_spec()) runner = make_runner(pipeline) @@ -232,7 +367,10 @@ def test_forward_exception_releases_pending_allocation_and_model_state(monkeypat raise RuntimeError("layer exploded") monkeypatch.setattr(DiffusionModelRunner, "execute_model", boom) - request = SimpleNamespace(sampling_params=SimpleNamespace(extra_args={"session_id": "broken"})) + request = SimpleNamespace( + request_id="broken-request", + sampling_params=SimpleNamespace(extra_args={"session_id": "broken"}), + ) with pytest.raises(RuntimeError, match="layer exploded"): runner.execute_model(request) @@ -263,7 +401,10 @@ def test_synchronize_exception_uses_forward_cleanup_path(monkeypatch): monkeypatch.setattr(DiffusionModelRunner, "execute_model", return_after_allocation) monkeypatch.setattr(torch.accelerator, "synchronize", synchronize_boom) runner.device = torch.device("cuda") - request = SimpleNamespace(sampling_params=SimpleNamespace(extra_args={"session_id": "broken"})) + request = SimpleNamespace( + request_id="broken-request", + sampling_params=SimpleNamespace(extra_args={"session_id": "broken"}), + ) with pytest.raises(RuntimeError, match="asynchronous kernel failed"): runner.execute_model(request) diff --git a/tests/diffusion/ar_diffusion/test_consumer.py b/tests/diffusion/ar_diffusion/test_consumer.py new file mode 100644 index 000000000..d1197113c --- /dev/null +++ b/tests/diffusion/ar_diffusion/test_consumer.py @@ -0,0 +1,231 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from collections.abc import AsyncIterator, Sequence +from typing import Any + +import pytest + +from vllm_omni.experimental.ar_diffusion.consumer import ARDiffusionOmniTickConsumer +from vllm_omni.experimental.ar_diffusion.session import ( + ARDiffusionSession, + ARDiffusionSessionEvent, +) +from vllm_omni.experimental.ar_diffusion.tick_protocol import ( + ARDiffusionChunkMetadata, + ARDiffusionControlInput, + ARDiffusionTickRequest, +) +from vllm_omni.inputs.data import ( + OmniDiffusionSamplingParams, + OmniPromptType, + OmniSamplingParams, +) +from vllm_omni.outputs import OmniRequestOutput + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class FakeStagePool: + def __init__(self, num_replicas: int = 1) -> None: + self.num_replicas = num_replicas + + +class FakeEngine: + def __init__(self, num_replicas: int = 1) -> None: + self.stage_pools = [FakeStagePool(num_replicas)] + + +class FakeGenerateClient: + def __init__( + self, + *, + include_metadata: bool = True, + applied_event_ids: list[int] | None = None, + output_error: str | None = None, + num_replicas: int = 1, + ) -> None: + self.engine = FakeEngine(num_replicas) + self.include_metadata = include_metadata + self.applied_event_ids = applied_event_ids + self.output_error = output_error + self.calls: list[dict[str, Any]] = [] + + async def generate( + self, + prompt: OmniPromptType, + *, + request_id: str, + sampling_params_list: Sequence[OmniSamplingParams], + output_modalities: list[str] | None = None, + ) -> AsyncIterator[OmniRequestOutput]: + self.calls.append( + { + "prompt": prompt, + "request_id": request_id, + "sampling_params_list": sampling_params_list, + "output_modalities": output_modalities, + } + ) + params = sampling_params_list[0] + assert isinstance(params, OmniDiffusionSamplingParams) + tick = ARDiffusionTickRequest.from_extra_args(params.extra_args) + assert tick is not None + chunk_metadata = ARDiffusionChunkMetadata.from_tick(tick).to_dict() + if self.applied_event_ids is not None: + chunk_metadata["applied_event_ids"] = self.applied_event_ids + metadata = {"metadata": {"ar_diffusion": chunk_metadata}} if self.include_metadata else {} + yield OmniRequestOutput( + request_id=request_id, + stage_id=0, + final_output_type="video", + images=["chunk"], # type: ignore[list-item] + _multimodal_output=metadata, + finished=True, + error=self.output_error, + ) + + +class FakeLifecycle: + async def reset_session(self, session_id: str) -> None: + pass + + async def close_session(self, session_id: str) -> None: + pass + + +def make_tick() -> ARDiffusionTickRequest: + return ARDiffusionTickRequest( + session_id="world-1", + request_id="world-1-chunk-3", + chunk_index=3, + applied_event_ids=(7, 8), + prompt="turn left", + controls=( + ARDiffusionControlInput( + track="camera", + schema="example.camera.v1", + data={"samples": [{"x": 1.0}]}, + ), + ), + ) + + +def test_consumer_rejects_replicated_ar_diffusion_stage() -> None: + client = FakeGenerateClient(num_replicas=2) + + with pytest.raises( + ValueError, + match=r"require exactly one replica.*num_replicas=2", + ): + ARDiffusionOmniTickConsumer( + client, + prompt_provider=lambda tick: tick.prompt or "", + sampling_params_list=[OmniDiffusionSamplingParams()], + diffusion_stage_id=0, + ) + + assert client.calls == [] + + +@pytest.mark.asyncio +async def test_consumer_submits_typed_tick_and_returns_standard_output() -> None: + client = FakeGenerateClient() + template = OmniDiffusionSamplingParams( + output_type="latent", + extra_args={"deployment_option": "kept"}, + ) + consumer = ARDiffusionOmniTickConsumer( + client, + prompt_provider=lambda tick: { + "prompt": tick.prompt, + "multi_modal_data": {"image": "initial-image"}, + }, + sampling_params_list=[template], + diffusion_stage_id=0, + output_modalities=["video"], + ) + tick = make_tick() + + output = await consumer.execute_tick(tick) + + assert output.images == ["chunk"] + assert consumer.chunk_metadata(output) == ARDiffusionChunkMetadata.from_tick(tick) + assert len(client.calls) == 1 + call = client.calls[0] + assert call["request_id"] == tick.request_id + assert call["output_modalities"] == ["video"] + assert call["prompt"] == { + "prompt": "turn left", + "multi_modal_data": {"image": "initial-image"}, + } + submitted_params = call["sampling_params_list"][0] + assert submitted_params.extra_args["deployment_option"] == "kept" + assert ARDiffusionTickRequest.from_extra_args(submitted_params.extra_args) == tick + assert "ar_diffusion_tick" not in template.extra_args + + +@pytest.mark.asyncio +async def test_consumer_rejects_missing_chunk_metadata() -> None: + consumer = ARDiffusionOmniTickConsumer( + FakeGenerateClient(include_metadata=False), + prompt_provider=lambda tick: tick.prompt or "", + sampling_params_list=[OmniDiffusionSamplingParams()], + diffusion_stage_id=0, + ) + + with pytest.raises(ValueError, match="multimodal_output.metadata"): + await consumer.execute_tick(make_tick()) + + +@pytest.mark.asyncio +async def test_consumer_rejects_non_monotonic_applied_event_metadata() -> None: + consumer = ARDiffusionOmniTickConsumer( + FakeGenerateClient(applied_event_ids=[8, 7]), + prompt_provider=lambda tick: tick.prompt or "", + sampling_params_list=[OmniDiffusionSamplingParams()], + diffusion_stage_id=0, + ) + + with pytest.raises(ValueError, match="unique and strictly increasing"): + await consumer.execute_tick(make_tick()) + + +@pytest.mark.asyncio +async def test_consumer_preserves_engine_failure_before_metadata_validation() -> None: + consumer = ARDiffusionOmniTickConsumer( + FakeGenerateClient(include_metadata=False, output_error="model failed"), + prompt_provider=lambda tick: tick.prompt or "", + sampling_params_list=[OmniDiffusionSamplingParams()], + diffusion_stage_id=0, + ) + + with pytest.raises(RuntimeError, match="model failed"): + await consumer.execute_tick(make_tick()) + + +@pytest.mark.asyncio +async def test_session_commits_only_after_standard_engine_output_metadata() -> None: + client = FakeGenerateClient() + consumer = ARDiffusionOmniTickConsumer( + client, + prompt_provider=lambda tick: tick.prompt or "", + sampling_params_list=[OmniDiffusionSamplingParams()], + diffusion_stage_id=0, + ) + session = ARDiffusionSession( + "world-1", + tick_consumer=consumer, + lifecycle=FakeLifecycle(), + max_pending_events=4, + request_id_factory=lambda session_id, chunk_index: f"{session_id}-chunk-{chunk_index}", + ) + await session.accept_event(ARDiffusionSessionEvent(event_id=7, prompt="turn left")) + + output = await session.next_chunk() + + assert output.request_id == "world-1-chunk-0" + assert session.chunk_index == 1 + assert session.pending_event_count == 0 + assert client.calls[0]["request_id"] == "world-1-chunk-0" diff --git a/tests/diffusion/ar_diffusion/test_kv_cache.py b/tests/diffusion/ar_diffusion/test_kv_cache.py index d09c1c1b6..086666063 100644 --- a/tests/diffusion/ar_diffusion/test_kv_cache.py +++ b/tests/diffusion/ar_diffusion/test_kv_cache.py @@ -30,6 +30,8 @@ pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] BLOCK = 16 +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def make_spec(*, chunk_size=BLOCK, window_chunks=2, sink_chunks=0, reset_at_boundary=False): return ChunkWindowSpec( @@ -95,7 +97,6 @@ def test_sliding_replace_snaps_to_chunk_boundary(): def test_sink_chunks_protected(): # sink = 1 chunk (16 tokens) is never skipped. skip = chunk_window_skipped_tokens(80, chunk_size=16, sliding_window=32, sink_chunks=1, reset_at_boundary=False) - # raw overflow = 80 - 32 - 16 = 32 -> snapped 32 assert skip == 32 @@ -201,12 +202,149 @@ def test_cross_attn_pool_deducted_from_self_attn_budget(): kv_branches=(ARDiffusionKVBranchSpec("positive", 0), ARDiffusionKVBranchSpec("negative", 0)), session_capacity=1, cross_attention_lengths={"text": L}, - device=torch.device("cpu"), ) cross_bytes = 2 * 2 * L * 4 * 64 * torch.float16.itemsize * 2 # K+V, pos+neg, layers - expected = compute_num_blocks(avail - cross_bytes, 0.5, kv.spec.page_size_bytes * 2) + page_bytes = kv.spec.page_size_bytes * 2 + scratch_bytes = kv.scratch_num_blocks * page_bytes + expected = (int(avail * 0.5) - cross_bytes - scratch_bytes) // page_bytes assert kv.num_blocks == expected - assert expected > 1 * (2 + 1) + 2 # above the local-slot floor, so the cross deduction is what's tested + assert kv.cross_attention_reserved_bytes == cross_bytes + assert (kv.num_blocks_total * page_bytes) + cross_bytes <= int(avail * 0.5) + assert expected > 1 * (2 + 1) + 2 # above the local-slot minimum, so the cross deduction is what's tested + + +def _make_tiny_capacity_kv( + *, + requested_capacity: int, + available_bytes: int, + gpu_memory_fraction: float = 1.0, + model_owned_state_bytes_per_session: int = 0, +) -> ARDiffusionKVCache: + return ARDiffusionKVCache( + ARDiffusionKVConfig( + enable=True, + chunk_size=1, + window_chunks=3, + sink_chunks=3, + gpu_memory_fraction=gpu_memory_fraction, + ), + num_layers=1, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + block_size=1, + max_model_len=128, + available_bytes=available_bytes, + kv_branches=(ARDiffusionKVBranchSpec("main", 0),), + session_capacity=requested_capacity, + cross_attention_lengths={"text": 2}, + frames_per_block=3, + model_owned_state_bytes_per_session=model_owned_state_bytes_per_session, + device=torch.device("cpu"), + ) + + +def test_capacity_two_retains_both_windows_and_allocates_next_block(): + # LingBot-like single-branch geometry: page=8 bytes; capacity=2 requires + # 17 managed + 3 scratch blocks, plus two 16-byte cross-attention + # reservations: 192 bytes exactly. + kv = _make_tiny_capacity_kv(requested_capacity=2, available_bytes=192) + assert kv.requested_session_capacity == 2 + assert kv.session_capacity == 2 + assert kv.managed_num_blocks == 17 + + adapters = [kv.begin_request(session_id) for session_id in ("first", "second")] + for adapter in adapters: + for _ in range(6): # three sink + three window blocks + kv.allocate_token_slots(adapter, 1) + kv.commit_chunk(adapter) + assert len(kv.window_block_ids(adapter)) == 6 + + # Only one request is in flight, so its three-frame block is counted once. + kv.allocate_token_slots(adapters[0], 3) + + +def test_requested_capacity_is_capped_and_cross_reservation_uses_effective_capacity(): + kv = _make_tiny_capacity_kv(requested_capacity=64, available_bytes=192) + + assert kv.requested_session_capacity == 64 + assert kv.session_capacity == 2 + assert kv.cross_attention_bytes_per_session == 16 + assert kv.cross_attention_reserved_bytes == 32 + assert kv.num_blocks_total * 8 + kv.cross_attention_reserved_bytes == kv.memory_budget_bytes + + +def test_capacity_rejects_budget_that_cannot_fit_one_session(): + # One session needs 11 managed + 3 scratch blocks and 16 cross bytes: + # 14 * 8 + 16 = 128 bytes. + with pytest.raises(ValueError, match="cannot fit one session.*available=127 bytes.*required=128 bytes"): + _make_tiny_capacity_kv(requested_capacity=1, available_bytes=127) + + +def test_fraction_is_soft_floor_when_one_session_fits_actual_memory(): + kv = _make_tiny_capacity_kv( + requested_capacity=2, + available_bytes=1280, + gpu_memory_fraction=0.05, + ) + + assert kv.configured_memory_budget_bytes == 64 + assert kv.memory_budget_bytes == 128 + assert kv.session_capacity == 1 + + +def test_model_owned_state_reduces_effective_session_capacity(): + kv = _make_tiny_capacity_kv( + requested_capacity=2, + available_bytes=200, + model_owned_state_bytes_per_session=16, + ) + + assert kv.session_capacity == 1 + assert kv.model_owned_state_reserved_bytes == 16 + assert ( + kv.num_blocks_total * 8 + kv.cross_attention_reserved_bytes + kv.model_owned_state_reserved_bytes + <= kv.memory_budget_bytes + ) + + +def _make_shipped_lingbot_geometry(*, gpu_memory_fraction: float) -> ARDiffusionKVCache: + return ARDiffusionKVCache( + ARDiffusionKVConfig( + enable=True, + chunk_size=1560, + window_chunks=9, + sink_chunks=9, + gpu_memory_fraction=gpu_memory_fraction, + ), + num_layers=40, + num_kv_heads=40, + head_size=128, + dtype=torch.bfloat16, + block_size=1560, + max_model_len=1 << 20, + available_bytes=100 * (1 << 30), + kv_branches=(ARDiffusionKVBranchSpec("main", 0),), + session_capacity=2, + cross_attention_lengths={"text": 512}, + frames_per_block=3, + model_owned_state_bytes_per_session=7_488_000, + ) + + +def test_shipped_lingbot_geometry_default_fraction_admits_one_session(): + kv = _make_shipped_lingbot_geometry(gpu_memory_fraction=0.1) + + assert kv.configured_memory_budget_bytes == 10 * (1 << 30) + assert kv.memory_budget_bytes == 33_653_670_400 + assert kv.session_capacity == 1 + + +def test_shipped_lingbot_geometry_tuned_fraction_admits_two_sessions(): + kv = _make_shipped_lingbot_geometry(gpu_memory_fraction=0.6) + + assert kv.memory_budget_bytes == 60 * (1 << 30) + assert kv.session_capacity == 2 def _make_kv( @@ -221,15 +359,24 @@ def _make_kv( if local_branches == 1 else (ARDiffusionKVBranchSpec("positive", 0), ARDiffusionKVBranchSpec("negative", 1)) ) + page_bytes = 2 * BLOCK * 4 * 64 * torch.float32.itemsize + declared_scratch_blocks = (max_scratch_tokens_per_branch + BLOCK - 1) // BLOCK + scratch_blocks = local_branches * (num_frame_per_block + declared_scratch_blocks) + managed_blocks = local_branches * (window_chunks + num_frame_per_block) + 2 return ARDiffusionKVCache( - ARDiffusionKVConfig(enable=True, chunk_size=BLOCK, window_chunks=window_chunks), + ARDiffusionKVConfig( + enable=True, + chunk_size=BLOCK, + window_chunks=window_chunks, + gpu_memory_fraction=1.0, + ), num_layers=1, num_kv_heads=4, head_size=64, dtype=torch.float32, block_size=BLOCK, max_model_len=4096, - available_bytes=1 << 16, # tiny -> the floor binds + available_bytes=(managed_blocks + scratch_blocks) * page_bytes, device=torch.device("cpu"), kv_branches=kv_branches, session_capacity=1, diff --git a/tests/diffusion/ar_diffusion/test_kv_orchestration.py b/tests/diffusion/ar_diffusion/test_kv_orchestration.py index 32603bf88..4482fa6fb 100644 --- a/tests/diffusion/ar_diffusion/test_kv_orchestration.py +++ b/tests/diffusion/ar_diffusion/test_kv_orchestration.py @@ -54,13 +54,15 @@ def test_full_request_lifecycle_and_eviction(): used = {int(s) // kv.block_size for s in slots} assert kv.null_block_id not in used assert used <= set(block_table) - # Resident window stays bounded by window + the in-flight chunk. + # Allocation may transiently add one in-flight chunk. assert len(kv.window_block_ids(adapter)) <= kv.spec.window_chunks + 1 - free_after.append(kv.manager.block_pool.get_num_free_blocks()) kv.commit_chunk(adapter) + # A successful commit immediately prunes physical ownership. + assert len(kv.window_block_ids(adapter)) <= kv.spec.window_chunks + free_after.append(kv.manager.block_pool.get_num_free_blocks()) # Pool memory plateaus once the window is full (eviction recycles blocks). - assert free_after[-1] == free_after[kv.spec.window_chunks] + assert free_after[-1] == free_after[kv.spec.window_chunks - 1] kv.end_request(adapter) assert kv.manager.block_pool.get_num_free_blocks() == free_total diff --git a/tests/diffusion/ar_diffusion/test_paged_attention.py b/tests/diffusion/ar_diffusion/test_paged_attention.py index 27135874f..603a2a175 100644 --- a/tests/diffusion/ar_diffusion/test_paged_attention.py +++ b/tests/diffusion/ar_diffusion/test_paged_attention.py @@ -332,7 +332,7 @@ def test_block_table_padded_to_fixed_width(): def test_prepare_is_idempotent_and_layers_share_metadata(): device = torch.device("cpu") - _, st = make_state(num_layers=2) + kv, st = make_state(num_layers=2) contexts = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=False) fctx = contexts[0].forward_ctx fctx.prepare(device=device, action_len=0, query_len=BLOCK) @@ -349,6 +349,10 @@ def test_prepare_is_idempotent_and_layers_share_metadata(): assert i0.block_table is i1.block_table assert i0.seq_lens is i1.seq_lens assert i0.video_slots is i1.video_slots + assert i0.key_pool is kv._k_pools[0] + assert i0.value_pool is kv._v_pools[0] + assert i1.key_pool is kv._k_pools[1] + assert i1.value_pool is kv._v_pools[1] def test_layer_inputs_before_prepare_raises(): diff --git a/tests/diffusion/ar_diffusion/test_pipeline_bridge.py b/tests/diffusion/ar_diffusion/test_pipeline_bridge.py index ec60290b9..ed5233800 100644 --- a/tests/diffusion/ar_diffusion/test_pipeline_bridge.py +++ b/tests/diffusion/ar_diffusion/test_pipeline_bridge.py @@ -174,6 +174,19 @@ def test_window_reset_can_keep_named_cross_attention_cache(): assert not st.is_cross_attention_populated(POS, "text") +def test_clear_cross_attention_preserves_self_attention_history(): + kv, st = make_state(num_layers=1, cross_attn_length=8) + _prepare_and_commit(st, POS, 2) + k = torch.randn(1, 8, N_HEADS, HEAD_DIM) + st.populate_cross_attention(POS, "text", [(k, k)]) + resident_before = kv.window_block_ids(st.adapter(POS)) + + st.clear_cross_attention() + + assert kv.window_block_ids(st.adapter(POS)) == resident_before + assert not st.is_cross_attention_populated(POS, "text") + + def test_cross_attention_uses_logical_branch_when_local_slot_is_shared(): _, st = make_state(num_layers=2, cross_attn_length=8, shared_local_index=True) assert st.kv_cache.num_local_kv_branches == 1 diff --git a/tests/diffusion/ar_diffusion/test_session.py b/tests/diffusion/ar_diffusion/test_session.py new file mode 100644 index 000000000..a31be388b --- /dev/null +++ b/tests/diffusion/ar_diffusion/test_session.py @@ -0,0 +1,505 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from vllm_omni.experimental.ar_diffusion.session import ( + ARDiffusionChunkMetadataError, + ARDiffusionEventAcceptanceStatus, + ARDiffusionEventQueueFullError, + ARDiffusionPreparedControls, + ARDiffusionSession, + ARDiffusionSessionError, + ARDiffusionSessionEvent, + ARDiffusionSessionManager, + ARDiffusionSessionStateError, + ARDiffusionSessionStatus, + ARDiffusionStaleEventError, + ARDiffusionWorkerLifecycle, +) +from vllm_omni.experimental.ar_diffusion.tick_protocol import ( + ARDiffusionChunkMetadata, + ARDiffusionControlInput, + ARDiffusionTickRequest, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class FakeTickConsumer: + def __init__(self) -> None: + self.ticks: list[ARDiffusionTickRequest] = [] + self.failures_remaining = 0 + self.started: asyncio.Event | None = None + self.release: asyncio.Event | None = None + self.metadata_override: ARDiffusionChunkMetadata | None = None + + async def execute_tick(self, tick: ARDiffusionTickRequest) -> ARDiffusionChunkMetadata: + self.ticks.append(tick) + if self.started is not None: + self.started.set() + if self.release is not None: + await self.release.wait() + if self.failures_remaining: + self.failures_remaining -= 1 + raise RuntimeError("fake tick failed") + return self.metadata_override or ARDiffusionChunkMetadata.from_tick(tick) + + def chunk_metadata(self, output: ARDiffusionChunkMetadata) -> ARDiffusionChunkMetadata: + return output + + +class FakeLifecycle: + def __init__(self) -> None: + self.resets: list[str] = [] + self.closes: list[str] = [] + self.close_failures_remaining = 0 + + async def reset_session(self, session_id: str) -> None: + self.resets.append(session_id) + + async def close_session(self, session_id: str) -> None: + self.closes.append(session_id) + if self.close_failures_remaining: + self.close_failures_remaining -= 1 + raise RuntimeError("fake close failed") + + +class FailingCommitReducer: + def __init__(self) -> None: + self.commit_calls = 0 + self.reset_calls = 0 + + def prepare( + self, + *, + current_controls: dict[str, ARDiffusionControlInput], + events: tuple[ARDiffusionSessionEvent, ...], + chunk_index: int, + ) -> ARDiffusionPreparedControls: + del current_controls, events, chunk_index + return ARDiffusionPreparedControls(controls=()) + + def commit(self, prepared: ARDiffusionPreparedControls) -> None: + del prepared + self.commit_calls += 1 + raise RuntimeError("fake reducer commit failed") + + def reset(self) -> None: + self.reset_calls += 1 + + +def make_session( + *, + max_pending_events: int = 8, + consumer: FakeTickConsumer | None = None, + lifecycle: FakeLifecycle | None = None, +) -> tuple[ARDiffusionSession, FakeTickConsumer, FakeLifecycle]: + resolved_consumer = consumer or FakeTickConsumer() + resolved_lifecycle = lifecycle or FakeLifecycle() + session = ARDiffusionSession( + "world-1", + tick_consumer=resolved_consumer, + lifecycle=resolved_lifecycle, + max_pending_events=max_pending_events, + request_id_factory=lambda session_id, chunk_index: f"{session_id}-request-{chunk_index}", + ) + return session, resolved_consumer, resolved_lifecycle + + +@pytest.mark.asyncio +async def test_events_are_accepted_then_sorted_and_applied_at_chunk_boundary() -> None: + session, consumer, _ = make_session() + event_two = ARDiffusionSessionEvent(event_id=2, prompt="turn") + event_one = ARDiffusionSessionEvent( + event_id=1, + controls=( + ARDiffusionControlInput( + track="camera", + schema="example.camera.v1", + data={"samples": [{"x": 1.0}]}, + ), + ), + ) + + accepted_two = await session.accept_event(event_two) + accepted_one = await session.accept_event(event_one) + duplicate = await session.accept_event(event_two) + + assert accepted_two.status is ARDiffusionEventAcceptanceStatus.ACCEPTED + assert accepted_one.status is ARDiffusionEventAcceptanceStatus.ACCEPTED + assert duplicate.status is ARDiffusionEventAcceptanceStatus.DUPLICATE + assert consumer.ticks == [] + + metadata = await session.next_chunk() + + assert metadata.applied_event_ids == (1, 2) + assert consumer.ticks[0] == ARDiffusionTickRequest( + session_id="world-1", + request_id="world-1-request-0", + chunk_index=0, + applied_event_ids=(1, 2), + prompt="turn", + controls=( + ARDiffusionControlInput( + track="camera", + schema="example.camera.v1", + data={"samples": [{"x": 1.0}]}, + ), + ), + ) + assert session.pending_event_count == 0 + + applied_duplicate = await session.accept_event(event_two) + assert applied_duplicate.status is ARDiffusionEventAcceptanceStatus.DUPLICATE + + next_metadata = await session.next_chunk() + assert next_metadata.chunk_index == 1 + assert next_metadata.applied_event_ids == () + assert consumer.ticks[1].prompt == "turn" + assert consumer.ticks[1].controls == consumer.ticks[0].controls + + +@pytest.mark.asyncio +async def test_composite_event_is_isolated_and_queue_admission_is_atomic() -> None: + session, consumer, _ = make_session(max_pending_events=1) + payload = {"samples": [{"x": 1.0}]} + composite = ARDiffusionSessionEvent( + event_id=5, + prompt="new prompt", + controls=( + ARDiffusionControlInput( + track="camera", + schema="example.camera.v1", + data=payload, + ), + ARDiffusionControlInput( + track="action", + schema="example.action.v1", + data={"keys": ["left"]}, + ), + ), + ) + await session.accept_event(composite) + payload["samples"][0]["x"] = 99.0 + + duplicate = await session.accept_event(composite) + assert duplicate.status is ARDiffusionEventAcceptanceStatus.DUPLICATE + with pytest.raises(ARDiffusionEventQueueFullError, match="queue is full"): + await session.accept_event(ARDiffusionSessionEvent(event_id=6, prompt="rejected")) + + await session.next_chunk() + assert consumer.ticks[0].applied_event_ids == (5,) + assert consumer.ticks[0].prompt == "new prompt" + assert consumer.ticks[0].controls[0].data["keys"] == ("left",) + assert consumer.ticks[0].controls[1].data["samples"] == ({"x": 1.0},) + assert consumer.ticks[0].controls[0].to_dict()["data"] == {"keys": ["left"]} + assert consumer.ticks[0].controls[1].to_dict()["data"] == {"samples": [{"x": 1.0}]} + + +def test_event_rejects_duplicate_tracks_and_non_transport_payloads() -> None: + with pytest.raises(ValueError, match="at most once"): + ARDiffusionSessionEvent( + event_id=1, + controls=( + ARDiffusionControlInput(track="camera", schema="one", data={}), + ARDiffusionControlInput(track="camera", schema="two", data={}), + ), + ) + + with pytest.raises(ValueError, match="transport-safe"): + ARDiffusionControlInput( + track="camera", + schema="example.camera.v1", + data={"object": object()}, + ) + + +@pytest.mark.asyncio +async def test_failed_tick_closes_worker_and_requires_reset() -> None: + consumer = FakeTickConsumer() + consumer.failures_remaining = 1 + lifecycle = FakeLifecycle() + session, _, _ = make_session(consumer=consumer, lifecycle=lifecycle) + await session.accept_event(ARDiffusionSessionEvent(event_id=3, prompt="retry me")) + + with pytest.raises(RuntimeError, match="fake tick failed"): + await session.next_chunk() + + assert session.status is ARDiffusionSessionStatus.FAILED + assert session.chunk_index == 0 + assert session.pending_event_count == 1 + assert lifecycle.closes == ["world-1"] + with pytest.raises(ARDiffusionSessionStateError, match="failed"): + await session.next_chunk() + + await session.reset() + assert session.status is ARDiffusionSessionStatus.ACTIVE + assert session.pending_event_count == 0 + assert lifecycle.resets == ["world-1"] + await session.accept_event(ARDiffusionSessionEvent(event_id=4, prompt="after reset")) + metadata = await session.next_chunk() + assert metadata.chunk_index == 0 + assert metadata.applied_event_ids == (4,) + assert [tick.applied_event_ids for tick in consumer.ticks] == [(3,), (4,)] + + +@pytest.mark.asyncio +async def test_inflight_snapshot_is_immutable_and_new_event_waits_for_next_chunk() -> None: + consumer = FakeTickConsumer() + consumer.started = asyncio.Event() + consumer.release = asyncio.Event() + session, _, _ = make_session(consumer=consumer) + await session.accept_event(ARDiffusionSessionEvent(event_id=10, prompt="first")) + + first_chunk = asyncio.create_task(session.next_chunk()) + await consumer.started.wait() + accepted = await session.accept_event(ARDiffusionSessionEvent(event_id=11, prompt="second")) + assert accepted.status is ARDiffusionEventAcceptanceStatus.ACCEPTED + assert consumer.ticks[0].prompt == "first" + assert consumer.ticks[0].applied_event_ids == (10,) + + with pytest.raises(ARDiffusionStaleEventError, match="event floor is 10"): + await session.accept_event(ARDiffusionSessionEvent(event_id=9, prompt="stale")) + + consumer.release.set() + first_metadata = await first_chunk + second_metadata = await session.next_chunk() + assert first_metadata.applied_event_ids == (10,) + assert second_metadata.applied_event_ids == (11,) + assert consumer.ticks[1].prompt == "second" + + +@pytest.mark.asyncio +async def test_metadata_mismatch_does_not_commit_snapshot() -> None: + consumer = FakeTickConsumer() + consumer.metadata_override = ARDiffusionChunkMetadata( + session_id="another-world", + request_id="wrong", + chunk_index=0, + ) + lifecycle = FakeLifecycle() + session, _, _ = make_session(consumer=consumer, lifecycle=lifecycle) + await session.accept_event(ARDiffusionSessionEvent(event_id=1, prompt="pending")) + + with pytest.raises(ARDiffusionChunkMetadataError, match="does not match"): + await session.next_chunk() + + assert session.chunk_index == 0 + assert session.pending_event_count == 1 + assert session.status is ARDiffusionSessionStatus.FAILED + assert lifecycle.closes == ["world-1"] + + +@pytest.mark.asyncio +async def test_reducer_commit_failure_is_fail_closed_without_partial_session_commit() -> None: + consumer = FakeTickConsumer() + lifecycle = FakeLifecycle() + reducer = FailingCommitReducer() + session = ARDiffusionSession( + "world-1", + tick_consumer=consumer, + lifecycle=lifecycle, + max_pending_events=4, + request_id_factory=lambda session_id, chunk_index: f"{session_id}-request-{chunk_index}", + control_reducer=reducer, + ) + await session.accept_event(ARDiffusionSessionEvent(event_id=1, prompt="pending")) + + with pytest.raises(RuntimeError, match="fake reducer commit failed"): + await session.next_chunk() + + assert reducer.commit_calls == 1 + assert session.status is ARDiffusionSessionStatus.FAILED + assert session.chunk_index == 0 + assert session.pending_event_count == 1 + assert lifecycle.closes == ["world-1"] + with pytest.raises(ARDiffusionSessionStateError, match="failed"): + await session.next_chunk() + + await session.reset() + assert reducer.reset_calls == 1 + assert session.status is ARDiffusionSessionStatus.ACTIVE + assert session.chunk_index == 0 + assert session.pending_event_count == 0 + + +@pytest.mark.asyncio +async def test_reset_close_and_disconnect_reach_lifecycle_boundary() -> None: + consumer = FakeTickConsumer() + lifecycle = FakeLifecycle() + manager = ARDiffusionSessionManager( + tick_consumer=consumer, + lifecycle=lifecycle, + max_pending_events=4, + request_id_factory=lambda session_id, chunk_index: f"{session_id}-{chunk_index}", + ) + session = await manager.create_session("world-reset") + await session.accept_event(ARDiffusionSessionEvent(event_id=4, prompt="before reset")) + await session.next_chunk() + await manager.reset_session("world-reset") + + assert lifecycle.resets == ["world-reset"] + assert session.status is ARDiffusionSessionStatus.ACTIVE + assert session.chunk_index == 0 + duplicate = await session.accept_event(ARDiffusionSessionEvent(event_id=4, prompt="duplicate")) + assert duplicate.status is ARDiffusionEventAcceptanceStatus.DUPLICATE + + reset_tick = await session.next_chunk() + assert reset_tick.chunk_index == 0 + assert consumer.ticks[-1].prompt is None + assert consumer.ticks[-1].controls == () + + await manager.close_session("world-reset") + assert lifecycle.closes == ["world-reset"] + assert session.status is ARDiffusionSessionStatus.CLOSED + with pytest.raises(ARDiffusionSessionStateError, match="does not exist"): + await manager.get_session("world-reset") + + disconnected = await manager.create_session("world-disconnect") + await manager.disconnect("world-disconnect") + assert disconnected.status is ARDiffusionSessionStatus.CLOSED + assert lifecycle.closes[-1] == "world-disconnect" + + +@pytest.mark.asyncio +async def test_manager_keeps_interleaved_sessions_independent() -> None: + consumer = FakeTickConsumer() + manager = ARDiffusionSessionManager( + tick_consumer=consumer, + lifecycle=FakeLifecycle(), + max_pending_events=4, + request_id_factory=lambda session_id, chunk_index: f"{session_id}-request-{chunk_index}", + ) + session_a = await manager.create_session("world-a") + session_b = await manager.create_session("world-b") + + await session_a.accept_event(ARDiffusionSessionEvent(event_id=10, prompt="a0")) + await session_a.next_chunk() + await session_b.accept_event(ARDiffusionSessionEvent(event_id=1, prompt="b0")) + await session_b.next_chunk() + await session_a.accept_event(ARDiffusionSessionEvent(event_id=11, prompt="a1")) + await session_a.next_chunk() + + assert [ + ( + tick.session_id, + tick.request_id, + tick.chunk_index, + tick.applied_event_ids, + ) + for tick in consumer.ticks + ] == [ + ("world-a", "world-a-request-0", 0, (10,)), + ("world-b", "world-b-request-0", 0, (1,)), + ("world-a", "world-a-request-1", 1, (11,)), + ] + assert session_a.chunk_index == 2 + assert session_b.chunk_index == 1 + with pytest.raises(ARDiffusionStaleEventError, match="event floor is 11"): + await session_a.accept_event(ARDiffusionSessionEvent(event_id=2, prompt="stale only for a")) + assert ( + await session_b.accept_event(ARDiffusionSessionEvent(event_id=2, prompt="b1")) + ).status is ARDiffusionEventAcceptanceStatus.ACCEPTED + + +@pytest.mark.asyncio +async def test_failed_close_retains_tombstone_until_cleanup_retry_succeeds() -> None: + consumer = FakeTickConsumer() + lifecycle = FakeLifecycle() + lifecycle.close_failures_remaining = 1 + manager = ARDiffusionSessionManager( + tick_consumer=consumer, + lifecycle=lifecycle, + max_pending_events=4, + ) + session = await manager.create_session("world-close-retry") + + with pytest.raises(RuntimeError, match="fake close failed"): + await manager.close_session("world-close-retry") + + assert session.status is ARDiffusionSessionStatus.CLEANUP_FAILED + assert await manager.get_session("world-close-retry") is session + with pytest.raises(ARDiffusionSessionStateError, match="already exists"): + await manager.create_session("world-close-retry") + with pytest.raises(ARDiffusionSessionStateError, match="cleanup_failed"): + await manager.reset_session("world-close-retry") + + await manager.close_session("world-close-retry") + + assert lifecycle.closes == ["world-close-retry", "world-close-retry"] + assert session.status is ARDiffusionSessionStatus.CLOSED + with pytest.raises(ARDiffusionSessionStateError, match="does not exist"): + await manager.get_session("world-close-retry") + recreated = await manager.create_session("world-close-retry") + assert recreated is not session + + +@pytest.mark.asyncio +async def test_failed_disconnect_retains_session_for_explicit_cleanup_retry() -> None: + lifecycle = FakeLifecycle() + lifecycle.close_failures_remaining = 1 + manager = ARDiffusionSessionManager( + tick_consumer=FakeTickConsumer(), + lifecycle=lifecycle, + max_pending_events=4, + ) + session = await manager.create_session("world-disconnect-retry") + + with pytest.raises(RuntimeError, match="fake close failed"): + await manager.disconnect("world-disconnect-retry") + + assert session.status is ARDiffusionSessionStatus.CLEANUP_FAILED + assert await manager.get_session("world-disconnect-retry") is session + + await manager.close_session("world-disconnect-retry") + assert session.status is ARDiffusionSessionStatus.CLOSED + + +class FakeRPCClient: + def __init__(self, result: list[Any]) -> None: + self.result = result + self.calls: list[dict[str, Any]] = [] + + async def collective_rpc(self, **kwargs: Any) -> list[Any]: + self.calls.append(kwargs) + return self.result + + +@pytest.mark.asyncio +async def test_worker_lifecycle_routes_to_selected_stage_rpc() -> None: + client = FakeRPCClient([[True]]) + lifecycle = ARDiffusionWorkerLifecycle(client, stage_ids=[2], timeout=3.0) + + await lifecycle.reset_session("world-1") + await lifecycle.close_session("world-1") + + assert client.calls == [ + { + "method": "reset_ar_diffusion_session", + "timeout": 3.0, + "args": ("world-1",), + "stage_ids": [2], + }, + { + "method": "close_ar_diffusion_session", + "timeout": 3.0, + "args": ("world-1",), + "stage_ids": [2], + }, + ] + + +@pytest.mark.asyncio +async def test_worker_lifecycle_rejects_unsupported_stage_result() -> None: + lifecycle = ARDiffusionWorkerLifecycle( + FakeRPCClient([{"supported": False, "error": "not an AR stage"}]), + stage_ids=[1], + ) + + with pytest.raises(ARDiffusionSessionError, match="not an AR stage"): + await lifecycle.close_session("world-1") diff --git a/tests/diffusion/ar_diffusion/test_tick_protocol.py b/tests/diffusion/ar_diffusion/test_tick_protocol.py new file mode 100644 index 000000000..70e71ba80 --- /dev/null +++ b/tests/diffusion/ar_diffusion/test_tick_protocol.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from vllm_omni.experimental.ar_diffusion.tick_protocol import ( + ARDiffusionChunkMetadata, + ARDiffusionControlInput, + ARDiffusionTickRequest, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_tick_round_trip_preserves_identity_and_control_snapshot() -> None: + tick = ARDiffusionTickRequest( + session_id="world-1", + request_id="chunk-request-7", + chunk_index=7, + applied_event_ids=(10, 11), + prompt="turn left", + controls=( + ARDiffusionControlInput( + track="camera", + schema="lingbot.camera_trajectory.v1", + data={"poses": [[1.0, 0.0]], "intrinsics": [[2.0]]}, + ), + ), + ) + + restored = ARDiffusionTickRequest.from_extra_args(tick.to_extra_args()) + + assert restored == tick + assert ARDiffusionChunkMetadata.from_tick(tick).to_dict() == { + "session_id": "world-1", + "request_id": "chunk-request-7", + "chunk_index": 7, + "applied_event_ids": [10, 11], + } + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("session_id", "", "session_id"), + ("request_id", "", "request_id"), + ("chunk_index", -1, "chunk_index"), + ("applied_event_ids", (2, 1), "strictly increasing"), + ("applied_event_ids", (1, 1), "strictly increasing"), + ], +) +def test_tick_rejects_invalid_identity( + field: str, + value: object, + message: str, +) -> None: + values = { + "session_id": "world-1", + "request_id": "request-1", + "chunk_index": 0, + "applied_event_ids": (), + } + values[field] = value + + with pytest.raises(ValueError, match=message): + ARDiffusionTickRequest(**values) + + +def test_tick_snapshot_is_detached_and_deeply_immutable() -> None: + payload = {"samples": [{"keys": ["w"]}]} + event_ids = [1, 2] + controls = [ + ARDiffusionControlInput( + track="camera", + schema="example.camera.v1", + data=payload, + ) + ] + tick = ARDiffusionTickRequest( + session_id="world-1", + request_id="request-a", + chunk_index=0, + applied_event_ids=event_ids, # type: ignore[arg-type] + controls=controls, # type: ignore[arg-type] + ) + serialized = tick.to_extra_args() + + payload["samples"][0]["keys"].append("j") + event_ids.append(3) + controls.clear() + serialized["ar_diffusion_tick"]["controls"][0]["data"]["samples"][0]["keys"].append("k") + + assert tick.applied_event_ids == (1, 2) + assert isinstance(tick.controls, tuple) + assert tick.to_extra_args()["ar_diffusion_tick"]["controls"] == [ + { + "track": "camera", + "schema": "example.camera.v1", + "data": {"samples": [{"keys": ["w"]}]}, + } + ] + with pytest.raises(TypeError): + tick.controls[0].data["samples"] = () # type: ignore[index] + with pytest.raises(AttributeError): + tick.controls[0].data["samples"][0]["keys"].append("j") + + +def test_legacy_request_has_no_typed_tick() -> None: + assert ARDiffusionTickRequest.from_extra_args({"session_id": "legacy"}) is None diff --git a/tests/diffusion/ar_diffusion/test_worker_lifecycle.py b/tests/diffusion/ar_diffusion/test_worker_lifecycle.py new file mode 100644 index 000000000..42204049c --- /dev/null +++ b/tests/diffusion/ar_diffusion/test_worker_lifecycle.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from vllm_omni.diffusion.worker.diffusion_worker import DiffusionWorker + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class FakeARDiffusionRunner: + def __init__(self) -> None: + self.resets: list[str] = [] + self.closes: list[str] = [] + + def reset_session(self, session_id: str) -> None: + self.resets.append(session_id) + + def close_session(self, session_id: str) -> None: + self.closes.append(session_id) + + +class FakeRegularRunner: + pass + + +def make_worker(runner: object) -> DiffusionWorker: + worker = object.__new__(DiffusionWorker) + worker.model_runner = runner + return worker + + +def test_worker_delegates_ar_session_lifecycle_to_runner() -> None: + runner = FakeARDiffusionRunner() + worker = make_worker(runner) + + assert worker.reset_ar_diffusion_session("world-1") is True + assert worker.close_ar_diffusion_session("world-1") is True + assert runner.resets == ["world-1"] + assert runner.closes == ["world-1"] + + +def test_worker_reports_unsupported_runner_without_model_assumptions() -> None: + worker = make_worker(FakeRegularRunner()) + + assert worker.reset_ar_diffusion_session("world-1") is False + assert worker.close_ar_diffusion_session("world-1") is False + + +def test_worker_rejects_empty_session_id() -> None: + worker = make_worker(FakeARDiffusionRunner()) + + with pytest.raises(ValueError, match="session_id"): + worker.close_ar_diffusion_session("") diff --git a/tests/diffusion/models/dreamzero/test_pipeline_state.py b/tests/diffusion/models/dreamzero/test_pipeline_state.py index 366876ef7..ff349d6af 100644 --- a/tests/diffusion/models/dreamzero/test_pipeline_state.py +++ b/tests/diffusion/models/dreamzero/test_pipeline_state.py @@ -6,7 +6,10 @@ from types import SimpleNamespace import pytest -from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import DreamZeroPipeline +from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import ( + DREAMZERO_MODEL_OWNED_STATE_BYTES_PER_SESSION, + DreamZeroPipeline, +) from vllm_omni.diffusion.models.dreamzero.state_dreamzero import DreamZeroState pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -32,6 +35,37 @@ def test_dreamzero_pipeline_state_is_session_keyed() -> None: assert session_b.call_count == 3 +def test_dreamzero_kv_spec_accounts_for_model_owned_cuda_state(monkeypatch) -> None: + pipeline = DreamZeroPipeline.__new__(DreamZeroPipeline) + pipeline.transformer = SimpleNamespace( + frame_seqlen=16, + blocks=[ + SimpleNamespace( + self_attn=SimpleNamespace( + max_attention_size=96, + tp_num_heads=4, + ) + ) + ], + text_len=8, + model_type="t2v", + num_layers=2, + num_heads=4, + dim=256, + num_frame_per_block=4, + num_action_per_block=2, + num_state_per_block=1, + ) + monkeypatch.setattr( + "vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero.get_classifier_free_guidance_world_size", + lambda: 1, + ) + + spec = pipeline.ar_diffusion_kv_cache_spec() + + assert spec.model_owned_state_bytes_per_session == DREAMZERO_MODEL_OWNED_STATE_BYTES_PER_SESSION + + def test_dreamzero_pipeline_state_follows_runner_lifecycle_notifications() -> None: pipeline = _empty_pipeline() diff --git a/tests/diffusion/models/lingbot_world/__init__.py b/tests/diffusion/models/lingbot_world/__init__.py new file mode 100644 index 000000000..988131360 --- /dev/null +++ b/tests/diffusion/models/lingbot_world/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/tests/diffusion/models/lingbot_world/fixtures/lingbot_world_model_index.json.fixture b/tests/diffusion/models/lingbot_world/fixtures/lingbot_world_model_index.json.fixture new file mode 100644 index 000000000..015a5aa91 --- /dev/null +++ b/tests/diffusion/models/lingbot_world/fixtures/lingbot_world_model_index.json.fixture @@ -0,0 +1,13 @@ +{ + "_class_name": "LingBotWorldCausalDMDPipeline", + "_diffusers_version": "0.35.0.dev0", + "boundary_ratio": 0.9, + "image_encoder": [null, null], + "image_processor": [null, null], + "scheduler": ["diffusers", "UniPCMultistepScheduler"], + "text_encoder": ["transformers", "UMT5EncoderModel"], + "tokenizer": ["transformers", "T5TokenizerFast"], + "transformer": ["diffusers", "CausalLingBotWorldTransformer3DModel"], + "transformer_2": [null, null], + "vae": ["diffusers", "AutoencoderKLWan"] +} diff --git a/tests/diffusion/models/lingbot_world/fixtures/lingbot_world_official_shapes.json.fixture b/tests/diffusion/models/lingbot_world/fixtures/lingbot_world_official_shapes.json.fixture new file mode 100644 index 000000000..d07408a04 --- /dev/null +++ b/tests/diffusion/models/lingbot_world/fixtures/lingbot_world_official_shapes.json.fixture @@ -0,0 +1,42 @@ +{ + "sources": { + "config": "https://huggingface.co/robbyant/lingbot-world-v2-14b-causal-fast-diffusers/resolve/main/transformer/config.json", + "shard_1_header": "https://huggingface.co/robbyant/lingbot-world-v2-14b-causal-fast-diffusers/resolve/main/transformer/diffusion_pytorch_model-00001-of-00008.safetensors", + "shard_8_header": "https://huggingface.co/robbyant/lingbot-world-v2-14b-causal-fast-diffusers/resolve/main/transformer/diffusion_pytorch_model-00008-of-00008.safetensors" + }, + "dtype": "F32", + "shard_representatives": { + "diffusion_pytorch_model-00001-of-00008.safetensors": "blocks.0.cam_injector_layer1.bias", + "diffusion_pytorch_model-00002-of-00008.safetensors": "blocks.4.cam_injector_layer1.bias", + "diffusion_pytorch_model-00003-of-00008.safetensors": "blocks.10.cam_injector_layer1.bias", + "diffusion_pytorch_model-00004-of-00008.safetensors": "blocks.14.cam_scale_layer.bias", + "diffusion_pytorch_model-00005-of-00008.safetensors": "blocks.20.cam_injector_layer1.bias", + "diffusion_pytorch_model-00006-of-00008.safetensors": "blocks.25.cam_injector_layer1.bias", + "diffusion_pytorch_model-00007-of-00008.safetensors": "blocks.30.cam_injector_layer1.bias", + "diffusion_pytorch_model-00008-of-00008.safetensors": "blocks.35.cam_injector_layer1.bias" + }, + "parameter_shapes": { + "blocks.0.cam_injector_layer1.bias": [5120], + "blocks.4.cam_injector_layer1.bias": [5120], + "blocks.10.cam_injector_layer1.bias": [5120], + "blocks.14.cam_scale_layer.bias": [5120], + "blocks.20.cam_injector_layer1.bias": [5120], + "blocks.25.cam_injector_layer1.bias": [5120], + "blocks.30.cam_injector_layer1.bias": [5120], + "blocks.35.cam_injector_layer1.bias": [5120], + "patch_embedding.weight": [5120, 36, 1, 2, 2], + "patch_embedding_wancamctrl.weight": [5120, 1536], + "c2ws_hidden_states_layer1.weight": [5120, 5120], + "c2ws_hidden_states_layer2.weight": [5120, 5120], + "blocks.0.modulation": [1, 6, 5120], + "blocks.0.self_attn.q.weight": [5120, 5120], + "blocks.0.ffn.0.weight": [13824, 5120], + "blocks.0.ffn.2.weight": [5120, 13824], + "blocks.0.cam_injector_layer1.weight": [5120, 5120], + "blocks.0.cam_injector_layer2.weight": [5120, 5120], + "blocks.0.cam_scale_layer.weight": [5120, 5120], + "blocks.0.cam_shift_layer.weight": [5120, 5120], + "head.modulation": [1, 2, 5120], + "head.head.weight": [64, 5120] + } +} diff --git a/tests/diffusion/models/lingbot_world/fixtures/lingbot_world_scheduler_config.json.fixture b/tests/diffusion/models/lingbot_world/fixtures/lingbot_world_scheduler_config.json.fixture new file mode 100644 index 000000000..950d26fae --- /dev/null +++ b/tests/diffusion/models/lingbot_world/fixtures/lingbot_world_scheduler_config.json.fixture @@ -0,0 +1,30 @@ +{ + "_class_name": "UniPCMultistepScheduler", + "_diffusers_version": "0.35.0.dev0", + "beta_end": 0.02, + "beta_schedule": "linear", + "beta_start": 0.0001, + "disable_corrector": [], + "dynamic_thresholding_ratio": 0.995, + "final_sigmas_type": "zero", + "flow_shift": 5.0, + "lower_order_final": true, + "num_train_timesteps": 1000, + "predict_x0": true, + "prediction_type": "flow_prediction", + "rescale_betas_zero_snr": false, + "sample_max_value": 1.0, + "solver_order": 2, + "solver_p": null, + "solver_type": "bh2", + "steps_offset": 0, + "thresholding": false, + "time_shift_type": "exponential", + "timestep_spacing": "linspace", + "trained_betas": null, + "use_beta_sigmas": false, + "use_dynamic_shifting": false, + "use_exponential_sigmas": false, + "use_flow_sigmas": true, + "use_karras_sigmas": false +} diff --git a/tests/diffusion/models/lingbot_world/fixtures/lingbot_world_weight_index_names.json.fixture b/tests/diffusion/models/lingbot_world/fixtures/lingbot_world_weight_index_names.json.fixture new file mode 100644 index 000000000..2a2c93efd --- /dev/null +++ b/tests/diffusion/models/lingbot_world/fixtures/lingbot_world_weight_index_names.json.fixture @@ -0,0 +1,112 @@ +{ + "source": "https://huggingface.co/robbyant/lingbot-world-v2-14b-causal-fast-diffusers/resolve/main/transformer/diffusion_pytorch_model.safetensors.index.json", + "index_parameter_count": 1421, + "canonical_name_set_sha256": "5d31b7f64fd5e5e271b789b2dbe50fb7bbf146bd8b750646bdc9abb4e5b8139d", + "canonical_name_to_shard_sha256": "e1addc27d2b1fad6f10226a23a265ee3e3c61e74dce4d5859727bdf180a67992", + "canonical_serialization": "sorted lines formatted as parameter_name=shard_filename with LF separators and no trailing LF", + "shards": { + "diffusion_pytorch_model-00001-of-00008.safetensors": { + "parameter_count": 180, + "parameter_names_sha256": "81a3d4778de604e75609aae7dac3d8dcbc081811e7748267fe953bbeb375e79d" + }, + "diffusion_pytorch_model-00002-of-00008.safetensors": { + "parameter_count": 180, + "parameter_names_sha256": "4e36e2596a6468f29179fa5795396f2e9a7f7d8debc912b34a6f6c2c7c7c610d" + }, + "diffusion_pytorch_model-00003-of-00008.safetensors": { + "parameter_count": 180, + "parameter_names_sha256": "6f74a7b2cf7444c2daa2b3fc32bd3787589b35543c230c5c13b89f0f0f2a0998" + }, + "diffusion_pytorch_model-00004-of-00008.safetensors": { + "parameter_count": 180, + "parameter_names_sha256": "ec5c967ed5271ba7834337c4ee93a73f8ed0eead4492d4920624251c65c058ac" + }, + "diffusion_pytorch_model-00005-of-00008.safetensors": { + "parameter_count": 180, + "parameter_names_sha256": "ad1fdab5830dd9ce43ea007f112b4be223327b7c8197c766d9cf9219e74d619b" + }, + "diffusion_pytorch_model-00006-of-00008.safetensors": { + "parameter_count": 180, + "parameter_names_sha256": "fd58f0d86be42d8afcf55d3e07d8a22f7f6ef5064da931a600c890b71b6dfbae" + }, + "diffusion_pytorch_model-00007-of-00008.safetensors": { + "parameter_count": 180, + "parameter_names_sha256": "305a1714b5f4fb54d5877c593b451273ab1490d90fd0896a299d96e140295047" + }, + "diffusion_pytorch_model-00008-of-00008.safetensors": { + "parameter_count": 161, + "parameter_names_sha256": "4c18d94d96168682eb212f93e3cdf7313928776d8d9e6e4de0255fe899decb25" + } + }, + "top_level_modules": [ + "blocks", + "c2ws_hidden_states_layer1", + "c2ws_hidden_states_layer2", + "head", + "patch_embedding", + "patch_embedding_wancamctrl", + "text_embedding", + "time_embedding", + "time_projection" + ], + "non_block_parameters": [ + "c2ws_hidden_states_layer1.bias", + "c2ws_hidden_states_layer1.weight", + "c2ws_hidden_states_layer2.bias", + "c2ws_hidden_states_layer2.weight", + "head.head.bias", + "head.head.weight", + "head.modulation", + "patch_embedding.bias", + "patch_embedding.weight", + "patch_embedding_wancamctrl.bias", + "patch_embedding_wancamctrl.weight", + "text_embedding.0.bias", + "text_embedding.0.weight", + "text_embedding.2.bias", + "text_embedding.2.weight", + "time_embedding.0.bias", + "time_embedding.0.weight", + "time_embedding.2.bias", + "time_embedding.2.weight", + "time_projection.1.bias", + "time_projection.1.weight" + ], + "block_parameter_suffixes": [ + "cam_injector_layer1.bias", + "cam_injector_layer1.weight", + "cam_injector_layer2.bias", + "cam_injector_layer2.weight", + "cam_scale_layer.bias", + "cam_scale_layer.weight", + "cam_shift_layer.bias", + "cam_shift_layer.weight", + "cross_attn.k.bias", + "cross_attn.k.weight", + "cross_attn.norm_k.weight", + "cross_attn.norm_q.weight", + "cross_attn.o.bias", + "cross_attn.o.weight", + "cross_attn.q.bias", + "cross_attn.q.weight", + "cross_attn.v.bias", + "cross_attn.v.weight", + "ffn.0.bias", + "ffn.0.weight", + "ffn.2.bias", + "ffn.2.weight", + "modulation", + "norm3.bias", + "norm3.weight", + "self_attn.k.bias", + "self_attn.k.weight", + "self_attn.norm_k.weight", + "self_attn.norm_q.weight", + "self_attn.o.bias", + "self_attn.o.weight", + "self_attn.q.bias", + "self_attn.q.weight", + "self_attn.v.bias", + "self_attn.v.weight" + ] +} diff --git a/tests/diffusion/models/lingbot_world/test_lingbot_world_actions.py b/tests/diffusion/models/lingbot_world/test_lingbot_world_actions.py new file mode 100644 index 000000000..df012777f --- /dev/null +++ b/tests/diffusion/models/lingbot_world/test_lingbot_world_actions.py @@ -0,0 +1,294 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio + +import pytest +import torch + +from vllm_omni.diffusion.models.lingbot_world.actions import ( + LINGBOT_CAMERA_ACTION_SCHEMA, + LINGBOT_CAMERA_TRAJECTORY_SCHEMA, + LingBotCameraControlReducer, + integrate_lingbot_camera_actions, + parse_lingbot_camera_action_frames, +) +from vllm_omni.experimental.ar_diffusion.session import ( + ARDiffusionSession, + ARDiffusionSessionEvent, +) +from vllm_omni.experimental.ar_diffusion.tick_protocol import ( + ARDiffusionChunkMetadata, + ARDiffusionControlInput, + ARDiffusionTickRequest, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + +def _action_control(mode: str, **data) -> ARDiffusionControlInput: + return ARDiffusionControlInput( + track="camera", + schema=LINGBOT_CAMERA_ACTION_SCHEMA, + data={"mode": mode, **data}, + ) + + +def _prepared_frames(prepared) -> tuple[tuple[str, ...], ...]: + assert len(prepared.controls) == 1 + return prepared.controls[0].data["frames"] + + +def test_state_controls_hold_release_and_preserve_short_pulse() -> None: + reducer = LingBotCameraControlReducer() + pressed = reducer.prepare( + current_controls={}, + events=( + ARDiffusionSessionEvent( + event_id=1, + controls=( + _action_control( + "state", + transitions=[{"actions": ["W"], "client_ts_ms": 10}], + ), + ), + ), + ), + chunk_index=0, + ) + assert _prepared_frames(pressed) == (("w",), ("w",), ("w",)) + assert pressed.controls[0].to_dict()["data"]["frames"] == [["w"], ["w"], ["w"]] + reducer.commit(pressed) + + held = reducer.prepare( + current_controls={"camera": pressed.controls[0]}, + events=(), + chunk_index=1, + ) + assert _prepared_frames(held) == (("w",), ("w",), ("w",)) + reducer.commit(held) + + pulse = reducer.prepare( + current_controls={"camera": held.controls[0]}, + events=( + ARDiffusionSessionEvent( + event_id=2, + controls=( + _action_control( + "state", + transitions=[ + {"actions": ["j"], "client_ts_ms": 20}, + {"actions": [], "client_ts_ms": 21}, + ], + ), + ), + ), + ), + chunk_index=2, + ) + assert _prepared_frames(pulse) == (("j",), (), ()) + reducer.commit(pulse) + + released = reducer.prepare( + current_controls={"camera": pulse.controls[0]}, + events=(), + chunk_index=3, + ) + assert _prepared_frames(released) == ((), (), ()) + + +def test_script_is_transactional_consumed_once_and_neutral_padded() -> None: + reducer = LingBotCameraControlReducer() + event = ARDiffusionSessionEvent( + event_id=1, + controls=( + _action_control( + "script", + frames=[["w"], ["d"], ["l"], ["i"]], + ), + ), + ) + first_attempt = reducer.prepare( + current_controls={}, + events=(event,), + chunk_index=0, + ) + assert _prepared_frames(first_attempt) == (("w",), ("d",), ("l",)) + + # An uncommitted failed attempt must not consume the reducer's script. + second_attempt = reducer.prepare( + current_controls={}, + events=(event,), + chunk_index=0, + ) + assert _prepared_frames(second_attempt) == (("w",), ("d",), ("l",)) + reducer.commit(second_attempt) + + tail = reducer.prepare( + current_controls={"camera": second_attempt.controls[0]}, + events=(), + chunk_index=1, + ) + assert _prepared_frames(tail) == (("i",), (), ()) + reducer.commit(tail) + exhausted = reducer.prepare( + current_controls={"camera": tail.controls[0]}, + events=(), + chunk_index=2, + ) + assert _prepared_frames(exhausted) == ((), (), ()) + + +def test_trajectory_control_remains_an_expert_override() -> None: + reducer = LingBotCameraControlReducer() + trajectory = ARDiffusionControlInput( + track="camera", + schema=LINGBOT_CAMERA_TRAJECTORY_SCHEMA, + data={"poses": [[[1.0]]], "intrinsics": [[1.0]]}, + ) + prepared = reducer.prepare( + current_controls={}, + events=( + ARDiffusionSessionEvent( + event_id=1, + controls=(trajectory,), + ), + ), + chunk_index=0, + ) + assert prepared.controls == (trajectory,) + + +def test_action_integrator_matches_lingbot_motion_and_calibration() -> None: + trajectory, pitch = integrate_lingbot_camera_actions( + [["w"], ["d"], ["l"]], + width=832, + height=480, + ) + + assert pitch == 0.0 + torch.testing.assert_close( + trajectory.poses[0, :3, 3], + torch.tensor([0.0, 0.0, 0.05], dtype=torch.float64), + atol=1e-6, + rtol=0, + ) + torch.testing.assert_close( + trajectory.poses[1, :3, 3], + torch.tensor([0.05, 0.0, 0.05], dtype=torch.float64), + atol=1e-6, + rtol=0, + ) + expected_yaw = torch.tensor( + [ + [0.9945219, 0.0, 0.10452846], + [0.0, 1.0, 0.0], + [-0.10452846, 0.0, 0.9945219], + ], + dtype=torch.float64, + ) + torch.testing.assert_close( + trajectory.poses[2, :3, :3], + expected_yaw, + atol=1e-6, + rtol=0, + ) + torch.testing.assert_close( + trajectory.intrinsics[0], + torch.tensor([500.0, 500.0, 416.0, 240.0]), + ) + + resized, _ = integrate_lingbot_camera_actions( + [[]], + width=416, + height=240, + ) + scaled = resized.intrinsics[0] * torch.tensor([416 / 832, 240 / 480, 416 / 832, 240 / 480]) + torch.testing.assert_close( + scaled, + torch.tensor([500.0, 500.0, 208.0, 120.0]), + ) + + +@pytest.mark.parametrize( + ("data", "message"), + [ + ({"mode": "frames", "frames": [["x"], [], []]}, "W/A/S/D/I/J/K/L"), + ({"mode": "frames", "frames": [["w"]]}, "exactly one action list"), + ({"mode": "state", "frames": [[], [], []]}, "mode='frames'"), + ], +) +def test_model_tick_action_payload_validation(data, message: str) -> None: + with pytest.raises(ValueError, match=message): + parse_lingbot_camera_action_frames(data, expected_frames=3) + + +class _TickConsumer: + def __init__(self) -> None: + self.ticks: list[ARDiffusionTickRequest] = [] + + async def execute_tick(self, tick: ARDiffusionTickRequest) -> ARDiffusionChunkMetadata: + self.ticks.append(tick) + return ARDiffusionChunkMetadata.from_tick(tick) + + def chunk_metadata(self, output: ARDiffusionChunkMetadata) -> ARDiffusionChunkMetadata: + return output + + +class _Lifecycle: + async def reset_session(self, session_id: str) -> None: + del session_id + + async def close_session(self, session_id: str) -> None: + del session_id + + +def test_session_reducer_keeps_prompt_and_actions_atomic_and_resets() -> None: + async def exercise() -> None: + consumer = _TickConsumer() + reducer = LingBotCameraControlReducer() + session = ARDiffusionSession( + "world-actions", + tick_consumer=consumer, + lifecycle=_Lifecycle(), + max_pending_events=8, + request_id_factory=lambda session_id, chunk_index: f"{session_id}-{chunk_index}", + control_reducer=reducer, + ) + await session.accept_event( + ARDiffusionSessionEvent( + event_id=1, + prompt="turn toward the window", + controls=( + _action_control( + "state", + transitions=[{"actions": ["d"]}], + ), + ), + ) + ) + await session.next_chunk() + + tick = consumer.ticks[0] + assert tick.prompt == "turn toward the window" + assert tick.applied_event_ids == (1,) + assert tick.controls[0].data["frames"] == (("d",), ("d",), ("d",)) + + await session.reset() + await session.accept_event( + ARDiffusionSessionEvent( + event_id=2, + controls=( + _action_control( + "state", + transitions=[{"actions": []}], + ), + ), + ) + ) + await session.next_chunk() + assert consumer.ticks[1].chunk_index == 0 + assert consumer.ticks[1].controls[0].data["frames"] == ((), (), ()) + + asyncio.run(exercise()) diff --git a/tests/diffusion/models/lingbot_world/test_lingbot_world_attention.py b/tests/diffusion/models/lingbot_world/test_lingbot_world_attention.py new file mode 100644 index 000000000..f5567a706 --- /dev/null +++ b/tests/diffusion/models/lingbot_world/test_lingbot_world_attention.py @@ -0,0 +1,543 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import importlib.util +import math +import sys +from pathlib import Path +from types import ModuleType + +import pytest +import torch +import torch.nn.functional as F +from torch import nn + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + +_MODULE_PATH = Path(__file__).parents[4] / "vllm_omni/diffusion/models/lingbot_world/transformer.py" +_MODULE_NAME = "_lingbot_world_attention_under_test" +_MISSING_MODULE = object() +_STUBBED_MODULE_NAMES = ( + "vllm", + "vllm.distributed", + "vllm.model_executor", + "vllm.model_executor.layers", + "vllm.model_executor.layers.conv", + "vllm.model_executor.layers.linear", + "vllm.model_executor.model_loader", + "vllm.model_executor.model_loader.weight_utils", + "vllm.model_executor.utils", + "vllm_omni", + "vllm_omni.diffusion", + "vllm_omni.diffusion.attention", + "vllm_omni.diffusion.attention.layer", + "vllm_omni.diffusion.layers", + "vllm_omni.diffusion.layers.norm", + "vllm_omni.diffusion.layers.rope", + "vllm_omni.experimental", + "vllm_omni.experimental.ar_diffusion", + "vllm_omni.experimental.ar_diffusion.kv_cache", + "vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention", +) + + +@pytest.fixture(autouse=True) +def _inference_context(): + with torch.inference_mode(): + yield + + +def _install_vllm_stubs() -> None: + for name in _STUBBED_MODULE_NAMES: + sys.modules[name] = ModuleType(name) + + distributed = sys.modules["vllm.distributed"] + distributed.get_tensor_model_parallel_rank = lambda: 0 + distributed.get_tensor_model_parallel_world_size = lambda: 1 + distributed.tensor_model_parallel_all_reduce = lambda value: value + + def set_weight_attrs(weight: torch.Tensor, attrs: dict) -> None: + for name, value in attrs.items(): + setattr(weight, name, value) + + sys.modules["vllm.model_executor.utils"].set_weight_attrs = set_weight_attrs + sys.modules["vllm.model_executor.model_loader.weight_utils"].default_weight_loader = lambda param, loaded_weight: ( + param.data.copy_(loaded_weight) + ) + + class _Linear(nn.Module): + def __init__( + self, + input_size: int, + output_size: int, + *, + bias: bool = True, + return_bias: bool = False, + **kwargs, + ) -> None: + super().__init__() + del kwargs + self.return_bias = return_bias + self.weight = nn.Parameter(torch.empty(output_size, input_size)) + self.bias = nn.Parameter(torch.empty(output_size)) if bias else None + nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if self.bias is not None: + nn.init.zeros_(self.bias) + self.calls = 0 + + def forward(self, value: torch.Tensor): + self.calls += 1 + output = F.linear(value, self.weight, self.bias) + return (output, self.bias) if self.return_bias else output + + linear = sys.modules["vllm.model_executor.layers.linear"] + linear.ColumnParallelLinear = _Linear + linear.RowParallelLinear = _Linear + + class _QKVLinear(nn.Module): + def __init__( + self, + hidden_size: int, + head_size: int, + total_num_heads: int, + total_num_kv_heads: int | None = None, + *, + bias: bool = True, + **kwargs, + ) -> None: + super().__init__() + del kwargs + self.num_heads = total_num_heads + self.num_kv_heads = total_num_kv_heads or total_num_heads + output_size = (self.num_heads + 2 * self.num_kv_heads) * head_size + self.weight = nn.Parameter(torch.empty(output_size, hidden_size)) + self.bias = nn.Parameter(torch.empty(output_size)) if bias else None + nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if self.bias is not None: + nn.init.zeros_(self.bias) + self.calls = 0 + + def load_shard(param: torch.Tensor, loaded_weight: torch.Tensor, shard_id: str) -> None: + shard_index = {"q": 0, "k": 1, "v": 2}[shard_id] + shard_size = param.shape[0] // 3 + param.data.narrow(0, shard_index * shard_size, shard_size).copy_(loaded_weight) + + self.weight.weight_loader = load_shard + if self.bias is not None: + self.bias.weight_loader = load_shard + + def forward(self, value: torch.Tensor): + self.calls += 1 + return F.linear(value, self.weight, self.bias), None + + linear.QKVParallelLinear = _QKVLinear + + class _Conv3dLayer(nn.Conv3d): + def __init__(self, in_channels: int, out_channels: int, kernel_size, stride, **kwargs) -> None: + super().__init__(in_channels, out_channels, kernel_size=kernel_size, stride=stride, **kwargs) + + sys.modules["vllm.model_executor.layers.conv"].Conv3dLayer = _Conv3dLayer + + class _LayerNorm(nn.LayerNorm): + def __init__(self, dim: int, eps: float = 1e-6, elementwise_affine: bool = True) -> None: + super().__init__(dim, eps=eps, elementwise_affine=elementwise_affine) + + sys.modules["vllm_omni.diffusion.layers.norm"].LayerNorm = _LayerNorm + + class _Attention(nn.Module): + def __init__(self, *args, softmax_scale: float, **kwargs) -> None: + super().__init__() + del args + self.softmax_scale = softmax_scale + self.causal = kwargs["causal"] + self.calls: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = [] + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata=None, + ) -> torch.Tensor: + del attn_metadata + self.calls.append((query.detach().clone(), key.detach().clone(), value.detach().clone())) + scores = torch.einsum("bqhd,bkhd->bhqk", query, key) * self.softmax_scale + weights = scores.softmax(dim=-1) + return torch.einsum("bhqk,bkhd->bqhd", weights, value) + + sys.modules["vllm_omni.diffusion.attention.layer"].Attention = _Attention + + class _RotaryEmbeddingWan(nn.Module): + def __init__(self, *args, **kwargs) -> None: + super().__init__() + del args, kwargs + self.calls: list[tuple[torch.Tensor, torch.Tensor]] = [] + + def forward(self, value: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + self.calls.append((cos.detach().clone(), sin.detach().clone())) + return value + + sys.modules["vllm_omni.diffusion.layers.rope"].RotaryEmbeddingWan = _RotaryEmbeddingWan + + paged_attention = sys.modules["vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention"] + + class _PagedLayerContext: + pass + + class _PagedLayerInputs: + pass + + paged_attention.ARDiffusionPagedLayerContext = _PagedLayerContext + paged_attention.ARDiffusionPagedLayerInputs = _PagedLayerInputs + paged_attention.ar_diffusion_paged_attention = lambda *args, **kwargs: (_ for _ in ()).throw( + AssertionError("CUDA paged attention was not expected in this CPU test") + ) + paged_attention.paged_write_attn = lambda *args, **kwargs: (_ for _ in ()).throw( + AssertionError("paged path was not expected in this direct-cache test") + ) + + +def _load_module(): + assert _MODULE_PATH.exists(), "LingBot attention module has not been implemented" + module_names = (*_STUBBED_MODULE_NAMES, _MODULE_NAME) + previous = {name: sys.modules.get(name, _MISSING_MODULE) for name in module_names} + try: + _install_vllm_stubs() + spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + finally: + for name, original in previous.items(): + if original is _MISSING_MODULE: + sys.modules.pop(name, None) + else: + sys.modules[name] = original + + +def test_load_module_restores_process_module_state() -> None: + module_names = (*_STUBBED_MODULE_NAMES, _MODULE_NAME) + previous = {name: sys.modules.get(name, _MISSING_MODULE) for name in module_names} + + _load_module() + + for name, original in previous.items(): + assert sys.modules.get(name, _MISSING_MODULE) is original + + +def test_allocate_lingbot_cache_creates_request_local_layer_storage() -> None: + module = _load_module() + + cache = module.allocate_lingbot_cache( + batch_size=2, + num_layers=3, + max_tokens=7, + num_local_heads=2, + head_dim=4, + device=torch.device("cpu"), + dtype=torch.float32, + ) + + assert len(cache.self_attention) == 3 + assert len(cache.cross_attention) == 3 + assert cache.cross_attention == [None, None, None] + assert cache.self_attention[0].key.shape == (2, 7, 2, 4) + assert cache.self_attention[0].value.shape == (2, 7, 2, 4) + assert cache.self_attention[0].key.dtype == torch.float32 + assert cache.self_attention[0].end == 0 + assert cache.self_attention[0].key.data_ptr() != cache.self_attention[1].key.data_ptr() + + +def _allocate_single_layer(module, *, max_tokens: int): + return module.allocate_lingbot_cache( + batch_size=1, + num_layers=1, + max_tokens=max_tokens, + num_local_heads=1, + head_dim=2, + device=torch.device("cpu"), + dtype=torch.float32, + ) + + +def _set_identity_attention(attention: nn.Module) -> None: + qkv = getattr(attention, "qkv", None) + if qkv is not None: + projections = (qkv, attention.o) + else: + projections = (attention.q, attention.k, attention.v, attention.o) + for projection in projections: + with torch.no_grad(): + identity = torch.eye(2) + projection.weight.copy_(torch.cat((identity, identity, identity)) if projection is qkv else identity) + if projection.bias is not None: + projection.bias.zero_() + attention.norm_q = nn.Identity() + attention.norm_k = nn.Identity() + + +def _tokens(*values: float) -> torch.Tensor: + return torch.tensor([[[value, 0.0] for value in values]]) + + +def _record_inputs(module: nn.Module) -> list[tuple]: + records: list[tuple] = [] + module.register_forward_pre_hook(lambda _module, args: records.append(args)) + return records + + +def _record_outputs(module: nn.Module) -> list[torch.Tensor]: + records: list[torch.Tensor] = [] + + def record(_module, _args, output) -> None: + records.append(output[0] if isinstance(output, tuple) else output) + + module.register_forward_hook(record) + return records + + +def _cache_snapshot(cache) -> tuple[torch.Tensor, torch.Tensor, tuple[int, int, int | None, int]]: + return ( + cache.key.detach().view(torch.uint8).clone(), + cache.value.detach().view(torch.uint8).clone(), + (cache.end, cache.absolute_end, cache.last_start, cache.sink_end), + ) + + +def _assert_cache_unchanged(cache, snapshot) -> None: + key, value, metadata = snapshot + assert torch.equal(cache.key.detach().view(torch.uint8), key) + assert torch.equal(cache.value.detach().view(torch.uint8), value) + assert (cache.end, cache.absolute_end, cache.last_start, cache.sink_end) == metadata + + +def test_self_attention_repeated_offset_overwrites_then_later_offset_appends() -> None: + module = _load_module() + attention = module.LingBotSelfAttention(dim=2, num_heads=1) + _set_identity_attention(attention) + cache = _allocate_single_layer(module, max_tokens=6).self_attention[0] + + attention(_tokens(1, 2), cache=cache, current_start=0, sink_tokens=0) + attention(_tokens(10, 20), cache=cache, current_start=0, sink_tokens=0) + + assert cache.end == 2 + assert cache.absolute_end == 2 + torch.testing.assert_close(cache.key[0, : cache.end, 0, 0], torch.tensor([10.0, 20.0])) + + attention(_tokens(3, 4), cache=cache, current_start=2, sink_tokens=0) + + assert cache.end == 4 + assert cache.absolute_end == 4 + torch.testing.assert_close(cache.key[0, : cache.end, 0, 0], torch.tensor([10.0, 20.0, 3.0, 4.0])) + + +def test_self_attention_uses_one_fused_qkv_projection() -> None: + module = _load_module() + attention = module.LingBotSelfAttention(dim=2, num_heads=1) + cache = _allocate_single_layer(module, max_tokens=2).self_attention[0] + + attention(_tokens(1, 2), cache=cache, current_start=0, sink_tokens=0) + + assert attention.qkv.calls == 1 + + +def test_self_attention_routes_paged_inputs_through_engine_owned_cache() -> None: + module = _load_module() + attention = module.LingBotSelfAttention(dim=2, num_heads=1) + _set_identity_attention(attention) + paged_inputs = module.ARDiffusionPagedLayerInputs() + calls = [] + + def paged_write(inputs, query, key, value, k_act, v_act, scale): + calls.append((inputs, query.clone(), key.clone(), value.clone(), k_act, v_act, scale)) + return query + + module.paged_write_attn = paged_write + output = attention( + _tokens(1, 2), + cache=paged_inputs, + current_start=0, + sink_tokens=0, + update_cache=False, + ) + + assert output.shape == (1, 2, 2) + assert len(calls) == 1 + assert calls[0][0] is paged_inputs + assert calls[0][4:6] == (None, None) + assert attention.attn.calls == [] + assert not any(hasattr(attention, name) for name in ("q", "k", "v")) + + +def test_self_attention_is_chunk_causal_without_masking_inside_current_chunk() -> None: + module = _load_module() + attention = module.LingBotSelfAttention(dim=2, num_heads=1) + _set_identity_attention(attention) + cache = _allocate_single_layer(module, max_tokens=8).self_attention[0] + attention_inputs = _record_inputs(attention.attn) + + attention(_tokens(1, 2), cache=cache, current_start=0, sink_tokens=0) + first_keys = attention_inputs[-1][1].detach().clone() + attention(_tokens(3, 4), cache=cache, current_start=2, sink_tokens=0) + second_keys = attention_inputs[-1][1].detach().clone() + + assert attention.attn.causal is False + torch.testing.assert_close(first_keys[0, :, 0, 0], torch.tensor([1.0, 2.0])) + torch.testing.assert_close(second_keys[0, :, 0, 0], torch.tensor([1.0, 2.0, 3.0, 4.0])) + + +def test_self_attention_retains_sink_and_latest_local_history_after_eviction() -> None: + module = _load_module() + attention = module.LingBotSelfAttention(dim=2, num_heads=1) + _set_identity_attention(attention) + cache = _allocate_single_layer(module, max_tokens=4).self_attention[0] + attention_inputs = _record_inputs(attention.attn) + + attention(_tokens(1, 2), cache=cache, current_start=0, sink_tokens=1) + attention(_tokens(3, 4), cache=cache, current_start=2, sink_tokens=1) + attention(_tokens(5, 6), cache=cache, current_start=4, sink_tokens=1) + + visible_keys = attention_inputs[-1][1].detach().clone() + assert cache.end == 4 + assert cache.absolute_end == 6 + torch.testing.assert_close(visible_keys[0, :, 0, 0], torch.tensor([1.0, 4.0, 5.0, 6.0])) + torch.testing.assert_close(cache.key[0, : cache.end, 0, 0], torch.tensor([1.0, 4.0, 5.0, 6.0])) + + +def test_cross_attention_projects_encoder_kv_once_per_request() -> None: + module = _load_module() + attention = module.LingBotCrossAttention(dim=2, num_heads=1) + _set_identity_attention(attention) + key_outputs = _record_outputs(attention.k) + value_outputs = _record_outputs(attention.v) + attention_inputs = _record_inputs(attention.attn) + + output, cache = attention(_tokens(1), _tokens(2, 3), cache=None) + second_output, reused_cache = attention(_tokens(4), _tokens(20, 30), cache=cache) + + assert output.shape == second_output.shape == (1, 1, 2) + assert reused_cache is cache + assert len(key_outputs) == 1 + assert len(value_outputs) == 1 + assert cache.end == 2 + torch.testing.assert_close(cache.key[0, :, 0, 0], torch.tensor([2.0, 3.0])) + torch.testing.assert_close(attention_inputs[-1][1], cache.key) + + +def test_self_attention_cache_is_isolated_between_requests() -> None: + module = _load_module() + attention = module.LingBotSelfAttention(dim=2, num_heads=1) + _set_identity_attention(attention) + first = _allocate_single_layer(module, max_tokens=4).self_attention[0] + second = _allocate_single_layer(module, max_tokens=4).self_attention[0] + + attention(_tokens(1, 2), cache=first, current_start=0, sink_tokens=0) + + assert first.end == 2 + assert second.end == 0 + assert torch.count_nonzero(second.key) == 0 + attention(_tokens(8), cache=second, current_start=0, sink_tokens=0) + torch.testing.assert_close(first.key[0, : first.end, 0, 0], torch.tensor([1.0, 2.0])) + torch.testing.assert_close(second.key[0, : second.end, 0, 0], torch.tensor([8.0])) + + +def test_cross_attention_keeps_checkpoint_qkvo_and_norm_parameter_names() -> None: + module = _load_module() + attention = module.LingBotCrossAttention(dim=4, num_heads=2) + + names = set(attention.state_dict()) + + assert { + "q.weight", + "k.weight", + "v.weight", + "o.weight", + "norm_q.weight", + "norm_k.weight", + } <= names + assert callable(attention.norm_q.weight.weight_loader) + + +def test_tp_world_size_one_attention_output_shapes() -> None: + module = _load_module() + self_attention = module.LingBotSelfAttention(dim=4, num_heads=2) + cross_attention = module.LingBotCrossAttention(dim=4, num_heads=2) + cache = module.allocate_lingbot_cache( + batch_size=2, + num_layers=1, + max_tokens=5, + num_local_heads=2, + head_dim=2, + device=torch.device("cpu"), + dtype=torch.float32, + ) + hidden_states = torch.randn(2, 3, 4) + encoder_hidden_states = torch.randn(2, 5, 4) + + self_output = self_attention(hidden_states, cache=cache.self_attention[0], current_start=0, sink_tokens=0) + cross_output, cache.cross_attention[0] = cross_attention( + hidden_states, + encoder_hidden_states, + cache=cache.cross_attention[0], + ) + + assert self_output.shape == hidden_states.shape + assert cross_output.shape == hidden_states.shape + assert cache.cross_attention[0] is not None + + +def test_self_attention_applies_rotary_embedding_to_current_query_and_key() -> None: + module = _load_module() + attention = module.LingBotSelfAttention(dim=2, num_heads=1) + cache = _allocate_single_layer(module, max_tokens=2).self_attention[0] + cos = torch.ones(2, 1) + sin = torch.zeros(2, 1) + rotary_inputs = _record_inputs(attention.rotary_embedding) + + attention(_tokens(1, 2), cache=cache, current_start=0, rotary_emb=(cos, sin), sink_tokens=0) + + assert len(rotary_inputs) == 2 + torch.testing.assert_close(rotary_inputs[0][1], cos) + torch.testing.assert_close(rotary_inputs[1][2], sin) + + +def test_attention_block_owns_checkpoint_parent_namespaces() -> None: + module = _load_module() + block = module.LingBotAttentionBlock(dim=4, num_heads=2, prefix="blocks.0") + + names = set(block.state_dict()) + expected = { + *(f"self_attn.{child}.weight" for child in ("qkv", "o", "norm_q", "norm_k")), + *(f"cross_attn.{child}.weight" for child in ("q", "k", "v", "o", "norm_q", "norm_k")), + } + + assert expected <= names + + +def test_cross_attention_caches_are_isolated_between_requests() -> None: + module = _load_module() + attention = module.LingBotCrossAttention(dim=2, num_heads=1) + + _, first = attention(_tokens(1), _tokens(2, 3), cache=None) + _, second = attention(_tokens(1), _tokens(8, 9), cache=None) + + assert first.key.untyped_storage().data_ptr() != second.key.untyped_storage().data_ptr() + assert first.value.untyped_storage().data_ptr() != second.value.untyped_storage().data_ptr() + second_key = second.key.clone() + first.key.zero_() + torch.testing.assert_close(second.key, second_key) + + +def test_tp_rmsnorm_weight_loader_selects_rank_shard(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_module() + monkeypatch.setattr(module, "get_tensor_model_parallel_world_size", lambda: 2) + monkeypatch.setattr(module, "get_tensor_model_parallel_rank", lambda: 1) + norm = module._LingBotRMSNorm(2, eps=1e-6) + + norm.weight.weight_loader(norm.weight, torch.tensor([10.0, 20.0, 30.0, 40.0])) + + torch.testing.assert_close(norm.weight, torch.tensor([30.0, 40.0])) diff --git a/tests/diffusion/models/lingbot_world/test_lingbot_world_camera.py b/tests/diffusion/models/lingbot_world/test_lingbot_world_camera.py new file mode 100644 index 000000000..8b30237c4 --- /dev/null +++ b/tests/diffusion/models/lingbot_world/test_lingbot_world_camera.py @@ -0,0 +1,525 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import importlib.util +import math +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest +import torch + +_MODULE_PATH = Path(__file__).parents[4] / "vllm_omni/diffusion/models/lingbot_world/camera.py" +_SPEC = importlib.util.spec_from_file_location("_lingbot_world_camera_under_test", _MODULE_PATH) +assert _SPEC is not None and _SPEC.loader is not None +_CAMERA_MODULE = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _CAMERA_MODULE +_SPEC.loader.exec_module(_CAMERA_MODULE) + +CameraTrajectory = _CAMERA_MODULE.CameraTrajectory +build_plucker_embedding = _CAMERA_MODULE.build_plucker_embedding +interpolate_camera_trajectory = _CAMERA_MODULE.interpolate_camera_trajectory +load_camera_trajectory = _CAMERA_MODULE.load_camera_trajectory +resolve_trusted_action_directory = _CAMERA_MODULE.resolve_trusted_action_directory + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + +def _identity_poses(num_frames: int) -> np.ndarray: + return np.repeat(np.eye(4, dtype=np.float64)[None], num_frames, axis=0) + + +def _write_trajectory( + directory: Path, + poses: np.ndarray, + intrinsics: np.ndarray, +) -> None: + np.save(directory / "poses.npy", poses) + np.save(directory / "intrinsics.npy", intrinsics) + + +def _trusted(directory: Path): + return resolve_trusted_action_directory(".", directory) + + +def _trajectory( + poses: torch.Tensor, + intrinsics: torch.Tensor | None = None, +) -> CameraTrajectory: + if intrinsics is None: + intrinsics = torch.tensor([[832.0 / 3.0, 160.0, 416.0, 240.0]]).repeat(poses.shape[0], 1) + return CameraTrajectory(poses=poses, intrinsics=intrinsics) + + +@pytest.mark.parametrize("present_file", [None, "poses.npy"]) +def test_load_camera_trajectory_rejects_missing_files(tmp_path: Path, present_file: str | None) -> None: + if present_file == "poses.npy": + np.save(tmp_path / present_file, _identity_poses(1)) + + missing_file = "poses.npy" if present_file is None else "intrinsics.npy" + with pytest.raises(FileNotFoundError, match=missing_file): + load_camera_trajectory(_trusted(tmp_path)) + + +@pytest.mark.parametrize( + ("poses", "intrinsics", "message"), + [ + (np.zeros((2, 3, 4)), np.zeros((2, 4)), "poses.npy"), + (_identity_poses(2), np.zeros((2, 3)), "intrinsics.npy"), + (_identity_poses(0), np.zeros((0, 4)), "1 and 4096 source frames"), + (_identity_poses(2), np.zeros((1, 4)), "same number of frames"), + ], +) +def test_load_camera_trajectory_rejects_invalid_shapes( + tmp_path: Path, + poses: np.ndarray, + intrinsics: np.ndarray, + message: str, +) -> None: + _write_trajectory(tmp_path, poses, intrinsics) + + with pytest.raises(ValueError, match=message): + load_camera_trajectory(_trusted(tmp_path)) + + +@pytest.mark.parametrize("array_name", ["poses", "intrinsics"]) +def test_load_camera_trajectory_rejects_non_finite_values(tmp_path: Path, array_name: str) -> None: + poses = _identity_poses(1) + intrinsics = np.ones((1, 4), dtype=np.float64) + if array_name == "poses": + poses[0, 0, 0] = np.nan + else: + intrinsics[0, 0] = np.inf + _write_trajectory(tmp_path, poses, intrinsics) + + with pytest.raises(ValueError, match="finite"): + load_camera_trajectory(_trusted(tmp_path)) + + +def test_load_camera_trajectory_preserves_raw_c2w_poses(tmp_path: Path) -> None: + first_c2w = np.array( + [ + [0.0, -1.0, 0.0, 2.0], + [1.0, 0.0, 0.0, 3.0], + [0.0, 0.0, 1.0, 4.0], + [0.0, 0.0, 0.0, 1.0], + ] + ) + second_c2w = np.array( + [ + [0.0, 0.0, 1.0, 5.0], + [0.0, 1.0, 0.0, 4.0], + [-1.0, 0.0, 0.0, 6.0], + [0.0, 0.0, 0.0, 1.0], + ] + ) + poses = np.stack((first_c2w, second_c2w)) + intrinsics = np.array([[100.0, 110.0, 120.0, 130.0]] * 2) + _write_trajectory(tmp_path, poses, intrinsics) + + trajectory = load_camera_trajectory(_trusted(tmp_path)) + + assert trajectory.poses.dtype == torch.float32 + assert trajectory.intrinsics.dtype == torch.float32 + torch.testing.assert_close(trajectory.poses, torch.from_numpy(poses).float()) + torch.testing.assert_close(trajectory.intrinsics, torch.from_numpy(intrinsics).float()) + + +def test_load_camera_trajectory_rejects_object_dtype_before_materializing(tmp_path: Path) -> None: + poses = _identity_poses(1).astype(object) + intrinsics = np.ones((1, 4), dtype=np.float32) + _write_trajectory(tmp_path, poses, intrinsics) + + with pytest.raises(ValueError, match="poses.npy.*real numeric"): + load_camera_trajectory(_trusted(tmp_path)) + + +def test_load_camera_trajectory_accepts_official_length_and_materializes_request_prefix(tmp_path: Path) -> None: + _write_trajectory( + tmp_path, + _identity_poses(269), + np.arange(269 * 4, dtype=np.float32).reshape(269, 4), + ) + + trajectory = load_camera_trajectory(_trusted(tmp_path)) + + assert trajectory.poses.shape == (117, 4, 4) + assert trajectory.intrinsics.shape == (117, 4) + np.testing.assert_array_equal( + trajectory.intrinsics.numpy(), + np.arange(269 * 4, dtype=np.float32).reshape(269, 4)[:117], + ) + + +def test_load_camera_trajectory_rejects_unbounded_source_frame_count(tmp_path: Path) -> None: + _write_trajectory( + tmp_path, + _identity_poses(4097), + np.ones((4097, 4), dtype=np.float32), + ) + + with pytest.raises(ValueError, match="poses.npy.*bounded camera-array limit|4096 source frames"): + load_camera_trajectory(_trusted(tmp_path)) + + +def test_load_camera_trajectory_rejects_truncated_payload_before_loading(tmp_path: Path) -> None: + _write_trajectory(tmp_path, _identity_poses(1), np.ones((1, 4), dtype=np.float32)) + poses_path = tmp_path / "poses.npy" + poses_path.write_bytes(poses_path.read_bytes()[:-1]) + + with pytest.raises(ValueError, match="poses.npy byte size"): + load_camera_trajectory(_trusted(tmp_path)) + + +def test_load_camera_trajectory_rejects_oversized_npy_header(tmp_path: Path) -> None: + header_size = 20_000 + (tmp_path / "poses.npy").write_bytes( + np.lib.format.magic(2, 0) + header_size.to_bytes(4, "little") + b" " * (header_size - 1) + b"\n" + ) + np.save(tmp_path / "intrinsics.npy", np.ones((1, 4), dtype=np.float32)) + + with pytest.raises(ValueError, match="poses.npy.*oversized NPY header"): + load_camera_trajectory(_trusted(tmp_path)) + + +def test_load_camera_trajectory_rejects_symlink_swap_after_resolution(tmp_path: Path) -> None: + trusted_root = tmp_path / "trusted" + action_directory = trusted_root / "action" + outside = tmp_path / "outside" + action_directory.mkdir(parents=True) + outside.mkdir() + _write_trajectory(action_directory, _identity_poses(1), np.ones((1, 4), dtype=np.float32)) + _write_trajectory(outside, _identity_poses(1), np.ones((1, 4), dtype=np.float32)) + resolved = resolve_trusted_action_directory("action", trusted_root) + action_directory.rename(trusted_root / "original-action") + action_directory.symlink_to(outside, target_is_directory=True) + + with pytest.raises(ValueError, match="securely open.*action directory"): + load_camera_trajectory(resolved) + + +def test_load_camera_trajectory_rejects_symlinked_camera_file(tmp_path: Path) -> None: + trusted_root = tmp_path / "trusted" + action_directory = trusted_root / "action" + outside = tmp_path / "outside" + action_directory.mkdir(parents=True) + outside.mkdir() + _write_trajectory(action_directory, _identity_poses(1), np.ones((1, 4), dtype=np.float32)) + np.save(outside / "poses.npy", _identity_poses(1)) + resolved = resolve_trusted_action_directory("action", trusted_root) + (action_directory / "poses.npy").unlink() + (action_directory / "poses.npy").symlink_to(outside / "poses.npy") + + with pytest.raises(ValueError, match="poses.npy.*opened securely"): + load_camera_trajectory(resolved) + + +@pytest.mark.skipif(not hasattr(__import__("os"), "mkfifo"), reason="requires POSIX FIFO support") +def test_load_camera_trajectory_rejects_fifo_without_blocking(tmp_path: Path) -> None: + import os + + os.mkfifo(tmp_path / "poses.npy") + np.save(tmp_path / "intrinsics.npy", np.ones((1, 4), dtype=np.float32)) + script = """ +import importlib.util +import sys + +spec = importlib.util.spec_from_file_location("_lingbot_camera_fifo_test", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +action = module.resolve_trusted_action_directory(".", sys.argv[2]) +try: + module.load_camera_trajectory(action) +except ValueError as error: + print(error) +else: + raise AssertionError("FIFO camera input was accepted") +""" + + completed = subprocess.run( + [sys.executable, "-c", script, str(_MODULE_PATH), str(tmp_path)], + capture_output=True, + text=True, + timeout=3, + check=False, + ) + + assert completed.returncode == 0, completed.stderr + assert "poses.npy must be a regular NPY file" in completed.stdout + + +def test_load_camera_trajectory_decodes_the_validated_file_snapshot(tmp_path: Path, monkeypatch) -> None: + original_poses = _identity_poses(1) + replacement_poses = original_poses.copy() + replacement_poses[0, 0, 3] = 123.0 + _write_trajectory(tmp_path, original_poses, np.ones((1, 4), dtype=np.float32)) + poses_path = tmp_path / "poses.npy" + original_load = _CAMERA_MODULE.np.load + mutation_count = 0 + + def mutate_source_then_decode(source, *args, **kwargs): + nonlocal mutation_count + if mutation_count == 0: + mutation_count += 1 + np.save(poses_path, replacement_poses) + return original_load(source, *args, **kwargs) + + monkeypatch.setattr(_CAMERA_MODULE.np, "load", mutate_source_then_decode) + + trajectory = load_camera_trajectory(_trusted(tmp_path)) + + assert mutation_count == 1 + torch.testing.assert_close(trajectory.poses, torch.from_numpy(original_poses).float()) + + +def test_load_camera_trajectory_ignores_non_camera_event_files(tmp_path: Path) -> None: + poses = _identity_poses(1) + intrinsics = np.ones((1, 4), dtype=np.float32) + _write_trajectory(tmp_path, poses, intrinsics) + np.save(tmp_path / "wasd_events.npy", np.array([{"key": "w"}], dtype=object)) + + trajectory = load_camera_trajectory(_trusted(tmp_path)) + + torch.testing.assert_close(trajectory.poses, torch.from_numpy(poses).float()) + + +def test_interpolate_camera_trajectory_uses_linear_and_spherical_interpolation() -> None: + poses = torch.eye(4).repeat(2, 1, 1) + poses[1, :3, :3] = torch.tensor( + [ + [0.0, -1.0, 0.0], + [1.0, 0.0, 0.0], + [0.0, 0.0, 1.0], + ] + ) + poses[1, :3, 3] = torch.tensor([2.0, 4.0, 6.0]) + intrinsics = torch.tensor([[10.0, 20.0, 30.0, 40.0], [20.0, 40.0, 60.0, 80.0]]) + trajectory = CameraTrajectory(poses=poses, intrinsics=intrinsics) + + result = interpolate_camera_trajectory(trajectory, 3) + + torch.testing.assert_close(result.poses[0], poses[0]) + torch.testing.assert_close(result.poses[-1], poses[-1]) + torch.testing.assert_close(result.poses[1, :3, 3], torch.tensor([1.0, 2.0, 3.0])) + expected_mid_rotation = torch.tensor( + [ + [math.sqrt(0.5), -math.sqrt(0.5), 0.0], + [math.sqrt(0.5), math.sqrt(0.5), 0.0], + [0.0, 0.0, 1.0], + ] + ) + torch.testing.assert_close(result.poses[1, :3, :3], expected_mid_rotation) + torch.testing.assert_close(result.intrinsics[1], torch.tensor([15.0, 30.0, 45.0, 60.0])) + + +def test_build_plucker_embedding_has_forward_center_ray() -> None: + trajectory = _trajectory(torch.eye(4).unsqueeze(0)) + + result = build_plucker_embedding( + trajectory, + height=3, + width=3, + target_height=3, + target_width=3, + device=torch.device("cpu"), + dtype=torch.float32, + ) + + torch.testing.assert_close(result[0, :3, 1, 1], torch.zeros(3)) + torch.testing.assert_close(result[0, 3:, 1, 1], torch.tensor([0.0, 0.0, 1.0])) + + +def test_build_plucker_embedding_scales_reference_intrinsics() -> None: + trajectory = _trajectory( + torch.eye(4).unsqueeze(0), + torch.tensor([[416.0, 240.0, 416.0, 240.0]]), + ) + + result = build_plucker_embedding( + trajectory, + height=4, + width=8, + target_height=4, + target_width=8, + device=torch.device("cpu"), + dtype=torch.float32, + ) + + expected_corner_direction = torch.tensor([-0.875, -0.75, 1.0]) + expected_corner_direction /= torch.linalg.vector_norm(expected_corner_direction) + torch.testing.assert_close(result[0, 3:, 0, 0], expected_corner_direction) + + +def test_build_plucker_embedding_samples_target_pixel_centers() -> None: + trajectory = _trajectory( + torch.eye(4).unsqueeze(0), + torch.tensor([[416.0, 240.0, 416.0, 240.0]]), + ) + + result = build_plucker_embedding( + trajectory, + height=6, + width=10, + target_height=2, + target_width=5, + device=torch.device("cpu"), + dtype=torch.float32, + ) + + # Target centers map through the source scale: (0.5 * 2, 0.5 * 3). + expected_direction = torch.tensor([-0.8, -0.5, 1.0]) + expected_direction /= torch.linalg.vector_norm(expected_direction) + torch.testing.assert_close(result[0, 3:, 0, 0], expected_direction) + + +def test_build_plucker_embedding_uses_framewise_deltas_for_rotations_and_translations() -> None: + poses = torch.eye(4).repeat(3, 1, 1) + poses[0, :3, 3] = torch.tensor([5.0, 7.0, 11.0]) + poses[1, :3, :3] = torch.tensor( + [ + [0.0, -1.0, 0.0], + [1.0, 0.0, 0.0], + [0.0, 0.0, 1.0], + ] + ) + poses[1, :3, 3] = torch.tensor([5.0, 9.0, 11.0]) + poses[2, :3, :3] = torch.tensor( + [ + [-1.0, 0.0, 0.0], + [0.0, -1.0, 0.0], + [0.0, 0.0, 1.0], + ] + ) + poses[2, :3, 3] = torch.tensor([3.0, 9.0, 11.0]) + trajectory = _trajectory( + poses, + torch.tensor([[416.0, 480.0, 208.0, 240.0]]).repeat(3, 1), + ) + + result = build_plucker_embedding( + trajectory, + height=1, + width=2, + target_height=1, + target_width=2, + device=torch.device("cpu"), + dtype=torch.float32, + ) + + expected_direction = torch.tensor([0.0, 1.0, 1.0]) / math.sqrt(2.0) + torch.testing.assert_close(result[0, :3, 0, 1], torch.zeros(3)) + torch.testing.assert_close(result[1, :3, 0, 1], torch.tensor([0.0, 1.0, 0.0])) + torch.testing.assert_close(result[2, :3, 0, 1], torch.tensor([0.0, 1.0, 0.0])) + torch.testing.assert_close(result[1, 3:, 0, 1], expected_direction) + torch.testing.assert_close(result[2, 3:, 0, 1], expected_direction) + + +def test_build_plucker_embedding_normalizes_framewise_translation_by_max_norm() -> None: + poses = torch.eye(4).repeat(3, 1, 1) + poses[:, :3, 3] = torch.tensor( + [ + [10.0, -2.0, 5.0], + [13.0, 2.0, 5.0], + [19.0, 10.0, 5.0], + ] + ) + + result = build_plucker_embedding( + _trajectory(poses), + height=1, + width=1, + target_height=1, + target_width=1, + device=torch.device("cpu"), + dtype=torch.float32, + ) + + expected_origins = torch.tensor([[0.0, 0.0, 0.0], [0.3, 0.4, 0.0], [0.6, 0.8, 0.0]]) + torch.testing.assert_close(result[:, :3, 0, 0], expected_origins) + + +def test_build_plucker_embedding_interpolates_raw_poses_before_framewise_conversion() -> None: + poses = torch.eye(4).repeat(2, 1, 1) + poses[0, :3, 3] = torch.tensor([20.0, 4.0, -2.0]) + poses[1, :3, 3] = torch.tensor([26.0, 12.0, -2.0]) + raw = _trajectory(poses) + + interpolated = interpolate_camera_trajectory(raw, 3) + result = build_plucker_embedding( + interpolated, + height=1, + width=1, + target_height=1, + target_width=1, + device=torch.device("cpu"), + dtype=torch.float32, + ) + + expected_delta = torch.tensor([0.6, 0.8, 0.0]) + torch.testing.assert_close(result[0, :3, 0, 0], torch.zeros(3)) + torch.testing.assert_close(result[1, :3, 0, 0], expected_delta) + torch.testing.assert_close(result[2, :3, 0, 0], expected_delta) + + +def test_build_plucker_embedding_uses_half_pixel_centers_and_origin_direction_channel_order() -> None: + poses = torch.eye(4).repeat(2, 1, 1) + poses[1, :3, 3] = torch.tensor([2.0, 0.0, 0.0]) + trajectory = _trajectory( + poses, + torch.tensor([[832.0, 480.0, 0.0, 0.0]]).repeat(2, 1), + ) + + result = build_plucker_embedding( + trajectory, + height=1, + width=2, + target_height=1, + target_width=2, + device=torch.device("cpu"), + dtype=torch.float32, + ) + + expected_direction = torch.tensor([0.25, 0.5, 1.0]) + expected_direction /= torch.linalg.vector_norm(expected_direction) + torch.testing.assert_close(result[1, :3, 0, 0], torch.tensor([1.0, 0.0, 0.0])) + torch.testing.assert_close(result[1, 3:, 0, 0], expected_direction) + + +def test_build_plucker_embedding_honors_shape_dtype_and_device() -> None: + trajectory = _trajectory(torch.eye(4).repeat(2, 1, 1)) + + result = build_plucker_embedding( + trajectory, + height=12, + width=20, + target_height=3, + target_width=5, + device=torch.device("cpu"), + dtype=torch.float64, + ) + + assert result.shape == (2, 6, 3, 5) + assert result.dtype == torch.float64 + assert result.device == torch.device("cpu") + + +def test_build_plucker_embedding_is_deterministic() -> None: + trajectory = _trajectory(torch.eye(4).repeat(2, 1, 1)) + kwargs = { + "height": 12, + "width": 20, + "target_height": 3, + "target_width": 5, + "device": torch.device("cpu"), + "dtype": torch.float32, + } + + first = build_plucker_embedding(trajectory, **kwargs) + second = build_plucker_embedding(trajectory, **kwargs) + + torch.testing.assert_close(first, second, rtol=0.0, atol=0.0) diff --git a/tests/diffusion/models/lingbot_world/test_lingbot_world_transformer.py b/tests/diffusion/models/lingbot_world/test_lingbot_world_transformer.py new file mode 100644 index 000000000..e2e2a08b6 --- /dev/null +++ b/tests/diffusion/models/lingbot_world/test_lingbot_world_transformer.py @@ -0,0 +1,685 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import hashlib +import inspect +import json +import math +from pathlib import Path + +import pytest +import torch + +from tests.diffusion.models.lingbot_world import test_lingbot_world_attention as attention_tests + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + +_FIXTURE_PATH = Path(__file__).with_name("fixtures") / "lingbot_world_weight_index_names.json.fixture" +_SHAPE_FIXTURE_PATH = Path(__file__).with_name("fixtures") / "lingbot_world_official_shapes.json.fixture" + + +@pytest.fixture(autouse=True) +def _inference_context(): + with torch.inference_mode(): + yield + + +def _tiny_model( + module, + *, + num_layers: int = 2, + num_frames_per_block: int = 1, + sliding_window_num_frames: int = 3, +): + return module.CausalLingBotWorldTransformer3DModel( + patch_size=(1, 2, 2), + num_attention_heads=2, + attention_head_dim=2, + in_channels=36, + out_channels=2, + text_dim=6, + freq_dim=4, + ffn_dim=8, + num_layers=num_layers, + cross_attn_norm=True, + eps=1e-6, + rope_max_seq_len=16, + sink_size=1, + num_frames_per_block=num_frames_per_block, + sliding_window_num_frames=sliding_window_num_frames, + local_attn_size=-1, + ) + + +def _cache(module, model, *, max_tokens: int | None = None): + if max_tokens is None: + window_frames = ( + model.config.local_attn_size + if model.config.local_attn_size != -1 + else model.config.sliding_window_num_frames + ) + max_tokens = window_frames * 2 * 2 + return module.allocate_lingbot_cache( + batch_size=1, + num_layers=len(model.blocks), + max_tokens=max_tokens, + num_local_heads=2, + head_dim=2, + device=torch.device("cpu"), + dtype=torch.float32, + ) + + +def _checkpoint_parameter_specs(model) -> dict[str, tuple[torch.Size, torch.dtype]]: + specs: dict[str, tuple[torch.Size, torch.dtype]] = {} + for name, parameter in model.named_parameters(): + marker = ".self_attn.qkv." + if marker not in name: + specs[name] = (parameter.shape, parameter.dtype) + continue + shard_shape = torch.Size((parameter.shape[0] // 3, *parameter.shape[1:])) + for projection in ("q", "k", "v"): + specs[name.replace(marker, f".self_attn.{projection}.")] = (shard_shape, parameter.dtype) + return specs + + +def _checkpoint_weights_for_model(model) -> tuple[dict[str, torch.Tensor], dict[str, torch.Tensor]]: + checkpoint_weights: dict[str, torch.Tensor] = {} + expected_parameters: dict[str, torch.Tensor] = {} + next_value = 1 + for name, parameter in model.named_parameters(): + marker = ".self_attn.qkv." + if marker not in name: + value = torch.full_like(parameter, next_value) + checkpoint_weights[name] = value + expected_parameters[name] = value + next_value += 1 + continue + shards = [] + shard_shape = (parameter.shape[0] // 3, *parameter.shape[1:]) + for projection in ("q", "k", "v"): + value = torch.full(shard_shape, next_value, dtype=parameter.dtype, device=parameter.device) + checkpoint_weights[name.replace(marker, f".self_attn.{projection}.")] = value + shards.append(value) + next_value += 1 + expected_parameters[name] = torch.cat(shards) + return checkpoint_weights, expected_parameters + + +def _self_cache_snapshot(cache) -> list[tuple[torch.Tensor, torch.Tensor, tuple[int, int, int | None, int]]]: + return [attention_tests._cache_snapshot(layer_cache) for layer_cache in cache.self_attention] + + +def _assert_self_cache_unchanged(cache, snapshot) -> None: + for layer_cache, layer_snapshot in zip(cache.self_attention, snapshot, strict=True): + attention_tests._assert_cache_unchanged(layer_cache, layer_snapshot) + + +def test_tiny_transformer_runs_four_chunks_with_explicit_cache_commit_and_camera_path() -> None: + torch.manual_seed(7) + module = attention_tests._load_module() + model = _tiny_model(module).eval() + cache = _cache(module, model) + encoder_hidden_states = torch.randn(1, 3, 6) + camera_hidden_states = torch.randn(1, 6 * 8 * 8, 1, 4, 4) + cross_key_outputs = [attention_tests._record_outputs(block.cross_attn.k) for block in model.blocks] + text_projection_outputs = attention_tests._record_outputs(model.text_embedding) + outputs = [] + + assert model.patch_embedding.in_channels == 36 + assert model.patch_embedding_wancamctrl.in_features == 6 * 8 * 8 * 1 * 2 * 2 + + for start_frame in range(4): + hidden_states = torch.randn(1, 36, 1, 4, 4) + timestep = torch.tensor([float(start_frame + 1)]) + snapshot = _self_cache_snapshot(cache) + + transient_output = model( + hidden_states, + timestep, + encoder_hidden_states, + camera_hidden_states, + cache=cache, + start_frame=start_frame, + update_cache=False, + ) + + assert transient_output.shape == (1, 2, 1, 4, 4) + _assert_self_cache_unchanged(cache, snapshot) + assert all(layer_cache is not None for layer_cache in cache.cross_attention) + + committed_output = model( + hidden_states, + timestep, + encoder_hidden_states, + camera_hidden_states, + cache=cache, + start_frame=start_frame, + update_cache=True, + ) + torch.testing.assert_close(committed_output, transient_output) + outputs.append(committed_output) + + expected_token_end = (start_frame + 1) * 2 * 2 + assert all(layer_cache.absolute_end == expected_token_end for layer_cache in cache.self_attention) + + assert len(outputs) == 4 + assert all(len(outputs) == 1 for outputs in cross_key_outputs) + assert len(text_projection_outputs) == 1 + + alternate_cache = _cache(module, model) + alternate_output = model( + torch.zeros(1, 36, 1, 4, 4), + torch.tensor([1.0]), + encoder_hidden_states, + torch.zeros_like(camera_hidden_states), + cache=alternate_cache, + start_frame=0, + update_cache=False, + ) + camera_output = model( + torch.zeros(1, 36, 1, 4, 4), + torch.tensor([1.0]), + encoder_hidden_states, + torch.ones_like(camera_hidden_states), + cache=_cache(module, model), + start_frame=0, + update_cache=False, + ) + assert not torch.equal(camera_output, alternate_output) + + +def test_partial_cross_attention_cache_projects_text_only_for_missing_layers() -> None: + module = attention_tests._load_module() + model = _tiny_model(module, num_layers=2).eval() + cache = _cache(module, model) + hidden_states = torch.randn(1, 36, 1, 4, 4) + timestep = torch.tensor([1.0]) + encoder_hidden_states = torch.randn(1, 3, 6) + camera_hidden_states = torch.randn(1, 6 * 8 * 8, 1, 4, 4) + + model( + hidden_states, + timestep, + encoder_hidden_states, + camera_hidden_states, + cache=cache, + start_frame=0, + update_cache=False, + ) + cache.cross_attention[1] = None + text_projection_outputs = attention_tests._record_outputs(model.text_embedding) + cross_key_outputs = [attention_tests._record_outputs(block.cross_attn.k) for block in model.blocks] + + model( + hidden_states, + timestep, + encoder_hidden_states, + camera_hidden_states, + cache=cache, + start_frame=0, + update_cache=False, + ) + + assert len(text_projection_outputs) == 1 + assert [len(outputs) for outputs in cross_key_outputs] == [0, 1] + assert all(layer_cache is not None for layer_cache in cache.cross_attention) + + +@pytest.mark.parametrize("frames", [1, 6], ids=("partial", "multiple_blocks")) +def test_forward_rejects_chunks_that_do_not_equal_configured_block_size(frames: int) -> None: + module = attention_tests._load_module() + model = _tiny_model( + module, + num_layers=1, + num_frames_per_block=3, + sliding_window_num_frames=6, + ) + cache = _cache(module, model) + snapshot = _self_cache_snapshot(cache) + patch_inputs = attention_tests._record_inputs(model.patch_embedding) + + with pytest.raises(ValueError, match="exactly 3 post-patch frames"): + model( + torch.randn(1, 36, frames, 4, 4), + torch.tensor([1.0]), + torch.randn(1, 3, 6), + torch.randn(1, 6 * 8 * 8, frames, 4, 4), + cache=cache, + start_frame=0, + update_cache=True, + ) + + assert patch_inputs == [] + _assert_self_cache_unchanged(cache, snapshot) + assert cache.cross_attention == [None] + + +def test_forward_accepts_exactly_one_configured_frame_block() -> None: + module = attention_tests._load_module() + model = _tiny_model( + module, + num_layers=1, + num_frames_per_block=3, + sliding_window_num_frames=6, + ) + + output = model( + torch.randn(1, 36, 3, 4, 4), + torch.tensor([1.0]), + torch.randn(1, 3, 6), + torch.randn(1, 6 * 8 * 8, 3, 4, 4), + cache=_cache(module, model), + start_frame=0, + update_cache=False, + ) + + assert output.shape == (1, 2, 3, 4, 4) + + +def test_transformer_allocates_request_cache_from_its_configured_geometry() -> None: + module = attention_tests._load_module() + model = _tiny_model( + module, + num_layers=2, + num_frames_per_block=3, + sliding_window_num_frames=6, + ) + + cache = model.allocate_cache( + batch_size=2, + latent_height=8, + latent_width=12, + device=torch.device("cpu"), + dtype=torch.float32, + ) + + assert len(cache.self_attention) == 2 + assert len(cache.cross_attention) == 2 + assert cache.cross_attention == [None, None] + assert all(layer.key.shape == (2, 6 * 4 * 6, 2, 2) for layer in cache.self_attention) + assert all(layer.value.shape == (2, 6 * 4 * 6, 2, 2) for layer in cache.self_attention) + + +def test_video_patch_embedding_uses_temporal_height_width_token_order() -> None: + module = attention_tests._load_module() + model = _tiny_model(module, num_layers=1, num_frames_per_block=2) + hidden_states = torch.zeros(1, 36, 2, 2, 4) + hidden_states[0, 0] = torch.tensor( + [ + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0]], + [[9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]], + ] + ) + with torch.no_grad(): + model.patch_embedding.weight.zero_() + model.patch_embedding.bias.zero_() + model.patch_embedding.weight[0, 0, 0, 0, 0] = 1 + + tokens = model.patch_embedding(hidden_states).flatten(2).transpose(1, 2) + + torch.testing.assert_close(tokens[0, :, 0], torch.tensor([1.0, 3.0, 9.0, 11.0])) + + +def test_unpatchify_restores_two_frame_channel_and_spatial_order() -> None: + module = attention_tests._load_module() + model = _tiny_model(module, num_layers=1, num_frames_per_block=2) + hidden_states = ( + torch.stack( + ( + torch.arange(0, 8), + torch.arange(10, 18), + torch.arange(20, 28), + torch.arange(30, 38), + ) + ) + .unsqueeze(0) + .float() + ) + expected = torch.tensor( + [ + [ + [[0, 2, 10, 12], [4, 6, 14, 16]], + [[20, 22, 30, 32], [24, 26, 34, 36]], + ], + [ + [[1, 3, 11, 13], [5, 7, 15, 17]], + [[21, 23, 31, 33], [25, 27, 35, 37]], + ], + ], + dtype=torch.float32, + ).unsqueeze(0) + + output = model._unpatchify( + hidden_states, + batch_size=1, + frames=2, + height=1, + width=2, + ) + + torch.testing.assert_close(output, expected) + + +def test_head_modulation_broadcasts_distinct_condition_per_frame() -> None: + module = attention_tests._load_module() + model = _tiny_model(module, num_layers=1, num_frames_per_block=2) + model.head.norm = torch.nn.Identity() + with torch.no_grad(): + model.head.modulation.zero_() + model.head.head.weight.zero_() + model.head.head.bias.zero_() + model.head.head.weight[:, 0] = 1 + hidden_states = torch.ones(1, 4, model.dim) + timestep_embedding = torch.tensor([[[10.0, 0.0, 0.0, 0.0], [20.0, 0.0, 0.0, 0.0]]]) + expected = torch.tensor([21.0, 21.0, 41.0, 41.0]).view(1, 4, 1).expand(1, 4, 8) + + output = model.head(hidden_states, timestep_embedding) + + torch.testing.assert_close(output, expected) + + +def test_constructor_defaults_match_official_checkpoint_config() -> None: + module = attention_tests._load_module() + parameters = inspect.signature(module.CausalLingBotWorldTransformer3DModel.__init__).parameters + expected = { + "in_channels": 36, + "out_channels": 16, + "num_layers": 40, + "num_attention_heads": 40, + "attention_head_dim": 128, + "patch_size": (1, 2, 2), + "text_dim": 4096, + "ffn_dim": 13824, + "freq_dim": 256, + "rope_max_seq_len": 1024, + "eps": 1e-6, + "cross_attn_norm": True, + "sink_size": 9, + "num_frames_per_block": 3, + "sliding_window_num_frames": 18, + "local_attn_size": -1, + } + + assert {name: parameters[name].default for name in expected} == expected + + +def test_transformer_exposes_parameter_dtype_for_pipeline_runtime() -> None: + module = attention_tests._load_module() + model = _tiny_model(module) + + assert model.dtype == next(model.parameters()).dtype + + +def test_transformer_declares_regional_compile_block() -> None: + module = attention_tests._load_module() + + assert module.CausalLingBotWorldTransformer3DModel._repeated_blocks == ["LingBotAttentionBlock"] + + +def test_lingbot_rms_norm_uses_global_tp_square_mean(monkeypatch) -> None: + module = attention_tests._load_module() + reduced_values: list[torch.Tensor] = [] + + monkeypatch.setattr(module, "get_tensor_model_parallel_world_size", lambda: 2) + + def all_reduce(value: torch.Tensor) -> torch.Tensor: + reduced_values.append(value.detach().clone()) + return value * 2 + + monkeypatch.setattr(module, "tensor_model_parallel_all_reduce", all_reduce) + norm = module._LingBotRMSNorm(hidden_size=2, eps=0.0) + value = torch.tensor([[3.0, 4.0]]) + + output = norm(value) + + torch.testing.assert_close(output, value / math.sqrt(12.5)) + assert len(reduced_values) == 1 + torch.testing.assert_close(reduced_values[0], torch.tensor([[25.0]])) + + +def test_attention_rejects_heads_not_divisible_by_tp_size(monkeypatch) -> None: + module = attention_tests._load_module() + monkeypatch.setattr(module, "get_tensor_model_parallel_world_size", lambda: 3) + + with pytest.raises(ValueError, match="num_heads=2.*tp_size=3"): + module.LingBotSelfAttention(dim=4, num_heads=2) + + +def test_constructor_rejects_unsupported_qk_norm_with_config_error() -> None: + module = attention_tests._load_module() + + with pytest.raises(ValueError, match="qk_norm.*rms_norm_across_heads"): + module.CausalLingBotWorldTransformer3DModel(qk_norm="layer_norm") + + +@pytest.mark.parametrize("field", ["image_dim", "added_kv_proj_dim", "pos_embed_seq_len"]) +def test_constructor_rejects_non_null_image_embedding_fields(field: str) -> None: + module = attention_tests._load_module() + + with pytest.raises(ValueError, match=field): + module.CausalLingBotWorldTransformer3DModel(**{field: 4}) + + +def test_constructor_rejects_unsupported_quantization_with_runtime_error() -> None: + module = attention_tests._load_module() + + with pytest.raises(RuntimeError, match="quant_config.*not supported"): + module.CausalLingBotWorldTransformer3DModel(quant_config=object()) + + +def test_from_config_accepts_diffusers_metadata_and_normalizes_patch_size() -> None: + module = attention_tests._load_module() + + class ConfigProbe(module.CausalLingBotWorldTransformer3DModel): + def __init__(self, **kwargs) -> None: + self.received_kwargs = kwargs + + config = { + "_class_name": "CausalLingBotWorldTransformer3DModel", + "_diffusers_version": "0.35.0.dev0", + "patch_size": [1, 2, 2], + "num_attention_heads": 40, + "attention_head_dim": 128, + "in_channels": 36, + "out_channels": 16, + "text_dim": 4096, + "freq_dim": 256, + "ffn_dim": 13824, + "num_layers": 40, + "cross_attn_norm": True, + "eps": 1e-6, + "image_dim": None, + "added_kv_proj_dim": None, + "rope_max_seq_len": 1024, + "pos_embed_seq_len": None, + "qk_norm": "rms_norm_across_heads", + "sink_size": 9, + "num_frames_per_block": 3, + "sliding_window_num_frames": 18, + "local_attn_size": -1, + } + + model = ConfigProbe.from_config(config, prefix="transformer") + + assert model.received_kwargs["patch_size"] == (1, 2, 2) + assert model.received_kwargs["num_layers"] == 40 + assert model.received_kwargs["prefix"] == "transformer" + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("in_channels", 35), + ("num_layers", 39), + ("num_attention_heads", 32), + ("attention_head_dim", 64), + ("ffn_dim", 10240), + ("num_frames_per_block", 4), + ("sliding_window_num_frames", 24), + ("sink_size", 0), + ], +) +def test_from_config_rejects_checkpoint_topology_drift(field: str, value: object) -> None: + module = attention_tests._load_module() + config = { + "_class_name": "CausalLingBotWorldTransformer3DModel", + "_diffusers_version": "0.35.0.dev0", + "patch_size": [1, 2, 2], + "num_attention_heads": 40, + "attention_head_dim": 128, + "in_channels": 36, + "out_channels": 16, + "text_dim": 4096, + "freq_dim": 256, + "ffn_dim": 13824, + "num_layers": 40, + "cross_attn_norm": True, + "eps": 1e-6, + "image_dim": None, + "added_kv_proj_dim": None, + "rope_max_seq_len": 1024, + "pos_embed_seq_len": None, + "qk_norm": "rms_norm_across_heads", + "sink_size": 9, + "num_frames_per_block": 3, + "sliding_window_num_frames": 18, + "local_attn_size": -1, + } + config[field] = value + + with pytest.raises(ValueError, match=field): + module.CausalLingBotWorldTransformer3DModel.from_config(config) + + +def test_from_config_ignores_non_semantic_checkpoint_metadata() -> None: + module = attention_tests._load_module() + config = { + "_class_name": "CausalLingBotWorldTransformer3DModel", + "architectures": ["CausalLingBotWorldTransformer3DModel"], + "future_diffusers_metadata": {"version": 1}, + } + + with torch.device("meta"): + model = module.CausalLingBotWorldTransformer3DModel.from_config(config) + + assert model.config.in_channels == 36 + assert model.config.num_layers == 40 + + +def test_load_weights_uses_parameter_loaders_and_rejects_unknown_model_keys() -> None: + module = attention_tests._load_module() + model = _tiny_model(module, num_layers=1) + checkpoint_weights, expected_parameters = _checkpoint_weights_for_model(model) + loader_calls: list[str] = [] + qkv_weight = model.blocks[0].self_attn.qkv.weight + + def record_loader(param: torch.Tensor, loaded_weight: torch.Tensor, shard_id: str) -> None: + loader_calls.append(shard_id) + shard_index = {"q": 0, "k": 1, "v": 2}[shard_id] + shard_size = param.shape[0] // 3 + param.data.narrow(0, shard_index * shard_size, shard_size).copy_(loaded_weight) + + qkv_weight.weight_loader = record_loader + loaded = model.load_weights(iter(checkpoint_weights.items())) + + assert set(checkpoint_weights) <= loaded + assert set(dict(model.named_parameters())) <= loaded + assert loader_calls == ["q", "k", "v"] + for name, param in model.named_parameters(): + torch.testing.assert_close(param, expected_parameters[name]) + + first_name = next(iter(checkpoint_weights)) + partial = _tiny_model(module, num_layers=1) + partial_loaded = partial.load_weights([(first_name, checkpoint_weights[first_name])]) + expected_partial = {first_name} + for projection in ("q", "k", "v"): + marker = f".self_attn.{projection}." + if marker in first_name: + expected_partial.add(first_name.replace(marker, ".self_attn.qkv.")) + break + assert partial_loaded == expected_partial + + with pytest.raises(KeyError, match="unexpected_model.weight"): + model.load_weights([("unexpected_model.weight", torch.ones(1))]) + + +def test_load_weights_consumes_checkpoint_iterator_incrementally() -> None: + module = attention_tests._load_module() + model = _tiny_model(module, num_layers=1) + first_name, first_param = next(iter(model.named_parameters())) + loaded_weight = torch.full_like(first_param, 17) + + def weights(): + yield first_name, loaded_weight + torch.testing.assert_close(first_param, loaded_weight) + + assert model.load_weights(weights()) == {first_name} + + +def test_real_auto_weights_loader_covers_fused_model_parameter_namespace() -> None: + utils = pytest.importorskip("vllm.model_executor.models.utils") + module = attention_tests._load_module() + transformer = _tiny_model(module, num_layers=1) + checkpoint_weights, _ = _checkpoint_weights_for_model(transformer) + + class Pipeline(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.transformer = transformer + + pipeline = Pipeline() + loaded = utils.AutoWeightsLoader(pipeline).load_weights( + (f"transformer.{name}", value) for name, value in checkpoint_weights.items() + ) + + assert set(dict(pipeline.named_parameters())) <= loaded + + +def test_checkpoint_weight_index_fixture_matches_model_namespaces() -> None: + module = attention_tests._load_module() + fixture = json.loads(_FIXTURE_PATH.read_text()) + model = _tiny_model(module, num_layers=1) + parameter_names = set(_checkpoint_parameter_specs(model)) + + assert {name.split(".", 1)[0] for name in parameter_names} == set(fixture["top_level_modules"]) + assert {name for name in parameter_names if not name.startswith("blocks.")} == set(fixture["non_block_parameters"]) + assert {name.removeprefix("blocks.0.") for name in parameter_names if name.startswith("blocks.0.")} == set( + fixture["block_parameter_suffixes"] + ) + expanded_names = set(fixture["non_block_parameters"]) + expanded_names.update( + f"blocks.{layer}.{suffix}" for layer in range(40) for suffix in fixture["block_parameter_suffixes"] + ) + canonical_names = "\n".join(sorted(expanded_names)) + + assert len(expanded_names) == fixture["index_parameter_count"] == 1421 + assert hashlib.sha256(canonical_names.encode()).hexdigest() == fixture["canonical_name_set_sha256"] + assert fixture["canonical_name_to_shard_sha256"] == ( + "e1addc27d2b1fad6f10226a23a265ee3e3c61e74dce4d5859727bdf180a67992" + ) + assert list(fixture["shards"]) == [ + f"diffusion_pytorch_model-{index:05}-of-00008.safetensors" for index in range(1, 9) + ] + assert sum(shard["parameter_count"] for shard in fixture["shards"].values()) == 1421 + assert all(len(shard["parameter_names_sha256"]) == 64 for shard in fixture["shards"].values()) + + +def test_official_default_shapes_match_public_safetensors_header_fixture() -> None: + module = attention_tests._load_module() + fixture = json.loads(_SHAPE_FIXTURE_PATH.read_text()) + + with torch.device("meta"): + model = module.CausalLingBotWorldTransformer3DModel() + parameter_specs = _checkpoint_parameter_specs(model) + expected_dtype = {"F32": torch.float32}[fixture["dtype"]] + + assert len(fixture["shard_representatives"]) == 8 + assert set(fixture["shard_representatives"].values()) <= set(fixture["parameter_shapes"]) + + for name, expected_shape in fixture["parameter_shapes"].items(): + shape, dtype = parameter_specs[name] + assert tuple(shape) == tuple(expected_shape), name + assert dtype == expected_dtype diff --git a/tests/diffusion/models/lingbot_world/test_lingbot_world_typing.py b/tests/diffusion/models/lingbot_world/test_lingbot_world_typing.py new file mode 100644 index 000000000..56c25fee3 --- /dev/null +++ b/tests/diffusion/models/lingbot_world/test_lingbot_world_typing.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + +_ROOT = Path(__file__).parents[4] +_PRODUCTION_FILES = ( + _ROOT / "vllm_omni/diffusion/models/lingbot_world/camera.py", + _ROOT / "vllm_omni/diffusion/models/lingbot_world/transformer.py", + _ROOT / "vllm_omni/diffusion/models/lingbot_world/pipeline.py", +) + + +def test_all_lingbot_production_functions_have_complete_annotations() -> None: + missing: list[str] = [] + for path in _PRODUCTION_FILES: + tree = ast.parse(path.read_text()) + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + arguments = (*node.args.posonlyargs, *node.args.args, *node.args.kwonlyargs) + for argument in arguments: + if argument.arg not in {"self", "cls"} and argument.annotation is None: + missing.append(f"{path.name}:{node.lineno}:{node.name}:{argument.arg}") + if node.args.vararg is not None and node.args.vararg.annotation is None: + missing.append(f"{path.name}:{node.lineno}:{node.name}:*{node.args.vararg.arg}") + if node.args.kwarg is not None and node.args.kwarg.annotation is None: + missing.append(f"{path.name}:{node.lineno}:{node.name}:**{node.args.kwarg.arg}") + if node.returns is None: + missing.append(f"{path.name}:{node.lineno}:{node.name}:return") + + assert missing == [] diff --git a/tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py b/tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py new file mode 100644 index 000000000..1a08e8270 --- /dev/null +++ b/tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py @@ -0,0 +1,1765 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import gc +import weakref +from pathlib import Path +from threading import Lock +from types import SimpleNamespace + +import pytest +import torch +from diffusers.utils.torch_utils import randn_tensor as _diffusers_randn_tensor +from PIL import Image +from torch import nn + +import vllm_omni.diffusion.models.lingbot_world.pipeline as lingbot_pipeline +from tests.diffusion.models.wan2_2.conftest import noop_progress_bar +from vllm_omni.diffusion.models.lingbot_world.actions import ( + integrate_lingbot_camera_actions, +) +from vllm_omni.diffusion.models.lingbot_world.camera import CameraTrajectory as _CameraTrajectory +from vllm_omni.diffusion.models.lingbot_world.camera import ( + build_plucker_embedding as _real_build_plucker_embedding, +) +from vllm_omni.experimental.ar_diffusion.tick_protocol import ( + ARDiffusionControlInput, + ARDiffusionTickRequest, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + +_ROOT = Path(__file__).parents[4] +_LINGBOT_INIT_PATH = _ROOT / "vllm_omni/diffusion/models/lingbot_world/__init__.py" +_MODEL_INDEX_FIXTURE = Path(__file__).with_name("fixtures") / "lingbot_world_model_index.json.fixture" +_SCHEDULER_FIXTURE = Path(__file__).with_name("fixtures") / "lingbot_world_scheduler_config.json.fixture" + + +def _scheduler_config(**overrides): + values = { + "_class_name": "UniPCMultistepScheduler", + "num_train_timesteps": 1000, + "flow_shift": 5.0, + "prediction_type": "flow_prediction", + "predict_x0": True, + "use_flow_sigmas": True, + "use_dynamic_shifting": False, + "use_beta_sigmas": False, + "use_exponential_sigmas": False, + "use_karras_sigmas": False, + "final_sigmas_type": "zero", + "timestep_spacing": "linspace", + "solver_order": 2, + "solver_type": "bh2", + "lower_order_final": True, + "disable_corrector": [], + "time_shift_type": "exponential", + } + values.update(overrides) + return values + + +class _AutoWeightsLoader: + def __init__(self, module): + self.module = module + + def load_weights(self, weights): + loaded = self.module.transformer.load_weights( + (name.removeprefix("transformer."), value) for name, value in weights if name.startswith("transformer.") + ) + return {f"transformer.{name}" for name in loaded} + + +class _FakePretrained: + @classmethod + def from_pretrained(cls, *args, **kwargs): + del args, kwargs + return cls() + + def to(self, *args, **kwargs): + self.to_calls = getattr(self, "to_calls", []) + self.to_calls.append((args, kwargs)) + return self + + +class _FakeScheduler: + def __init__(self, **kwargs): + self.config = SimpleNamespace(**kwargs) + + +class _FakeTransformerFactory: + @classmethod + def from_config(cls, config, *, quant_config=None, prefix=""): + cls.last_call = (config, quant_config, prefix) + return _RecordingTransformer() + + +class _Cache: + def __init__(self, *, num_layers: int, max_tokens: int, num_local_heads: int, head_dim: int): + shape = (1, max_tokens, num_local_heads, head_dim) + self.self_attention = [ + SimpleNamespace(key=torch.zeros(shape), value=torch.zeros(shape)) for _ in range(num_layers) + ] + self.cross_attention = [None] * num_layers + + +class _RecordingTransformer(nn.Module): + def __init__(self, *, raise_on_call: int | None = None, dtype: torch.dtype = torch.float32): + super().__init__() + self.config = SimpleNamespace( + patch_size=(1, 2, 2), + in_channels=36, + out_channels=16, + text_dim=8, + num_layers=2, + num_attention_heads=2, + attention_head_dim=4, + num_frames_per_block=3, + sliding_window_num_frames=6, + local_attn_size=-1, + sink_size=3, + ) + self.blocks = nn.ModuleList([nn.Identity(), nn.Identity()]) + for block in self.blocks: + block.self_attn = SimpleNamespace(num_local_heads=2, head_dim=4) + self.calls: list[dict] = [] + self.cache_allocations: list[dict] = [] + self.raise_on_call = raise_on_call + self._dtype = dtype + self.loaded_weights: list[tuple[str, torch.Tensor]] = [] + + @property + def dtype(self) -> torch.dtype: + return self._dtype + + def forward(self, **kwargs): + call = { + "hidden_states": kwargs["hidden_states"].detach().clone(), + "timestep": kwargs["timestep"].detach().clone(), + "encoder_hidden_states": kwargs["encoder_hidden_states"].detach().clone(), + "camera_hidden_states": kwargs["camera_hidden_states"].detach().clone(), + "cache_id": id(kwargs["cache"]), + "start_frame": kwargs["start_frame"], + "update_cache": kwargs["update_cache"], + } + self.calls.append(call) + if self.raise_on_call == len(self.calls): + raise RuntimeError("forced transformer failure") + return torch.ones_like(kwargs["hidden_states"][:, :16]) + + def allocate_cache(self, **kwargs): + self.cache_allocations.append(dict(kwargs)) + patch_height, patch_width = self.config.patch_size[1:] + window_frames = ( + self.config.local_attn_size if self.config.local_attn_size != -1 else self.config.sliding_window_num_frames + ) + max_tokens = window_frames * (kwargs["latent_height"] // patch_height) * (kwargs["latent_width"] // patch_width) + return _Cache( + num_layers=self.config.num_layers, + max_tokens=max_tokens, + num_local_heads=2, + head_dim=4, + ) + + def load_weights(self, weights): + self.loaded_weights = list(weights) + return {name for name, _ in self.loaded_weights} + + +class _StubVAE(_FakePretrained): + dtype = torch.float32 + + def __init__(self): + self.config = SimpleNamespace( + z_dim=16, + scale_factor_temporal=4, + scale_factor_spatial=8, + latents_mean=[float(index) - 3.0 for index in range(16)], + latents_std=[1.0 + index / 4.0 for index in range(16)], + ) + self.encode_inputs: list[torch.Tensor] = [] + self.decode_inputs: list[torch.Tensor] = [] + self.on_decode = None + + def encode(self, video: torch.Tensor): + self.encode_inputs.append(video.detach().clone()) + latent_frames = (video.shape[2] - 1) // 4 + 1 + latents = torch.zeros( + video.shape[0], + 16, + latent_frames, + video.shape[-2] // 8, + video.shape[-1] // 8, + dtype=video.dtype, + device=video.device, + ) + latents[:, :, 0] = 2.0 + return SimpleNamespace(latents=latents) + + def decode(self, latents: torch.Tensor, return_dict: bool = False): + del return_dict + if self.on_decode is not None: + self.on_decode() + self.decode_inputs.append(latents.detach().clone()) + pixel_frames = (latents.shape[2] - 1) * 4 + 1 + decoded = torch.zeros( + latents.shape[0], + 3, + pixel_frames, + latents.shape[-2] * 8, + latents.shape[-1] * 8, + dtype=latents.dtype, + device=latents.device, + ) + return (decoded,) + + +class _SamplingParams: + def __init__( + self, + *, + height: int | None = 16, + width: int | None = 16, + num_frames: int = 9, + num_inference_steps: int | None = 4, + num_outputs_per_prompt: int = 1, + seed: int | None = 17, + generator: torch.Generator | None = None, + output_type: str | None = "latent", + max_sequence_length: int | None = 512, + extra_args: dict | None = None, + include_action: bool = True, + ) -> None: + self.height = height + self.width = width + self.num_frames = num_frames + self.num_inference_steps = num_inference_steps + self.num_outputs_per_prompt = num_outputs_per_prompt + self.seed = seed + self.generator = ( + generator + if generator is not None + else (torch.Generator(device="cpu").manual_seed(seed) if seed is not None else None) + ) + self.output_type = output_type + self.max_sequence_length = max_sequence_length + self.extra_args = {"action_path": "."} if include_action else {} + self.extra_args["_lingbot_camera_trajectory"] = _CameraTrajectory( + poses=torch.eye(4).repeat(32, 1, 1), + intrinsics=torch.tensor([[100.0, 100.0, 8.0, 8.0]]).repeat(32, 1), + ) + if extra_args is not None: + self.extra_args.update(extra_args) + self.latents = None + + +class _RequestBatch: + def __init__(self, prompt, sampling_params: _SamplingParams, *, num_reqs: int = 1): + self._prompts = [prompt] * num_reqs + self._sampling = sampling_params + self.num_reqs = num_reqs + + @property + def prompts(self): + return self._prompts + + @property + def sampling_params(self): + return self._sampling + + @property + def request_id(self): + return "legacy-lingbot-request" + + +def _load_pipeline_module(): + return lingbot_pipeline + + +@pytest.fixture(autouse=True) +def _stub_pipeline_dependencies(monkeypatch: pytest.MonkeyPatch) -> None: + """Replace only heavyweight component construction, not the import graph.""" + + loader_state = SimpleNamespace(prefetch_calls=[]) + + def prefetch_subfolders(model, subfolders, *, local_files_only): + loader_state.prefetch_calls.append((model, tuple(subfolders), local_files_only)) + + def from_pretrained_with_prefetch(callable_, model, **kwargs): + kwargs.pop("prefetch_list", None) + return callable_(model, **kwargs) + + def load_transformer_config(model, subfolder, local_files_only): + del model, subfolder, local_files_only + return { + "_class_name": "CausalLingBotWorldTransformer3DModel", + "patch_size": [1, 2, 2], + "in_channels": 36, + "out_channels": 16, + "text_dim": 8, + "num_layers": 2, + "num_attention_heads": 2, + "attention_head_dim": 4, + "num_frames_per_block": 3, + "sliding_window_num_frames": 6, + "local_attn_size": -1, + } + + def retrieve_latents(value, sample_mode="argmax"): + assert sample_mode == "argmax" + return value.latents + + def load_json(model, filename, local_files_only): + del model, local_files_only + assert filename == "scheduler/scheduler_config.json" + return _scheduler_config() + + trajectory = _CameraTrajectory( + poses=torch.eye(4).repeat(32, 1, 1), + intrinsics=torch.tensor([[100.0, 100.0, 8.0, 8.0]]).repeat(32, 1), + ) + + def load_camera_trajectory(action_path): + assert action_path + return trajectory + + def interpolate_camera_trajectory(value, num_frames): + return SimpleNamespace( + poses=value.poses[:num_frames], + intrinsics=value.intrinsics[:num_frames], + ) + + def build_plucker_embedding(value, *, height, width, target_height, target_width, device, dtype): + del target_height, target_width + frames = value.poses.shape[0] + data = torch.arange(frames * 6 * height * width, device=device, dtype=torch.float32) + return data.reshape(frames, 6, height, width).to(dtype=dtype) + + replacements = { + "AutoTokenizer": _FakePretrained, + "UMT5EncoderModel": _FakePretrained, + "AutoWeightsLoader": _AutoWeightsLoader, + "DistributedAutoencoderKLWan": _StubVAE, + "FlowUniPCMultistepScheduler": _FakeScheduler, + "CausalLingBotWorldTransformer3DModel": _FakeTransformerFactory, + "get_local_device": lambda: torch.device("cpu"), + "set_forward_context_denoise_step_idx": lambda index: None, + "prefetch_subfolders": prefetch_subfolders, + "from_pretrained_with_prefetch": from_pretrained_with_prefetch, + "load_transformer_config": load_transformer_config, + "retrieve_latents": retrieve_latents, + "_load_json": load_json, + "load_camera_trajectory": load_camera_trajectory, + "interpolate_camera_trajectory": interpolate_camera_trajectory, + "build_plucker_embedding": build_plucker_embedding, + "randn_tensor": _diffusers_randn_tensor, + } + for name, value in replacements.items(): + monkeypatch.setattr(lingbot_pipeline, name, value) + monkeypatch.setattr(lingbot_pipeline, "_loader_state", loader_state, raising=False) + + +def _od_config(**overrides): + values = { + "model": "checkpoint", + "dtype": torch.float32, + "flow_shift": None, + "quantization_config": None, + "enable_diffusion_pipeline_profiler": False, + "model_config": { + "lingbot_action_root": str(_ROOT), + "ar_diffusion_height": 16, + "ar_diffusion_width": 16, + }, + "enable_cpu_offload": False, + "enable_layerwise_offload": False, + "enforce_eager": True, + "parallel_config": SimpleNamespace( + pipeline_parallel_size=1, + sequence_parallel_size=1, + cfg_parallel_size=1, + vae_patch_parallel_size=1, + use_hsdp=False, + enable_expert_parallel=False, + ), + } + values.update(overrides) + return SimpleNamespace(**values) + + +def _prompt(*, action_path: str | None = None, images=None): + if images is None: + images = Image.new("RGB", (16, 16), color=(255, 128, 0)) + prompt = { + "prompt": "move through the room", + "multi_modal_data": {"image": images}, + "additional_information": {}, + } + if action_path is not None: + prompt["additional_information"]["action_path"] = action_path + return prompt + + +def _pipeline(module, *, transformer=None, od_config=None): + pipeline = module.LingBotWorldCausalDMDPipeline(od_config=od_config or _od_config()) + if transformer is not None: + pipeline.transformer = transformer + pipeline.encode_prompt = lambda *args, **kwargs: torch.ones(1, 512, 8) + pipeline.progress_bar = noop_progress_bar + return pipeline + + +def _preprocess_request(module, *, prompt=None, sampling=None, od_config=None): + request = SimpleNamespace( + prompt=_prompt() if prompt is None else prompt, + sampling_params=_SamplingParams() if sampling is None else sampling, + ) + preprocess = module.get_lingbot_world_pre_process_func(od_config or _od_config()) + return preprocess(request) + + +@pytest.mark.parametrize("offload_field", ["enable_cpu_offload", "enable_layerwise_offload"]) +def test_pipeline_respects_loader_managed_component_placement(offload_field: str) -> None: + module = _load_pipeline_module() + + pipeline = _pipeline(module, od_config=_od_config(**{offload_field: True})) + + assert getattr(pipeline.text_encoder, "to_calls", []) == [] + assert getattr(pipeline.vae, "to_calls", []) == [] + + +def test_ar_diffusion_capability_uses_fixed_tp_local_lingbot_geometry() -> None: + module = _load_pipeline_module() + module.get_tensor_model_parallel_world_size = lambda: 1 + pipeline = _pipeline(module) + + spec = pipeline.ar_diffusion_kv_cache_spec() + + assert spec.num_layers == 2 + assert spec.num_kv_heads == 2 + assert spec.head_size == 4 + assert spec.tokens_per_frame == 1 + assert spec.frames_per_block == 3 + assert spec.window_frames == 3 + assert spec.sink_frames == 3 + assert [(branch.name, branch.local_index) for branch in spec.kv_branches] == [("main", 0)] + assert spec.cross_attention_lengths == {"text": 512} + assert spec.model_owned_state_bytes_per_session == 9_600 + + +def test_preprocess_materializes_external_inputs_before_worker_execution(tmp_path: Path) -> None: + module = _load_pipeline_module() + action_root = tmp_path / "trusted-actions" + action_dir = action_root / "forward" + action_dir.mkdir(parents=True) + image_path = tmp_path / "first-frame.png" + Image.new("RGBA", (16, 16), color=(1, 2, 3, 4)).save(image_path) + sampling = _SamplingParams(extra_args={"action_path": "forward"}) + sampling.extra_args.pop("_lingbot_camera_trajectory") + request = SimpleNamespace( + prompt={"prompt": "move", "multi_modal_data": {"image": str(image_path)}}, + sampling_params=sampling, + ) + trajectory = _CameraTrajectory( + poses=torch.eye(4).repeat(9, 1, 1), + intrinsics=torch.ones(9, 4), + ) + load_calls = [] + module.load_camera_trajectory = lambda action: load_calls.append(action) or trajectory + + preprocess = module.get_lingbot_world_pre_process_func( + _od_config(model_config={"lingbot_action_root": str(action_root)}) + ) + result = preprocess(request) + + assert result is request + assert isinstance(request.prompt["multi_modal_data"]["image"], Image.Image) + assert request.prompt["multi_modal_data"]["image"].mode == "RGB" + assert sampling.extra_args["action_path"] == "forward" + assert sampling.extra_args["_lingbot_camera_trajectory"] is trajectory + assert len(load_calls) == 1 + + +def test_preprocess_materializes_camera_from_typed_tick_without_action_path() -> None: + module = _load_pipeline_module() + poses = torch.eye(4).repeat(9, 1, 1) + intrinsics = torch.tensor([[100.0, 100.0, 8.0, 8.0]]).repeat(9, 1) + tick = ARDiffusionTickRequest( + session_id="world-1", + request_id="tick-request-0", + chunk_index=0, + controls=( + ARDiffusionControlInput( + track="camera", + schema="lingbot.camera_trajectory.v1", + data={ + "poses": poses.tolist(), + "intrinsics": intrinsics.tolist(), + }, + ), + ), + ) + sampling = _SamplingParams(include_action=False) + sampling.extra_args.update(tick.to_extra_args()) + request = SimpleNamespace( + request_id="tick-request-0", + prompt=_prompt(), + sampling_params=sampling, + ) + + result = module.get_lingbot_world_pre_process_func(_od_config())(request) + + trajectory = result.sampling_params.extra_args["_lingbot_camera_trajectory"] + torch.testing.assert_close(trajectory.poses, poses) + torch.testing.assert_close(trajectory.intrinsics, intrinsics) + + +def test_preprocess_materializes_chunk_sized_actions_from_typed_tick() -> None: + module = _load_pipeline_module() + tick = ARDiffusionTickRequest( + session_id="world-actions", + request_id="tick-request-0", + chunk_index=0, + controls=( + ARDiffusionControlInput( + track="camera", + schema="lingbot.camera_actions.v1", + data={ + "mode": "frames", + "frames": [["w"], ["w", "j"], []], + }, + ), + ), + ) + sampling = _SamplingParams(include_action=False) + sampling.extra_args.update(tick.to_extra_args()) + request = SimpleNamespace( + request_id="tick-request-0", + prompt=_prompt(), + sampling_params=sampling, + ) + + result = module.get_lingbot_world_pre_process_func(_od_config())(request) + + assert result.sampling_params.extra_args["_lingbot_camera_trajectory"] is None + assert result.sampling_params.extra_args["_lingbot_camera_actions"] == ( + ("w",), + ("w", "j"), + (), + ) + + +def _request(*, sampling=None, prompt=None, num_reqs: int = 1): + return _RequestBatch( + _prompt() if prompt is None else prompt, + _SamplingParams() if sampling is None else sampling, + num_reqs=num_reqs, + ) + + +def _resolve_pipeline_through_real_registry(pipeline_module): + from vllm_omni.diffusion import registry as registry_module + + resolved = registry_module.DiffusionModelRegistry._try_load_model_cls("LingBotWorldCausalDMDPipeline") + entry = registry_module._DIFFUSION_MODELS["LingBotWorldCausalDMDPipeline"] + cache_acceleration_disabled = "LingBotWorldCausalDMDPipeline" in registry_module._NO_CACHE_ACCELERATION + preprocess_name = registry_module._DIFFUSION_PRE_PROCESS_FUNCS["LingBotWorldCausalDMDPipeline"] + preprocess = registry_module.get_diffusion_pre_process_func( + SimpleNamespace( + model_class_name="LingBotWorldCausalDMDPipeline", + model_config={"lingbot_action_root": str(_ROOT)}, + ) + ) + return resolved, entry, cache_acceleration_disabled, preprocess_name, preprocess + + +def test_component_discovery_uses_official_checkpoint_contract() -> None: + module = _load_pipeline_module() + pipeline = module.LingBotWorldCausalDMDPipeline(od_config=_od_config()) + + assert pipeline._dit_modules == ["transformer"] + assert pipeline._encoder_modules == ["text_encoder"] + assert pipeline._vae_modules == ["vae"] + assert pipeline.dummy_run_num_frames == 0 + assert pipeline.weights_sources == [ + module.DiffusersPipelineLoader.ComponentSource("checkpoint", "transformer", None, "transformer.", True) + ] + assert _FakeTransformerFactory.last_call[1:] == (None, "transformer") + assert pipeline.scheduler.config.shift == 5.0 + assert pipeline.scheduler.config.num_train_timesteps == 1000 + assert module._loader_state.prefetch_calls == [("checkpoint", ("tokenizer", "text_encoder", "vae"), False)] + + +@pytest.mark.parametrize( + ("field", "value", "feature"), + [ + ("pipeline_parallel_size", 2, "pipeline parallelism"), + ("sequence_parallel_size", 2, "sequence parallelism"), + ("cfg_parallel_size", 2, "CFG parallelism"), + ("vae_patch_parallel_size", 2, "VAE parallelism"), + ("use_hsdp", True, "HSDP"), + ("enable_expert_parallel", True, "expert parallelism"), + ], +) +def test_unsupported_parallel_modes_fail_before_component_loading(field: str, value: object, feature: str) -> None: + module = _load_pipeline_module() + parallel_config = _od_config().parallel_config + setattr(parallel_config, field, value) + + with pytest.raises(NotImplementedError, match=feature): + module.LingBotWorldCausalDMDPipeline(od_config=_od_config(parallel_config=parallel_config)) + + assert module._loader_state.prefetch_calls == [] + + +def test_unsupported_quantization_fails_before_component_loading() -> None: + module = _load_pipeline_module() + + with pytest.raises(NotImplementedError, match="quantization"): + module.LingBotWorldCausalDMDPipeline(od_config=_od_config(quantization_config=object())) + + assert module._loader_state.prefetch_calls == [] + + +def test_official_scheduler_config_matches_fixed_dmd_contract() -> None: + import json + + module = _load_pipeline_module() + scheduler_config = json.loads(_SCHEDULER_FIXTURE.read_text()) + module._load_json = lambda *args, **kwargs: scheduler_config.copy() + + pipeline = _pipeline(module) + + assert pipeline.scheduler.config.num_train_timesteps == 1000 + assert pipeline.scheduler.config.shift == 5.0 + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("_class_name", "FlowMatchEulerDiscreteScheduler"), + ("num_train_timesteps", 999), + ("prediction_type", "epsilon"), + ("predict_x0", False), + ("use_flow_sigmas", False), + ("use_dynamic_shifting", True), + ("final_sigmas_type", "sigma_min"), + ], +) +def test_scheduler_config_rejects_semantic_drift(field: str, value: object) -> None: + module = _load_pipeline_module() + module._load_json = lambda *args, **kwargs: _scheduler_config(**{field: value}) + + with pytest.raises(ValueError, match=field): + _pipeline(module) + + +def test_official_model_index_discovers_only_declared_components() -> None: + import json + + module = _load_pipeline_module() + model_index = json.loads(_MODEL_INDEX_FIXTURE.read_text()) + resolved, entry, cache_acceleration_disabled, preprocess_name, preprocess = _resolve_pipeline_through_real_registry( + module + ) + + assert model_index["_class_name"] == "LingBotWorldCausalDMDPipeline" + assert resolved is module.LingBotWorldCausalDMDPipeline + assert entry == ("lingbot_world", "pipeline", "LingBotWorldCausalDMDPipeline") + assert cache_acceleration_disabled + assert preprocess_name == "get_lingbot_world_pre_process_func" + assert callable(preprocess) + assert model_index["tokenizer"] == ["transformers", "T5TokenizerFast"] + assert model_index["text_encoder"] == ["transformers", "UMT5EncoderModel"] + assert model_index["vae"] == ["diffusers", "AutoencoderKLWan"] + assert model_index["scheduler"] == ["diffusers", "UniPCMultistepScheduler"] + assert model_index["transformer"] == ["diffusers", "CausalLingBotWorldTransformer3DModel"] + assert model_index["image_encoder"] == [None, None] + assert model_index["image_processor"] == [None, None] + assert model_index["transformer_2"] == [None, None] + + +def test_postprocess_returns_latents_without_video_conversion() -> None: + module = _load_pipeline_module() + postprocess = module.get_lingbot_world_post_process_func(_od_config()) + latents = torch.randn(1, 16, 3, 2, 2) + + output = postprocess(latents, sampling_params=SimpleNamespace(output_type="latent")) + + assert output is latents + + +def test_postprocess_uses_the_standard_diffusion_output_envelope(monkeypatch) -> None: + import diffusers.video_processor as video_processor_module + + module = _load_pipeline_module() + processed_video = object() + calls = [] + + class VideoProcessor: + def __init__(self, *, vae_scale_factor): + assert vae_scale_factor == 8 + + def postprocess_video(self, video, *, output_type): + calls.append((video, output_type)) + return processed_video + + monkeypatch.setattr(video_processor_module, "VideoProcessor", VideoProcessor) + postprocess = module.get_lingbot_world_post_process_func(_od_config()) + video = torch.randn(1, 3, 9, 8, 8) + + output = postprocess(video, output_type="np") + + assert output == {"payload": {"video": processed_video}, "metadata": {}} + assert calls == [(video, "np")] + + +def test_forward_propagates_pipeline_profiler_stage_durations() -> None: + module = _load_pipeline_module() + pipeline = _pipeline(module) + pipeline._profiler_lock = Lock() + pipeline._stage_durations = {"_generate_block": 0.25, "vae.decode": 0.5} + + output = pipeline(_request()) + + assert output.stage_durations == pipeline.stage_durations + + +def test_pipeline_weight_loader_preserves_component_parameter_prefixes() -> None: + module = _load_pipeline_module() + pipeline = _pipeline(module) + weight = torch.randn(2, 2) + + loaded = pipeline.load_weights([("transformer.blocks.0.weight", weight)]) + + assert loaded == {"transformer.blocks.0.weight"} + assert pipeline.transformer.loaded_weights == [("blocks.0.weight", weight)] + + +def test_request_requires_runner_provided_generator() -> None: + module = _load_pipeline_module() + sampling = _SamplingParams(seed=None, generator=None) + + with pytest.raises(ValueError, match="runner-provided torch.Generator"): + _pipeline(module)._parse_request(_RequestBatch(_prompt(), sampling)) + + +@pytest.mark.parametrize("unsupported", ["generator-list", "caller-latents"]) +def test_request_rejects_unsupported_sampling_state(unsupported: str) -> None: + module = _load_pipeline_module() + sampling = _SamplingParams() + if unsupported == "generator-list": + sampling.generator = [sampling.generator] + message = "generator list" + else: + sampling.latents = torch.zeros(1, 16, 3, 2, 2) + message = "caller-provided latents" + + with pytest.raises(ValueError, match=message): + _pipeline(module)._parse_request(_RequestBatch(_prompt(), sampling)) + + +def test_denoise_state_stays_fp32_while_transformer_inputs_use_model_dtype() -> None: + module = _load_pipeline_module() + transformer = _RecordingTransformer(dtype=torch.bfloat16) + pipeline = _pipeline(module, transformer=transformer) + requested_noise_dtypes: list[torch.dtype] = [] + + def randn(shape, *, generator, device, dtype): + del generator + assert device == pipeline.device + requested_noise_dtypes.append(dtype) + return torch.zeros(shape, device=device, dtype=dtype) + + module.randn_tensor = randn + result = pipeline(_request()) + + assert requested_noise_dtypes == [torch.float32] * 4 + assert result.output.dtype == torch.float32 + assert all(call["hidden_states"].dtype == torch.bfloat16 for call in transformer.calls) + + +def test_path_image_rejects_oversized_source_before_decode_or_convert(monkeypatch, tmp_path: Path) -> None: + module = _load_pipeline_module() + source_path = tmp_path / "oversized-compressed.png" + Image.new("1", (4097, 4097)).save(source_path) + decode_calls: list[str] = [] + + def forbidden_convert(*args, **kwargs): + del args, kwargs + decode_calls.append("convert") + raise AssertionError("oversized source reached convert") + + def forbidden_load(*args, **kwargs): + del args, kwargs + decode_calls.append("load") + raise AssertionError("oversized source reached load") + + monkeypatch.setattr(module.PIL.Image.Image, "convert", forbidden_convert) + monkeypatch.setattr(module.PIL.Image.Image, "load", forbidden_load) + + with pytest.raises(ValueError, match="source image.*4096.*4096"): + _preprocess_request(module, prompt=_prompt(images=str(source_path))) + + assert decode_calls == [] + + +def test_normal_path_image_is_decoded_after_source_size_validation(tmp_path: Path) -> None: + module = _load_pipeline_module() + source_path = tmp_path / "normal.png" + Image.new("RGBA", (32, 24), color=(1, 2, 3, 128)).save(source_path) + + request = _preprocess_request(module, prompt=_prompt(images=str(source_path))) + parsed = _pipeline(module)._parse_request(_RequestBatch(request.prompt, request.sampling_params)) + + assert module._MAX_SOURCE_IMAGE_PIXELS == 4096 * 4096 + assert isinstance(parsed.image, Image.Image) + assert parsed.image.mode == "RGB" + assert parsed.image.size == (32, 24) + + +@pytest.mark.parametrize( + ("source_size", "expected_size"), + [ + ((832, 480), (464, 832)), + ((480, 832), (832, 480)), + ((512, 512), (624, 624)), + ], +) +def test_default_resolution_matches_official_480p_aspect_derivation(source_size, expected_size) -> None: + module = _load_pipeline_module() + sampling = _SamplingParams(height=None, width=None) + + parsed = _pipeline(module)._parse_request(_RequestBatch(_prompt(images=Image.new("RGB", source_size)), sampling)) + + assert (parsed.height, parsed.width) == expected_size + assert parsed.height % 16 == 0 + assert parsed.width % 16 == 0 + assert parsed.height * parsed.width <= 480 * 832 + + +def test_pil_and_tensor_images_share_official_bicubic_preprocess() -> None: + module = _load_pipeline_module() + height, width = 7, 11 + source = torch.arange(3 * height * width, dtype=torch.uint8).reshape(3, height, width) + pil_image = Image.fromarray(source.permute(1, 2, 0).numpy(), mode="RGB") + sampling = _SamplingParams(height=16, width=32) + + pil_pipeline = _pipeline(module) + tensor_pipeline = _pipeline(module) + pil_inputs = pil_pipeline._parse_request(_RequestBatch(_prompt(images=pil_image), sampling)) + tensor_inputs = tensor_pipeline._parse_request( + _RequestBatch(_prompt(images=source.clone()), _SamplingParams(height=16, width=32)) + ) + pil_condition = pil_pipeline._prepare_image_tensor(pil_inputs.image, height=16, width=32) + tensor_condition = tensor_pipeline._prepare_image_tensor(tensor_inputs.image, height=16, width=32) + expected = torch.nn.functional.interpolate( + source.unsqueeze(0).float() / 255.0, + size=(16, 32), + mode="bicubic", + align_corners=False, + ) + expected = expected * 2.0 - 1.0 + + torch.testing.assert_close(pil_condition, expected) + torch.testing.assert_close(tensor_condition, expected) + + +def test_path_image_decode_error_is_sanitized(monkeypatch) -> None: + module = _load_pipeline_module() + unsafe_path = "/private/source/customer-secret.png" + monkeypatch.setattr( + module.PIL.Image, + "open", + lambda path: (_ for _ in ()).throw(module.PIL.Image.DecompressionBombError(f"unsafe {path}")), + ) + + with pytest.raises(ValueError, match="Unable to load multi_modal_data.image") as exc_info: + _preprocess_request(module, prompt=_prompt(images=unsafe_path)) + + assert unsafe_path not in str(exc_info.value) + assert exc_info.value.__cause__ is None + + +@pytest.mark.parametrize("source_size", [(4096, 4096), (4097, 4097)]) +def test_supplied_pil_image_obeys_documented_source_pixel_ceiling(source_size) -> None: + module = _load_pipeline_module() + source_image = Image.new("1", source_size) + close_calls: list[bool] = [] + source_image.close = lambda: close_calls.append(True) + + if source_size[0] * source_size[1] <= 4096 * 4096: + request = _preprocess_request(module, prompt=_prompt(images=source_image)) + parsed = _pipeline(module)._parse_request(_RequestBatch(request.prompt, request.sampling_params)) + assert parsed.image is not source_image + assert parsed.image.mode == "RGB" + assert close_calls == [] + else: + with pytest.raises(ValueError, match="source image.*4096.*4096"): + _preprocess_request(module, prompt=_prompt(images=source_image)) + assert close_calls == [] + + +def test_supplied_pil_decode_error_is_sanitized_without_closing_caller(monkeypatch) -> None: + module = _load_pipeline_module() + source_image = Image.new("RGB", (16, 16)) + unsafe_detail = "/private/source/customer-secret.png" + close_calls: list[bool] = [] + monkeypatch.setattr(source_image, "convert", lambda mode: (_ for _ in ()).throw(OSError(unsafe_detail))) + monkeypatch.setattr(source_image, "close", lambda: close_calls.append(True)) + + with pytest.raises(ValueError, match="Unable to load multi_modal_data.image") as exc_info: + _preprocess_request(module, prompt=_prompt(images=source_image)) + + assert unsafe_detail not in str(exc_info.value) + assert exc_info.value.__cause__ is None + assert close_calls == [] + + +def test_action_path_resolves_from_sampling_extra_args(tmp_path: Path) -> None: + module = _load_pipeline_module() + action_root = tmp_path / "trusted-actions" + contained_action = action_root / "actions" + contained_action.mkdir(parents=True) + sampling = _SamplingParams(extra_args={"action_path": "actions"}) + prompt = _prompt() + original_prompt = prompt.copy() + trajectory = _CameraTrajectory(torch.eye(4).repeat(9, 1, 1), torch.ones(9, 4)) + resolved_actions = [] + module.load_camera_trajectory = lambda action: resolved_actions.append(action) or trajectory + + request = _preprocess_request( + module, + prompt=prompt, + sampling=sampling, + od_config=_od_config(model_config={"lingbot_action_root": str(action_root)}), + ) + parsed = _pipeline(module)._parse_request(_RequestBatch(request.prompt, request.sampling_params)) + + assert resolved_actions[0].root == action_root.resolve() + assert resolved_actions[0].relative == Path("actions") + assert parsed.camera_trajectory is trajectory + assert sampling.extra_args["action_path"] == "actions" + assert sampling.extra_args["_lingbot_camera_trajectory"] is trajectory + assert prompt == original_prompt + + +@pytest.mark.parametrize("with_extra", [False, True], ids=["additional-only", "extra-and-additional"]) +def test_additional_information_action_path_is_not_an_input_source(with_extra: bool) -> None: + module = _load_pipeline_module() + sampling = _SamplingParams( + extra_args={"action_path": "."} if with_extra else None, + include_action=with_extra, + ) + + with pytest.raises(ValueError, match="additional_information.*action_path.*not supported"): + _preprocess_request(module, prompt=_prompt(action_path="legacy-actions"), sampling=sampling) + + +def test_action_path_is_required_in_sampling_extra_args() -> None: + module = _load_pipeline_module() + + with pytest.raises(ValueError, match="sampling_params.extra_args.action_path"): + _preprocess_request(module, sampling=_SamplingParams(include_action=False)) + + +@pytest.mark.parametrize("escape_kind", ["traversal", "absolute", "symlink"]) +def test_online_action_path_rejects_escape_from_trusted_root(escape_kind: str, tmp_path: Path) -> None: + module = _load_pipeline_module() + root = tmp_path / "trusted" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + if escape_kind == "traversal": + action_path = "../outside" + elif escape_kind == "absolute": + action_path = str(outside) + else: + (root / "escape-link").symlink_to(outside, target_is_directory=True) + action_path = "escape-link" + with pytest.raises(ValueError, match="trusted.*root|contained"): + _preprocess_request( + module, + sampling=_SamplingParams(extra_args={"action_path": action_path}), + od_config=_od_config(model_config={"lingbot_action_root": str(root)}), + ) + + +def test_online_action_path_uses_environment_root_fallback(monkeypatch, tmp_path: Path) -> None: + module = _load_pipeline_module() + root = tmp_path / "trusted" + action_dir = root / "forward" + action_dir.mkdir(parents=True) + monkeypatch.setenv("VLLM_OMNI_LINGBOT_ACTION_ROOT", str(root)) + resolved_actions = [] + module.load_camera_trajectory = lambda action: ( + resolved_actions.append(action) or _CameraTrajectory(torch.eye(4).repeat(9, 1, 1), torch.ones(9, 4)) + ) + + request = _preprocess_request( + module, + sampling=_SamplingParams(extra_args={"action_path": "forward"}), + od_config=_od_config(model_config={}), + ) + + assert request.sampling_params.extra_args["_lingbot_camera_trajectory"] is not None + assert resolved_actions[0].root == root.resolve() + assert resolved_actions[0].relative == Path("forward") + + +def test_online_action_path_error_suppresses_path_bearing_filesystem_cause(tmp_path: Path) -> None: + module = _load_pipeline_module() + root = tmp_path / "trusted" + root.mkdir() + + with pytest.raises(ValueError, match="trusted action root") as exc_info: + _preprocess_request( + module, + sampling=_SamplingParams(extra_args={"action_path": "does-not-exist"}), + od_config=_od_config(model_config={"lingbot_action_root": str(root)}), + ) + + assert str(root) not in str(exc_info.value) + assert exc_info.value.__cause__ is None + + +@pytest.mark.parametrize("source", ["root", "candidate"]) +def test_online_action_path_unknown_user_is_sanitized(source: str, tmp_path: Path) -> None: + module = _load_pipeline_module() + unknown_user_path = "~__vllm_omni_user_that_does_not_exist__/actions" + root = tmp_path / "trusted" + root.mkdir() + configured_root = unknown_user_path if source == "root" else str(root) + action_path = "actions" if source == "root" else unknown_user_path + + with pytest.raises(ValueError, match="trusted action root") as exc_info: + _preprocess_request( + module, + sampling=_SamplingParams(extra_args={"action_path": action_path}), + od_config=_od_config(model_config={"lingbot_action_root": configured_root}), + ) + + assert "__vllm_omni_user_that_does_not_exist__" not in str(exc_info.value) + assert exc_info.value.__cause__ is None + + +def test_action_path_requires_a_trusted_root_for_every_request() -> None: + module = _load_pipeline_module() + + with pytest.raises(ValueError, match="lingbot_action_root|VLLM_OMNI_LINGBOT_ACTION_ROOT"): + _preprocess_request(module, od_config=_od_config(model_config={})) + + +@pytest.mark.parametrize( + ("request_batch", "message"), + [ + (_request(num_reqs=2), "single prompt"), + (_request(prompt=_prompt(images=[Image.new("RGB", (16, 16))])), "image.*list"), + (_request(prompt=_prompt(images=[])), "image.*list"), + ( + _request(prompt=_prompt(images=[Image.new("RGB", (16, 16)), Image.new("RGB", (16, 16))])), + "image.*list", + ), + (_request(sampling=_SamplingParams(num_outputs_per_prompt=2)), "num_outputs_per_prompt"), + (_request(sampling=_SamplingParams(num_inference_steps=5)), "num_inference_steps"), + (_request(sampling=_SamplingParams(height=15)), "height.*divisible"), + (_request(sampling=_SamplingParams(width=15)), "width.*divisible"), + (_request(sampling=_SamplingParams(height=None, width=16)), "height and width.*both"), + (_request(sampling=_SamplingParams(num_frames=13)), "num_frames.*three-frame"), + ], +) +def test_request_validation_rejects_unsupported_contracts(request_batch, message: str) -> None: + module = _load_pipeline_module() + + with pytest.raises(ValueError, match=message): + _pipeline(module)._parse_request(request_batch) + + +def test_resource_limits_accept_exact_documented_boundaries() -> None: + module = _load_pipeline_module() + sampling = _SamplingParams(height=480, width=832, num_frames=117, max_sequence_length=512) + + parsed = _pipeline(module)._parse_request(_RequestBatch(_prompt(), sampling)) + + assert (parsed.height, parsed.width, parsed.num_frames, parsed.max_sequence_length) == (480, 832, 117, 512) + + +@pytest.mark.parametrize( + ("sampling", "message"), + [ + (_SamplingParams(height=480, width=848), "pixel area|480.*832"), + (_SamplingParams(num_frames=129), "num_frames.*117"), + (_SamplingParams(max_sequence_length=511), "max_sequence_length.*512"), + (_SamplingParams(max_sequence_length=513), "max_sequence_length.*512"), + (_SamplingParams(max_sequence_length=512.0), "max_sequence_length.*512"), + ], +) +def test_resource_limits_reject_oversize_before_any_component_call(sampling, message: str) -> None: + module = _load_pipeline_module() + pipeline = _pipeline(module) + calls: list[str] = [] + pipeline.encode_prompt = lambda *args, **kwargs: calls.append("text") + pipeline._prepare_condition = lambda *args, **kwargs: calls.append("vae") + pipeline._prepare_camera = lambda *args, **kwargs: calls.append("camera") + pipeline.transformer.allocate_cache = lambda *args, **kwargs: calls.append("cache") + + with pytest.raises(ValueError, match=message): + pipeline(_RequestBatch(_prompt(), sampling)) + + assert calls == [] + + +def test_request_validation_rejects_insufficient_camera_frames() -> None: + module = _load_pipeline_module() + pipeline = _pipeline(module) + short_trajectory = _CameraTrajectory( + poses=torch.eye(4).repeat(8, 1, 1), + intrinsics=torch.ones(8, 4), + ) + sampling = _SamplingParams(extra_args={"_lingbot_camera_trajectory": short_trajectory}) + + with pytest.raises(ValueError, match="camera.*frames.*num_frames"): + pipeline(_request(sampling=sampling)) + + +def test_camera_load_error_is_actionable_without_echoing_path_contents() -> None: + module = _load_pipeline_module() + module.load_camera_trajectory = lambda path: (_ for _ in ()).throw(FileNotFoundError(f"missing {path}/poses.npy")) + + with pytest.raises(ValueError, match="camera trajectory.*action_path") as exc_info: + _preprocess_request(module) + + assert str(_ROOT) not in str(exc_info.value) + assert exc_info.value.__cause__ is None + + +def test_first_frame_condition_and_camera_fold_match_transformer_contract() -> None: + module = _load_pipeline_module() + transformer = _RecordingTransformer() + pipeline = _pipeline(module, transformer=transformer) + module.randn_tensor = lambda shape, **kwargs: torch.full( + shape, + -99.0, + device=kwargs["device"], + dtype=kwargs["dtype"], + ) + + result = pipeline(_request()) + + assert result.output.shape == (1, 16, 3, 2, 2) + first_input = transformer.calls[0]["hidden_states"] + assert first_input.shape == (1, 36, 3, 2, 2) + mean = torch.tensor(pipeline.vae.config.latents_mean).view(1, 16, 1, 1) + std = torch.tensor(pipeline.vae.config.latents_std).view(1, 16, 1, 1) + expected_first = (torch.full((1, 16, 2, 2), 2.0) - mean) / std + expected_future = (torch.zeros(1, 16, 2, 2, 2) - mean.unsqueeze(2)) / std.unsqueeze(2) + torch.testing.assert_close(first_input[:, :16], torch.full((1, 16, 3, 2, 2), -99.0)) + torch.testing.assert_close(first_input[:, 16:20, 0], torch.ones(1, 4, 2, 2)) + torch.testing.assert_close(first_input[:, 16:20, 1:], torch.zeros(1, 4, 2, 2, 2)) + torch.testing.assert_close(first_input[:, 20:36, 0], expected_first) + torch.testing.assert_close(first_input[:, 20:36, 1:], expected_future) + + raw_camera = module.build_plucker_embedding( + SimpleNamespace(poses=torch.empty(3, 4, 4)), + height=16, + width=16, + target_height=16, + target_width=16, + device=torch.device("cpu"), + dtype=torch.float32, + ) + expected_camera = module._fold_camera_embedding(raw_camera) + reference_camera = torch.nn.functional.pixel_unshuffle(raw_camera, 8).permute(1, 0, 2, 3).unsqueeze(0) + torch.testing.assert_close(expected_camera, reference_camera) + torch.testing.assert_close(transformer.calls[0]["camera_hidden_states"], expected_camera) + assert expected_camera.shape == (1, 384, 3, 2, 2) + + +def test_fixed_dmd_transition_and_cache_commit_trace() -> None: + module = _load_pipeline_module() + transformer = _RecordingTransformer() + pipeline = _pipeline(module, transformer=transformer) + + result = pipeline(_request()) + + generator = torch.Generator(device="cpu").manual_seed(17) + current = torch.randn((1, 16, 3, 2, 2), generator=generator) + warped_schedule = ( + (1000.0, 1.0), + (937.5, 0.9375), + (2500.0 / 3.0, 5.0 / 6.0), + (625.0, 0.625), + ) + for index, (_, sigma) in enumerate(warped_schedule): + x0 = current - sigma + if index + 1 < len(module.LINGBOT_DMD_TIMESTEPS): + next_sigma = warped_schedule[index + 1][1] + noise = torch.randn(current.shape, generator=generator) + current = (1.0 - next_sigma) * x0 + next_sigma * noise + else: + current = x0 + torch.testing.assert_close(result.output, current) + + torch.testing.assert_close( + torch.cat([call["timestep"] for call in transformer.calls]), + torch.tensor([*(timestep for timestep, _ in warped_schedule), 0.0]), + ) + assert [call["update_cache"] for call in transformer.calls] == [False, False, False, False, True] + assert [call["start_frame"] for call in transformer.calls] == [0, 0, 0, 0, 0] + assert len({call["cache_id"] for call in transformer.calls}) == 1 + torch.testing.assert_close(transformer.calls[-1]["hidden_states"][:, :16], result.output) + + +@pytest.mark.parametrize( + ("flow_shift", "expected_timesteps"), + [ + (5.0, (1000.0, 937.5, 2500.0 / 3.0, 625.0)), + (2.5, (1000.0, 15000.0 / 17.0, 5000.0 / 7.0, 5000.0 / 11.0)), + ], +) +def test_request_flow_shift_warps_transformer_timesteps(flow_shift: float, expected_timesteps) -> None: + module = _load_pipeline_module() + transformer = _RecordingTransformer() + pipeline = _pipeline(module, transformer=transformer) + + pipeline( + _request( + sampling=_SamplingParams(extra_args={"action_path": ".", "flow_shift": flow_shift}), + ) + ) + + torch.testing.assert_close( + torch.cat([call["timestep"] for call in transformer.calls[:4]]), + torch.tensor(expected_timesteps), + ) + + +def test_request_tiny_positive_flow_shift_keeps_schedule_finite() -> None: + module = _load_pipeline_module() + transformer = _RecordingTransformer() + pipeline = _pipeline(module, transformer=transformer) + + pipeline( + _request( + sampling=_SamplingParams(extra_args={"action_path": ".", "flow_shift": 1e-20}), + ) + ) + + timesteps = torch.cat([call["timestep"] for call in transformer.calls[:4]]) + assert torch.isfinite(timesteps).all() + assert timesteps[0].item() == 1000.0 + + +def test_request_flow_shift_override_has_precedence_without_mutating_scheduler() -> None: + module = _load_pipeline_module() + pipeline = _pipeline(module, od_config=_od_config(flow_shift=3.0)) + + default_inputs = pipeline._parse_request(_request()) + sampling = _SamplingParams(extra_args={"flow_shift": 2.0}) + override_inputs = pipeline._parse_request(_RequestBatch(_prompt(), sampling)) + + assert default_inputs.flow_shift == 3.0 + assert override_inputs.flow_shift == 2.0 + assert sampling.extra_args["action_path"] == "." + assert sampling.extra_args["flow_shift"] == 2.0 + assert pipeline.scheduler.config.shift == 3.0 + + +def test_checkpoint_flow_shift_default_and_engine_request_override_precedence() -> None: + module = _load_pipeline_module() + checkpoint_scheduler_config = _scheduler_config(flow_shift=6.25, shift=7.5) + module._load_json = lambda *args, **kwargs: checkpoint_scheduler_config.copy() + + checkpoint_default = _pipeline(module) + engine_override = _pipeline(module, od_config=_od_config(flow_shift=3.5)) + request_override = checkpoint_default._parse_request( + _RequestBatch( + _prompt(), + _SamplingParams(extra_args={"flow_shift": 2.25}), + ) + ) + + assert checkpoint_default.scheduler.config.shift == 6.25 + assert checkpoint_default._parse_request(_request()).flow_shift == 6.25 + assert engine_override.scheduler.config.shift == 3.5 + assert engine_override._parse_request(_request()).flow_shift == 3.5 + assert request_override.flow_shift == 2.25 + + +@pytest.mark.parametrize( + ("scheduler_config", "expected"), + [ + ({"flow_shift": 6.0, "shift": 7.0}, 6.0), + ({"shift": 7.0}, 7.0), + ({}, 5.0), + ], +) +def test_checkpoint_scheduler_shift_fallback_order(scheduler_config, expected: float) -> None: + module = _load_pipeline_module() + checkpoint_config = _scheduler_config() + checkpoint_config.pop("flow_shift") + checkpoint_config.update(scheduler_config) + module._load_json = lambda *args, **kwargs: checkpoint_config.copy() + + pipeline = _pipeline(module) + + assert pipeline.scheduler.config.shift == expected + + +@pytest.mark.parametrize("flow_shift", [0, -1, float("nan"), float("inf"), "invalid"]) +def test_request_flow_shift_must_be_positive_and_finite(flow_shift) -> None: + module = _load_pipeline_module() + sampling = _SamplingParams(extra_args={"flow_shift": flow_shift}) + + with pytest.raises(ValueError, match="flow_shift.*positive.*finite"): + _pipeline(module)._parse_request(_RequestBatch(_prompt(), sampling)) + + +@pytest.mark.parametrize( + "flow_shift", + [True, False, 10**10000], + ids=["true", "false", "overflowing-integer"], +) +def test_request_flow_shift_rejects_booleans_and_normalizes_integer_overflow(flow_shift) -> None: + module = _load_pipeline_module() + sampling = _SamplingParams(extra_args={"flow_shift": flow_shift}) + + with pytest.raises(ValueError, match="flow_shift.*positive.*finite"): + _pipeline(module)._parse_request(_RequestBatch(_prompt(), sampling)) + + +def test_flow_shift_is_request_local_without_mutating_scheduler() -> None: + module = _load_pipeline_module() + pipeline = _pipeline(module) + + def generate(flow_shift=None): + extra_args = {} if flow_shift is None else {"flow_shift": flow_shift} + return pipeline( + _RequestBatch( + _prompt(), + _SamplingParams(num_frames=21, extra_args=extra_args), + ) + ).output + + default_before = generate() + shifted_two = generate(2.0) + shifted_seven = generate(7.0) + default_after = generate() + + torch.testing.assert_close(default_before, default_after) + assert not torch.equal(default_before, shifted_two) + assert not torch.equal(shifted_two, shifted_seven) + assert pipeline.scheduler.config.shift == 5.0 + + +def test_encode_prompt_zeroes_padded_umt5_states_to_exactly_512_tokens() -> None: + module = _load_pipeline_module() + pipeline = _pipeline(module) + attention_mask = torch.zeros(1, 512, dtype=torch.long) + attention_mask[:, :3] = 1 + pipeline.tokenizer = lambda *args, **kwargs: SimpleNamespace( + input_ids=torch.arange(512).view(1, 512), + attention_mask=attention_mask, + ) + raw_states = torch.arange(512 * 8, dtype=torch.float32).view(1, 512, 8) + 1.0 + pipeline.text_encoder = lambda input_ids, mask: SimpleNamespace(last_hidden_state=raw_states.clone()) + + encoded = module.LingBotWorldCausalDMDPipeline.encode_prompt( + pipeline, + "move", + max_sequence_length=512, + dtype=torch.float32, + ) + + assert encoded.shape == (1, 512, 8) + torch.testing.assert_close(encoded[:, :3], raw_states[:, :3]) + torch.testing.assert_close(encoded[:, 3:], torch.zeros_like(encoded[:, 3:])) + + +def test_multi_chunk_generation_uses_one_request_local_cache_and_decodes_accumulated_latents() -> None: + module = _load_pipeline_module() + transformer = _RecordingTransformer() + pipeline = _pipeline(module, transformer=transformer) + allocations = [] + original_allocate = transformer.allocate_cache + + def allocate(**kwargs): + cache = original_allocate(**kwargs) + allocations.append((kwargs, weakref.ref(cache))) + return cache + + transformer.allocate_cache = allocate + sampling = _SamplingParams(num_frames=21, output_type="np") + + result = pipeline(_request(sampling=sampling)) + + assert result.output.shape == (1, 3, 21, 16, 16) + assert len(transformer.calls) == 10 + assert [call["start_frame"] for call in transformer.calls] == [0] * 5 + [3] * 5 + expected_timesteps = torch.tensor([1000.0, 937.5, 2500.0 / 3.0, 625.0, 0.0] * 2) + torch.testing.assert_close( + torch.cat([call["timestep"] for call in transformer.calls]), + expected_timesteps, + ) + assert len(allocations) == 1 + kwargs, cache_ref = allocations[0] + assert kwargs == { + "batch_size": 1, + "latent_height": 2, + "latent_width": 2, + "device": torch.device("cpu"), + "dtype": torch.float32, + } + assert not hasattr(pipeline, "cache") + assert not hasattr(pipeline, "transformer_cache") + assert pipeline.vae.decode_inputs[0].shape == (1, 16, 6, 2, 2) + normalized_latents = torch.cat( + (transformer.calls[4]["hidden_states"][:, :16], transformer.calls[9]["hidden_states"][:, :16]), + dim=2, + ) + mean = torch.tensor(pipeline.vae.config.latents_mean).view(1, 16, 1, 1, 1) + std = torch.tensor(pipeline.vae.config.latents_std).view(1, 16, 1, 1, 1) + torch.testing.assert_close(pipeline.vae.decode_inputs[0], normalized_latents * std + mean) + transformer.calls.clear() + gc.collect() + assert cache_ref() is None + + +def _tick_extra_args(*, chunk_index: int, prompt: str = "move through the room"): + return ARDiffusionTickRequest( + session_id="world-1", + request_id="legacy-lingbot-request", + chunk_index=chunk_index, + applied_event_ids=(chunk_index,), + prompt=prompt, + controls=( + ARDiffusionControlInput( + track="camera", + schema="lingbot.camera_trajectory.v1", + data={"poses": [], "intrinsics": []}, + ), + ), + ).to_extra_args() + + +def test_typed_ticks_generate_one_global_block_and_return_standard_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_pipeline_module() + pipeline = _pipeline(module) + pipeline._ar_height = 16 + pipeline._ar_width = 16 + pipeline._ar_diffusion_kv_state = object() + cross_calls = [] + generated = [] + + def ar_text_caches(prompt_embeds, *, invalidate): + del prompt_embeds + cross_calls.append(invalidate) + return [SimpleNamespace()] + + def generate_block(**kwargs): + block = torch.randn( + (1, 16, 3, 2, 2), + generator=kwargs["generator"], + ) + generated.append( + { + "start_frame": kwargs["start_frame"], + "condition": kwargs["condition"].clone(), + "block": block.clone(), + } + ) + return block + + monkeypatch.setattr(pipeline, "_ar_text_caches", ar_text_caches) + monkeypatch.setattr(pipeline, "_generate_block", generate_block) + + first_sampling = _SamplingParams(extra_args=_tick_extra_args(chunk_index=0)) + first = pipeline(_request(sampling=first_sampling)) + switched_prompt = _prompt() + switched_prompt["prompt"] = "enter the snowy valley" + second_sampling = _SamplingParams( + extra_args=_tick_extra_args( + chunk_index=1, + prompt=switched_prompt["prompt"], + ) + ) + second = pipeline(_request(sampling=second_sampling, prompt=switched_prompt)) + + assert generated[0]["start_frame"] == 0 + assert generated[1]["start_frame"] == 3 + assert torch.count_nonzero(generated[0]["condition"]) > 0 + torch.testing.assert_close( + generated[1]["condition"][:, :4], + torch.zeros_like(generated[1]["condition"][:, :4]), + ) + assert torch.count_nonzero(generated[1]["condition"][:, 4:]) > 0 + assert cross_calls == [False, True] + assert first.output["payload"]["latents"].shape == (1, 16, 3, 2, 2) + assert second.output["metadata"]["ar_diffusion"] == { + "session_id": "world-1", + "request_id": "legacy-lingbot-request", + "chunk_index": 1, + "applied_event_ids": [1], + } + assert pipeline._ar_sessions["world-1"].next_chunk_index == 2 + assert pipeline._ar_sessions["world-1"].prompt == "enter the snowy valley" + assert pipeline._ar_sessions["world-1"].camera_tail is not None + assert pipeline._ar_sessions["world-1"].camera_tail.poses.shape == (1, 4, 4) + expected_generator = torch.Generator(device="cpu").manual_seed(17) + expected_first = torch.randn((1, 16, 3, 2, 2), generator=expected_generator) + expected_second = torch.randn((1, 16, 3, 2, 2), generator=expected_generator) + torch.testing.assert_close(generated[0]["block"], expected_first) + torch.testing.assert_close(generated[1]["block"], expected_second) + + +def test_typed_action_ticks_integrate_camera_across_chunks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_pipeline_module() + pipeline = _pipeline(module) + pipeline._ar_height = 16 + pipeline._ar_width = 16 + pipeline._ar_diffusion_kv_state = object() + monkeypatch.setattr( + pipeline, + "_ar_text_caches", + lambda *args, **kwargs: [SimpleNamespace()], + ) + monkeypatch.setattr( + pipeline, + "_generate_block", + lambda **kwargs: torch.zeros_like(kwargs["condition"][:, :16]), + ) + + def action_sampling(chunk_index: int, frames: list[list[str]]) -> _SamplingParams: + tick = ARDiffusionTickRequest( + session_id="world-actions", + request_id="legacy-lingbot-request", + chunk_index=chunk_index, + controls=( + ARDiffusionControlInput( + track="camera", + schema="lingbot.camera_actions.v1", + data={"mode": "frames", "frames": frames}, + ), + ), + ) + sampling = _SamplingParams(extra_args=tick.to_extra_args()) + sampling.extra_args["_lingbot_camera_trajectory"] = None + sampling.extra_args["_lingbot_camera_actions"] = tuple(tuple(frame) for frame in frames) + return sampling + + pipeline(_request(sampling=action_sampling(0, [["w"], ["w"], ["w"]]))) + first_tail = pipeline._ar_sessions["world-actions"].camera_tail.poses.clone() + pipeline(_request(sampling=action_sampling(1, [["d"], ["d"], ["d"]]))) + state = pipeline._ar_sessions["world-actions"] + + assert state.next_chunk_index == 2 + assert state.camera_tail is not None + torch.testing.assert_close(state.camera_tail.poses[0, 2, 3], first_tail[0, 2, 3]) + assert state.camera_tail.poses[0, 0, 3] > first_tail[0, 0, 3] + + +def test_first_typed_yaw_action_uses_pre_action_identity_anchor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_pipeline_module() + pipeline = _pipeline(module) + monkeypatch.setattr( + module, + "build_plucker_embedding", + _real_build_plucker_embedding, + ) + action_trajectory, _ = integrate_lingbot_camera_actions( + [["j"], [], []], + width=16, + height=16, + ) + neutral_trajectory, _ = integrate_lingbot_camera_actions( + [[], [], []], + width=16, + height=16, + ) + + def inputs(trajectory, actions): + return SimpleNamespace( + camera_trajectory=trajectory, + camera_actions=actions, + num_latent_frames=3, + num_frames=9, + height=16, + width=16, + ) + + action_embedding, tail = pipeline._prepare_camera( + inputs(action_trajectory, (("j",), (), ())), + dtype=torch.float32, + ) + neutral_embedding, _ = pipeline._prepare_camera( + inputs(neutral_trajectory, ((), (), ())), + dtype=torch.float32, + ) + + identity_anchor = torch.eye(4, dtype=action_trajectory.poses.dtype).unsqueeze(0) + explicit_trajectory = _CameraTrajectory( + poses=torch.cat((identity_anchor, action_trajectory.poses), dim=0), + intrinsics=torch.cat( + (action_trajectory.intrinsics[:1], action_trajectory.intrinsics), + dim=0, + ), + ) + explicit = _real_build_plucker_embedding( + explicit_trajectory, + height=16, + width=16, + target_height=16, + target_width=16, + device=torch.device("cpu"), + dtype=torch.float32, + )[1:] + + assert not torch.equal(action_embedding, neutral_embedding) + torch.testing.assert_close(action_embedding, module._fold_camera_embedding(explicit)) + torch.testing.assert_close(tail.poses, action_trajectory.poses[-1:]) + + +def test_typed_tick_rejects_non_contiguous_chunk_index( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_pipeline_module() + pipeline = _pipeline(module) + pipeline._ar_height = 16 + pipeline._ar_width = 16 + pipeline._ar_diffusion_kv_state = object() + monkeypatch.setattr( + pipeline, + "_ar_text_caches", + lambda *args, **kwargs: [SimpleNamespace()], + ) + + sampling = _SamplingParams(extra_args=_tick_extra_args(chunk_index=2)) + with pytest.raises(ValueError, match="must be contiguous"): + pipeline(_request(sampling=sampling)) + + +def test_typed_tick_rejects_chunk_beyond_realtime_condition_horizon( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_pipeline_module() + pipeline = _pipeline(module) + pipeline._ar_height = 16 + pipeline._ar_width = 16 + pipeline._ar_diffusion_kv_state = object() + pipeline._ar_sessions["world-1"] = module._LingBotARSessionState( + next_chunk_index=10, + ) + monkeypatch.setattr( + pipeline, + "encode_prompt", + lambda *args, **kwargs: pytest.fail("the realtime horizon must be validated before prompt encoding"), + ) + + sampling = _SamplingParams(extra_args=_tick_extra_args(chunk_index=10)) + with pytest.raises( + ValueError, + match=r"at most 10 ticks.*chunk_index 0 through 9.*117 pixel frames", + ): + pipeline(_request(sampling=sampling)) + + +def test_request_cache_is_released_before_vae_decode() -> None: + module = _load_pipeline_module() + transformer = _RecordingTransformer() + pipeline = _pipeline(module, transformer=transformer) + cache_refs: list[weakref.ReferenceType] = [] + original_allocate = transformer.allocate_cache + + def allocate(**kwargs): + cache = original_allocate(**kwargs) + cache_refs.append(weakref.ref(cache)) + return cache + + transformer.allocate_cache = allocate + + def assert_cache_released() -> None: + gc.collect() + assert len(cache_refs) == 1 + assert cache_refs[0]() is None + + pipeline.vae.on_decode = assert_cache_released + + result = pipeline(_request(sampling=_SamplingParams(num_frames=21, output_type="np"))) + + assert result.output.shape[2] == 21 + + +def test_117_frame_request_generates_ten_complete_latent_blocks() -> None: + module = _load_pipeline_module() + transformer = _RecordingTransformer() + pipeline = _pipeline(module, transformer=transformer) + trajectory = _CameraTrajectory( + poses=torch.eye(4).repeat(117, 1, 1), + intrinsics=torch.tensor([[100.0, 100.0, 8.0, 8.0]]).repeat(117, 1), + ) + sampling = _SamplingParams( + num_frames=117, + output_type="latent", + extra_args={"_lingbot_camera_trajectory": trajectory}, + ) + + result = pipeline(_request(sampling=sampling)) + + assert result.output.shape == (1, 16, 30, 2, 2) + assert len(transformer.calls) == 50 + assert [call["start_frame"] for call in transformer.calls[::5]] == list(range(0, 30, 3)) + + +def test_request_cache_becomes_unreachable_after_transformer_error() -> None: + module = _load_pipeline_module() + transformer = _RecordingTransformer(raise_on_call=2) + pipeline = _pipeline(module, transformer=transformer) + cache_refs = [] + original_allocate = transformer.allocate_cache + + def allocate(**kwargs): + cache = original_allocate(**kwargs) + cache_refs.append(weakref.ref(cache)) + return cache + + transformer.allocate_cache = allocate + + with pytest.raises(RuntimeError, match="forced transformer failure") as exc_info: + pipeline(_request()) + + exc_info.value.__traceback__ = None + del exc_info + transformer.calls.clear() + gc.collect() + assert len(cache_refs) == 1 + assert cache_refs[0]() is None + assert not hasattr(pipeline, "cache") + assert not hasattr(pipeline, "transformer_cache") + + +def test_registry_and_model_exports_resolve_official_pipeline_class_name() -> None: + module = _load_pipeline_module() + resolved, entry, cache_acceleration_disabled, preprocess_name, preprocess = _resolve_pipeline_through_real_registry( + module + ) + + assert entry == ("lingbot_world", "pipeline", "LingBotWorldCausalDMDPipeline") + assert resolved is module.LingBotWorldCausalDMDPipeline + assert cache_acceleration_disabled + assert preprocess_name == "get_lingbot_world_pre_process_func" + request = SimpleNamespace(prompt=_prompt(), sampling_params=_SamplingParams()) + assert preprocess(request) is request + assert isinstance(request.sampling_params.extra_args["_lingbot_camera_trajectory"], _CameraTrajectory) + assert module.LingBotWorldCausalDMDPipeline.__name__ == "LingBotWorldCausalDMDPipeline" + lingbot_init = _LINGBOT_INIT_PATH.read_text() + assert "from .pipeline import" in lingbot_init + assert '"LingBotWorldCausalDMDPipeline"' in lingbot_init + assert '"CausalLingBotWorldTransformer3DModel"' in lingbot_init diff --git a/tests/e2e/offline_inference/test_lingbot_world_v2.py b/tests/e2e/offline_inference/test_lingbot_world_v2.py new file mode 100644 index 000000000..39d9b9d95 --- /dev/null +++ b/tests/e2e/offline_inference/test_lingbot_world_v2.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""End-to-end smoke test for LingBot-World v2 generation.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import numpy as np +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniRunnerHandler +from vllm_omni.diffusion.data import DiffusionParallelConfig +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +MODEL = os.environ.get( + "VLLM_OMNI_LINGBOT_WORLD_V2_CHECKPOINT_PATH", + "robbyant/lingbot-world-v2-14b-causal-fast-diffusers", +) +_IMAGE_PATH = os.environ.get("VLLM_OMNI_LINGBOT_WORLD_V2_IMAGE_PATH") +_ACTION_DIR_VALUE = os.environ.get("VLLM_OMNI_LINGBOT_WORLD_V2_ACTION_DIR") +_ACTION_DIR = Path(_ACTION_DIR_VALUE).expanduser().resolve() if _ACTION_DIR_VALUE else None +_HAS_INPUT_ASSETS = _IMAGE_PATH is not None and _ACTION_DIR is not None + +_OMNI_RUNNER_PARAM = ( + MODEL, + None, + { + "flow_shift": 5.0, + "parallel_config": DiffusionParallelConfig(tensor_parallel_size=1), + "model_config": { + "lingbot_action_root": str(_ACTION_DIR.parent) if _ACTION_DIR is not None else "", + }, + }, +) + +pytestmark = [ + pytest.mark.slow, + pytest.mark.diffusion, + pytest.mark.skipif( + not _HAS_INPUT_ASSETS, + reason=("VLLM_OMNI_LINGBOT_WORLD_V2_IMAGE_PATH and VLLM_OMNI_LINGBOT_WORLD_V2_ACTION_DIR are required"), + ), + pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), +] + + +@hardware_test(res={"cuda": "H100"}, num_cards=1) +def test_lingbot_world_v2_tp1_one_block(omni_runner_handler: OmniRunnerHandler) -> None: + """Load the real checkpoint and generate one causal block on one GPU.""" + + assert _IMAGE_PATH is not None + assert _ACTION_DIR is not None + response = omni_runner_handler.send_diffusion_request( + { + "model": omni_runner_handler.runner.model_name, + "prompt": "The camera moves slowly forward through the scene.", + "multi_modal_data": {"image": str(Path(_IMAGE_PATH).expanduser().resolve())}, + "sampling_params": OmniDiffusionSamplingParams( + height=480, + width=832, + num_frames=9, + num_inference_steps=4, + max_sequence_length=512, + seed=42, + fps=16, + extra_args={ + "action_path": _ACTION_DIR.name, + "flow_shift": 5.0, + }, + ), + } + ) + + assert response.success + assert response.images is not None + assert len(response.images) == 1 + video = np.asarray(response.images[0]) + assert video.shape == (1, 9, 480, 832, 3) + assert np.isfinite(video).all() + assert float(np.ptp(video)) > 1e-6 + assert float(np.mean(np.abs(np.diff(video, axis=1)))) > 1e-6 diff --git a/tests/entrypoints/test_utils.py b/tests/entrypoints/test_utils.py index ccc6f69e8..803097dcf 100644 --- a/tests/entrypoints/test_utils.py +++ b/tests/entrypoints/test_utils.py @@ -417,8 +417,9 @@ class TestTryResolveOmniModelType: class TestLoadAndResolveStageConfigs: def test_load_and_resolve_with_kwargs(self, mocker: MockerFixture): - """Ensure that dtype survives default stage creation.""" - kwargs = {"dtype": torch.float32} + """Ensure that kwargs survive default stage creation.""" + engine_backend = "vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine" + kwargs = {"dtype": torch.float32, "engine_backend": engine_backend} mocker.patch("vllm_omni.entrypoints.utils.resolve_model_config_path", return_value=None) mocker.patch("vllm_omni.entrypoints.utils.load_stage_configs_from_model", return_value=([], None)) @@ -431,6 +432,7 @@ class TestLoadAndResolveStageConfigs: assert config_path is None assert len(stage_configs) == 1 assert "dtype" in stage_configs[0]["engine_args"] + assert stage_configs[0]["engine_args"]["engine_backend"] == engine_backend def test_deploy_config_preserves_cli_overrides_and_replicas(self, tmp_path, mocker: MockerFixture): deploy_path = tmp_path / "qwen3_multi.yaml" diff --git a/tests/model_tests/diffusion/test_alignment.py b/tests/model_tests/diffusion/test_alignment.py index fd13fd482..423ccc7a8 100644 --- a/tests/model_tests/diffusion/test_alignment.py +++ b/tests/model_tests/diffusion/test_alignment.py @@ -64,6 +64,9 @@ EXCLUDED_MODELS = [ "HunyuanVideo15Pipeline", "HunyuanVideo15ImageToVideoPipeline", "LingBotVideoPipeline", + # Requires camera/action assets and persistent AR state; covered by its + # dedicated contract tests and real-checkpoint E2E smoke instead. + "LingBotWorldCausalDMDPipeline", "MagiHumanPipeline", "OmniVoicePipeline", "OmniVoice", diff --git a/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py b/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py index b2668b0a5..693766332 100644 --- a/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py +++ b/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py @@ -70,6 +70,9 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams logger = logging.getLogger(__name__) MAX_DREAMZERO_SESSIONS = 64 +# Shipped DreamZero geometry retains 24 Wan VAE causal-convolution cache +# entries. This is the measured persistent CUDA upper bound per live session. +DREAMZERO_MODEL_OWNED_STATE_BYTES_PER_SESSION = 603 * 1024 * 1024 # The pipeline's per-session state is a bespoke ``DreamZeroState`` by default, or # a ``DreamZeroStateAdapter`` view when the opt-in session manager is enabled. @@ -149,6 +152,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): session_capacity=MAX_DREAMZERO_SESSIONS, cross_attention=tuple(cross_attention), max_scratch_tokens_per_branch=int(transformer.num_action_per_block + transformer.num_state_per_block), + model_owned_state_bytes_per_session=DREAMZERO_MODEL_OWNED_STATE_BYTES_PER_SESSION, ) @contextmanager diff --git a/vllm_omni/diffusion/models/lingbot_world/__init__.py b/vllm_omni/diffusion/models/lingbot_world/__init__.py new file mode 100644 index 000000000..a7fe05807 --- /dev/null +++ b/vllm_omni/diffusion/models/lingbot_world/__init__.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from .actions import LingBotCameraControlReducer +from .pipeline import ( + LingBotWorldCausalDMDPipeline, + get_lingbot_world_post_process_func, + get_lingbot_world_pre_process_func, +) +from .transformer import CausalLingBotWorldTransformer3DModel + +__all__ = [ + "CausalLingBotWorldTransformer3DModel", + "LingBotCameraControlReducer", + "LingBotWorldCausalDMDPipeline", + "get_lingbot_world_post_process_func", + "get_lingbot_world_pre_process_func", +] diff --git a/vllm_omni/diffusion/models/lingbot_world/actions.py b/vllm_omni/diffusion/models/lingbot_world/actions.py new file mode 100644 index 000000000..e725c698a --- /dev/null +++ b/vllm_omni/diffusion/models/lingbot_world/actions.py @@ -0,0 +1,302 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Realtime keyboard controls for LingBot World 2.0.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +import numpy as np +import torch + +from vllm_omni.diffusion.models.lingbot_world.camera import ( + CameraTrajectory, +) +from vllm_omni.experimental.ar_diffusion.session import ( + ARDiffusionPreparedControls, + ARDiffusionSessionEvent, +) +from vllm_omni.experimental.ar_diffusion.tick_protocol import ( + ARDiffusionControlInput, +) + +LINGBOT_CAMERA_ACTION_SCHEMA = "lingbot.camera_actions.v1" +LINGBOT_CAMERA_TRAJECTORY_SCHEMA = "lingbot.camera_trajectory.v1" +_ACTION_ORDER = ("w", "a", "s", "d", "i", "j", "k", "l") +_VALID_ACTIONS = frozenset(_ACTION_ORDER) +_REFERENCE_HEIGHT = 480 +_REFERENCE_WIDTH = 832 + + +def _normalize_actions(value: object, *, field: str) -> tuple[str, ...]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise ValueError(f"{field} must be a sequence of LingBot action keys.") + actions: set[str] = set() + for item in value: + if not isinstance(item, str) or item.lower() not in _VALID_ACTIONS: + raise ValueError(f"{field} supports only W/A/S/D/I/J/K/L action keys.") + actions.add(item.lower()) + return tuple(action for action in _ACTION_ORDER if action in actions) + + +def _normalize_frames(value: object, *, field: str) -> tuple[tuple[str, ...], ...]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise ValueError(f"{field} must be a sequence of per-frame action lists.") + return tuple(_normalize_actions(actions, field=f"{field}[{index}]") for index, actions in enumerate(value)) + + +def parse_lingbot_camera_action_frames( + data: Mapping[str, Any], + *, + expected_frames: int, +) -> tuple[tuple[str, ...], ...]: + """Validate the chunk-sized payload emitted by the session reducer.""" + + if data.get("mode") != "frames": + raise ValueError("lingbot.camera_actions.v1 model ticks require mode='frames'.") + frames = _normalize_frames(data.get("frames"), field="camera action frames") + if len(frames) != expected_frames: + raise ValueError( + "lingbot.camera_actions.v1 must contain exactly one action list per " + f"latent frame; expected {expected_frames}, got {len(frames)}." + ) + return frames + + +@dataclass(frozen=True) +class _LingBotCameraReducerState: + mode: str | None = None + held_actions: tuple[str, ...] = () + script_frames: tuple[tuple[str, ...], ...] = () + + +class LingBotCameraControlReducer: + """Sample live state/script controls into one LingBot AR block. + + State mode preserves held keys across chunks and guarantees a one-frame + pulse when a press and release both arrive before the next chunk. Script + mode is a finite FIFO padded with neutral frames after exhaustion. + """ + + def __init__(self, *, frames_per_block: int = 3) -> None: + if isinstance(frames_per_block, bool) or not isinstance(frames_per_block, int) or frames_per_block <= 0: + raise ValueError("frames_per_block must be a positive integer.") + self._frames_per_block = frames_per_block + self._state = _LingBotCameraReducerState() + + @staticmethod + def _state_transitions(data: Mapping[str, Any]) -> tuple[tuple[str, ...], ...]: + transitions = data.get("transitions") + if not isinstance(transitions, Sequence) or isinstance(transitions, (str, bytes)) or not transitions: + raise ValueError("LingBot state-mode camera actions require non-empty transitions.") + normalized: list[tuple[str, ...]] = [] + previous_timestamp: int | None = None + for index, transition in enumerate(transitions): + if not isinstance(transition, Mapping): + raise ValueError("Each LingBot camera state transition must be a mapping.") + timestamp = transition.get("client_ts_ms") + if timestamp is not None: + if isinstance(timestamp, bool) or not isinstance(timestamp, int) or timestamp < 0: + raise ValueError("client_ts_ms must be a non-negative integer when provided.") + if previous_timestamp is not None and timestamp < previous_timestamp: + raise ValueError("client_ts_ms values must be monotonic within one event.") + previous_timestamp = timestamp + normalized.append( + _normalize_actions( + transition.get("actions"), + field=f"camera state transitions[{index}].actions", + ) + ) + return tuple(normalized) + + @staticmethod + def _script(data: Mapping[str, Any]) -> tuple[tuple[str, ...], ...]: + return _normalize_frames(data.get("frames"), field="camera action script frames") + + def prepare( + self, + *, + current_controls: Mapping[str, ARDiffusionControlInput], + events: Sequence[ARDiffusionSessionEvent], + chunk_index: int, + ) -> ARDiffusionPreparedControls: + del chunk_index + controls = { + track: control + for track, control in current_controls.items() + if not (track == "camera" and control.schema == LINGBOT_CAMERA_ACTION_SCHEMA) + } + state = self._state + pending_transitions: tuple[tuple[str, ...], ...] = () + + for event in events: + for control in event.controls: + if control.track != "camera" or control.schema != LINGBOT_CAMERA_ACTION_SCHEMA: + controls[control.track] = control + if control.track == "camera": + state = _LingBotCameraReducerState() + pending_transitions = () + continue + + mode = control.data.get("mode") + controls.pop("camera", None) + if mode == "script": + state = _LingBotCameraReducerState( + mode="script", + script_frames=self._script(control.data), + ) + pending_transitions = () + elif mode == "state": + transitions = self._state_transitions(control.data) + if state.mode != "state": + state = _LingBotCameraReducerState(mode="state") + else: + state = _LingBotCameraReducerState( + mode="state", + held_actions=state.held_actions, + ) + pending_transitions += transitions + else: + raise ValueError("lingbot.camera_actions.v1 events require mode='state' or mode='script'.") + + if state.mode == "script": + frames = state.script_frames[: self._frames_per_block] + frames += ((),) * (self._frames_per_block - len(frames)) + state = _LingBotCameraReducerState( + mode="script", + script_frames=state.script_frames[self._frames_per_block :], + ) + elif state.mode == "state": + final_actions = pending_transitions[-1] if pending_transitions else state.held_actions + pulse = next( + (actions for actions in reversed(pending_transitions) if actions), + None, + ) + if pulse is not None and pulse != final_actions: + frames = (pulse,) + (final_actions,) * (self._frames_per_block - 1) + else: + frames = (final_actions,) * self._frames_per_block + state = _LingBotCameraReducerState( + mode="state", + held_actions=final_actions, + ) + else: + frames = () + + if frames: + controls["camera"] = ARDiffusionControlInput( + track="camera", + schema=LINGBOT_CAMERA_ACTION_SCHEMA, + data={ + "mode": "frames", + "frames": [list(actions) for actions in frames], + }, + ) + return ARDiffusionPreparedControls( + controls=tuple(controls[track] for track in sorted(controls)), + private_state=state, + ) + + def commit(self, prepared: ARDiffusionPreparedControls) -> None: + if not isinstance(prepared.private_state, _LingBotCameraReducerState): + raise TypeError("LingBot reducer prepared state has an unexpected type.") + self._state = prepared.private_state + + def reset(self) -> None: + self._state = _LingBotCameraReducerState() + + +def _rotation_matrix(axis: str, angle: float) -> np.ndarray: + cosine = math.cos(angle) + sine = math.sin(angle) + if axis == "x": + return np.array( + ((1.0, 0.0, 0.0), (0.0, cosine, -sine), (0.0, sine, cosine)), + dtype=np.float64, + ) + return np.array( + ((cosine, 0.0, sine), (0.0, 1.0, 0.0), (-sine, 0.0, cosine)), + dtype=np.float64, + ) + + +def integrate_lingbot_camera_actions( + frames: Sequence[Sequence[str]], + *, + width: int, + height: int, + initial_pose: torch.Tensor | None = None, + initial_pitch: float = 0.0, +) -> tuple[CameraTrajectory, float]: + """Convert latent-frame WASD/IJKL controls to cumulative C2W poses. + + Calibration and motion constants intentionally match SGLang's LingBot + adapter: movement 0.05, pitch 4 degrees, yaw 6 degrees, pitch clamp 85 + degrees, and target-resolution focal lengths of 500 pixels. + """ + + normalized_frames = tuple( + _normalize_actions(actions, field=f"camera action frames[{index}]") for index, actions in enumerate(frames) + ) + if not normalized_frames: + raise ValueError("camera action frames must not be empty.") + if width <= 0 or height <= 0: + raise ValueError("camera action resolution must be positive.") + + if initial_pose is None: + current_pose = np.eye(4, dtype=np.float64) + else: + if initial_pose.shape != (4, 4) or not torch.isfinite(initial_pose).all(): + raise ValueError("initial_pose must be one finite 4x4 camera-to-world matrix.") + current_pose = initial_pose.detach().cpu().double().numpy().copy() + current_pitch = float(initial_pitch) + pitch_limit = math.radians(85.0) + poses: list[np.ndarray] = [] + + for actions in normalized_frames: + rotation = current_pose[:3, :3] + translation = current_pose[:3, 3] + pitch_delta = math.radians(4.0) * (("i" in actions) - ("k" in actions)) + if not -pitch_limit <= current_pitch + pitch_delta <= pitch_limit: + pitch_delta = 0.0 + else: + current_pitch += pitch_delta + yaw_delta = math.radians(6.0) * (("l" in actions) - ("j" in actions)) + new_rotation = _rotation_matrix("y", yaw_delta) @ rotation @ _rotation_matrix("x", pitch_delta) + + forward = np.array((new_rotation[0, 2], 0.0, new_rotation[2, 2])) + right = np.array((new_rotation[0, 0], 0.0, new_rotation[2, 0])) + forward_norm = np.linalg.norm(forward) + right_norm = np.linalg.norm(right) + if forward_norm > 0: + forward /= forward_norm + 1e-6 + if right_norm > 0: + right /= right_norm + 1e-6 + + movement = np.zeros(3, dtype=np.float64) + movement += forward * 0.05 * (("w" in actions) - ("s" in actions)) + movement += right * 0.05 * (("d" in actions) - ("a" in actions)) + current_pose = np.eye(4, dtype=np.float64) + current_pose[:3, :3] = new_rotation + current_pose[:3, 3] = translation + movement + poses.append(current_pose) + + # build_plucker_embedding expects intrinsics in the 832x480 reference + # coordinate system. These values become SGLang's [500, 500, W/2, H/2] + # after that function scales them to the requested resolution. + reference_intrinsics = ( + 500.0 * _REFERENCE_WIDTH / width, + 500.0 * _REFERENCE_HEIGHT / height, + _REFERENCE_WIDTH / 2, + _REFERENCE_HEIGHT / 2, + ) + trajectory = CameraTrajectory( + # Keep the cumulative controller pose in FP64 between chunks, matching + # SGLang's NumPy integration and avoiding long-session drift. + poses=torch.from_numpy(np.stack(poses)), + intrinsics=torch.tensor(reference_intrinsics, dtype=torch.float32).repeat(len(poses), 1), + ) + return trajectory, current_pitch diff --git a/vllm_omni/diffusion/models/lingbot_world/camera.py b/vllm_omni/diffusion/models/lingbot_world/camera.py new file mode 100644 index 000000000..0e78e53e6 --- /dev/null +++ b/vllm_omni/diffusion/models/lingbot_world/camera.py @@ -0,0 +1,481 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Camera trajectory loading and geometry for LingBot World v2 conditioning.""" + +from __future__ import annotations + +import io +import math +import os +import stat +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO + +import numpy as np +import torch + +_REFERENCE_HEIGHT = 480 +_REFERENCE_WIDTH = 832 +_MAX_ACTION_FRAMES = 117 +_MAX_SOURCE_ACTION_FRAMES = 4096 +_MAX_NPY_HEADER_BYTES = 10_000 +_DIRECTORY_OPEN_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW +_FILE_OPEN_FLAGS = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK + + +@dataclass(frozen=True) +class CameraTrajectory: + """Camera-to-world poses and ``(fx, fy, cx, cy)`` intrinsics by frame.""" + + poses: torch.Tensor + intrinsics: torch.Tensor + + +@dataclass(frozen=True) +class TrustedActionDirectory: + """Canonical action location tied to the trusted root's filesystem identity.""" + + root: Path + relative: Path + root_device: int + root_inode: int + + +def resolve_trusted_action_directory( + action_path: str | os.PathLike[str], + trusted_root: str | os.PathLike[str], +) -> TrustedActionDirectory: + """Resolve an action directory beneath a trusted root without exposing paths.""" + + try: + root = Path(trusted_root).expanduser().resolve(strict=True) + except (OSError, RuntimeError): + raise ValueError("The configured LingBot trusted action root is unavailable.") from None + if not root.is_dir(): + raise ValueError("The configured LingBot trusted action root must be a directory.") + + try: + candidate = Path(action_path).expanduser() + if not candidate.is_absolute(): + candidate = root / candidate + candidate = candidate.resolve(strict=True) + relative = candidate.relative_to(root) + except (OSError, RuntimeError, ValueError): + raise ValueError( + "sampling_params.extra_args.action_path must be contained by the trusted action root." + ) from None + if not candidate.is_dir(): + raise ValueError("sampling_params.extra_args.action_path must identify a directory in the trusted action root.") + + try: + root_stat = root.stat() + except OSError: + raise ValueError("The configured LingBot trusted action root is unavailable.") from None + return TrustedActionDirectory( + root=root, + relative=relative, + root_device=root_stat.st_dev, + root_inode=root_stat.st_ino, + ) + + +def _validate_trajectory( + trajectory: CameraTrajectory, + *, + file_suffix: str = "", +) -> None: + poses = trajectory.poses + intrinsics = trajectory.intrinsics + if poses.ndim != 3 or poses.shape[1:] != (4, 4): + raise ValueError(f"poses{file_suffix} must have shape [frames, 4, 4], got {tuple(poses.shape)}") + if intrinsics.ndim != 2 or intrinsics.shape[1:] != (4,): + raise ValueError( + f"intrinsics{file_suffix} must have shape [frames, 4] ordered as fx, fy, cx, cy, " + f"got {tuple(intrinsics.shape)}" + ) + if poses.shape[0] == 0: + raise ValueError("camera trajectory must contain at least one frame") + if poses.shape[0] != intrinsics.shape[0]: + raise ValueError("poses and intrinsics must contain the same number of frames") + if not torch.isfinite(poses).all() or not torch.isfinite(intrinsics).all(): + raise ValueError("camera trajectory values must all be finite") + + +@contextmanager +def _open_action_directory(action_directory: TrustedActionDirectory) -> Iterator[int]: + root_fd: int | None = None + action_fd: int | None = None + try: + root_fd = os.open(action_directory.root, _DIRECTORY_OPEN_FLAGS) + root_stat = os.fstat(root_fd) + if (root_stat.st_dev, root_stat.st_ino) != ( + action_directory.root_device, + action_directory.root_inode, + ): + raise ValueError("The configured LingBot trusted action root changed before camera files were opened.") + + action_fd = os.dup(root_fd) + for component in action_directory.relative.parts: + next_fd = os.open(component, _DIRECTORY_OPEN_FLAGS, dir_fd=action_fd) + os.close(action_fd) + action_fd = next_fd + except OSError: + if action_fd is not None: + os.close(action_fd) + if root_fd is not None: + os.close(root_fd) + raise ValueError("Unable to securely open the LingBot action directory within the trusted root.") from None + except ValueError: + if action_fd is not None: + os.close(action_fd) + if root_fd is not None: + os.close(root_fd) + raise + assert action_fd is not None + assert root_fd is not None + try: + yield action_fd + finally: + os.close(action_fd) + os.close(root_fd) + + +@contextmanager +def _open_camera_file(action_fd: int, filename: str) -> Iterator[tuple[BinaryIO, os.stat_result]]: + file_fd = -1 + try: + file_fd = os.open(filename, _FILE_OPEN_FLAGS, dir_fd=action_fd) + file_stat = os.fstat(file_fd) + if not stat.S_ISREG(file_stat.st_mode): + raise ValueError(f"{filename} must be a regular NPY file.") + with os.fdopen(file_fd, "rb", closefd=True) as file_handle: + file_fd = -1 + yield file_handle, file_stat + except FileNotFoundError: + raise FileNotFoundError(f"camera trajectory file not found: {filename}") from None + except OSError: + raise ValueError(f"{filename} could not be opened securely from the trusted action directory.") from None + finally: + if file_fd >= 0: + os.close(file_fd) + + +def _load_bounded_npy( + action_fd: int, + filename: str, + *, + trailing_shape: tuple[int, ...], +) -> np.ndarray: + with _open_camera_file(action_fd, filename) as (file_handle, file_stat): + max_payload_bytes = _MAX_SOURCE_ACTION_FRAMES * math.prod(trailing_shape) * 8 + max_file_bytes = _MAX_NPY_HEADER_BYTES + max_payload_bytes + 32 + if file_stat.st_size > max_file_bytes: + raise ValueError(f"{filename} byte size exceeds the bounded camera-array limit.") + snapshot = file_handle.read(max_file_bytes + 1) + if len(snapshot) != file_stat.st_size: + raise ValueError(f"{filename} changed while its bounded contents were read.") + snapshot_file = io.BytesIO(snapshot) + try: + version = np.lib.format.read_magic(snapshot_file) + if version == (1, 0): + shape, _fortran_order, dtype = np.lib.format.read_array_header_1_0(snapshot_file) + elif version == (2, 0): + shape, _fortran_order, dtype = np.lib.format.read_array_header_2_0(snapshot_file) + else: + raise ValueError(f"{filename} must use NPY format version 1.0 or 2.0.") + except (EOFError, TypeError, ValueError): + raise ValueError(f"{filename} has an invalid or oversized NPY header.") from None + + if len(shape) != len(trailing_shape) + 1 or tuple(shape[1:]) != trailing_shape: + expected = ", ".join(str(value) for value in trailing_shape) + raise ValueError(f"{filename} must have shape [frames, {expected}], got {shape}.") + frames = shape[0] + if isinstance(frames, bool) or not isinstance(frames, int) or not 1 <= frames <= _MAX_SOURCE_ACTION_FRAMES: + raise ValueError(f"{filename} must contain between 1 and {_MAX_SOURCE_ACTION_FRAMES} source frames.") + if dtype.kind not in "fiu" or dtype.hasobject or dtype.itemsize > 8: + raise ValueError(f"{filename} must contain real numeric values with at most 8 bytes per element.") + + data_offset = snapshot_file.tell() + expected_bytes = math.prod(shape) * dtype.itemsize + if len(snapshot) != data_offset + expected_bytes: + raise ValueError(f"{filename} byte size does not match its NPY header.") + snapshot_file.seek(0) + try: + array = np.load(snapshot_file, allow_pickle=False) + except (EOFError, TypeError, ValueError): + raise ValueError(f"{filename} does not contain a valid numeric NPY array.") from None + # Official LingBot trajectories may be longer than one request. Only + # materialize the bounded prefix that any supported request can consume. + result: np.ndarray = np.asarray(array[:_MAX_ACTION_FRAMES], dtype=np.float32) + return result + + +def load_camera_trajectory(action_directory: TrustedActionDirectory) -> CameraTrajectory: + """Load bounded numeric camera arrays through a trusted directory handle.""" + + if not isinstance(action_directory, TrustedActionDirectory): + raise TypeError("action_directory must be resolved against a trusted action root before loading.") + with _open_action_directory(action_directory) as action_fd: + poses_array = _load_bounded_npy(action_fd, "poses.npy", trailing_shape=(4, 4)) + intrinsics_array = _load_bounded_npy(action_fd, "intrinsics.npy", trailing_shape=(4,)) + + trajectory = CameraTrajectory( + poses=torch.from_numpy(poses_array), + intrinsics=torch.from_numpy(intrinsics_array), + ) + _validate_trajectory(trajectory, file_suffix=".npy") + return trajectory + + +def _linear_interpolate(values: torch.Tensor, target_times: np.ndarray) -> torch.Tensor: + source_times = np.linspace(0.0, 1.0, values.shape[0]) + values_array = values.detach().cpu().double().numpy() + columns = [ + np.interp(target_times, source_times, values_array[:, column]) for column in range(values_array.shape[1]) + ] + interpolated = torch.from_numpy(np.stack(columns, axis=1)) + return interpolated.to(device=values.device, dtype=values.dtype) + + +def _rotation_matrices_to_quaternions(matrices: np.ndarray) -> np.ndarray: + """Convert rotation matrices to normalized ``(w, x, y, z)`` quaternions.""" + + quaternions: np.ndarray = np.empty((matrices.shape[0], 4), dtype=np.float64) + for index, matrix in enumerate(matrices): + trace = float(np.trace(matrix)) + if trace > 0.0: + scale = math.sqrt(trace + 1.0) * 2.0 + quaternion = np.array( + [ + 0.25 * scale, + (matrix[2, 1] - matrix[1, 2]) / scale, + (matrix[0, 2] - matrix[2, 0]) / scale, + (matrix[1, 0] - matrix[0, 1]) / scale, + ] + ) + else: + axis = int(np.argmax(np.diag(matrix))) + if axis == 0: + scale = math.sqrt(max(0.0, 1.0 + matrix[0, 0] - matrix[1, 1] - matrix[2, 2])) * 2.0 + quaternion = np.array( + [ + (matrix[2, 1] - matrix[1, 2]) / scale, + 0.25 * scale, + (matrix[0, 1] + matrix[1, 0]) / scale, + (matrix[0, 2] + matrix[2, 0]) / scale, + ] + ) + elif axis == 1: + scale = math.sqrt(max(0.0, 1.0 + matrix[1, 1] - matrix[0, 0] - matrix[2, 2])) * 2.0 + quaternion = np.array( + [ + (matrix[0, 2] - matrix[2, 0]) / scale, + (matrix[0, 1] + matrix[1, 0]) / scale, + 0.25 * scale, + (matrix[1, 2] + matrix[2, 1]) / scale, + ] + ) + else: + scale = math.sqrt(max(0.0, 1.0 + matrix[2, 2] - matrix[0, 0] - matrix[1, 1])) * 2.0 + quaternion = np.array( + [ + (matrix[1, 0] - matrix[0, 1]) / scale, + (matrix[0, 2] + matrix[2, 0]) / scale, + (matrix[1, 2] + matrix[2, 1]) / scale, + 0.25 * scale, + ] + ) + norm = np.linalg.norm(quaternion) + if not np.isfinite(norm) or norm <= np.finfo(np.float64).eps: + raise ValueError("camera rotation matrices must describe valid rotations") + quaternions[index] = quaternion / norm + return quaternions + + +def _quaternions_to_rotation_matrices(quaternions: np.ndarray) -> np.ndarray: + quaternions = quaternions / np.linalg.norm(quaternions, axis=1, keepdims=True) + w, x, y, z = quaternions.T + matrices: np.ndarray = np.stack( + ( + 1 - 2 * (y * y + z * z), + 2 * (x * y - z * w), + 2 * (x * z + y * w), + 2 * (x * y + z * w), + 1 - 2 * (x * x + z * z), + 2 * (y * z - x * w), + 2 * (x * z - y * w), + 2 * (y * z + x * w), + 1 - 2 * (x * x + y * y), + ), + axis=1, + ).reshape(-1, 3, 3) + return matrices + + +def _slerp_rotations(matrices: np.ndarray, target_times: np.ndarray) -> np.ndarray: + quaternions = _rotation_matrices_to_quaternions(matrices) + interpolated = np.empty((target_times.shape[0], 4), dtype=np.float64) + for target_index, target_time in enumerate(target_times): + scaled_time = float(target_time) * (matrices.shape[0] - 1) + left = min(max(math.floor(scaled_time), 0), matrices.shape[0] - 2) + fraction = scaled_time - left + first = quaternions[left] + second = quaternions[left + 1] + cosine = float(np.dot(first, second)) + if cosine < 0.0: + second = -second + cosine = -cosine + cosine = min(cosine, 1.0) + if cosine > 0.9995: + quaternion = first + fraction * (second - first) + interpolated[target_index] = quaternion / np.linalg.norm(quaternion) + continue + angle = math.acos(cosine) + denominator = math.sin(angle) + interpolated[target_index] = ( + math.sin((1.0 - fraction) * angle) / denominator * first + math.sin(fraction * angle) / denominator * second + ) + return _quaternions_to_rotation_matrices(interpolated) + + +def interpolate_camera_trajectory( + trajectory: CameraTrajectory, + num_frames: int, +) -> CameraTrajectory: + """Resample translations/intrinsics linearly and rotations with quaternion SLERP.""" + + _validate_trajectory(trajectory) + if num_frames <= 0: + raise ValueError(f"num_frames must be positive, got {num_frames}") + + source_frames = trajectory.poses.shape[0] + if num_frames == source_frames: + return trajectory + if source_frames == 1: + return CameraTrajectory( + poses=trajectory.poses.repeat(num_frames, 1, 1), + intrinsics=trajectory.intrinsics.repeat(num_frames, 1), + ) + + # Rotations use quaternion SLERP; translations and intrinsics are linear. + target_times = np.linspace(0.0, 1.0, num_frames) + rotation_matrices = trajectory.poses[:, :3, :3].detach().cpu().double().numpy() + interpolated_rotations = _slerp_rotations(rotation_matrices, target_times) + translations = _linear_interpolate(trajectory.poses[:, :3, 3], target_times) + intrinsics = _linear_interpolate(trajectory.intrinsics, target_times) + + poses = torch.eye( + 4, + device=trajectory.poses.device, + dtype=trajectory.poses.dtype, + ).repeat(num_frames, 1, 1) + poses[:, :3, :3] = torch.from_numpy(interpolated_rotations).to( + device=trajectory.poses.device, + dtype=trajectory.poses.dtype, + ) + poses[:, :3, 3] = translations + return CameraTrajectory(poses=poses, intrinsics=intrinsics) + + +def _invert_camera_poses(poses: torch.Tensor) -> torch.Tensor: + """Invert rigid transforms using R^-1 = R^T and t^-1 = -R^T t.""" + + rotations = poses[:, :3, :3] + translations = poses[:, :3, 3:] + inverse_rotations = rotations.transpose(-1, -2) + inverse = torch.eye(4, device=poses.device, dtype=poses.dtype).repeat(poses.shape[0], 1, 1) + inverse[:, :3, :3] = inverse_rotations + inverse[:, :3, 3:] = -torch.bmm(inverse_rotations, translations) + return inverse + + +def _prepare_framewise_poses(raw_poses: torch.Tensor) -> torch.Tensor: + """Convert raw C2W poses to normalized first-relative framewise deltas.""" + + # Remove global placement, then convert the sequence to framewise deltas. + first_world_to_camera = _invert_camera_poses(raw_poses[:1]) + relative_poses = torch.matmul(first_world_to_camera, raw_poses).clone() + relative_poses[0] = torch.eye(4, device=raw_poses.device, dtype=raw_poses.dtype) + if relative_poses.shape[0] > 1: + relative_poses[1:] = torch.bmm( + _invert_camera_poses(relative_poses[:-1]), + relative_poses[1:], + ) + + # Only relative direction/magnitude matters to the checkpoint; normalizing + # by the largest step removes the trajectory's arbitrary translation scale. + translations = relative_poses[:, :3, 3] + max_translation_norm = torch.linalg.vector_norm(translations, dim=-1).max() + if max_translation_norm > 0: + relative_poses[:, :3, 3] = translations / max_translation_norm + return relative_poses + + +def build_plucker_embedding( + trajectory: CameraTrajectory, + *, + height: int, + width: int, + target_height: int, + target_width: int, + device: torch.device, + dtype: torch.dtype, +) -> torch.Tensor: + """Build checkpoint-ordered ``(ray origin, ray direction)`` channels.""" + + _validate_trajectory(trajectory) + dimensions = { + "height": height, + "width": width, + "target_height": target_height, + "target_width": target_width, + } + for name, value in dimensions.items(): + if value <= 0: + raise ValueError(f"{name} must be positive, got {value}") + if dtype not in (torch.float16, torch.bfloat16, torch.float32, torch.float64): + raise ValueError(f"dtype must be floating point, got {dtype}") + + # Compute geometry in at least FP32, then cast at the model boundary. + compute_dtype = torch.float32 if dtype in (torch.float16, torch.bfloat16) else dtype + poses = _prepare_framewise_poses(trajectory.poses.to(device=device, dtype=compute_dtype)) + intrinsics = trajectory.intrinsics.to(device=device, dtype=compute_dtype).clone() + intrinsics[:, (0, 2)] *= width / _REFERENCE_WIDTH + intrinsics[:, (1, 3)] *= height / _REFERENCE_HEIGHT + if torch.any(intrinsics[:, :2] == 0): + raise ValueError("camera focal lengths fx and fy must be non-zero") + + # Sample requested-pixel coordinates directly at the conditioning resolution. + x_coordinates = (torch.arange(target_width, device=device, dtype=compute_dtype) + 0.5) * (width / target_width) + y_coordinates = (torch.arange(target_height, device=device, dtype=compute_dtype) + 0.5) * (height / target_height) + grid_y, grid_x = torch.meshgrid(y_coordinates, x_coordinates, indexing="ij") + grid_x = grid_x.unsqueeze(0) + grid_y = grid_y.unsqueeze(0) + + # Pinhole back-projection: pixel (x, y) becomes the camera-space direction + # ((x-cx)/fx, (y-cy)/fy, 1), which is normalized before rotation. + fx, fy, cx, cy = intrinsics.unbind(dim=1) + camera_x = (grid_x - cx[:, None, None]) / fx[:, None, None] + camera_y = (grid_y - cy[:, None, None]) / fy[:, None, None] + camera_directions = torch.stack( + (camera_x, camera_y, torch.ones_like(camera_x)), + dim=-1, + ) + camera_directions = camera_directions / torch.linalg.vector_norm(camera_directions, dim=-1, keepdim=True) + + # A ray is represented by its camera center (origin) and unit direction in + # the prepared relative coordinate frame. Channel order is checkpoint + # data, not an implementation preference: [origin_xyz, direction_xyz]. + world_directions = torch.einsum( + "fij,fhwj->fhwi", + poses[:, :3, :3], + camera_directions, + ) + world_directions = world_directions / torch.linalg.vector_norm(world_directions, dim=-1, keepdim=True) + ray_origins = poses[:, None, None, :3, 3].expand_as(world_directions) + + embedding = torch.cat((ray_origins, world_directions), dim=-1) + return embedding.permute(0, 3, 1, 2).contiguous().to(dtype=dtype) diff --git a/vllm_omni/diffusion/models/lingbot_world/pipeline.py b/vllm_omni/diffusion/models/lingbot_world/pipeline.py new file mode 100644 index 000000000..5f71979e8 --- /dev/null +++ b/vllm_omni/diffusion/models/lingbot_world/pipeline.py @@ -0,0 +1,1292 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Request-scoped LingBot-World v2 causal DMD pipeline.""" + +from __future__ import annotations + +import math +import os +from collections.abc import Callable, Iterable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, Any, ClassVar, cast + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from diffusers.utils.torch_utils import randn_tensor +from torch import nn +from transformers import AutoTokenizer, UMT5EncoderModel +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.model_executor.models.utils import AutoWeightsLoader + +from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import DistributedAutoencoderKLWan +from vllm_omni.diffusion.distributed.utils import get_local_device +from vllm_omni.diffusion.forward_context import set_forward_context_denoise_step_idx +from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader +from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders +from vllm_omni.diffusion.models.interface import SupportImageInput, SupportsComponentDiscovery +from vllm_omni.diffusion.models.lingbot_world.actions import ( + LINGBOT_CAMERA_ACTION_SCHEMA, + LINGBOT_CAMERA_TRAJECTORY_SCHEMA, + integrate_lingbot_camera_actions, + parse_lingbot_camera_action_frames, +) +from vllm_omni.diffusion.models.lingbot_world.camera import ( + CameraTrajectory, + build_plucker_embedding, + interpolate_camera_trajectory, + load_camera_trajectory, + resolve_trusted_action_directory, +) +from vllm_omni.diffusion.models.lingbot_world.transformer import ( + CausalLingBotWorldTransformer3DModel, + LingBotAttentionCache, + LingBotTransformerCache, +) +from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin +from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler +from vllm_omni.diffusion.models.utils import _load_json +from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import load_transformer_config, retrieve_latents +from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin +from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.experimental.ar_diffusion.capability import ( + ARDiffusionCrossAttentionKVSpec, + ARDiffusionKVBranchSpec, + ARDiffusionKVCacheSpec, +) +from vllm_omni.experimental.ar_diffusion.tick_protocol import ( + ARDiffusionChunkMetadata, + ARDiffusionTickRequest, +) + +if TYPE_CHECKING: + from tqdm.std import tqdm as TqdmProgressBar + + from vllm_omni.experimental.ar_diffusion.kv_cache.state import ( + ARDiffusionKVState, + ) + +LINGBOT_DMD_TIMESTEPS = (1000, 750, 500, 250) +_CAMERA_SPATIAL_FOLD = 8 +_MAX_PIXEL_AREA = 480 * 832 +_MAX_SOURCE_IMAGE_PIXELS = 4096 * 4096 +_MAX_RAW_FRAMES = 117 +_MAX_SEQUENCE_LENGTH = 512 +_ACTION_ROOT_ENV = "VLLM_OMNI_LINGBOT_ACTION_ROOT" +_PREPROCESSED_CAMERA_KEY = "_lingbot_camera_trajectory" +_PREPROCESSED_CAMERA_ACTIONS_KEY = "_lingbot_camera_actions" +_SOURCE_IMAGE_ERROR = ( + "Unable to load multi_modal_data.image; expected a decodable image within 4096 * 4096 source pixels." +) + + +@dataclass(frozen=True) +class _LingBotRequestInputs: + """Validated model-specific request boundary.""" + + prompt: str + image: PIL.Image.Image | torch.Tensor + camera_trajectory: CameraTrajectory | None + camera_actions: tuple[tuple[str, ...], ...] | None + height: int + width: int + num_frames: int + num_latent_frames: int + output_type: str + max_sequence_length: int + flow_shift: float + generator: torch.Generator + + +@dataclass +class _LingBotARSessionState: + """Small model-owned state; attention tensors remain runner-owned.""" + + next_chunk_index: int = 0 + prompt: str | None = None + generator_state: torch.Tensor | None = None + image_condition: torch.Tensor | None = None + camera_tail: CameraTrajectory | None = None + camera_pitch: float = 0.0 + + +def _positive_finite_flow_shift(value: Any) -> float: + if isinstance(value, bool): + raise ValueError("flow_shift must be a positive finite number.") + try: + flow_shift = float(value) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError("flow_shift must be a positive finite number.") from exc + if not math.isfinite(flow_shift) or flow_shift <= 0: + raise ValueError("flow_shift must be a positive finite number.") + return flow_shift + + +def _build_shifted_flow_schedule( + *, + flow_shift: float, +) -> tuple[tuple[float, float], ...]: + """Map checkpoint DMD labels to request-local warped timestep/sigma pairs.""" + + # The checkpoint labels index the unshifted [1, ..., 1 / N] training + # lattice. Apply the request's flow shift before using the value both as + # the transformer timestep and the sampling sigma. + base_sigmas = torch.tensor(LINGBOT_DMD_TIMESTEPS, dtype=torch.float64) / 1000 + shifted_numerators = flow_shift * base_sigmas + shifted_sigmas = shifted_numerators / ((1.0 - base_sigmas) + shifted_numerators) + warped_timesteps = shifted_sigmas * 1000 + return tuple( + (float(timestep), float(sigma)) + for timestep, sigma in zip(warped_timesteps.tolist(), shifted_sigmas.tolist(), strict=True) + ) + + +def _validate_scheduler_config(config: dict[str, Any]) -> None: + contract = { + "_class_name": "UniPCMultistepScheduler", + "num_train_timesteps": 1000, + "prediction_type": "flow_prediction", + "predict_x0": True, + "use_flow_sigmas": True, + "use_dynamic_shifting": False, + "use_beta_sigmas": False, + "use_exponential_sigmas": False, + "use_karras_sigmas": False, + "final_sigmas_type": "zero", + "timestep_spacing": "linspace", + } + for name, expected in contract.items(): + if config.get(name) != expected: + raise ValueError(f"LingBot scheduler config {name} must be {expected!r}, got {config.get(name)!r}.") + + +def _validate_parallel_config(od_config: OmniDiffusionConfig) -> None: + if getattr(od_config, "quantization_config", None) is not None: + raise NotImplementedError("LingBot World v1 does not support quantization.") + parallel_config = getattr(od_config, "parallel_config", None) + if parallel_config is None: + return + unsupported_sizes = { + "pipeline_parallel_size": "pipeline parallelism", + "sequence_parallel_size": "sequence parallelism", + "cfg_parallel_size": "CFG parallelism", + "vae_patch_parallel_size": "VAE parallelism", + } + for field, feature in unsupported_sizes.items(): + size = getattr(parallel_config, field, 1) or 1 + if size > 1: + raise NotImplementedError(f"LingBot World v1 does not support {feature} ({field}={size}).") + if getattr(parallel_config, "use_hsdp", False): + raise NotImplementedError("LingBot World v1 does not support HSDP.") + if getattr(parallel_config, "enable_expert_parallel", False): + raise NotImplementedError("LingBot World v1 does not support expert parallelism.") + + +def _validate_source_image_size(image: PIL.Image.Image) -> None: + width, height = image.size + if ( + isinstance(width, bool) + or not isinstance(width, int) + or width <= 0 + or isinstance(height, bool) + or not isinstance(height, int) + or height <= 0 + ): + raise ValueError("source image width and height must be positive integers.") + if width * height > _MAX_SOURCE_IMAGE_PIXELS: + raise ValueError("source image pixel count must not exceed 4096 * 4096.") + + +def _decode_source_image(image: PIL.Image.Image) -> PIL.Image.Image: + _validate_source_image_size(image) + try: + return image.convert("RGB") + except (OSError, SyntaxError, ValueError, PIL.Image.DecompressionBombError): + raise ValueError(_SOURCE_IMAGE_ERROR) from None + + +def _load_source_image(path: str | os.PathLike[str]) -> PIL.Image.Image: + try: + source_image = PIL.Image.open(path) + except (OSError, SyntaxError, ValueError, PIL.Image.DecompressionBombError): + raise ValueError(_SOURCE_IMAGE_ERROR) from None + try: + return _decode_source_image(source_image) + finally: + source_image.close() + + +def get_lingbot_world_pre_process_func( + od_config: OmniDiffusionConfig, +) -> Callable[[OmniDiffusionRequest], OmniDiffusionRequest]: + """Materialize request-local files once before dispatching GPU workers.""" + + model_config = getattr(od_config, "model_config", None) or {} + configured_action_root = model_config.get("lingbot_action_root") or os.environ.get(_ACTION_ROOT_ENV) + + def pre_process_func(request: OmniDiffusionRequest) -> OmniDiffusionRequest: + prompt = request.prompt + if not isinstance(prompt, dict): + raise ValueError("LingBot World requires a prompt mapping containing multi_modal_data.image.") + multi_modal_data = prompt.get("multi_modal_data") or {} + additional_information = prompt.get("additional_information") or {} + if not isinstance(multi_modal_data, dict): + raise ValueError("prompt.multi_modal_data must be a mapping containing image.") + if not isinstance(additional_information, dict): + raise ValueError("prompt.additional_information must be a mapping.") + if "action_path" in additional_information: + raise ValueError( + "prompt.additional_information.action_path is not supported; use " + "sampling_params.extra_args.action_path." + ) + + image = multi_modal_data.get("image") + if isinstance(image, list): + raise ValueError("LingBot World requires one image and does not accept an image list.") + if image is None: + raise ValueError("LingBot World requires exactly one image in multi_modal_data.image.") + if isinstance(image, (str, os.PathLike)): + image = _load_source_image(image) + elif isinstance(image, PIL.Image.Image): + image = _decode_source_image(image) + elif not isinstance(image, torch.Tensor): + raise ValueError("multi_modal_data.image must be a PIL image, tensor, or file path.") + + extra_args = getattr(request.sampling_params, "extra_args", None) or {} + if not isinstance(extra_args, dict): + raise ValueError("sampling_params.extra_args must be a mapping.") + tick = ARDiffusionTickRequest.from_extra_args(extra_args) + if tick is not None: + camera_controls = [control for control in tick.controls if control.track == "camera"] + if len(camera_controls) != 1: + raise ValueError("LingBot realtime ticks require exactly one camera control.") + camera_control = camera_controls[0] + if camera_control.schema == LINGBOT_CAMERA_TRAJECTORY_SCHEMA: + camera_data = camera_control.data + try: + poses = torch.as_tensor(camera_data["poses"], dtype=torch.float32) + intrinsics = torch.as_tensor( + camera_data["intrinsics"], + dtype=torch.float32, + ) + except (KeyError, TypeError, ValueError): + raise ValueError( + "lingbot.camera_trajectory.v1 requires numeric poses and intrinsics fields." + ) from None + if poses.ndim != 3 or poses.shape[1:] != (4, 4): + raise ValueError("camera control poses must have shape [frames, 4, 4].") + if intrinsics.ndim != 2 or intrinsics.shape[1:] != (4,): + raise ValueError("camera control intrinsics must have shape [frames, 4].") + if poses.shape[0] != intrinsics.shape[0] or poses.shape[0] == 0: + raise ValueError( + "camera control poses and intrinsics must contain the same positive number of frames." + ) + if not torch.isfinite(poses).all() or not torch.isfinite(intrinsics).all(): + raise ValueError("camera control values must all be finite.") + trajectory = CameraTrajectory( + poses=poses, + intrinsics=intrinsics, + ) + camera_actions = None + elif camera_control.schema == LINGBOT_CAMERA_ACTION_SCHEMA: + trajectory = None + camera_actions = parse_lingbot_camera_action_frames( + camera_control.data, + expected_frames=3, + ) + else: + raise ValueError( + "LingBot realtime camera controls require schema " + "'lingbot.camera_trajectory.v1' or 'lingbot.camera_actions.v1'." + ) + else: + action_path = extra_args.get("action_path") + if not isinstance(action_path, (str, os.PathLike)) or not str(action_path): + raise ValueError("action_path is required in sampling_params.extra_args.action_path.") + if not configured_action_root: + raise ValueError( + "sampling_params.extra_args.action_path requires a trusted action root configured by " + f"model_config.lingbot_action_root or {_ACTION_ROOT_ENV}." + ) + action_directory = resolve_trusted_action_directory( + action_path, + configured_action_root, + ) + try: + trajectory = load_camera_trajectory(action_directory) + except OSError: + raise ValueError( + "Unable to load camera trajectory from action_path; expected poses.npy and intrinsics.npy." + ) from None + camera_actions = None + + updated_prompt = dict(prompt) + updated_multi_modal_data = dict(multi_modal_data) + updated_multi_modal_data["image"] = image + updated_prompt["multi_modal_data"] = updated_multi_modal_data + request.prompt = updated_prompt + request.sampling_params.extra_args = { + **extra_args, + _PREPROCESSED_CAMERA_KEY: trajectory, + _PREPROCESSED_CAMERA_ACTIONS_KEY: camera_actions, + } + return request + + return pre_process_func + + +def _fold_camera_embedding( + camera_embedding: torch.Tensor, +) -> torch.Tensor: + """Pixel-unshuffle ``[frames, 6, H, W]`` onto the Wan latent grid.""" + + folded = F.pixel_unshuffle(camera_embedding, _CAMERA_SPATIAL_FOLD) + return folded.permute(1, 0, 2, 3).unsqueeze(0).contiguous() + + +def get_lingbot_world_post_process_func(od_config: OmniDiffusionConfig) -> Callable[..., Any]: + del od_config + from diffusers.video_processor import VideoProcessor + + video_processor = VideoProcessor(vae_scale_factor=8) + + def post_process_func( + video: torch.Tensor, + output_type: str = "np", + sampling_params: Any | None = None, + ) -> Any: + if isinstance(video, dict) and isinstance(video.get("payload"), dict): + return video + if sampling_params is not None: + output_type = getattr(sampling_params, "output_type", None) or output_type + if output_type == "latent": + return video + return { + "payload": {"video": video_processor.postprocess_video(video, output_type=output_type)}, + "metadata": {}, + } + + return post_process_func + + +class LingBotWorldCausalDMDPipeline( + nn.Module, + SupportImageInput, + SupportsComponentDiscovery, + ProgressBarMixin, + DiffusionPipelineProfilerMixin, +): + """LingBot-World v2 I2V generation with a request-local causal cache.""" + + _dit_modules: ClassVar[list[str]] = ["transformer"] + _encoder_modules: ClassVar[list[str]] = ["text_encoder"] + _vae_modules: ClassVar[list[str]] = ["vae"] + # Generic warmup cannot synthesize the required camera action directory. + dummy_run_num_frames: ClassVar[int] = 0 + _AR_BRANCH = "main" + _AR_TEXT_CACHE = "text" + + def __init__( + self, + *, + od_config: OmniDiffusionConfig, + prefix: str = "", + ) -> None: + super().__init__() + del prefix + _validate_parallel_config(od_config) + self.od_config = od_config + self.device = get_local_device() + dtype = getattr(od_config, "dtype", torch.bfloat16) + model = od_config.model + local_files_only = os.path.exists(model) + managed_component_placement = bool( + getattr(od_config, "enable_cpu_offload", False) or getattr(od_config, "enable_layerwise_offload", False) + ) + + # Standard components use from_pretrained; the custom transformer uses the loader. + self.weights_sources = [ + DiffusersPipelineLoader.ComponentSource( + model_or_path=model, + subfolder="transformer", + revision=None, + prefix="transformer.", + fall_back_to_pt=True, + ) + ] + + subfolders = ["tokenizer", "text_encoder", "vae"] + prefetch_subfolders(model, subfolders, local_files_only=local_files_only) + self.tokenizer = from_pretrained_with_prefetch( + AutoTokenizer.from_pretrained, + model, + subfolder="tokenizer", + prefetch_list=subfolders, + local_files_only=local_files_only, + ) + self.text_encoder = from_pretrained_with_prefetch( + UMT5EncoderModel.from_pretrained, + model, + subfolder="text_encoder", + prefetch_list=subfolders, + local_files_only=local_files_only, + torch_dtype=dtype, + ) + if not managed_component_placement: + self.text_encoder = self.text_encoder.to(self.device) + self.vae = from_pretrained_with_prefetch( + DistributedAutoencoderKLWan.from_pretrained, + model, + subfolder="vae", + prefetch_list=subfolders, + local_files_only=local_files_only, + torch_dtype=dtype, + ) + if not managed_component_placement: + self.vae = self.vae.to(self.device) + + transformer_config = load_transformer_config(model, "transformer", local_files_only) + self.transformer = CausalLingBotWorldTransformer3DModel.from_config( + transformer_config, + quant_config=getattr(od_config, "quantization_config", None), + prefix="transformer", + ) + + scheduler_config = _load_json(model, "scheduler/scheduler_config.json", local_files_only) + _validate_scheduler_config(scheduler_config) + configured_shift = getattr(od_config, "flow_shift", None) + checkpoint_shift = scheduler_config.get("flow_shift", scheduler_config.get("shift", 5.0)) + scheduler_shift = _positive_finite_flow_shift( + checkpoint_shift if configured_shift is None else configured_shift + ) + scheduler_keys = { + "num_train_timesteps", + "solver_order", + "prediction_type", + "use_dynamic_shifting", + "thresholding", + "dynamic_thresholding_ratio", + "sample_max_value", + "predict_x0", + "solver_type", + "lower_order_final", + "disable_corrector", + "timestep_spacing", + "steps_offset", + "final_sigmas_type", + } + scheduler_kwargs = {name: value for name, value in scheduler_config.items() if name in scheduler_keys} + scheduler_kwargs["shift"] = scheduler_shift + self.scheduler = FlowUniPCMultistepScheduler(**scheduler_kwargs) + + self.vae_scale_factor_temporal = int(getattr(self.vae.config, "scale_factor_temporal", 4)) + self.vae_scale_factor_spatial = int(getattr(self.vae.config, "scale_factor_spatial", 8)) + model_config = getattr(od_config, "model_config", None) or {} + self._ar_height = int(model_config.get("ar_diffusion_height", 480)) + self._ar_width = int(model_config.get("ar_diffusion_width", 832)) + self._ar_diffusion_kv_state: ARDiffusionKVState | None = None + self._ar_sessions: dict[str, _LingBotARSessionState] = {} + self.setup_diffusion_pipeline_profiler( + profiler_targets=[ + "vae.encode", + "vae.decode", + "_generate_block", + "text_encoder.forward", + "tokenizer.forward", + ], + enable_diffusion_pipeline_profiler=od_config.enable_diffusion_pipeline_profiler, + ) + + def ar_diffusion_kv_cache_spec(self) -> ARDiffusionKVCacheSpec: + """Describe the fixed worker-local cache geometry for realtime ticks.""" + patch_frames, patch_height, patch_width = self.transformer.config.patch_size + if patch_frames != 1: + raise ValueError("LingBot AR-Diffusion requires temporal patch size 1.") + spatial = self.vae_scale_factor_spatial + if self._ar_height % (spatial * patch_height) or self._ar_width % (spatial * patch_width): + raise ValueError("ar_diffusion_height/width must align with VAE and DiT patch sizes.") + latent_height = self._ar_height // spatial + latent_width = self._ar_width // spatial + tokens_per_frame = (latent_height // patch_height) * (latent_width // patch_width) + tp_size = get_tensor_model_parallel_world_size() + num_local_heads = int(self.transformer.config.num_attention_heads) // tp_size + total_window_frames = ( + int(self.transformer.config.local_attn_size) + if int(self.transformer.config.local_attn_size) != -1 + else int(self.transformer.config.sliding_window_num_frames) + ) + sink_frames = int(self.transformer.config.sink_size) + recent_window_frames = total_window_frames - sink_frames + if recent_window_frames <= 0: + raise ValueError("LingBot AR-Diffusion cache needs a positive recent window after reserving sink frames.") + horizon_latent_frames = (_MAX_RAW_FRAMES - 1) // self.vae_scale_factor_temporal + 1 + condition_channels = self.vae_scale_factor_temporal + int(self.transformer.config.out_channels) + condition_bytes_per_session = ( + condition_channels + * horizon_latent_frames + * latent_height + * latent_width + * torch.empty((), dtype=self.transformer.dtype).element_size() + ) + return ARDiffusionKVCacheSpec( + num_layers=int(self.transformer.config.num_layers), + num_kv_heads=num_local_heads, + head_size=int(self.transformer.config.attention_head_dim), + tokens_per_frame=tokens_per_frame, + frames_per_block=int(self.transformer.config.num_frames_per_block), + window_frames=recent_window_frames, + sink_frames=sink_frames, + kv_branches=(ARDiffusionKVBranchSpec(self._AR_BRANCH, 0),), + session_capacity=2, + cross_attention=( + ARDiffusionCrossAttentionKVSpec( + self._AR_TEXT_CACHE, + _MAX_SEQUENCE_LENGTH, + ), + ), + model_owned_state_bytes_per_session=condition_bytes_per_session, + ) + + @contextmanager + def bind_ar_diffusion_state( + self, + session_id: str, + state: ARDiffusionKVState, + ) -> Iterator[None]: + if self._ar_diffusion_kv_state is not None: + raise RuntimeError("LingBot AR-Diffusion state is already bound.") + if state.session_id != session_id: + raise ValueError(f"LingBot bound session mismatch: {state.session_id!r} != {session_id!r}.") + self._ar_diffusion_kv_state = state + try: + yield + finally: + self._ar_diffusion_kv_state = None + + def reset_ar_diffusion_session(self, session_id: str) -> None: + self._ar_sessions.pop(session_id, None) + + def close_ar_diffusion_session(self, session_id: str) -> None: + self._ar_sessions.pop(session_id, None) + + def _parse_request(self, req: DiffusionRequestBatch) -> _LingBotRequestInputs: + if req.num_reqs != 1 or len(req.prompts) != 1: + raise ValueError("LingBot World supports a single prompt request, not request batching.") + sampling = req.sampling_params + if int(sampling.num_outputs_per_prompt or 1) != 1: + raise ValueError("LingBot World requires num_outputs_per_prompt=1.") + generator = getattr(sampling, "generator", None) + if isinstance(generator, list): + raise ValueError("LingBot World accepts one torch.Generator, not a generator list.") + if not isinstance(generator, torch.Generator): + raise ValueError("LingBot World requires the runner-provided torch.Generator.") + if getattr(sampling, "latents", None) is not None: + raise ValueError("LingBot World does not support caller-provided latents.") + + prompt_value = req.prompts[0] + multi_modal_data: dict[str, Any] + if isinstance(prompt_value, str): + prompt = prompt_value + multi_modal_data = {} + elif isinstance(prompt_value, dict): + prompt = prompt_value.get("prompt") or "" + multi_modal_data = prompt_value.get("multi_modal_data") or {} + else: + raise ValueError("prompt must be a string or prompt mapping.") + if not isinstance(prompt, str) or not prompt.strip(): + raise ValueError("prompt must contain non-empty text.") + if not isinstance(multi_modal_data, dict): + raise ValueError("prompt.multi_modal_data must be a mapping containing image.") + + image = multi_modal_data.get("image") + if isinstance(image, list): + raise ValueError("LingBot World requires one image and does not accept an image list.") + if image is None: + raise ValueError("LingBot World requires exactly one image in multi_modal_data.image.") + if isinstance(image, (str, os.PathLike)): + raise ValueError("file-path images must be materialized by the LingBot pre-process function.") + if not isinstance(image, (PIL.Image.Image, torch.Tensor)): + raise ValueError("multi_modal_data.image must be a PIL image, tensor, or file path.") + + extra_args = getattr(sampling, "extra_args", None) or {} + if not isinstance(extra_args, dict): + raise ValueError("sampling_params.extra_args must be a mapping.") + camera_trajectory = extra_args.get(_PREPROCESSED_CAMERA_KEY) + camera_actions = extra_args.get(_PREPROCESSED_CAMERA_ACTIONS_KEY) + if camera_trajectory is not None and not isinstance(camera_trajectory, CameraTrajectory): + raise ValueError("LingBot camera trajectory must be materialized by the pre-process function.") + if camera_actions is not None: + camera_actions = tuple(tuple(actions) for actions in camera_actions) + if (camera_trajectory is None) == (camera_actions is None): + raise ValueError("LingBot pre-processing must materialize exactly one camera input.") + + request_flow_shift = ( + extra_args["flow_shift"] if "flow_shift" in extra_args else getattr(self.scheduler.config, "shift", 5.0) + ) + flow_shift = _positive_finite_flow_shift(request_flow_shift) + + height = getattr(sampling, "height", None) + width = getattr(sampling, "width", None) + if isinstance(image, PIL.Image.Image): + image_width, image_height = image.size + elif image.ndim == 3 and image.shape[0] == 3: + image_height, image_width = image.shape[-2:] + elif image.ndim == 4 and image.shape[:2] == (1, 3): + image_height, image_width = image.shape[-2:] + else: + raise ValueError("tensor image must have shape [3, height, width] or [1, 3, height, width].") + if image_height <= 0 or image_width <= 0: + raise ValueError("source image width and height must be positive integers.") + if image_height * image_width > _MAX_SOURCE_IMAGE_PIXELS: + raise ValueError("source image pixel count must not exceed 4096 * 4096.") + if (height is None) != (width is None): + raise ValueError("height and width must either both be provided or both be omitted.") + patch_size = tuple(self.transformer.config.patch_size) + if len(patch_size) != 3 or patch_size[0] != 1: + raise RuntimeError( + "transformer.config.patch_size must be a three-dimensional tuple with temporal patch size 1." + ) + height_divisor = self.vae_scale_factor_spatial * patch_size[1] + width_divisor = self.vae_scale_factor_spatial * patch_size[2] + if height is None or width is None: + aspect_ratio = image_height / image_width + latent_height = round( + math.sqrt(_MAX_PIXEL_AREA * aspect_ratio) + // self.vae_scale_factor_spatial + // patch_size[1] + * patch_size[1] + ) + latent_width = round( + math.sqrt(_MAX_PIXEL_AREA / aspect_ratio) + // self.vae_scale_factor_spatial + // patch_size[2] + * patch_size[2] + ) + height = latent_height * self.vae_scale_factor_spatial + width = latent_width * self.vae_scale_factor_spatial + if isinstance(height, bool) or not isinstance(height, int) or height <= 0: + raise ValueError(f"height must be a positive integer, got {height!r}.") + if isinstance(width, bool) or not isinstance(width, int) or width <= 0: + raise ValueError(f"width must be a positive integer, got {width!r}.") + if height % height_divisor: + raise ValueError(f"height must be divisible by {height_divisor}, got {height}.") + if width % width_divisor: + raise ValueError(f"width must be divisible by {width_divisor}, got {width}.") + if height * width > _MAX_PIXEL_AREA: + raise ValueError("height * width pixel area must not exceed 480 * 832.") + + num_inference_steps = getattr(sampling, "num_inference_steps", None) + if num_inference_steps is None: + num_inference_steps = len(LINGBOT_DMD_TIMESTEPS) + if num_inference_steps != len(LINGBOT_DMD_TIMESTEPS): + raise ValueError( + "num_inference_steps must be 4 for LingBot World causal DMD " + f"timesteps {list(LINGBOT_DMD_TIMESTEPS)}, got {num_inference_steps}." + ) + + num_frames = getattr(sampling, "num_frames", None) + if isinstance(num_frames, bool) or not isinstance(num_frames, int) or num_frames <= 0: + raise ValueError(f"num_frames must be a positive integer, got {num_frames!r}.") + if num_frames > _MAX_RAW_FRAMES: + raise ValueError(f"num_frames must not exceed {_MAX_RAW_FRAMES}.") + temporal_factor = self.vae_scale_factor_temporal + if (num_frames - 1) % temporal_factor: + raise ValueError( + "num_frames must satisfy the causal Wan VAE geometry " + f"(num_frames - 1) divisible by {temporal_factor}, got {num_frames}." + ) + num_latent_frames = (num_frames - 1) // temporal_factor + 1 + block_frames = int(self.transformer.config.num_frames_per_block) + if num_latent_frames % block_frames: + raise ValueError( + "num_frames must map to a whole number of configured three-frame latent blocks; " + f"got num_frames={num_frames}, latent_frames={num_latent_frames}, block_frames={block_frames}." + ) + + max_sequence_length = getattr(sampling, "max_sequence_length", None) + if max_sequence_length is None: + max_sequence_length = _MAX_SEQUENCE_LENGTH + if isinstance(max_sequence_length, bool) or not isinstance(max_sequence_length, int): + raise ValueError(f"max_sequence_length must be exactly {_MAX_SEQUENCE_LENGTH}.") + if max_sequence_length != _MAX_SEQUENCE_LENGTH: + raise ValueError(f"max_sequence_length must be exactly {_MAX_SEQUENCE_LENGTH}.") + + return _LingBotRequestInputs( + prompt=prompt.strip(), + image=image, + camera_trajectory=camera_trajectory, + camera_actions=camera_actions, + height=height, + width=width, + num_frames=num_frames, + num_latent_frames=num_latent_frames, + output_type=getattr(sampling, "output_type", None) or "np", + max_sequence_length=max_sequence_length, + flow_shift=flow_shift, + generator=generator, + ) + + def _prepare_image_tensor(self, image: PIL.Image.Image | torch.Tensor, *, height: int, width: int) -> torch.Tensor: + if isinstance(image, PIL.Image.Image): + array = np.asarray(image, dtype=np.float32).copy() + image_tensor = torch.from_numpy(array).permute(2, 0, 1).unsqueeze(0) / 255.0 + else: + image_tensor = image.detach() + if image_tensor.ndim == 3: + image_tensor = image_tensor.unsqueeze(0) + image_tensor = image_tensor.to(dtype=torch.float32) + if not torch.isfinite(image_tensor).all(): + raise ValueError("tensor image values must all be finite.") + minimum = image_tensor.min().item() + maximum = image_tensor.max().item() + if minimum >= 0.0 and maximum > 255.0: + raise ValueError("tensor image values must be in [0, 1], [0, 255], or [-1, 1].") + if minimum < -1.0 or (minimum < 0.0 and maximum > 1.0): + raise ValueError("tensor image values must be in [0, 1], [0, 255], or [-1, 1].") + if maximum > 1.0: + image_tensor = image_tensor / 255.0 + if image_tensor.min().item() >= 0.0: + image_tensor = image_tensor * 2.0 - 1.0 + if image_tensor.shape[-2:] != (height, width): + image_tensor = F.interpolate( + image_tensor, + size=(height, width), + mode="bicubic", + align_corners=False, + ) + return image_tensor.to(device=self.device, dtype=torch.float32) + + def _vae_latent_stats(self, reference: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + shape = (1, -1, 1, 1, 1) + latent_mean = torch.as_tensor( + self.vae.config.latents_mean, + device=reference.device, + dtype=reference.dtype, + ).view(*shape) + latent_std = torch.as_tensor( + self.vae.config.latents_std, + device=reference.device, + dtype=reference.dtype, + ).view(*shape) + return latent_mean, latent_std + + def _prepare_condition(self, inputs: _LingBotRequestInputs, *, dtype: torch.dtype) -> torch.Tensor: + """Encode the first frame as ``[mask4, image_latent16]``.""" + + image = self._prepare_image_tensor(inputs.image, height=inputs.height, width=inputs.width) + video_condition = image.new_zeros(1, 3, inputs.num_frames, inputs.height, inputs.width) + video_condition[:, :, 0] = image + latent_condition = retrieve_latents( + self.vae.encode(video_condition.to(dtype=self.vae.dtype)), + sample_mode="argmax", + ) + if latent_condition.shape != ( + 1, + self.transformer.config.out_channels, + inputs.num_latent_frames, + inputs.height // self.vae_scale_factor_spatial, + inputs.width // self.vae_scale_factor_spatial, + ): + raise RuntimeError( + f"vae.encode returned an incompatible image latent shape: got {tuple(latent_condition.shape)}." + ) + latent_mean, latent_std = self._vae_latent_stats(latent_condition) + latent_condition = (latent_condition - latent_mean) / latent_std + temporal_mask = latent_condition.new_zeros( + 1, + self.vae_scale_factor_temporal, + inputs.num_latent_frames, + latent_condition.shape[-2], + latent_condition.shape[-1], + ) + temporal_mask[:, :, 0] = 1 + condition = torch.cat((temporal_mask, latent_condition), dim=1).to(dtype=dtype) + if condition.shape[1] != 20: + raise RuntimeError( + "LingBot image condition must contain 4 temporal-mask then 16 image-latent channels, " + f"got {condition.shape[1]}." + ) + return condition + + def _prepare_camera( + self, + inputs: _LingBotRequestInputs, + *, + dtype: torch.dtype, + previous: CameraTrajectory | None = None, + ) -> tuple[torch.Tensor, CameraTrajectory]: + """Convert raw camera frames to a latent-aligned ray tensor.""" + + trajectory = inputs.camera_trajectory + if trajectory is None: + raise RuntimeError("LingBot camera trajectory was not prepared before camera embedding.") + available_frames = int(trajectory.poses.shape[0]) + if inputs.camera_actions is not None: + if available_frames != inputs.num_latent_frames: + raise ValueError( + "camera actions must produce exactly one pose per latent frame; " + f"got camera_frames={available_frames}, latent_frames={inputs.num_latent_frames}." + ) + else: + if available_frames < inputs.num_frames: + raise ValueError( + "camera trajectory frames must be at least num_frames; " + f"got camera_frames={available_frames}, num_frames={inputs.num_frames}." + ) + trajectory = CameraTrajectory( + poses=trajectory.poses[: inputs.num_frames], + intrinsics=trajectory.intrinsics[: inputs.num_frames], + ) + trajectory = interpolate_camera_trajectory(trajectory, inputs.num_latent_frames) + embedding_trajectory = trajectory + drop_anchor = False + if previous is not None: + embedding_trajectory = CameraTrajectory( + poses=torch.cat((previous.poses, trajectory.poses), dim=0), + intrinsics=torch.cat( + (previous.intrinsics, trajectory.intrinsics), + dim=0, + ), + ) + drop_anchor = True + elif inputs.camera_actions is not None: + # The action integrator returns post-action poses. Keep the first + # action visible to framewise-delta conditioning by prepending its + # known pre-action state (identity for a new realtime session). + identity = torch.eye( + 4, + device=trajectory.poses.device, + dtype=trajectory.poses.dtype, + ).unsqueeze(0) + embedding_trajectory = CameraTrajectory( + poses=torch.cat((identity, trajectory.poses), dim=0), + intrinsics=torch.cat( + (trajectory.intrinsics[:1], trajectory.intrinsics), + dim=0, + ), + ) + drop_anchor = True + camera_embedding = build_plucker_embedding( + embedding_trajectory, + height=inputs.height, + width=inputs.width, + target_height=inputs.height, + target_width=inputs.width, + device=self.device, + dtype=dtype, + ) + if drop_anchor: + camera_embedding = camera_embedding[1:] + tail = CameraTrajectory( + poses=trajectory.poses[-1:].clone(), + intrinsics=trajectory.intrinsics[-1:].clone(), + ) + return _fold_camera_embedding(camera_embedding), tail + + def _ar_text_caches( + self, + prompt_embeds: torch.Tensor, + *, + invalidate: bool, + ) -> list[LingBotAttentionCache]: + state = self._ar_diffusion_kv_state + if state is None: + raise RuntimeError("LingBot AR text cache requested without a bound state.") + if invalidate: + state.clear_cross_attention() + if not state.is_cross_attention_populated( + self._AR_BRANCH, + self._AR_TEXT_CACHE, + ): + projected_text = self.transformer.text_embedding(prompt_embeds) + + def layer_kv() -> Iterator[tuple[torch.Tensor, torch.Tensor]]: + for block in self.transformer.blocks: + cross_attention = block.cross_attn + key = cross_attention.norm_k(cross_attention.k(projected_text)).unflatten( + 2, + ( + cross_attention.num_local_heads, + cross_attention.head_dim, + ), + ) + value = cross_attention.v(projected_text).unflatten( + 2, + ( + cross_attention.num_local_heads, + cross_attention.head_dim, + ), + ) + yield key, value + + state.populate_cross_attention( + self._AR_BRANCH, + self._AR_TEXT_CACHE, + layer_kv(), + ) + return [ + LingBotAttentionCache( + key=layer["k"], + value=layer["v"], + end=prompt_embeds.shape[1], + absolute_end=prompt_embeds.shape[1], + last_start=0, + ) + for layer in state.get_cross_attention_kv( + self._AR_BRANCH, + self._AR_TEXT_CACHE, + ) + ] + + def _ar_transformer_cache( + self, + *, + condition: torch.Tensor, + cross_attention: list[LingBotAttentionCache], + commit_current: bool, + ) -> LingBotTransformerCache: + state = self._ar_diffusion_kv_state + if state is None: + raise RuntimeError("LingBot AR cache requested without a bound state.") + patch_frames, patch_height, patch_width = self.transformer.config.patch_size + frames = condition.shape[2] // patch_frames + height = condition.shape[3] // patch_height + width = condition.shape[4] // patch_width + seq_len = frames * height * width + return LingBotTransformerCache( + self_attention=state.get_kv_caches( + self._AR_BRANCH, + seq_len=seq_len, + commit_current=commit_current, + ), + cross_attention=cross_attention, + ) + + def _generate_block( + self, + *, + condition: torch.Tensor, + camera: torch.Tensor, + prompt_embeds: torch.Tensor, + cache: LingBotTransformerCache | None, + ar_cross_attention: list[LingBotAttentionCache] | None, + start_frame: int, + schedule: tuple[tuple[float, float], ...], + generator: torch.Generator, + progress_bar: TqdmProgressBar[Any], + ) -> torch.Tensor: + block_shape = ( + 1, + self.transformer.config.out_channels, + condition.shape[2], + condition.shape[3], + condition.shape[4], + ) + current_latents = randn_tensor( + block_shape, + generator=generator, + device=self.device, + dtype=torch.float32, + ) + for step_index, (timestep_value, sigma) in enumerate(schedule): + if not self.od_config.enforce_eager: + torch.compiler.cudagraph_mark_step_begin() + set_forward_context_denoise_step_idx(step_index) + timestep = torch.full((1,), float(timestep_value), device=self.device, dtype=torch.float32) + # Checkpoint channel contract: + # [noise/x_t(16), temporal_mask(4), image_latent(16)] -> 36. + model_input = torch.cat((current_latents.to(dtype=condition.dtype), condition), dim=1) + current_cache = ( + self._ar_transformer_cache( + condition=condition, + cross_attention=cast( + list[LingBotAttentionCache], + ar_cross_attention, + ), + commit_current=False, + ) + if ar_cross_attention is not None + else cast(LingBotTransformerCache, cache) + ) + flow_prediction = self.transformer( + hidden_states=model_input, + timestep=timestep, + encoder_hidden_states=prompt_embeds, + camera_hidden_states=camera, + cache=current_cache, + start_frame=start_frame, + update_cache=False, + ) + if flow_prediction.shape != current_latents.shape: + raise RuntimeError( + "transformer flow prediction shape must match the 16-channel noise latent, " + f"got {tuple(flow_prediction.shape)} and {tuple(current_latents.shape)}." + ) + # The checkpoint's flow parameterization is inverted by + # x0 = x_t - sigma * flow. Intermediate steps re-noise that x0 + # estimate at the next sigma; the final step keeps x0 as this + # block's generated latent. + x0 = current_latents - sigma * flow_prediction.float() + if step_index + 1 < len(schedule): + next_sigma = schedule[step_index + 1][1] + noise = randn_tensor( + current_latents.shape, + generator=generator, + device=self.device, + dtype=torch.float32, + ) + current_latents = (1.0 - next_sigma) * x0 + next_sigma * noise + else: + current_latents = x0 + progress_bar.update() + + # Commit K/V only for the final clean block, never for noisy probes. + cache_input = torch.cat((current_latents.to(dtype=condition.dtype), condition), dim=1) + commit_cache = ( + self._ar_transformer_cache( + condition=condition, + cross_attention=cast( + list[LingBotAttentionCache], + ar_cross_attention, + ), + commit_current=True, + ) + if ar_cross_attention is not None + else cast(LingBotTransformerCache, cache) + ) + if not self.od_config.enforce_eager: + torch.compiler.cudagraph_mark_step_begin() + self.transformer( + hidden_states=cache_input, + timestep=torch.zeros(1, device=self.device, dtype=torch.float32), + encoder_hidden_states=prompt_embeds, + camera_hidden_states=camera, + cache=commit_cache, + start_frame=start_frame, + update_cache=True, + ) + if ar_cross_attention is not None: + state = self._ar_diffusion_kv_state + if state is None: + raise RuntimeError("LingBot AR state disappeared before KV commit.") + state.commit_paged_context(self._AR_BRANCH) + return current_latents + + def encode_prompt( + self, + prompt: str, + *, + max_sequence_length: int, + dtype: torch.dtype, + ) -> torch.Tensor: + text_inputs = self.tokenizer( + [" ".join(prompt.strip().split())], + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_attention_mask=True, + return_tensors="pt", + ) + input_ids = text_inputs.input_ids.to(self.device) + attention_mask = text_inputs.attention_mask.to(self.device) + prompt_embeds = self.text_encoder(input_ids, attention_mask).last_hidden_state + prompt_embeds = prompt_embeds.to(device=self.device, dtype=dtype) + return prompt_embeds * attention_mask.unsqueeze(-1).to(dtype=prompt_embeds.dtype) + + def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput: + inputs = self._parse_request(req) + tick = ARDiffusionTickRequest.from_extra_args(req.sampling_params.extra_args) + if tick is not None and self._ar_diffusion_kv_state is None: + raise RuntimeError("LingBot typed ticks require ARDiffusionEngine session binding.") + if tick is None and self._ar_diffusion_kv_state is not None: + raise ValueError("LingBot ARDiffusionEngine requests must carry ar_diffusion_tick.") + session_state: _LingBotARSessionState | None = None + if tick is not None: + if tick.prompt is not None and " ".join(tick.prompt.split()) != " ".join(inputs.prompt.split()): + raise ValueError("ar_diffusion_tick.prompt must match the standard request prompt.") + block_frames = int(self.transformer.config.num_frames_per_block) + if inputs.num_latent_frames != block_frames: + raise ValueError("A LingBot AR-Diffusion request must generate exactly one three-latent-frame block.") + if (inputs.height, inputs.width) != ( + self._ar_height, + self._ar_width, + ): + raise ValueError( + "LingBot AR-Diffusion request resolution must match the " + "fixed cache geometry " + f"{self._ar_height}x{self._ar_width}." + ) + horizon_latent_frames = (_MAX_RAW_FRAMES - 1) // self.vae_scale_factor_temporal + 1 + max_realtime_ticks = horizon_latent_frames // block_frames + if tick.chunk_index >= max_realtime_ticks: + raise ValueError( + "LingBot realtime generation currently supports at most " + f"{max_realtime_ticks} ticks per generation epoch " + f"(chunk_index 0 through {max_realtime_ticks - 1}) because " + f"the image-condition horizon is {_MAX_RAW_FRAMES} pixel " + "frames; reset or create a session to start a new world." + ) + session_state = self._ar_sessions.setdefault( + tick.session_id, + _LingBotARSessionState(), + ) + if tick.chunk_index != session_state.next_chunk_index: + raise ValueError( + "LingBot chunk_index must be contiguous within a session: " + f"expected {session_state.next_chunk_index}, got " + f"{tick.chunk_index}." + ) + schedule = _build_shifted_flow_schedule( + flow_shift=inputs.flow_shift, + ) + dtype = self.transformer.dtype + # Phase 1: turn all three user inputs into DiT-ready conditions. + prompt_embeds = self.encode_prompt( + inputs.prompt, + max_sequence_length=inputs.max_sequence_length, + dtype=dtype, + ) + if tick is None: + condition = self._prepare_condition(inputs, dtype=dtype) + else: + assert session_state is not None + if session_state.image_condition is None: + horizon_latent_frames = (_MAX_RAW_FRAMES - 1) // self.vae_scale_factor_temporal + 1 + session_state.image_condition = self._prepare_condition( + replace( + inputs, + num_frames=_MAX_RAW_FRAMES, + num_latent_frames=horizon_latent_frames, + ), + dtype=dtype, + ) + block_frames = int(self.transformer.config.num_frames_per_block) + condition_start = tick.chunk_index * block_frames + condition_stop = condition_start + block_frames + if condition_stop > session_state.image_condition.shape[2]: + raise ValueError("LingBot chunk_index exceeds the configured causal image condition horizon.") + condition = session_state.image_condition[ + :, + :, + condition_start:condition_stop, + ] + camera_pitch: float | None = None + if inputs.camera_actions is not None: + assert session_state is not None + action_trajectory, camera_pitch = integrate_lingbot_camera_actions( + inputs.camera_actions, + width=inputs.width, + height=inputs.height, + initial_pose=(session_state.camera_tail.poses[-1] if session_state.camera_tail is not None else None), + initial_pitch=session_state.camera_pitch, + ) + inputs = replace(inputs, camera_trajectory=action_trajectory) + camera, camera_tail = self._prepare_camera( + inputs, + dtype=dtype, + previous=session_state.camera_tail if session_state is not None else None, + ) + if camera.shape[2:] != condition.shape[2:]: + raise RuntimeError( + "folded camera and image condition must share latent frame/height/width geometry; " + f"got camera={tuple(camera.shape)}, condition={tuple(condition.shape)}." + ) + + block_frames = int(self.transformer.config.num_frames_per_block) + cache: LingBotTransformerCache | None = None + ar_cross_attention: list[LingBotAttentionCache] | None = None + if tick is None: + # Direct/offline oracle keeps request-local contiguous K/V. + cache = self.transformer.allocate_cache( + batch_size=1, + latent_height=condition.shape[-2], + latent_width=condition.shape[-1], + device=self.device, + dtype=dtype, + ) + else: + assert session_state is not None + prompt_changed = session_state.prompt is not None and session_state.prompt != inputs.prompt + if session_state.generator_state is not None: + inputs.generator.set_state(session_state.generator_state) + ar_cross_attention = self._ar_text_caches( + prompt_embeds, + invalidate=prompt_changed, + ) + generated_blocks: list[torch.Tensor] = [] + total_steps = (inputs.num_latent_frames // block_frames) * len(LINGBOT_DMD_TIMESTEPS) + first_start_frame = tick.chunk_index * block_frames if tick is not None else 0 + with self.progress_bar(total=total_steps) as progress_bar: + # Phase 3: generate left-to-right in latent-frame blocks. The + # monotonically increasing ``start_frame`` is also the temporal + # position and cache offset used by the Transformer. + for local_start_frame in range( + 0, + inputs.num_latent_frames, + block_frames, + ): + start_frame = first_start_frame + local_start_frame + stop_frame = local_start_frame + block_frames + generated_blocks.append( + self._generate_block( + condition=condition[:, :, local_start_frame:stop_frame], + camera=camera[:, :, local_start_frame:stop_frame], + prompt_embeds=prompt_embeds, + cache=cache, + ar_cross_attention=ar_cross_attention, + start_frame=start_frame, + schedule=schedule, + generator=inputs.generator, + progress_bar=progress_bar, + ) + ) + generated_latents = torch.cat(generated_blocks, dim=2) + # The direct/offline cache is request-local and no longer needed once + # all latent blocks have been generated. Release it before VAE decode + # so the two large allocations do not overlap. + cache = None + if tick is not None: + assert session_state is not None + session_state.prompt = inputs.prompt + session_state.generator_state = inputs.generator.get_state() + session_state.camera_tail = camera_tail + if camera_pitch is not None: + session_state.camera_pitch = camera_pitch + session_state.next_chunk_index += 1 + + # Phase 4: either expose model-space latents or invert the checkpoint's + # latent normalization and decode to pixel-space video. + if tick is not None: + if inputs.output_type != "latent": + raise ValueError( + "LingBot realtime ticks currently require output_type='latent'; " + "stateful streaming VAE decode is a separate integration step." + ) + output = { + "payload": {"latents": generated_latents}, + "metadata": {"ar_diffusion": ARDiffusionChunkMetadata.from_tick(tick).to_dict()}, + } + elif inputs.output_type == "latent": + output = generated_latents + else: + latent_mean, latent_std = self._vae_latent_stats(generated_latents) + vae_latents = (generated_latents * latent_std + latent_mean).to(dtype=self.vae.dtype) + output = self.vae.decode(vae_latents, return_dict=False)[0] + if output.shape[2] != inputs.num_frames: + raise RuntimeError( + "vae.decode returned an incompatible temporal geometry: " + f"expected {inputs.num_frames} frames, got {output.shape[2]}." + ) + return DiffusionOutput( + output=output, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + ) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self) + return cast(set[str], loader.load_weights(weights)) diff --git a/vllm_omni/diffusion/models/lingbot_world/transformer.py b/vllm_omni/diffusion/models/lingbot_world/transformer.py new file mode 100644 index 000000000..35ec18801 --- /dev/null +++ b/vllm_omni/diffusion/models/lingbot_world/transformer.py @@ -0,0 +1,1201 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Checkpoint-compatible causal DiT for the LingBot World v2 model package.""" + +from __future__ import annotations + +import math +from collections.abc import Iterable +from dataclasses import dataclass +from types import SimpleNamespace +from typing import Any, Self, cast + +import torch +import torch.nn as nn +import torch.nn.functional as F +from vllm.distributed import ( + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce, +) +from vllm.model_executor.layers.conv import Conv3dLayer +from vllm.model_executor.layers.linear import ColumnParallelLinear, QKVParallelLinear, RowParallelLinear +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.utils import set_weight_attrs + +from vllm_omni.diffusion.attention.layer import Attention +from vllm_omni.diffusion.layers.norm import LayerNorm +from vllm_omni.diffusion.layers.rope import RotaryEmbeddingWan +from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import ( + ARDiffusionPagedLayerContext, + ARDiffusionPagedLayerInputs, + ar_diffusion_paged_attention, + paged_write_attn, +) + + +@dataclass +class LingBotAttentionCache: + """K/V storage plus its logical cursor metadata. + + ``end`` is occupied storage, ``absolute_end`` is the next global token + position, and ``sink_end`` separates permanently retained prefix tokens + from the sliding local window. ``last_start`` rejects overlapping or + out-of-order causal chunks. + """ + + key: torch.Tensor + value: torch.Tensor + end: int = 0 + absolute_end: int = 0 + last_start: int | None = None + sink_end: int = 0 + + +@dataclass +class LingBotTransformerCache: + """One request's per-layer video K/V and reusable text K/V.""" + + self_attention: list[LingBotAttentionCache | ARDiffusionPagedLayerContext | ARDiffusionPagedLayerInputs] + cross_attention: list[LingBotAttentionCache | None] + + +def allocate_lingbot_cache( + *, + batch_size: int, + num_layers: int, + max_tokens: int, + num_local_heads: int, + head_dim: int, + device: torch.device, + dtype: torch.dtype, +) -> LingBotTransformerCache: + # Cross-attention starts empty because its token count is known after text encoding. + shape = (batch_size, max_tokens, num_local_heads, head_dim) + self_attention = [ + LingBotAttentionCache( + key=torch.zeros(shape, device=device, dtype=dtype), + value=torch.zeros(shape, device=device, dtype=dtype), + ) + for _ in range(num_layers) + ] + return LingBotTransformerCache( + self_attention=self_attention, + cross_attention=[None for _ in range(num_layers)], + ) + + +class _LingBotRMSNorm(nn.Module): + """RMSNorm over a tensor-parallel sharded projection.""" + + def __init__(self, hidden_size: int, eps: float) -> None: + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.ones(hidden_size)) + set_weight_attrs(self.weight, {"weight_loader": self.weight_loader}) + + def weight_loader(self, param: torch.Tensor, loaded_weight: torch.Tensor) -> None: + if param.shape == loaded_weight.shape: + param.data.copy_(loaded_weight) + return + + tp_size = get_tensor_model_parallel_world_size() + if loaded_weight.shape[0] % tp_size != 0: + raise ValueError( + f"Cannot shard RMSNorm weight of shape {tuple(loaded_weight.shape)} across tp_size={tp_size}." + ) + shard_size = loaded_weight.shape[0] // tp_size + shard_start = get_tensor_model_parallel_rank() * shard_size + shard = loaded_weight.narrow(0, shard_start, shard_size) + if param.shape != shard.shape: + raise ValueError(f"RMSNorm shard shape mismatch: param={tuple(param.shape)}, shard={tuple(shard.shape)}.") + param.data.copy_(shard) + + def forward(self, value: torch.Tensor) -> torch.Tensor: + tp_size = get_tensor_model_parallel_world_size() + value_float = value.float() + sum_of_squares = value_float.pow(2).sum(dim=-1, keepdim=True) + element_count = value.shape[-1] + if tp_size > 1: + sum_of_squares = tensor_model_parallel_all_reduce(sum_of_squares) + element_count *= tp_size + rms = torch.sqrt(sum_of_squares / element_count + self.eps) + return (value_float / rms * self.weight.float()).to(value.dtype) + + +def _projection_prefix(prefix: str, name: str) -> str: + return f"{prefix}.{name}" if prefix else name + + +class LingBotSelfAttention(nn.Module): + """Block-causal self-attention over retained history and one full chunk.""" + + def __init__( + self, + dim: int, + num_heads: int, + *, + eps: float = 1e-6, + prefix: str = "", + ) -> None: + super().__init__() + if dim % num_heads != 0: + raise ValueError(f"dim={dim} must be divisible by num_heads={num_heads}.") + + tp_size = get_tensor_model_parallel_world_size() + if num_heads % tp_size != 0: + raise ValueError(f"num_heads={num_heads} must be divisible by tp_size={tp_size}.") + + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.qkv = QKVParallelLinear( + hidden_size=dim, + head_size=self.head_dim, + total_num_heads=num_heads, + bias=True, + prefix=_projection_prefix(prefix, "qkv"), + ) + self.num_local_heads = self.qkv.num_heads + self.tp_inner_dim = self.num_local_heads * self.head_dim + self.o = RowParallelLinear( + dim, + dim, + bias=True, + input_is_parallel=True, + return_bias=False, + prefix=_projection_prefix(prefix, "o"), + ) + self.norm_q = _LingBotRMSNorm(self.tp_inner_dim, eps) + self.norm_k = _LingBotRMSNorm(self.tp_inner_dim, eps) + self.rotary_embedding = RotaryEmbeddingWan(is_neox_style=False, half_head_dim=True) + self.attn = Attention( + num_heads=self.num_local_heads, + head_size=self.head_dim, + num_kv_heads=self.num_local_heads, + softmax_scale=self.head_dim**-0.5, + causal=False, + role="self", + qkv_layout="BSND", + prefix=prefix, + skip_sequence_parallel=True, + ) + + def _update_cache( + self, + cache: LingBotAttentionCache, + key: torch.Tensor, + value: torch.Tensor, + current_start: int, + *, + sink_tokens: int, + update_cache: bool, + ) -> tuple[torch.Tensor, torch.Tensor]: + chunk_tokens = key.shape[1] + capacity = cache.key.shape[1] + next_sink_end = cache.sink_end + + if cache.last_start is None: + if chunk_tokens > capacity: + raise ValueError( + f"Current chunk has {chunk_tokens} tokens but the cache can hold only {capacity}; " + "the full current chunk must remain visible." + ) + next_key = key + next_value = value + next_sink_end = max(0, min(chunk_tokens, sink_tokens - current_start)) + elif current_start == cache.last_start: + if current_start + chunk_tokens != cache.absolute_end: + raise ValueError("A repeated current_start must overwrite the same-size current chunk.") + if chunk_tokens > cache.end: + raise ValueError("The current chunk is no longer fully retained in the cache.") + prefix_end = cache.end - chunk_tokens + next_key = torch.cat((cache.key[:, :prefix_end], key), dim=1) + next_value = torch.cat((cache.value[:, :prefix_end], value), dim=1) + else: + if current_start < cache.last_start: + raise ValueError(f"current_start={current_start} precedes the latest chunk start {cache.last_start}.") + if current_start < cache.absolute_end: + raise ValueError( + f"current_start={current_start} overlaps cached tokens ending at {cache.absolute_end}." + ) + if current_start > cache.absolute_end: + raise ValueError( + f"New chunks must be contiguous: current_start={current_start}, expected {cache.absolute_end}." + ) + + incoming_sink_tokens = max(0, min(chunk_tokens, sink_tokens - current_start)) + old_sink_key = cache.key[:, : cache.sink_end] + old_sink_value = cache.value[:, : cache.sink_end] + new_sink_key = key[:, :incoming_sink_tokens] + new_sink_value = value[:, :incoming_sink_tokens] + next_sink_end = cache.sink_end + incoming_sink_tokens + + old_local_key = cache.key[:, cache.sink_end : cache.end] + old_local_value = cache.value[:, cache.sink_end : cache.end] + new_local_key = key[:, incoming_sink_tokens:] + new_local_value = value[:, incoming_sink_tokens:] + local_capacity = capacity - next_sink_end + if new_local_key.shape[1] > local_capacity: + raise ValueError("The configured cache cannot retain all sink tokens and the full current chunk.") + retained_local_tokens = min( + old_local_key.shape[1], + local_capacity - new_local_key.shape[1], + ) + if retained_local_tokens: + old_local_key = old_local_key[:, -retained_local_tokens:] + old_local_value = old_local_value[:, -retained_local_tokens:] + else: + old_local_key = old_local_key[:, :0] + old_local_value = old_local_value[:, :0] + + next_key = torch.cat((old_sink_key, new_sink_key, old_local_key, new_local_key), dim=1) + next_value = torch.cat((old_sink_value, new_sink_value, old_local_value, new_local_value), dim=1) + + next_end = next_key.shape[1] + if update_cache: + cache.key[:, :next_end].copy_(next_key) + cache.value[:, :next_end].copy_(next_value) + cache.end = next_end + cache.absolute_end = current_start + chunk_tokens + cache.last_start = current_start + cache.sink_end = next_sink_end + return next_key, next_value + + def forward( + self, + hidden_states: torch.Tensor, + *, + cache: LingBotAttentionCache | ARDiffusionPagedLayerInputs, + current_start: int, + rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None, + sink_tokens: int, + update_cache: bool = True, + ) -> torch.Tensor: + # Project logical [B, S, D] tokens into TP-local + # [B, S, num_local_heads, head_dim]. + qkv, _ = self.qkv(hidden_states) + query, key, value = qkv.split((self.tp_inner_dim, self.tp_inner_dim, self.tp_inner_dim), dim=-1) + query = self.norm_q(query) + key = self.norm_k(key) + + query = query.unflatten(2, (self.num_local_heads, self.head_dim)) + key = key.unflatten(2, (self.num_local_heads, self.head_dim)) + value = value.unflatten(2, (self.num_local_heads, self.head_dim)) + if rotary_emb is not None: + cos, sin = rotary_emb + query = self.rotary_embedding(query, cos, sin) + key = self.rotary_embedding(key, cos, sin) + + if isinstance(cache, ARDiffusionPagedLayerInputs): + if query.shape[0] != 1: + raise RuntimeError("LingBot AR-Diffusion paged attention requires batch_size=1.") + output = paged_write_attn( + cache, + query[0], + key[0], + value[0], + None, + None, + self.head_dim**-0.5, + ).unsqueeze(0) + else: + # The direct path sees history + current K/V even when the caller + # asks not to mutate persistent cache state. + visible_key, visible_value = self._update_cache( + cache, + key, + value, + current_start, + sink_tokens=sink_tokens, + update_cache=update_cache, + ) + if query.is_cuda and query.shape[0] == 1: + # Use the same block-table FlashAttention entry point as the + # realtime path so direct replay is a numerical oracle for + # paged execution, rather than a comparison between two + # different attention kernels. + block_size = key.shape[1] + key_cache = visible_key[0].unflatten(0, (-1, block_size)) + value_cache = visible_value[0].unflatten(0, (-1, block_size)) + block_count = key_cache.shape[0] + block_table = torch.arange( + block_count, + dtype=torch.int32, + device=query.device, + ).unsqueeze(0) + query_start_loc = torch.tensor( + [0, query.shape[1]], + dtype=torch.int32, + device=query.device, + ) + seq_lens = torch.tensor( + [visible_key.shape[1]], + dtype=torch.int32, + device=query.device, + ) + output = ar_diffusion_paged_attention( + query, + key_cache, + value_cache, + block_table=block_table, + query_start_loc=query_start_loc, + seq_lens=seq_lens, + max_query_len=query.shape[1], + max_seq_len=visible_key.shape[1], + softmax_scale=self.head_dim**-0.5, + ) + else: + output = self.attn(query, visible_key, visible_value) + return self.o(output.flatten(2, 3)) + + +class LingBotCrossAttention(nn.Module): + """Cross-attention with caller-owned request-local encoder K/V reuse.""" + + def __init__( + self, + dim: int, + num_heads: int, + *, + eps: float = 1e-6, + prefix: str = "", + ) -> None: + super().__init__() + if dim % num_heads != 0: + raise ValueError(f"dim={dim} must be divisible by num_heads={num_heads}.") + + tp_size = get_tensor_model_parallel_world_size() + if num_heads % tp_size != 0: + raise ValueError(f"num_heads={num_heads} must be divisible by tp_size={tp_size}.") + + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.num_local_heads = num_heads // tp_size + self.tp_inner_dim = self.num_local_heads * self.head_dim + + self.q = ColumnParallelLinear( + dim, + dim, + bias=True, + gather_output=False, + return_bias=False, + prefix=_projection_prefix(prefix, "q"), + ) + self.k = ColumnParallelLinear( + dim, + dim, + bias=True, + gather_output=False, + return_bias=False, + prefix=_projection_prefix(prefix, "k"), + ) + self.v = ColumnParallelLinear( + dim, + dim, + bias=True, + gather_output=False, + return_bias=False, + prefix=_projection_prefix(prefix, "v"), + ) + self.o = RowParallelLinear( + dim, + dim, + bias=True, + input_is_parallel=True, + return_bias=False, + prefix=_projection_prefix(prefix, "o"), + ) + self.norm_q = _LingBotRMSNorm(self.tp_inner_dim, eps) + self.norm_k = _LingBotRMSNorm(self.tp_inner_dim, eps) + self.attn = Attention( + num_heads=self.num_local_heads, + head_size=self.head_dim, + num_kv_heads=self.num_local_heads, + softmax_scale=self.head_dim**-0.5, + causal=False, + role="cross", + qkv_layout="BSND", + prefix=prefix, + skip_sequence_parallel=True, + disable_kv_quant=True, + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | None, + *, + cache: LingBotAttentionCache | None, + ) -> tuple[torch.Tensor, LingBotAttentionCache]: + query = self.norm_q(self.q(hidden_states)) + query = query.unflatten(2, (self.num_local_heads, self.head_dim)) + + # Text K/V is constant within a request and is projected once per layer. + if cache is None: + if encoder_hidden_states is None: + raise ValueError("encoder_hidden_states are required when the cross-attention cache is empty.") + key = self.norm_k(self.k(encoder_hidden_states)) + value = self.v(encoder_hidden_states) + key = key.unflatten(2, (self.num_local_heads, self.head_dim)) + value = value.unflatten(2, (self.num_local_heads, self.head_dim)) + cache = LingBotAttentionCache( + key=key, + value=value, + end=key.shape[1], + absolute_end=key.shape[1], + last_start=0, + ) + else: + key = cache.key[:, : cache.end] + value = cache.value[:, : cache.end] + + output = self.attn(query, key, value) + return self.o(output.flatten(2, 3)), cache + + +class LingBotAttentionBlock(nn.Module): + """Checkpoint-compatible LingBot block with causal video attention.""" + + def __init__( + self, + dim: int, + num_heads: int, + *, + ffn_dim: int | None = None, + cross_attn_norm: bool = True, + eps: float = 1e-6, + prefix: str = "", + ) -> None: + super().__init__() + ffn_dim = ffn_dim or dim * 4 + self.dim = dim + self.norm1 = LayerNorm(dim, eps=eps, elementwise_affine=False) + self.self_attn = LingBotSelfAttention( + dim, + num_heads, + eps=eps, + prefix=_projection_prefix(prefix, "self_attn"), + ) + self.cross_attn = LingBotCrossAttention( + dim, + num_heads, + eps=eps, + prefix=_projection_prefix(prefix, "cross_attn"), + ) + self.norm2 = LayerNorm(dim, eps=eps, elementwise_affine=False) + self.norm3 = LayerNorm(dim, eps=eps) if cross_attn_norm else nn.Identity() + self.ffn = nn.Sequential( + ColumnParallelLinear( + dim, + ffn_dim, + bias=True, + gather_output=False, + return_bias=False, + prefix=_projection_prefix(prefix, "ffn.0"), + ), + nn.GELU(approximate="tanh"), + RowParallelLinear( + ffn_dim, + dim, + bias=True, + input_is_parallel=True, + return_bias=False, + prefix=_projection_prefix(prefix, "ffn.2"), + ), + ) + self.modulation = nn.Parameter(torch.randn(1, 6, dim) / math.sqrt(dim)) + self.cam_injector_layer1 = ColumnParallelLinear( + dim, + dim, + bias=True, + gather_output=False, + return_bias=False, + prefix=_projection_prefix(prefix, "cam_injector_layer1"), + ) + self.cam_injector_layer2 = RowParallelLinear( + dim, + dim, + bias=True, + input_is_parallel=True, + return_bias=False, + prefix=_projection_prefix(prefix, "cam_injector_layer2"), + ) + self.cam_scale_layer = nn.Linear(dim, dim) + self.cam_shift_layer = nn.Linear(dim, dim) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | None, + timestep_projection: torch.Tensor, + camera_hidden_states: torch.Tensor, + *, + self_cache: LingBotAttentionCache | ARDiffusionPagedLayerInputs, + cross_cache: LingBotAttentionCache | None, + current_start: int, + sink_tokens: int, + update_cache: bool, + rotary_emb: tuple[torch.Tensor, torch.Tensor], + ) -> tuple[torch.Tensor, LingBotAttentionCache]: + batch_size, token_count, dim = hidden_states.shape + num_frames = timestep_projection.shape[1] + tokens_per_frame = token_count // num_frames + # Timestep, camera, and text remain separate conditioning paths. + modulation = self.modulation.unsqueeze(1) + timestep_projection.float() + shift_msa, scale_msa, gate_msa, shift_ffn, scale_ffn, gate_ffn = modulation.chunk(6, dim=2) + + hidden_grid = hidden_states.unflatten(1, (num_frames, tokens_per_frame)) + normalized = self.norm1(hidden_states.float()).unflatten(1, (num_frames, tokens_per_frame)) + normalized = (normalized * (1 + scale_msa) + shift_msa).flatten(1, 2).to(hidden_states.dtype) + attention_output = self.self_attn( + normalized, + cache=self_cache, + current_start=current_start, + rotary_emb=rotary_emb, + sink_tokens=sink_tokens, + update_cache=update_cache, + ) + hidden_grid = hidden_grid + attention_output.unflatten(1, (num_frames, tokens_per_frame)) * gate_msa + hidden_states = hidden_grid.flatten(1, 2).to(hidden_states.dtype) + + camera_features = self.cam_injector_layer2(F.silu(self.cam_injector_layer1(camera_hidden_states))) + camera_features = camera_features + camera_hidden_states + camera_scale = self.cam_scale_layer(camera_features) + camera_shift = self.cam_shift_layer(camera_features) + hidden_states = ((1 + camera_scale) * hidden_states + camera_shift).to(hidden_states.dtype) + + attention_output, cross_cache = self.cross_attn( + self.norm3(hidden_states), + encoder_hidden_states, + cache=cross_cache, + ) + hidden_states = hidden_states + attention_output + + hidden_grid = hidden_states.unflatten(1, (num_frames, tokens_per_frame)) + normalized = self.norm2(hidden_states.float()).unflatten(1, (num_frames, tokens_per_frame)) + normalized = (normalized * (1 + scale_ffn) + shift_ffn).flatten(1, 2).to(hidden_states.dtype) + ffn_output = self.ffn(normalized).unflatten(1, (num_frames, tokens_per_frame)) + hidden_states = (hidden_grid + ffn_output * gate_ffn).flatten(1, 2).to(hidden_states.dtype) + return hidden_states, cast(LingBotAttentionCache, cross_cache) + + +class _LingBotCameraPatchEmbedding(nn.Module): + """Linear camera patchifier matching the checkpoint's direct parameter names.""" + + def __init__( + self, + in_channels: int, + dim: int, + patch_size: tuple[int, int, int], + ) -> None: + super().__init__() + self.patch_size = patch_size + self.in_features = in_channels * math.prod(patch_size) + self.weight = nn.Parameter(torch.empty(dim, self.in_features)) + self.bias = nn.Parameter(torch.empty(dim)) + nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + bound = 1 / math.sqrt(self.in_features) + nn.init.uniform_(self.bias, -bound, bound) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + batch_size, channels, frames, height, width = hidden_states.shape + patch_frames, patch_height, patch_width = self.patch_size + # Preserve Conv3d patch order while retaining checkpoint weight/bias names. + hidden_states = hidden_states.reshape( + batch_size, + channels, + frames // patch_frames, + patch_frames, + height // patch_height, + patch_height, + width // patch_width, + patch_width, + ) + hidden_states = hidden_states.permute(0, 2, 4, 6, 1, 3, 5, 7) + hidden_states = hidden_states.reshape(batch_size, -1, self.in_features) + return F.linear(hidden_states, self.weight, self.bias) + + +class _LingBotHead(nn.Module): + def __init__( + self, + dim: int, + out_channels: int, + patch_size: tuple[int, int, int], + eps: float, + ) -> None: + super().__init__() + self.norm = LayerNorm(dim, eps=eps, elementwise_affine=False) + self.head = nn.Linear(dim, out_channels * math.prod(patch_size)) + self.modulation = nn.Parameter(torch.randn(1, 2, dim) / math.sqrt(dim)) + + def forward( + self, + hidden_states: torch.Tensor, + timestep_embedding: torch.Tensor, + ) -> torch.Tensor: + num_frames = timestep_embedding.shape[1] + tokens_per_frame = hidden_states.shape[1] // num_frames + modulation = self.modulation.unsqueeze(1) + timestep_embedding.unsqueeze(2).float() + shift, scale = modulation.chunk(2, dim=2) + normalized = self.norm(hidden_states.float()).unflatten(1, (num_frames, tokens_per_frame)) + normalized = (normalized * (1 + scale) + shift).flatten(1, 2).to(hidden_states.dtype) + return self.head(normalized) + + +def _sinusoidal_embedding(dim: int, timestep: torch.Tensor) -> torch.Tensor: + if dim % 2: + raise ValueError(f"freq_dim must be even, got {dim}.") + half_dim = dim // 2 + timestep = timestep.to(torch.float64) + frequencies = torch.pow( + 10000, + -torch.arange(half_dim, device=timestep.device, dtype=torch.float64) / half_dim, + ) + phase = torch.outer(timestep, frequencies) + return torch.cat((phase.cos(), phase.sin()), dim=1) + + +def _rope_axis(max_seq_len: int, dim: int) -> tuple[torch.Tensor, torch.Tensor]: + if dim == 0: + empty = torch.empty(max_seq_len, 0, dtype=torch.float32) + return empty, empty.clone() + if dim % 2: + raise ValueError(f"RoPE axis dimension must be even, got {dim}.") + frequencies = 1.0 / torch.pow( + 10000, + torch.arange(0, dim, 2, dtype=torch.float64) / dim, + ) + phase = torch.outer(torch.arange(max_seq_len, dtype=torch.float64), frequencies) + return phase.cos().float(), phase.sin().float() + + +class CausalLingBotWorldTransformer3DModel(nn.Module): + """Checkpoint-compatible causal LingBot World video transformer.""" + + _repeated_blocks = ["LingBotAttentionBlock"] + packed_modules_mapping = {"qkv": ["q", "k", "v"]} + _layerwise_offload_blocks_attrs = ["blocks"] + + def __init__( + self, + patch_size: tuple[int, int, int] = (1, 2, 2), + num_attention_heads: int = 40, + attention_head_dim: int = 128, + in_channels: int = 36, + out_channels: int = 16, + text_dim: int = 4096, + freq_dim: int = 256, + ffn_dim: int = 13824, + num_layers: int = 40, + cross_attn_norm: bool = True, + eps: float = 1e-6, + image_dim: int | None = None, + added_kv_proj_dim: int | None = None, + rope_max_seq_len: int = 1024, + pos_embed_seq_len: int | None = None, + qk_norm: str = "rms_norm_across_heads", + sink_size: int = 9, + num_frames_per_block: int = 3, + sliding_window_num_frames: int = 18, + local_attn_size: int = -1, + quant_config: object | None = None, + prefix: str = "", + ) -> None: + super().__init__() + if len(patch_size) != 3 or any(size <= 0 for size in patch_size): + raise ValueError("patch_size must contain three positive integers.") + if num_attention_heads <= 0 or attention_head_dim <= 0: + raise ValueError("Attention head counts and dimensions must be positive.") + if attention_head_dim % 2: + raise ValueError("attention_head_dim must be even for RoPE.") + if num_layers <= 0 or ffn_dim <= 0 or freq_dim <= 0: + raise ValueError("num_layers, ffn_dim, and freq_dim must be positive.") + if sink_size < 0 or num_frames_per_block <= 0 or sliding_window_num_frames <= 0: + raise ValueError("Causal frame-window sizes must be positive, except sink_size which may be zero.") + if local_attn_size != -1 and local_attn_size <= 0: + raise ValueError("local_attn_size must be -1 or a positive frame count.") + cache_window_frames = local_attn_size if local_attn_size != -1 else sliding_window_num_frames + if cache_window_frames < sink_size + num_frames_per_block: + raise ValueError("The causal cache window must fit all sink frames and one complete current block.") + if qk_norm != "rms_norm_across_heads": + raise ValueError( + f"qk_norm must be 'rms_norm_across_heads' for the LingBot World v2 checkpoint, got {qk_norm!r}." + ) + for field_name, field_value in ( + ("image_dim", image_dim), + ("added_kv_proj_dim", added_kv_proj_dim), + ("pos_embed_seq_len", pos_embed_seq_len), + ): + if field_value is not None: + raise ValueError(f"{field_name} must be None because LingBot World v2 has no image embedding path.") + if quant_config is not None: + raise RuntimeError( + "quant_config is not supported by the LingBot World transformer; construct the unquantized model." + ) + + dim = num_attention_heads * attention_head_dim + self.dim = dim + self.config = SimpleNamespace( + patch_size=patch_size, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + in_channels=in_channels, + out_channels=out_channels, + text_dim=text_dim, + freq_dim=freq_dim, + ffn_dim=ffn_dim, + num_layers=num_layers, + cross_attn_norm=cross_attn_norm, + eps=eps, + image_dim=image_dim, + added_kv_proj_dim=added_kv_proj_dim, + rope_max_seq_len=rope_max_seq_len, + pos_embed_seq_len=pos_embed_seq_len, + qk_norm=qk_norm, + sink_size=sink_size, + num_frames_per_block=num_frames_per_block, + sliding_window_num_frames=sliding_window_num_frames, + local_attn_size=local_attn_size, + ) + + self.patch_embedding = Conv3dLayer( + in_channels=in_channels, + out_channels=dim, + kernel_size=patch_size, + stride=patch_size, + ) + self.patch_embedding_wancamctrl = _LingBotCameraPatchEmbedding(6 * 8 * 8, dim, patch_size) + self.c2ws_hidden_states_layer1 = ColumnParallelLinear( + dim, + dim, + bias=True, + gather_output=False, + return_bias=False, + prefix=_projection_prefix(prefix, "c2ws_hidden_states_layer1"), + ) + self.c2ws_hidden_states_layer2 = RowParallelLinear( + dim, + dim, + bias=True, + input_is_parallel=True, + return_bias=False, + prefix=_projection_prefix(prefix, "c2ws_hidden_states_layer2"), + ) + self.text_embedding = nn.Sequential( + nn.Linear(text_dim, dim), + nn.GELU(approximate="tanh"), + nn.Linear(dim, dim), + ) + self.time_embedding = nn.Sequential( + nn.Linear(freq_dim, dim), + nn.SiLU(), + nn.Linear(dim, dim), + ) + self.time_projection = nn.Sequential( + nn.SiLU(), + nn.Linear(dim, dim * 6), + ) + self.blocks = nn.ModuleList( + [ + LingBotAttentionBlock( + dim, + num_attention_heads, + ffn_dim=ffn_dim, + cross_attn_norm=cross_attn_norm, + eps=eps, + prefix=_projection_prefix(prefix, f"blocks.{index}"), + ) + for index in range(num_layers) + ] + ) + self.head = _LingBotHead(dim, out_channels, patch_size, eps) + + temporal_dim = attention_head_dim - 4 * (attention_head_dim // 6) + height_dim = width_dim = 2 * (attention_head_dim // 6) + for axis, axis_dim in (("temporal", temporal_dim), ("height", height_dim), ("width", width_dim)): + cosine, sine = _rope_axis(rope_max_seq_len, axis_dim) + self.register_buffer(f"_rope_{axis}_cosine", cosine, persistent=False) + self.register_buffer(f"_rope_{axis}_sine", sine, persistent=False) + + @property + def dtype(self) -> torch.dtype: + """Return the dtype used by the transformer parameters.""" + + return next(self.parameters()).dtype + + @classmethod + def from_config( + cls, + config: dict[str, Any], + *, + quant_config: Any | None = None, + prefix: str = "", + ) -> Self: + checkpoint_contract = { + "patch_size": (1, 2, 2), + "num_attention_heads": 40, + "attention_head_dim": 128, + "in_channels": 36, + "out_channels": 16, + "text_dim": 4096, + "freq_dim": 256, + "ffn_dim": 13824, + "num_layers": 40, + "cross_attn_norm": True, + "eps": 1e-6, + "image_dim": None, + "added_kv_proj_dim": None, + "rope_max_seq_len": 1024, + "pos_embed_seq_len": None, + "qk_norm": "rms_norm_across_heads", + "sink_size": 9, + "num_frames_per_block": 3, + "sliding_window_num_frames": 18, + "local_attn_size": -1, + } + class_name = config.get("_class_name") + if class_name is not None and class_name != "CausalLingBotWorldTransformer3DModel": + raise ValueError( + "_class_name must be 'CausalLingBotWorldTransformer3DModel' for the LingBot World v2 checkpoint." + ) + kwargs = {name: value for name, value in config.items() if name in checkpoint_contract} + if "patch_size" in kwargs: + kwargs["patch_size"] = tuple(kwargs["patch_size"]) + for name, expected in checkpoint_contract.items(): + actual = kwargs.get(name, expected) + if actual != expected: + raise ValueError(f"LingBot checkpoint config {name} must be {expected!r}, got {actual!r}.") + return cls(**kwargs, quant_config=quant_config, prefix=prefix) + + def allocate_cache( + self, + *, + batch_size: int, + latent_height: int, + latent_width: int, + device: torch.device, + dtype: torch.dtype, + ) -> LingBotTransformerCache: + """Allocate one caller-owned cache using this transformer's geometry.""" + + if isinstance(batch_size, bool) or not isinstance(batch_size, int) or batch_size <= 0: + raise ValueError("batch_size must be a positive integer for LingBot cache allocation.") + if ( + isinstance(latent_height, bool) + or not isinstance(latent_height, int) + or latent_height <= 0 + or isinstance(latent_width, bool) + or not isinstance(latent_width, int) + or latent_width <= 0 + ): + raise ValueError("latent_height and latent_width must be positive integers.") + + # Cache capacity is measured in post-patch tokens, not raw/latent + # pixels: window_frames * (latent_height/patch_h) * (latent_width/patch_w). + patch_frames, patch_height, patch_width = self.config.patch_size + if patch_frames != 1 or latent_height % patch_height or latent_width % patch_width: + raise ValueError("latent height/width must align with the configured LingBot patch size.") + post_patch_height = latent_height // patch_height + post_patch_width = latent_width // patch_width + window_frames = ( + self.config.local_attn_size if self.config.local_attn_size != -1 else self.config.sliding_window_num_frames + ) + max_tokens = int(window_frames * post_patch_height * post_patch_width) + tp_size = get_tensor_model_parallel_world_size() + num_local_heads = self.config.num_attention_heads // tp_size + return allocate_lingbot_cache( + batch_size=batch_size, + num_layers=self.config.num_layers, + max_tokens=max_tokens, + num_local_heads=num_local_heads, + head_dim=self.config.attention_head_dim, + device=device, + dtype=dtype, + ) + + def _rotary_embedding( + self, + *, + frames: int, + height: int, + width: int, + start_frame: int, + dtype: torch.dtype, + device: torch.device, + ) -> tuple[torch.Tensor, torch.Tensor]: + if start_frame + frames > self.config.rope_max_seq_len: + raise ValueError("Temporal RoPE positions exceed rope_max_seq_len.") + if height > self.config.rope_max_seq_len or width > self.config.rope_max_seq_len: + raise ValueError("Spatial RoPE positions exceed rope_max_seq_len.") + + def expand_axis(table: torch.Tensor, axis: str) -> torch.Tensor: + if axis == "temporal": + return ( + table[start_frame : start_frame + frames].view(frames, 1, 1, -1).expand(frames, height, width, -1) + ) + if axis == "height": + return table[:height].view(1, height, 1, -1).expand(frames, height, width, -1) + return table[:width].view(1, 1, width, -1).expand(frames, height, width, -1) + + cosine = torch.cat( + ( + expand_axis(self._rope_temporal_cosine, "temporal"), + expand_axis(self._rope_height_cosine, "height"), + expand_axis(self._rope_width_cosine, "width"), + ), + dim=-1, + ) + sine = torch.cat( + ( + expand_axis(self._rope_temporal_sine, "temporal"), + expand_axis(self._rope_height_sine, "height"), + expand_axis(self._rope_width_sine, "width"), + ), + dim=-1, + ) + return ( + cosine.reshape(frames * height * width, -1).to(device=device, dtype=dtype), + sine.reshape(frames * height * width, -1).to(device=device, dtype=dtype), + ) + + def _validate_inputs( + self, + hidden_states: torch.Tensor, + timestep: torch.Tensor, + encoder_hidden_states: torch.Tensor, + camera_hidden_states: torch.Tensor, + cache: LingBotTransformerCache, + start_frame: int, + ) -> tuple[int, int, int, int, int]: + if hidden_states.ndim != 5: + raise ValueError("hidden_states must be rank 5 [batch, channels, frames, height, width].") + if hidden_states.shape[1] != self.config.in_channels: + raise ValueError(f"hidden_states must have {self.config.in_channels} channels.") + if not isinstance(start_frame, int) or isinstance(start_frame, bool) or start_frame < 0: + raise ValueError("start_frame must be a non-negative integer latent-frame offset.") + patch_frames, patch_height, patch_width = self.config.patch_size + batch_size, _, frames, height, width = hidden_states.shape + if frames % patch_frames or height % patch_height or width % patch_width: + raise ValueError("hidden_states dimensions must be divisible by patch_size.") + post_patch_frames = frames // patch_frames + if post_patch_frames != self.config.num_frames_per_block: + raise ValueError( + f"Each LingBot forward must contain exactly {self.config.num_frames_per_block} post-patch frames, " + f"got {post_patch_frames}." + ) + if start_frame % patch_frames: + raise ValueError("start_frame must align to the temporal patch size.") + if encoder_hidden_states.ndim != 3: + raise ValueError("encoder_hidden_states must be rank 3 [batch, tokens, text_dim].") + if encoder_hidden_states.shape[0] != batch_size or encoder_hidden_states.shape[2] != self.config.text_dim: + raise ValueError("encoder_hidden_states batch/width do not match the LingBot config.") + if encoder_hidden_states.shape[1] == 0: + raise ValueError("encoder_hidden_states must contain at least one token.") + expected_camera_shape = (batch_size, 6 * 8 * 8, frames, height, width) + if camera_hidden_states.shape != expected_camera_shape: + raise ValueError( + "camera_hidden_states must use folded [batch, 6*8*8, frames, height, width] layout " + f"matching the video; expected {expected_camera_shape}, got {tuple(camera_hidden_states.shape)}." + ) + if hidden_states.device != encoder_hidden_states.device or hidden_states.device != camera_hidden_states.device: + raise ValueError("Video, text, and camera tensors must be on the same device.") + if hidden_states.dtype != encoder_hidden_states.dtype or hidden_states.dtype != camera_hidden_states.dtype: + raise ValueError("Video, text, and camera tensors must use the same dtype.") + if timestep.device != hidden_states.device: + raise ValueError("timestep must be on the same device as hidden_states.") + if len(cache.self_attention) != self.config.num_layers or len(cache.cross_attention) != self.config.num_layers: + raise ValueError("LingBotTransformerCache layer counts must match num_layers.") + return batch_size, frames, height, width, patch_frames + + def _timestep_embeddings( + self, + timestep: torch.Tensor, + *, + batch_size: int, + frames: int, + dtype: torch.dtype, + ) -> tuple[torch.Tensor, torch.Tensor]: + if timestep.ndim == 0: + if batch_size != 1: + raise ValueError("A scalar timestep is only valid for batch_size=1.") + timestep = timestep.reshape(1) + if timestep.ndim == 1: + if timestep.shape[0] != batch_size: + raise ValueError("A rank-1 timestep must contain one value per batch item.") + timestep = timestep.unsqueeze(1).expand(batch_size, frames) + elif timestep.ndim == 2: + if timestep.shape != (batch_size, frames): + raise ValueError(f"A rank-2 timestep must have shape {(batch_size, frames)}.") + else: + raise ValueError("timestep must be scalar, rank 1 [batch], or rank 2 [batch, frames].") + + frequency_embedding = _sinusoidal_embedding(self.config.freq_dim, timestep.reshape(-1)).to(dtype=dtype) + timestep_embedding = self.time_embedding(frequency_embedding).unflatten(0, (batch_size, frames)) + timestep_projection = self.time_projection(timestep_embedding).unflatten(2, (6, self.dim)) + return timestep_embedding, timestep_projection + + def _unpatchify( + self, + hidden_states: torch.Tensor, + *, + batch_size: int, + frames: int, + height: int, + width: int, + ) -> torch.Tensor: + patch_frames, patch_height, patch_width = self.config.patch_size + hidden_states = hidden_states.reshape( + batch_size, + frames, + height, + width, + patch_frames, + patch_height, + patch_width, + self.config.out_channels, + ) + hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6) + return hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) + + def forward( + self, + hidden_states: torch.Tensor, + timestep: torch.Tensor, + encoder_hidden_states: torch.Tensor, + camera_hidden_states: torch.Tensor, + *, + cache: LingBotTransformerCache, + start_frame: int, + update_cache: bool, + ) -> torch.Tensor: + if not isinstance(update_cache, bool): + raise ValueError("update_cache must be a boolean.") + batch_size, frames, height, width, patch_frames = self._validate_inputs( + hidden_states, + timestep, + encoder_hidden_states, + camera_hidden_states, + cache, + start_frame, + ) + patch_height, patch_width = self.config.patch_size[1:] + patched_frames = frames // patch_frames + patched_height = height // patch_height + patched_width = width // patch_width + tokens_per_frame = patched_height * patched_width + patched_start_frame = start_frame // patch_frames + current_start = patched_start_frame * tokens_per_frame + sink_tokens = self.config.sink_size * tokens_per_frame + # Phase 1: create absolute 3D positions for this causal block. The + # spatial coordinates restart for every frame; temporal coordinates + # begin at ``start_frame`` so cached blocks retain their true ordering. + rotary_emb = self._rotary_embedding( + frames=patched_frames, + height=patched_height, + width=patched_width, + start_frame=patched_start_frame, + dtype=hidden_states.dtype, + device=hidden_states.device, + ) + # Phase 2: independently patchify video and camera grids to identical + # token layouts, then project timestep and text conditions. + hidden_states = self.patch_embedding(hidden_states).flatten(2).transpose(1, 2) + camera_hidden_states = self.patch_embedding_wancamctrl(camera_hidden_states) + camera_hidden_states = camera_hidden_states + self.c2ws_hidden_states_layer2( + F.silu(self.c2ws_hidden_states_layer1(camera_hidden_states)) + ) + if hidden_states.shape != camera_hidden_states.shape: + raise ValueError("Patched video and camera token shapes must match.") + + timestep_embedding, timestep_projection = self._timestep_embeddings( + timestep, + batch_size=batch_size, + frames=patched_frames, + dtype=hidden_states.dtype, + ) + projected_text = ( + self.text_embedding(encoder_hidden_states) + if any(value is None for value in cache.cross_attention) + else None + ) + if cache.self_attention and isinstance(cache.self_attention[0], ARDiffusionPagedLayerContext): + if batch_size != 1: + raise RuntimeError("LingBot AR-Diffusion paged attention requires batch_size=1.") + forward_context = cache.self_attention[0].forward_ctx + expected_seq_len = patched_frames * tokens_per_frame + if forward_context.seq_len != expected_seq_len: + raise RuntimeError( + "LingBot AR-Diffusion paged context token count does not " + f"match this block: {forward_context.seq_len} != " + f"{expected_seq_len}." + ) + forward_context.prepare( + device=hidden_states.device, + action_len=0, + query_len=hidden_states.shape[1], + ) + cache.self_attention = [layer_context.to_layer_inputs() for layer_context in cache.self_attention] + # Phase 3: each layer receives its own cache entry. Text K/V is passed + # only when absent; the returned cache is stored for subsequent DMD + # steps and causal blocks in this request. + for index, block in enumerate(self.blocks): + hidden_states, cross_cache = block( + hidden_states, + projected_text if cache.cross_attention[index] is None else None, + timestep_projection, + camera_hidden_states, + self_cache=cache.self_attention[index], + cross_cache=cache.cross_attention[index], + current_start=current_start, + sink_tokens=sink_tokens, + update_cache=update_cache, + rotary_emb=rotary_emb, + ) + cache.cross_attention[index] = cross_cache + + # Phase 4: map tokens to per-patch 16-channel flow values and restore + # [B, C, F, H, W] for the Pipeline's sampler update. + hidden_states = self.head(hidden_states, timestep_embedding) + return self._unpatchify( + hidden_states, + batch_size=batch_size, + frames=patched_frames, + height=patched_height, + width=patched_width, + ) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Load exact checkpoint names, delegating TP sharding to parameters.""" + + params = dict(self.named_parameters()) + loaded: set[str] = set() + qkv_mapping = (("q", "q"), ("k", "k"), ("v", "v")) + for checkpoint_name, loaded_weight in weights: + name = checkpoint_name + shard_id = None + for projection_name, projection_shard in qkv_mapping: + marker = f".self_attn.{projection_name}." + if marker in name: + name = name.replace(marker, ".self_attn.qkv.") + shard_id = projection_shard + break + if name not in params: + raise KeyError(f"Unexpected LingBot model weight name: {checkpoint_name}") + param = params[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + if shard_id is None: + weight_loader(param, loaded_weight) + else: + if weight_loader is default_weight_loader: + raise RuntimeError(f"Fused LingBot QKV parameter {name} has no stacked weight loader.") + weight_loader(param, loaded_weight, shard_id) + # DiffusersPipelineLoader compares this return value with the + # model parameter namespace. Packed Q/K/V checkpoint tensors + # therefore need to report both their source names and the fused + # model parameter they populated, matching Wan's loader contract. + loaded.add(checkpoint_name) + loaded.add(name) + return loaded diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index 9bac50323..2c4bffb4c 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -136,6 +136,11 @@ _DIFFUSION_MODELS = { "pipeline_wan2_2_i2v", "WanI2VDMD2Pipeline", ), + "LingBotWorldCausalDMDPipeline": ( + "lingbot_world", + "pipeline", + "LingBotWorldCausalDMDPipeline", + ), "LongCatImagePipeline": ( "longcat_image", "pipeline_longcat_image", @@ -353,6 +358,7 @@ _NO_CACHE_ACCELERATION = { # Pipelines that do not support cache acceleration (cache_dit / tea_cache). "NextStep11Pipeline", "AudioXPipeline", + "LingBotWorldCausalDMDPipeline", } @@ -547,6 +553,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = { "WanS2VPipeline": "get_wan22_s2v_post_process_func", "WanT2VDMD2Pipeline": "get_wan22_post_process_func", "WanI2VDMD2Pipeline": "get_wan22_i2v_post_process_func", + "LingBotWorldCausalDMDPipeline": "get_lingbot_world_post_process_func", "LongCatImagePipeline": "get_longcat_image_post_process_func", "LongCatVideoAvatarPipeline": "get_longcat_video_avatar_post_process_func", "BagelPipeline": "get_bagel_post_process_func", @@ -607,6 +614,7 @@ _DIFFUSION_PRE_PROCESS_FUNCS = { "WanS2VPipeline": "get_wan22_s2v_pre_process_func", "WanT2VDMD2Pipeline": "get_wan22_pre_process_func", "WanI2VDMD2Pipeline": "get_wan22_i2v_pre_process_func", + "LingBotWorldCausalDMDPipeline": "get_lingbot_world_pre_process_func", "OmniGen2Pipeline": "get_omnigen2_pre_process_func", "HeliosPipeline": "get_helios_pre_process_func", "HeliosPyramidPipeline": "get_helios_pre_process_func", diff --git a/vllm_omni/diffusion/worker/diffusion_worker.py b/vllm_omni/diffusion/worker/diffusion_worker.py index fea78bcf9..436fcca9a 100644 --- a/vllm_omni/diffusion/worker/diffusion_worker.py +++ b/vllm_omni/diffusion/worker/diffusion_worker.py @@ -444,6 +444,25 @@ class DiffusionWorker: else: profiler.stop() + def _run_ar_diffusion_session_lifecycle(self, method: str, session_id: str) -> bool: + """Delegate an optional AR session lifecycle call to the model runner.""" + if not isinstance(session_id, str) or not session_id.strip(): + raise ValueError("session_id must be a non-empty string.") + assert self.model_runner is not None, "Model runner not initialized" + lifecycle_method = getattr(self.model_runner, method, None) + if not callable(lifecycle_method): + return False + lifecycle_method(session_id) + return True + + def reset_ar_diffusion_session(self, session_id: str) -> bool: + """Reset runner-owned AR state through the collective RPC boundary.""" + return self._run_ar_diffusion_session_lifecycle("reset_session", session_id) + + def close_ar_diffusion_session(self, session_id: str) -> bool: + """Close runner-owned AR state through the collective RPC boundary.""" + return self._run_ar_diffusion_session_lifecycle("close_session", session_id) + def execute_model( self, req: OmniDiffusionRequest | list[OmniDiffusionRequest], diff --git a/vllm_omni/engine/async_omni_engine.py b/vllm_omni/engine/async_omni_engine.py index a613ebc80..c457b7052 100644 --- a/vllm_omni/engine/async_omni_engine.py +++ b/vllm_omni/engine/async_omni_engine.py @@ -982,6 +982,9 @@ class AsyncOmniEngine: stage_engine_args = { "max_num_seqs": kwargs.get("max_num_seqs") or 1, "parallel_config": parallel_config, + # Default-stage construction bypasses the structured projection. + # Runner selection remains owned by the selected engine/platform. + "engine_backend": kwargs.get("engine_backend", "default"), "model_class_name": kwargs.get("model_class_name", None), "task_type": kwargs.get("task_type", None), "model_config": kwargs.get("model_config", None), diff --git a/vllm_omni/experimental/ar_diffusion/__init__.py b/vllm_omni/experimental/ar_diffusion/__init__.py index b87ac1ccc..bd0f2eac2 100644 --- a/vllm_omni/experimental/ar_diffusion/__init__.py +++ b/vllm_omni/experimental/ar_diffusion/__init__.py @@ -13,12 +13,32 @@ from vllm_omni.experimental.ar_diffusion.capability import ( SupportsARDiffusionPipeline, SupportsARDiffusionWarmup, ) +from vllm_omni.experimental.ar_diffusion.consumer import ARDiffusionOmniTickConsumer from vllm_omni.experimental.ar_diffusion.engine import ARDiffusionEngine +from vllm_omni.experimental.ar_diffusion.session import ( + ARDiffusionSession, + ARDiffusionSessionManager, + ARDiffusionWorkerLifecycle, +) +from vllm_omni.experimental.ar_diffusion.tick_protocol import ( + AR_DIFFUSION_TICK_KEY, + ARDiffusionChunkMetadata, + ARDiffusionControlInput, + ARDiffusionTickRequest, +) __all__ = [ "ARDiffusionKVBranchSpec", "ARDiffusionCrossAttentionKVSpec", "ARDiffusionEngine", + "ARDiffusionOmniTickConsumer", + "ARDiffusionSession", + "ARDiffusionSessionManager", + "ARDiffusionWorkerLifecycle", + "AR_DIFFUSION_TICK_KEY", + "ARDiffusionChunkMetadata", + "ARDiffusionControlInput", + "ARDiffusionTickRequest", "ARDiffusionKVCacheSpec", "SupportsARDiffusionPipeline", "SupportsARDiffusionWarmup", diff --git a/vllm_omni/experimental/ar_diffusion/capability.py b/vllm_omni/experimental/ar_diffusion/capability.py index b30d10c1e..64eb62cf0 100644 --- a/vllm_omni/experimental/ar_diffusion/capability.py +++ b/vllm_omni/experimental/ar_diffusion/capability.py @@ -58,6 +58,9 @@ class ARDiffusionKVCacheSpec: ``max_scratch_tokens_per_branch`` is the maximum non-video KV (for example, action/state registers) that must coexist with an uncommitted video block. + ``model_owned_state_bytes_per_session`` is a conservative upper bound for + persistent per-session CUDA state outside runner-owned self/cross-attention + KV and scratch pools. ``session_capacity`` is the pipeline's upper bound; a runner may retain fewer sessions when its current memory and execution policy is stricter. @@ -79,6 +82,7 @@ class ARDiffusionKVCacheSpec: cross_attention: tuple[ARDiffusionCrossAttentionKVSpec, ...] = () max_model_len: int = 1 << 20 max_scratch_tokens_per_branch: int = 0 + model_owned_state_bytes_per_session: int = 0 def __post_init__(self) -> None: positive_fields = { @@ -101,6 +105,11 @@ class ARDiffusionKVCacheSpec: "AR-Diffusion max_scratch_tokens_per_branch must be non-negative, " f"got {self.max_scratch_tokens_per_branch}" ) + if self.model_owned_state_bytes_per_session < 0: + raise ValueError( + "AR-Diffusion model_owned_state_bytes_per_session must be non-negative, " + f"got {self.model_owned_state_bytes_per_session}" + ) if not self.kv_branches: raise ValueError("AR-Diffusion requires at least one KV branch") kv_branch_names = [kv_branch.name for kv_branch in self.kv_branches] diff --git a/vllm_omni/experimental/ar_diffusion/consumer.py b/vllm_omni/experimental/ar_diffusion/consumer.py new file mode 100644 index 000000000..a819e0b31 --- /dev/null +++ b/vllm_omni/experimental/ar_diffusion/consumer.py @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: Apache-2.0 +"""AsyncOmni consumer for typed realtime AR-Diffusion ticks.""" + +from __future__ import annotations + +import copy +from collections.abc import AsyncIterator, Callable, Mapping, Sequence +from typing import Any, Protocol + +from vllm_omni.experimental.ar_diffusion.tick_protocol import ( + ARDiffusionChunkMetadata, + ARDiffusionTickRequest, +) +from vllm_omni.inputs.data import ( + OmniDiffusionSamplingParams, + OmniPromptType, + OmniSamplingParams, +) +from vllm_omni.outputs import OmniRequestOutput + +AR_DIFFUSION_OUTPUT_METADATA_KEY = "ar_diffusion" + + +class ARDiffusionGenerateClient(Protocol): + """Internal subset of AsyncOmni used to execute one tick.""" + + def generate( + self, + prompt: OmniPromptType, + *, + request_id: str, + sampling_params_list: Sequence[OmniSamplingParams], + output_modalities: list[str] | None = None, + ) -> AsyncIterator[OmniRequestOutput]: ... + + +def _metadata_mapping(output: OmniRequestOutput) -> Mapping[str, Any]: + multimodal_output = output.multimodal_output + if not isinstance(multimodal_output, Mapping): + raise ValueError("AR-Diffusion output must contain a multimodal output mapping.") + metadata = multimodal_output.get("metadata") + if not isinstance(metadata, Mapping): + raise ValueError("AR-Diffusion output must contain multimodal_output.metadata.") + value = metadata.get(AR_DIFFUSION_OUTPUT_METADATA_KEY) + if not isinstance(value, Mapping): + raise ValueError(f"AR-Diffusion output metadata must contain {AR_DIFFUSION_OUTPUT_METADATA_KEY!r}.") + return value + + +def _parse_chunk_metadata(output: OmniRequestOutput) -> ARDiffusionChunkMetadata: + value = _metadata_mapping(output) + session_id = value.get("session_id") + request_id = value.get("request_id") + chunk_index = value.get("chunk_index") + applied_event_ids = value.get("applied_event_ids", ()) + if not isinstance(session_id, str) or not session_id.strip(): + raise ValueError("AR-Diffusion output metadata session_id must be a non-empty string.") + if not isinstance(request_id, str) or not request_id.strip(): + raise ValueError("AR-Diffusion output metadata request_id must be a non-empty string.") + if isinstance(chunk_index, bool) or not isinstance(chunk_index, int) or chunk_index < 0: + raise ValueError("AR-Diffusion output metadata chunk_index must be a non-negative integer.") + if not isinstance(applied_event_ids, Sequence) or isinstance(applied_event_ids, (str, bytes, bytearray)): + raise ValueError("AR-Diffusion output metadata applied_event_ids must be a sequence.") + event_ids = tuple(applied_event_ids) + if any(isinstance(event_id, bool) or not isinstance(event_id, int) or event_id < 0 for event_id in event_ids): + raise ValueError("AR-Diffusion output metadata applied_event_ids must contain non-negative integers.") + if event_ids != tuple(sorted(set(event_ids))): + raise ValueError("AR-Diffusion output metadata applied_event_ids must be unique and strictly increasing.") + return ARDiffusionChunkMetadata( + session_id=session_id, + request_id=request_id, + chunk_index=chunk_index, + applied_event_ids=event_ids, + ) + + +class ARDiffusionOmniTickConsumer: + """Converts a typed tick into one AsyncOmni diffusion request. + + ``prompt_provider`` packages session initialization data such as an initial + image into the normal Omni prompt. Control ``track/schema/data`` remains in + the typed tick and is not interpreted by this class. + """ + + def __init__( + self, + engine_client: ARDiffusionGenerateClient, + *, + prompt_provider: Callable[[ARDiffusionTickRequest], OmniPromptType], + sampling_params_list: Sequence[OmniSamplingParams], + diffusion_stage_id: int, + output_modalities: Sequence[str] | None = None, + ) -> None: + templates = tuple(sampling_params_list) + if not templates: + raise ValueError("sampling_params_list must not be empty.") + if ( + isinstance(diffusion_stage_id, bool) + or not isinstance(diffusion_stage_id, int) + or not 0 <= diffusion_stage_id < len(templates) + ): + raise ValueError("diffusion_stage_id must index sampling_params_list.") + if not isinstance(templates[diffusion_stage_id], OmniDiffusionSamplingParams): + raise TypeError("The selected diffusion stage must use OmniDiffusionSamplingParams.") + stage_pools = getattr(getattr(engine_client, "engine", None), "stage_pools", None) + if stage_pools is None or not 0 <= diffusion_stage_id < len(stage_pools): + raise ValueError( + "AR-Diffusion realtime execution requires an initialized AsyncOmni " + "engine with a matching diffusion stage pool." + ) + replica_count = getattr(stage_pools[diffusion_stage_id], "num_replicas", None) + if isinstance(replica_count, bool) or not isinstance(replica_count, int) or replica_count < 1: + raise ValueError("The selected AR-Diffusion stage must expose a positive num_replicas.") + if replica_count != 1: + raise ValueError( + "AR-Diffusion realtime sessions currently require exactly one " + "replica for the selected diffusion stage; session-affine " + f"routing is not implemented (got num_replicas={replica_count})." + ) + self._engine_client = engine_client + self._prompt_provider = prompt_provider + self._sampling_params_templates = templates + self._diffusion_stage_id = diffusion_stage_id + self._output_modalities = list(output_modalities) if output_modalities is not None else None + + def _sampling_params_for_tick( + self, + tick: ARDiffusionTickRequest, + ) -> list[OmniSamplingParams]: + params = [copy.copy(template) for template in self._sampling_params_templates] + diffusion_params = params[self._diffusion_stage_id] + assert isinstance(diffusion_params, OmniDiffusionSamplingParams) + diffusion_params.extra_args = { + **(diffusion_params.extra_args or {}), + **tick.to_extra_args(), + } + return params + + async def execute_tick(self, tick: ARDiffusionTickRequest) -> OmniRequestOutput: + prompt = self._prompt_provider(tick) + final_output: OmniRequestOutput | None = None + async for output in self._engine_client.generate( + prompt, + request_id=tick.request_id, + sampling_params_list=self._sampling_params_for_tick(tick), + output_modalities=self._output_modalities, + ): + if output.stage_id != self._diffusion_stage_id: + continue + if output.finished: + final_output = output + + if final_output is None: + raise RuntimeError( + f"AR-Diffusion tick {tick.request_id!r} completed without a final " + f"output from stage {self._diffusion_stage_id}." + ) + if final_output.error: + raise RuntimeError(f"AR-Diffusion tick {tick.request_id!r} failed: {final_output.error}") + if final_output.request_id != tick.request_id: + raise ValueError("AR-Diffusion output request_id does not match the submitted tick request_id.") + # Parse here so malformed/missing standard metadata fails at the engine + # boundary, before the session attempts to commit accepted events. + self.chunk_metadata(final_output) + return final_output + + def chunk_metadata(self, output: OmniRequestOutput) -> ARDiffusionChunkMetadata: + return _parse_chunk_metadata(output) diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/__init__.py b/vllm_omni/experimental/ar_diffusion/kv_cache/__init__.py index b407d24a3..94e6edf0d 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/__init__.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/__init__.py @@ -32,7 +32,6 @@ from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import ( ARDiffusionPagedLayerInputs, ar_diffusion_paged_attention, paged_write_attn, - set_current_paged_kv_cache, ) __all__ = [ @@ -53,5 +52,4 @@ __all__ = [ "paged_write_attn", "pool_write_chunk", "resident_block_ids", - "set_current_paged_kv_cache", ] diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/config.py b/vllm_omni/experimental/ar_diffusion/kv_cache/config.py index 07be2546e..ca91a8bb6 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/config.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/config.py @@ -23,7 +23,8 @@ class ARDiffusionKVConfig: sink_chunks: int = 0 # Boundary reset vs. sliding replacement. reset_at_boundary: bool = False - # Fraction of free device memory budgeted for the AR-Diffusion KV pool. + # Fraction of free device memory used to admit additional resident + # sessions. One session is admitted whenever it fits actual free memory. gpu_memory_fraction: float = 0.1 # When CUDA graph / torch.compile is on (not enforce_eager), pre-capture the # DiT graphs for every window-fill shape at load time via a synthetic rollout, diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py b/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py index 9ef3cca0c..da61afe4d 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py @@ -174,6 +174,7 @@ class ARDiffusionKVCache: device: torch.device | None = None, frames_per_block: int = 1, max_scratch_tokens_per_branch: int = 0, + model_owned_state_bytes_per_session: int = 0, ) -> None: if not config.enable: raise ValueError("ARDiffusionKVCache built with a disabled ARDiffusionKVConfig") @@ -197,13 +198,17 @@ class ARDiffusionKVCache: self.config = config self.kv_branches = kv_branches - self.session_capacity = session_capacity + self.requested_session_capacity = session_capacity self._kv_branch_local_indices = {kv_branch.name: kv_branch.local_index for kv_branch in kv_branches} self.num_local_kv_branches = max(local_indices) + 1 if frames_per_block <= 0: raise ValueError(f"frames_per_block must be positive, got {frames_per_block}") if max_scratch_tokens_per_branch < 0: raise ValueError(f"max_scratch_tokens_per_branch must be non-negative, got {max_scratch_tokens_per_branch}") + if model_owned_state_bytes_per_session < 0: + raise ValueError( + f"model_owned_state_bytes_per_session must be non-negative, got {model_owned_state_bytes_per_session}" + ) self.frames_per_block = int(frames_per_block) self.block_size = block_size self.num_layers = num_layers @@ -217,28 +222,11 @@ class ARDiffusionKVCache: self.device = device or torch.device("cpu") self._allocate_tensors = device is not None self._adapters: dict[str, ARDiffusionRequestAdapter] = {} - - # Named cross-attention K/V is allocated lazily per session and released - # with that session. Reserve its worst-case capacity when sizing the - # paged self-attention pool so the two stores share one memory budget. self._cross_sessions: dict[ str, dict[str, dict[str, tuple[list[torch.Tensor], list[torch.Tensor]]]], ] = {} - def _cross_pool_bytes(length: int) -> int: - return int(2 * len(self.kv_branches) * length * num_kv_heads * head_size * dtype.itemsize * num_layers) - - cross_bytes_per_session = sum(_cross_pool_bytes(length) for length in self.cross_attention_lengths.values()) - cross_total_bytes = cross_bytes_per_session * session_capacity - if cross_total_bytes: - _log.info( - "AR-Diffusion cross-attn reservation: %.1f MiB/session × %d sessions = %.1f MiB", - cross_bytes_per_session / (1024 * 1024), - session_capacity, - cross_total_bytes / (1024 * 1024), - ) - self.spec = ChunkWindowSpec( block_size=block_size, num_kv_heads=num_kv_heads, @@ -250,41 +238,7 @@ class ARDiffusionKVCache: sink_chunks=config.sink_chunks, reset_at_boundary=config.reset_at_boundary, ) - # Each pool block spans all layers' K/V, so size against the per-layer - # page size times the layer count. - # Size the self-attn pool against memory reserved for the maximum number - # of lazily allocated cross-attention sessions. - num_blocks = compute_num_blocks( - max(0, available_bytes - cross_total_bytes), - config.gpu_memory_fraction, - self.spec.page_size_bytes * num_layers, - ) - # Floor: one forward needs the resident window plus the in-flight chunk - # (frames_per_block frame-blocks) for every KV branch THIS rank runs, - # with a little eviction-transient headroom. The memory-fraction - # heuristic can under-size this once block_size grows — e.g. frame-granular - # paging at the true frame_seqlen makes each block larger and the pool - # fewer-blocks — so guarantee the minimum the rollout cannot run without, - # otherwise allocate_chunk hits an exhausted pool mid-forward. - resident_blocks = config.sink_chunks + config.window_chunks - min_blocks = self.num_local_kv_branches * (resident_blocks + self.frames_per_block) + 2 - if num_blocks < min_blocks: - _log.warning( - "AR-Diffusion KV pool: memory-fraction sizing gave %d blocks; raising to the %d-block " - "floor (%d local KV branch(es) x (sink_chunks=%d + window_chunks=%d " - "+ frames_per_block=%d) + 2 headroom)", - num_blocks, - min_blocks, - self.num_local_kv_branches, - config.sink_chunks, - config.window_chunks, - self.frames_per_block, - ) - num_blocks = min_blocks - layer_names = [f"ar_diffusion.layer.{i}" for i in range(num_layers)] - self.manager = build_kv_manager(self.spec, layer_names, num_blocks, max_model_len) - self.managed_num_blocks = num_blocks - self.num_blocks = num_blocks + # Scratch blocks are outside KVCacheManager ownership. A non-committing # forward needs one block per current frame plus space for any # model-declared action/state tokens that coexist with video KV. @@ -297,6 +251,101 @@ class ARDiffusionKVCache: scratch_per_kv_branch = max(minimum_scratch_blocks, override_blocks) self.scratch_blocks_per_kv_branch = scratch_per_kv_branch self.scratch_num_blocks = self.num_local_kv_branches * scratch_per_kv_branch + + # The self-attention pool, scratch pool, and lazily materialized + # cross-attention caches share one hard memory budget. Select the largest + # feasible resident-session count rather than raising a block-count floor + # past that budget. All resident sessions need a complete sink + window; + # only one request can be in flight, so frames_per_block is counted once. + page_size_bytes = self.spec.page_size_bytes * num_layers + self.available_memory_bytes = available_bytes + self.configured_memory_budget_bytes = int(available_bytes * config.gpu_memory_fraction) + # Reuse the public helper's validation for the memory fraction/page size. + compute_num_blocks(available_bytes, config.gpu_memory_fraction, page_size_bytes) + self.scratch_reserved_bytes = self.scratch_num_blocks * page_size_bytes + self.model_owned_state_bytes_per_session = model_owned_state_bytes_per_session + + def _cross_pool_bytes(length: int) -> int: + return int(2 * len(self.kv_branches) * length * num_kv_heads * head_size * dtype.itemsize * num_layers) + + self.cross_attention_bytes_per_session = sum( + _cross_pool_bytes(length) for length in self.cross_attention_lengths.values() + ) + + def _required_managed_blocks(capacity: int) -> int: + resident_per_session = config.sink_chunks + config.window_chunks + return self.num_local_kv_branches * (capacity * resident_per_session + self.frames_per_block) + 2 + + def _required_bytes(capacity: int) -> int: + return ( + self.scratch_reserved_bytes + + capacity * self.cross_attention_bytes_per_session + + capacity * self.model_owned_state_bytes_per_session + + _required_managed_blocks(capacity) * page_size_bytes + ) + + one_session_bytes = _required_bytes(1) + if one_session_bytes > available_bytes: + raise ValueError( + "AR-Diffusion available device memory cannot fit one session: " + f"available={available_bytes} bytes, required={one_session_bytes} bytes " + "(managed self-attention + cross-attention + scratch + model-owned state)." + ) + self.memory_budget_bytes = max(self.configured_memory_budget_bytes, one_session_bytes) + if self.memory_budget_bytes > self.configured_memory_budget_bytes: + _log.warning( + "AR-Diffusion raised the configured memory budget from %d to %d bytes " + "to admit one session that fits actual free device memory", + self.configured_memory_budget_bytes, + self.memory_budget_bytes, + ) + + effective_capacity = 0 + required_managed_blocks = 0 + for candidate in range(session_capacity, 0, -1): + candidate_managed_blocks = _required_managed_blocks(candidate) + if _required_bytes(candidate) <= self.memory_budget_bytes: + effective_capacity = candidate + required_managed_blocks = candidate_managed_blocks + break + assert effective_capacity > 0 + + self.session_capacity = effective_capacity + self.cross_attention_reserved_bytes = self.cross_attention_bytes_per_session * effective_capacity + self.model_owned_state_reserved_bytes = self.model_owned_state_bytes_per_session * effective_capacity + self_attn_budget_bytes = ( + self.memory_budget_bytes + - self.scratch_reserved_bytes + - self.cross_attention_reserved_bytes + - self.model_owned_state_reserved_bytes + ) + num_blocks = self_attn_budget_bytes // page_size_bytes + assert num_blocks >= required_managed_blocks + if effective_capacity < session_capacity: + _log.warning( + "AR-Diffusion resident session capacity reduced from %d to %d by the KV memory budget", + session_capacity, + effective_capacity, + ) + if self.cross_attention_reserved_bytes: + _log.info( + "AR-Diffusion cross-attn reservation: %.1f MiB/session × %d sessions = %.1f MiB", + self.cross_attention_bytes_per_session / (1024 * 1024), + effective_capacity, + self.cross_attention_reserved_bytes / (1024 * 1024), + ) + if self.model_owned_state_reserved_bytes: + _log.info( + "AR-Diffusion model-owned state reservation: %.1f MiB/session × %d sessions = %.1f MiB", + self.model_owned_state_bytes_per_session / (1024 * 1024), + effective_capacity, + self.model_owned_state_reserved_bytes / (1024 * 1024), + ) + + layer_names = [f"ar_diffusion.layer.{i}" for i in range(num_layers)] + self.manager = build_kv_manager(self.spec, layer_names, num_blocks, max_model_len) + self.managed_num_blocks = num_blocks + self.num_blocks = num_blocks self.num_blocks_total = self.managed_num_blocks + self.scratch_num_blocks self.null_block_id = self.manager.block_pool.null_block.block_id @@ -561,6 +610,11 @@ class ARDiffusionKVCache: """ _log.debug("AR-Diffusion commit: req=%s before=%d", adapter.request_id, adapter.completed_chunks) adapter.on_chunk_committed() + self.manager.remove_skipped_blocks( + adapter.request_id, + adapter.num_computed_tokens, + num_prompt_tokens=adapter.num_prompt_tokens, + ) _log.debug("AR-Diffusion commit: req=%s after=%d", adapter.request_id, adapter.completed_chunks) # -- pool-backed K/V access -------------------------------------------- diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py b/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py index f16ecc9e9..b2e19a95f 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py @@ -10,19 +10,6 @@ import torch from vllm_omni.experimental.ar_diffusion.kv_cache.paged import compute_slot_mapping -# Set by ARDiffusionPagedForwardContext.prepare() before each KV branch forward and -# read by the fused write+attend custom op below. The pools are process-lifetime -# allocations owned by one ARDiffusionKVCache per worker process (CFG-parallel / -# TP ranks are separate processes), mirroring vLLM's forward-context pattern for -# keeping multi-GiB cache tensors out of the compiled graph's inputs. -_CURRENT_PAGED_KV_CACHE: Any = None - - -def set_current_paged_kv_cache(kv_cache: Any) -> None: - global _CURRENT_PAGED_KV_CACHE - _CURRENT_PAGED_KV_CACHE = kv_cache - - _LAYER_IDX_TENSORS: dict[int, torch.Tensor] = {} @@ -50,6 +37,9 @@ class ARDiffusionPagedLayerInputs(NamedTuple): """ layer_idx: torch.Tensor + key_pool: torch.Tensor + value_pool: torch.Tensor + block_size: int seq_len: int video_slots: torch.Tensor action_slots: torch.Tensor @@ -228,7 +218,6 @@ class ARDiffusionPagedForwardContext: ) = self.build_block_table(action_len=action_len, query_len=query_len, device=device) if self.action_slot_mapping is None: self.action_slot_mapping = torch.empty(0, dtype=torch.long, device=device) - set_current_paged_kv_cache(self.kv_cache) self._prepared = True def layer_inputs(self, layer_idx: int) -> ARDiffusionPagedLayerInputs: @@ -236,6 +225,9 @@ class ARDiffusionPagedForwardContext: raise RuntimeError("ARDiffusionPagedForwardContext.layer_inputs() before prepare()") return ARDiffusionPagedLayerInputs( layer_idx=_layer_idx_tensor(layer_idx), + key_pool=self.kv_cache._k_pools[layer_idx], + value_pool=self.kv_cache._v_pools[layer_idx], + block_size=int(self.kv_cache.block_size), seq_len=int(self.seq_len), video_slots=self.current_video_slot_mapping, action_slots=self.action_slot_mapping, @@ -476,17 +468,18 @@ def ar_diffusion_paged_attention( # One opaque op per layer keeps the compiled DiT block fullgraph: dynamo treats # it as a single graph node (no eager island, no graph breaks), and the K/V slot # writes happen inside the op so write→read ordering with the block-table kernel -# is internal — no hidden-mutation ordering hazards against a separate reader op -# and no multi-GiB pool tensors as graph inputs (the flat and paged pool views -# alias the same storage, which AOTAutograd handles poorly as inputs). Pattern -# follows sage_attn3.py in-repo and vLLM's own unified attention ops. +# is internal. The flat pools are explicit mutable inputs: Inductor/CUDA Graph +# must track their storage lifetime instead of observing an undeclared mutation +# through a process-global registry. def _paged_write_attn_impl( query: torch.Tensor, k_curr: torch.Tensor, v_curr: torch.Tensor, k_act: torch.Tensor | None, v_act: torch.Tensor | None, - layer_idx: torch.Tensor, + key_pool: torch.Tensor, + value_pool: torch.Tensor, + block_size: int, video_slots: torch.Tensor, action_slots: torch.Tensor, block_table: torch.Tensor, @@ -496,21 +489,17 @@ def _paged_write_attn_impl( max_seq_len: int, softmax_scale: float, ) -> torch.Tensor: - kv = _CURRENT_PAGED_KV_CACHE - if kv is None: - raise RuntimeError("ar_diffusion_paged_write_attn called before prepare() set the KV pool registry") - layer_idx = int(layer_idx) - k_pool = kv._k_pools[layer_idx] - v_pool = kv._v_pools[layer_idx] - k_pool[video_slots] = k_curr.to(k_pool.dtype) - v_pool[video_slots] = v_curr.to(v_pool.dtype) + key_pool[video_slots] = k_curr.to(key_pool.dtype) + value_pool[video_slots] = v_curr.to(value_pool.dtype) if k_act is not None and v_act is not None and k_act.shape[0] > 0: - k_pool[action_slots] = k_act.to(k_pool.dtype) - v_pool[action_slots] = v_act.to(v_pool.dtype) + key_pool[action_slots] = k_act.to(key_pool.dtype) + value_pool[action_slots] = v_act.to(value_pool.dtype) + key_cache = key_pool.unflatten(0, (-1, block_size)) + value_cache = value_pool.unflatten(0, (-1, block_size)) return ar_diffusion_paged_attention( query, - kv.key_cache(layer_idx), - kv.value_cache(layer_idx), + key_cache, + value_cache, block_table=block_table, query_start_loc=query_start_loc, seq_lens=seq_lens, @@ -525,14 +514,19 @@ def _paged_write_attn_impl( # the module from sys.modules (same as sage_attn3.py). if not hasattr(torch.ops.vllm_omni, "ar_diffusion_paged_write_attn"): - @torch.library.custom_op("vllm_omni::ar_diffusion_paged_write_attn", mutates_args=()) + @torch.library.custom_op( + "vllm_omni::ar_diffusion_paged_write_attn", + mutates_args=("key_pool", "value_pool"), + ) def _paged_write_attn_op( query: torch.Tensor, k_curr: torch.Tensor, v_curr: torch.Tensor, k_act: torch.Tensor | None, v_act: torch.Tensor | None, - layer_idx: torch.Tensor, + key_pool: torch.Tensor, + value_pool: torch.Tensor, + block_size: int, video_slots: torch.Tensor, action_slots: torch.Tensor, block_table: torch.Tensor, @@ -548,7 +542,9 @@ if not hasattr(torch.ops.vllm_omni, "ar_diffusion_paged_write_attn"): v_curr, k_act, v_act, - layer_idx, + key_pool, + value_pool, + block_size, video_slots, action_slots, block_table, @@ -566,7 +562,9 @@ if not hasattr(torch.ops.vllm_omni, "ar_diffusion_paged_write_attn"): v_curr, k_act, v_act, - layer_idx, + key_pool, + value_pool, + block_size, video_slots, action_slots, block_table, @@ -589,7 +587,9 @@ def paged_write_attn( v_curr, k_act, v_act, - inputs.layer_idx, + inputs.key_pool, + inputs.value_pool, + inputs.block_size, inputs.video_slots, inputs.action_slots, inputs.block_table, diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/state.py b/vllm_omni/experimental/ar_diffusion/kv_cache/state.py index d114212bb..ec14a9177 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/state.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/state.py @@ -130,7 +130,7 @@ class ARDiffusionKVState: if ctx.commit_current and ctx._allocated_video: n_chunks = ctx.seq_len // self.kv_cache.spec.chunk_size for _ in range(n_chunks): - ctx.adapter.on_chunk_committed() + self.kv_cache.commit_chunk(ctx.adapter) self._committed[kv_branch] += ctx.seq_len _log.debug( "AR-Diffusion COMMIT [%s] new_tokens=%d chunks=%d resident=%d/%d", @@ -173,6 +173,17 @@ class ARDiffusionKVState: for i in range(self.num_layers) ] + def clear_cross_attention(self) -> None: + """Invalidate all named cross-attention caches without dropping self-KV. + + Prompt changes may preserve the autoregressive world history while + requiring fresh text projections. This operation deliberately leaves + branch adapters and their resident self-attention blocks untouched. + """ + if self._closed: + raise RuntimeError(f"AR-Diffusion session {self.session_id!r} is closed") + self.kv_cache.release_cross_attention(self.session_id) + def close(self) -> None: """Release all self- and cross-attention storage owned by this session.""" if self._closed: diff --git a/vllm_omni/experimental/ar_diffusion/runner.py b/vllm_omni/experimental/ar_diffusion/runner.py index 31a23eaf7..410c45fb1 100644 --- a/vllm_omni/experimental/ar_diffusion/runner.py +++ b/vllm_omni/experimental/ar_diffusion/runner.py @@ -21,6 +21,7 @@ from vllm_omni.experimental.ar_diffusion.capability import ( from vllm_omni.experimental.ar_diffusion.kv_cache.config import ARDiffusionKVConfig from vllm_omni.experimental.ar_diffusion.kv_cache.manager import ARDiffusionKVCache from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState +from vllm_omni.experimental.ar_diffusion.tick_protocol import ARDiffusionTickRequest logger = init_logger(__name__) @@ -51,7 +52,6 @@ class ARDiffusionModelRunner(DiffusionModelRunner): """ _WARMUP_SID = "__ardiffusion_warmup__" - _MAX_RESIDENT_SESSIONS = 1 def __init__(self, vllm_config: object, od_config: OmniDiffusionConfig, device: torch.device) -> None: super().__init__(vllm_config, od_config, device) @@ -123,7 +123,6 @@ class ARDiffusionModelRunner(DiffusionModelRunner): self.ar_diffusion_kv_config = config self._ar_diffusion_capability = capability self._ar_diffusion_kv_cache_spec = spec - self._session_capacity = min(spec.session_capacity, self._MAX_RESIDENT_SESSIONS) self.kv_cache = ARDiffusionKVCache( config, num_layers=spec.num_layers, @@ -134,12 +133,14 @@ class ARDiffusionModelRunner(DiffusionModelRunner): max_model_len=spec.max_model_len, available_bytes=self._available_memory_bytes() if available_bytes is None else available_bytes, kv_branches=spec.kv_branches, - session_capacity=self._session_capacity, + session_capacity=spec.session_capacity, cross_attention_lengths=spec.cross_attention_lengths, frames_per_block=spec.frames_per_block, max_scratch_tokens_per_branch=spec.max_scratch_tokens_per_branch, + model_owned_state_bytes_per_session=spec.model_owned_state_bytes_per_session, device=self.device, ) + self._session_capacity = self.kv_cache.session_capacity logger.info( "AR-Diffusion KV cache: blocks=%d layers=%d local_kv_heads=%d head_size=%d " "tokens/frame=%d frames/block=%d window=%d sink=%d kv_branches=%s cross=%s " @@ -230,9 +231,14 @@ class ARDiffusionModelRunner(DiffusionModelRunner): return state @staticmethod - def _request_session(req: OmniDiffusionRequest) -> tuple[str, dict]: + def _request_session( + req: OmniDiffusionRequest, + ) -> tuple[str, dict, ARDiffusionTickRequest | None]: extra_args = req.sampling_params.extra_args or {} - return str(extra_args.get("session_id") or "default"), extra_args + tick = ARDiffusionTickRequest.from_extra_args(extra_args) + if tick is not None: + return tick.session_id, extra_args, tick + return str(extra_args.get("session_id") or "default"), extra_args, None def execute_model( self, @@ -245,8 +251,10 @@ class ARDiffusionModelRunner(DiffusionModelRunner): if capability is None: raise RuntimeError("AR-Diffusion capability missing after KV cache initialization") - session_id, extra_args = self._request_session(req) - if extra_args.get("reset", False): + session_id, extra_args, tick = self._request_session(req) + reset = tick.reset if tick is not None else bool(extra_args.get("reset", False)) + close_session = tick.close_session if tick is not None else bool(extra_args.get("close_session", False)) + if reset: self.reset_session(session_id) state = self._get_or_create_session(session_id) started = time.perf_counter() @@ -268,7 +276,7 @@ class ARDiffusionModelRunner(DiffusionModelRunner): ) raise self._perf_e2e_times.append(time.perf_counter() - started) - if extra_args.get("close_session", False): + if close_session: self.close_session(session_id) return output diff --git a/vllm_omni/experimental/ar_diffusion/session.py b/vllm_omni/experimental/ar_diffusion/session.py new file mode 100644 index 000000000..6bb4b27d9 --- /dev/null +++ b/vllm_omni/experimental/ar_diffusion/session.py @@ -0,0 +1,578 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Internal realtime session control plane for AR-Diffusion. + +This module owns transport-level identity, event ordering, backpressure, and +chunk-boundary snapshots. Model-defined control payloads remain opaque here. +""" + +from __future__ import annotations + +import asyncio +import uuid +from collections import OrderedDict +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +from typing import Any, Generic, Protocol, TypeVar + +from vllm_omni.experimental.ar_diffusion.tick_protocol import ( + ARDiffusionChunkMetadata, + ARDiffusionControlInput, + ARDiffusionTickRequest, +) + + +class ARDiffusionSessionError(RuntimeError): + """Base error for realtime AR-Diffusion session operations.""" + + +class ARDiffusionSessionStateError(ARDiffusionSessionError): + """Raised when an operation is invalid for the current session state.""" + + +class ARDiffusionEventQueueFullError(ARDiffusionSessionError): + """Raised when accepting an event would exceed the bounded queue.""" + + +class ARDiffusionStaleEventError(ARDiffusionSessionError): + """Raised when an event is older than the current monotonic boundary.""" + + +class ARDiffusionChunkMetadataError(ARDiffusionSessionError): + """Raised when a tick consumer returns metadata for a different chunk.""" + + +class ARDiffusionEventAcceptanceStatus(str, Enum): + ACCEPTED = "accepted" + DUPLICATE = "duplicate" + + +class ARDiffusionSessionStatus(str, Enum): + ACTIVE = "active" + RESETTING = "resetting" + FAILED = "failed" + CLOSING = "closing" + CLEANUP_FAILED = "cleanup_failed" + CLOSED = "closed" + + +def _copy_control(control: ARDiffusionControlInput) -> ARDiffusionControlInput: + return ARDiffusionControlInput( + track=control.track, + schema=control.schema, + data=control.data, + ) + + +@dataclass(frozen=True) +class ARDiffusionSessionEvent: + """One atomically accepted prompt/control update. + + ``prompt=None`` means that this event does not update the prompt. Controls + are keyed by ``track`` when the chunk snapshot is assembled; their schema + and data are otherwise opaque to this layer. + """ + + event_id: int + prompt: str | None = None + controls: tuple[ARDiffusionControlInput, ...] = () + + def __post_init__(self) -> None: + if isinstance(self.event_id, bool) or not isinstance(self.event_id, int) or self.event_id < 0: + raise ValueError("event_id must be a non-negative integer.") + if self.prompt is not None and not isinstance(self.prompt, str): + raise ValueError("prompt must be a string or None.") + controls = tuple(self.controls) + if any(not isinstance(control, ARDiffusionControlInput) for control in controls): + raise ValueError("controls must contain ARDiffusionControlInput values.") + tracks = [control.track for control in controls] + if len(tracks) != len(set(tracks)): + raise ValueError("A composite event may update each control track at most once.") + if self.prompt is None and not controls: + raise ValueError("An event must update the prompt, one or more controls, or both.") + object.__setattr__(self, "controls", controls) + + +@dataclass(frozen=True) +class ARDiffusionEventAcceptance: + """Immediate queue acceptance result; this does not mean applied.""" + + session_id: str + event_id: int + status: ARDiffusionEventAcceptanceStatus + pending_event_count: int + + +TickOutputT = TypeVar("TickOutputT") + + +class ARDiffusionTickConsumer(Protocol[TickOutputT]): + """Consumes one immutable chunk-boundary tick.""" + + async def execute_tick(self, tick: ARDiffusionTickRequest) -> TickOutputT: ... + + def chunk_metadata(self, output: TickOutputT) -> ARDiffusionChunkMetadata: ... + + +@dataclass(frozen=True) +class ARDiffusionPreparedControls: + """One reducer-owned, transactional control snapshot. + + ``private_state`` is opaque to the session layer and becomes durable only + after the corresponding model tick succeeds. + """ + + controls: tuple[ARDiffusionControlInput, ...] + private_state: Any = None + + +class ARDiffusionControlReducer(Protocol): + """Optional model adapter for stateful controls sampled per AR block.""" + + def prepare( + self, + *, + current_controls: Mapping[str, ARDiffusionControlInput], + events: Sequence[ARDiffusionSessionEvent], + chunk_index: int, + ) -> ARDiffusionPreparedControls: ... + + def commit(self, prepared: ARDiffusionPreparedControls) -> None: ... + + def reset(self) -> None: ... + + +class ARDiffusionSessionLifecycle(Protocol): + """Worker-facing session lifecycle operations.""" + + async def reset_session(self, session_id: str) -> None: ... + + async def close_session(self, session_id: str) -> None: ... + + +class ARDiffusionCollectiveRPCClient(Protocol): + """Subset of the engine control RPC surface used by session lifecycle.""" + + async def collective_rpc( + self, + method: str, + timeout: float | None = None, + args: tuple[Any, ...] = (), + kwargs: dict[str, Any] | None = None, + stage_ids: list[int] | None = None, + ) -> list[Any]: ... + + +def _collect_rpc_support_results(value: Any, *, errors: list[str], supported: list[bool]) -> None: + if isinstance(value, bool): + supported.append(value) + return + if isinstance(value, (list, tuple)): + for item in value: + _collect_rpc_support_results(item, errors=errors, supported=supported) + return + if isinstance(value, Mapping): + if value.get("supported") is False: + errors.append(str(value.get("error") or "target stage does not support this lifecycle RPC")) + return + if "result" in value: + _collect_rpc_support_results(value["result"], errors=errors, supported=supported) + + +class ARDiffusionWorkerLifecycle: + """Routes reset/close through the existing orchestrator collective RPC.""" + + def __init__( + self, + rpc_client: ARDiffusionCollectiveRPCClient, + *, + stage_ids: Sequence[int], + timeout: float | None = None, + ) -> None: + normalized_stage_ids = tuple(stage_ids) + if not normalized_stage_ids: + raise ValueError("stage_ids must select at least one AR-Diffusion stage.") + if any(isinstance(stage_id, bool) or not isinstance(stage_id, int) or stage_id < 0 for stage_id in stage_ids): + raise ValueError("stage_ids must contain non-negative integers.") + self._rpc_client = rpc_client + self._stage_ids = normalized_stage_ids + self._timeout = timeout + + async def _invoke(self, method: str, session_id: str) -> None: + results = await self._rpc_client.collective_rpc( + method=method, + timeout=self._timeout, + args=(session_id,), + stage_ids=list(self._stage_ids), + ) + errors: list[str] = [] + supported: list[bool] = [] + _collect_rpc_support_results(results, errors=errors, supported=supported) + if errors: + raise ARDiffusionSessionError(f"{method} failed: {'; '.join(errors)}") + if not supported or not all(supported): + raise ARDiffusionSessionError(f"{method} is not supported by every selected worker.") + + async def reset_session(self, session_id: str) -> None: + await self._invoke("reset_ar_diffusion_session", session_id) + + async def close_session(self, session_id: str) -> None: + await self._invoke("close_ar_diffusion_session", session_id) + + +def _default_request_id(session_id: str, chunk_index: int) -> str: + return f"ar-{session_id}-{chunk_index}-{uuid.uuid4().hex}" + + +class ARDiffusionSession(Generic[TickOutputT]): + """Transactional event queue and chunk snapshot for one realtime world.""" + + def __init__( + self, + session_id: str, + *, + tick_consumer: ARDiffusionTickConsumer[TickOutputT], + lifecycle: ARDiffusionSessionLifecycle, + max_pending_events: int, + request_id_factory: Callable[[str, int], str] | None = None, + control_reducer: ARDiffusionControlReducer | None = None, + ) -> None: + if not isinstance(session_id, str) or not session_id.strip(): + raise ValueError("session_id must be a non-empty string.") + if isinstance(max_pending_events, bool) or not isinstance(max_pending_events, int) or max_pending_events <= 0: + raise ValueError("max_pending_events must be a positive integer.") + self.session_id = session_id + self._tick_consumer = tick_consumer + self._lifecycle = lifecycle + self._max_pending_events = max_pending_events + self._request_id_factory = request_id_factory or _default_request_id + self._control_reducer = control_reducer + + self._status = ARDiffusionSessionStatus.ACTIVE + self._pending_events: dict[int, ARDiffusionSessionEvent] = {} + self._recent_event_ids: OrderedDict[int, None] = OrderedDict() + self._event_floor = -1 + self._inflight_event_floor: int | None = None + self._chunk_index = 0 + self._prompt: str | None = None + self._controls: dict[str, ARDiffusionControlInput] = {} + + self._state_lock = asyncio.Lock() + self._tick_lock = asyncio.Lock() + + @property + def status(self) -> ARDiffusionSessionStatus: + return self._status + + @property + def chunk_index(self) -> int: + return self._chunk_index + + @property + def pending_event_count(self) -> int: + return len(self._pending_events) + + def _require_active(self) -> None: + if self._status is not ARDiffusionSessionStatus.ACTIVE: + raise ARDiffusionSessionStateError(f"AR-Diffusion session {self.session_id!r} is {self._status.value}.") + + def _require_resettable(self) -> None: + if self._status not in { + ARDiffusionSessionStatus.ACTIVE, + ARDiffusionSessionStatus.FAILED, + }: + raise ARDiffusionSessionStateError(f"AR-Diffusion session {self.session_id!r} is {self._status.value}.") + + async def accept_event(self, event: ARDiffusionSessionEvent) -> ARDiffusionEventAcceptance: + """Atomically enqueue an event or return an idempotent duplicate result.""" + if not isinstance(event, ARDiffusionSessionEvent): + raise TypeError("event must be an ARDiffusionSessionEvent.") + async with self._state_lock: + self._require_active() + if event.event_id in self._pending_events: + return ARDiffusionEventAcceptance( + session_id=self.session_id, + event_id=event.event_id, + status=ARDiffusionEventAcceptanceStatus.DUPLICATE, + pending_event_count=len(self._pending_events), + ) + if event.event_id in self._recent_event_ids: + return ARDiffusionEventAcceptance( + session_id=self.session_id, + event_id=event.event_id, + status=ARDiffusionEventAcceptanceStatus.DUPLICATE, + pending_event_count=len(self._pending_events), + ) + + event_floor = self._event_floor + if self._inflight_event_floor is not None: + event_floor = max(event_floor, self._inflight_event_floor) + if event.event_id <= event_floor: + raise ARDiffusionStaleEventError( + f"event_id {event.event_id} is stale; the session event floor is {event_floor}." + ) + if len(self._pending_events) >= self._max_pending_events: + raise ARDiffusionEventQueueFullError( + f"AR-Diffusion session {self.session_id!r} event queue is full " + f"({self._max_pending_events} pending)." + ) + + stored_event = ARDiffusionSessionEvent( + event_id=event.event_id, + prompt=event.prompt, + controls=tuple(_copy_control(control) for control in event.controls), + ) + self._pending_events[event.event_id] = stored_event + self._recent_event_ids[event.event_id] = None + while len(self._recent_event_ids) > self._max_pending_events: + self._recent_event_ids.popitem(last=False) + return ARDiffusionEventAcceptance( + session_id=self.session_id, + event_id=event.event_id, + status=ARDiffusionEventAcceptanceStatus.ACCEPTED, + pending_event_count=len(self._pending_events), + ) + + def _build_tick_locked( + self, + ) -> tuple[ + ARDiffusionTickRequest, + str | None, + dict[str, ARDiffusionControlInput], + ARDiffusionPreparedControls | None, + ]: + event_ids = tuple(sorted(self._pending_events)) + events = tuple(self._pending_events[event_id] for event_id in event_ids) + prompt = self._prompt + for event in events: + if event.prompt is not None: + prompt = event.prompt + + prepared_controls: ARDiffusionPreparedControls | None = None + if self._control_reducer is None: + controls = {track: _copy_control(control) for track, control in self._controls.items()} + for event in events: + for control in event.controls: + controls[control.track] = _copy_control(control) + else: + prepared_controls = self._control_reducer.prepare( + current_controls={track: _copy_control(control) for track, control in self._controls.items()}, + events=events, + chunk_index=self._chunk_index, + ) + if not isinstance(prepared_controls, ARDiffusionPreparedControls): + raise TypeError("control reducer prepare() must return ARDiffusionPreparedControls.") + controls = {} + for control in prepared_controls.controls: + if not isinstance(control, ARDiffusionControlInput): + raise TypeError("prepared controls must contain ARDiffusionControlInput values.") + if control.track in controls: + raise ValueError("prepared controls must contain at most one value per track.") + controls[control.track] = _copy_control(control) + + chunk_index = self._chunk_index + request_id = self._request_id_factory(self.session_id, chunk_index) + tick = ARDiffusionTickRequest( + session_id=self.session_id, + request_id=request_id, + chunk_index=chunk_index, + applied_event_ids=event_ids, + prompt=prompt, + controls=tuple(_copy_control(controls[track]) for track in sorted(controls)), + ) + self._inflight_event_floor = event_ids[-1] if event_ids else None + return tick, prompt, controls, prepared_controls + + async def _commit_tick( + self, + tick: ARDiffusionTickRequest, + prompt: str | None, + controls: dict[str, ARDiffusionControlInput], + prepared_controls: ARDiffusionPreparedControls | None, + ) -> None: + async with self._state_lock: + # Reducer state and the session snapshot form one logical commit. + # Run the only fallible operation before mutating session fields so + # a reducer failure cannot acknowledge events or advance the chunk. + if prepared_controls is not None: + assert self._control_reducer is not None + self._control_reducer.commit(prepared_controls) + for event_id in tick.applied_event_ids: + self._pending_events.pop(event_id, None) + if tick.applied_event_ids: + self._event_floor = tick.applied_event_ids[-1] + self._prompt = prompt + self._controls = controls + self._chunk_index += 1 + self._inflight_event_floor = None + + async def _fail_tick(self, tick_error: BaseException) -> None: + async with self._state_lock: + self._inflight_event_floor = None + self._status = ARDiffusionSessionStatus.FAILED + try: + # A model forward may already have released its state. This + # idempotent close also covers successful forwards whose metadata + # or local reducer commit could not be accepted. + await self._lifecycle.close_session(self.session_id) + except Exception as cleanup_error: + raise ARDiffusionSessionError( + "AR-Diffusion tick failed " + f"({type(tick_error).__name__}: {tick_error}) and worker " + f"session cleanup also failed: {cleanup_error}" + ) from tick_error + + async def next_chunk(self) -> TickOutputT: + """Execute and atomically commit one chunk-boundary snapshot.""" + async with self._tick_lock: + async with self._state_lock: + self._require_active() + tick, prompt, controls, prepared_controls = self._build_tick_locked() + + expected_metadata = ARDiffusionChunkMetadata.from_tick(tick) + try: + output = await self._tick_consumer.execute_tick(tick) + metadata = self._tick_consumer.chunk_metadata(output) + if not isinstance(metadata, ARDiffusionChunkMetadata) or metadata != expected_metadata: + raise ARDiffusionChunkMetadataError( + "Tick consumer metadata does not match the submitted AR-Diffusion tick." + ) + except BaseException as tick_error: + await self._fail_tick(tick_error) + raise + + commit_task = asyncio.create_task( + self._commit_tick( + tick, + prompt, + controls, + prepared_controls, + ) + ) + try: + await asyncio.shield(commit_task) + except asyncio.CancelledError: + # Once a consumer reports success, the local chunk/event state + # must commit even if the transport task is being cancelled. + try: + await commit_task + except BaseException as commit_error: + await self._fail_tick(commit_error) + raise + raise + except BaseException as commit_error: + await self._fail_tick(commit_error) + raise + return output + + async def reset(self) -> None: + """Reset worker state and discard queued/current transport state.""" + async with self._tick_lock: + async with self._state_lock: + self._require_resettable() + self._status = ARDiffusionSessionStatus.RESETTING + try: + await self._lifecycle.reset_session(self.session_id) + except BaseException: + async with self._state_lock: + self._status = ARDiffusionSessionStatus.FAILED + raise + async with self._state_lock: + if self._pending_events: + self._event_floor = max(self._event_floor, max(self._pending_events)) + self._pending_events.clear() + self._prompt = None + self._controls.clear() + if self._control_reducer is not None: + self._control_reducer.reset() + self._inflight_event_floor = None + self._chunk_index = 0 + self._status = ARDiffusionSessionStatus.ACTIVE + + async def close(self) -> None: + """Close worker state, retaining a tombstone until cleanup succeeds.""" + async with self._tick_lock: + async with self._state_lock: + if self._status is ARDiffusionSessionStatus.CLOSED: + return + self._status = ARDiffusionSessionStatus.CLOSING + try: + await self._lifecycle.close_session(self.session_id) + except BaseException: + async with self._state_lock: + self._status = ARDiffusionSessionStatus.CLEANUP_FAILED + raise + else: + async with self._state_lock: + self._pending_events.clear() + self._prompt = None + self._controls.clear() + self._inflight_event_floor = None + self._status = ARDiffusionSessionStatus.CLOSED + + async def disconnect(self) -> None: + """Disconnect uses the same owning-worker cleanup as explicit close.""" + await self.close() + + +class ARDiffusionSessionManager(Generic[TickOutputT]): + """Creates and owns transport sessions sharing one tick/lifecycle backend.""" + + def __init__( + self, + *, + tick_consumer: ARDiffusionTickConsumer[TickOutputT], + lifecycle: ARDiffusionSessionLifecycle, + max_pending_events: int, + request_id_factory: Callable[[str, int], str] | None = None, + session_id_factory: Callable[[], str] | None = None, + control_reducer_factory: Callable[[], ARDiffusionControlReducer] | None = None, + ) -> None: + self._tick_consumer = tick_consumer + self._lifecycle = lifecycle + self._max_pending_events = max_pending_events + self._request_id_factory = request_id_factory + self._session_id_factory = session_id_factory or (lambda: uuid.uuid4().hex) + self._control_reducer_factory = control_reducer_factory + self._sessions: dict[str, ARDiffusionSession[TickOutputT]] = {} + self._lock = asyncio.Lock() + + async def create_session(self, session_id: str | None = None) -> ARDiffusionSession[TickOutputT]: + resolved_session_id = session_id or self._session_id_factory() + async with self._lock: + if resolved_session_id in self._sessions: + raise ARDiffusionSessionStateError(f"AR-Diffusion session {resolved_session_id!r} already exists.") + session = ARDiffusionSession( + resolved_session_id, + tick_consumer=self._tick_consumer, + lifecycle=self._lifecycle, + max_pending_events=self._max_pending_events, + request_id_factory=self._request_id_factory, + control_reducer=( + self._control_reducer_factory() if self._control_reducer_factory is not None else None + ), + ) + self._sessions[resolved_session_id] = session + return session + + async def get_session(self, session_id: str) -> ARDiffusionSession[TickOutputT]: + async with self._lock: + try: + return self._sessions[session_id] + except KeyError as exc: + raise ARDiffusionSessionStateError(f"AR-Diffusion session {session_id!r} does not exist.") from exc + + async def reset_session(self, session_id: str) -> None: + session = await self.get_session(session_id) + await session.reset() + + async def close_session(self, session_id: str) -> None: + session = await self.get_session(session_id) + await session.close() + async with self._lock: + if self._sessions.get(session_id) is session: + self._sessions.pop(session_id) + + async def disconnect(self, session_id: str) -> None: + """Release the owning worker route when its transport disconnects.""" + await self.close_session(session_id) diff --git a/vllm_omni/experimental/ar_diffusion/tick_protocol.py b/vllm_omni/experimental/ar_diffusion/tick_protocol.py new file mode 100644 index 000000000..aa8315118 --- /dev/null +++ b/vllm_omni/experimental/ar_diffusion/tick_protocol.py @@ -0,0 +1,202 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Typed request boundary shared by AR-Diffusion models and serving APIs. + +The transport layer owns event ordering and snapshots. Model adapters consume +the resulting immutable tick without learning about WebSocket messages. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import Any + +AR_DIFFUSION_TICK_KEY = "ar_diffusion_tick" + + +def _non_empty_string(value: object, *, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field} must be a non-empty string.") + return value + + +def _non_negative_int(value: object, *, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{field} must be a non-negative integer.") + return value + + +def _freeze_transport_value(value: Any, *, path: str) -> Any: + """Copy JSON-like transport data into recursively immutable containers.""" + if value is None or isinstance(value, (str, bool, int, float)): + return value + if isinstance(value, Mapping): + frozen: dict[str, Any] = {} + for key, item in value.items(): + if not isinstance(key, str): + raise ValueError(f"{path} mapping keys must be strings.") + frozen[key] = _freeze_transport_value(item, path=f"{path}.{key}") + return MappingProxyType(frozen) + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return tuple(_freeze_transport_value(item, path=f"{path}[]") for item in value) + raise ValueError( + f"{path} must contain only transport-safe mappings, sequences, and scalar values; got {type(value).__name__}." + ) + + +def _thaw_transport_value(value: Any) -> Any: + """Return a detached JSON-serializable copy of an immutable snapshot.""" + if isinstance(value, Mapping): + return {key: _thaw_transport_value(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_thaw_transport_value(item) for item in value] + return value + + +@dataclass(frozen=True) +class ARDiffusionControlInput: + """One model-defined control track in a chunk-boundary snapshot.""" + + track: str + schema: str + data: Mapping[str, Any] + + def __post_init__(self) -> None: + _non_empty_string(self.track, field="control.track") + _non_empty_string(self.schema, field="control.schema") + if not isinstance(self.data, Mapping): + raise ValueError("control.data must be a mapping.") + object.__setattr__( + self, + "data", + _freeze_transport_value(self.data, path=f"control[{self.track!r}].data"), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "track": self.track, + "schema": self.schema, + "data": _thaw_transport_value(self.data), + } + + @classmethod + def from_dict(cls, value: object) -> ARDiffusionControlInput: + if not isinstance(value, Mapping): + raise ValueError("Each control must be a mapping.") + return cls( + track=value.get("track"), # type: ignore[arg-type] + schema=value.get("schema"), # type: ignore[arg-type] + data=value.get("data"), # type: ignore[arg-type] + ) + + +@dataclass(frozen=True) +class ARDiffusionTickRequest: + """A single atomic AR-block computation. + + ``request_id`` identifies this computation, ``session_id`` identifies the + persistent rollout, and ``applied_event_ids`` records which accepted + interaction events are represented by this immutable snapshot. + """ + + session_id: str + request_id: str + chunk_index: int + applied_event_ids: tuple[int, ...] = () + prompt: str | None = None + controls: tuple[ARDiffusionControlInput, ...] = () + reset: bool = False + close_session: bool = False + + def __post_init__(self) -> None: + _non_empty_string(self.session_id, field="session_id") + _non_empty_string(self.request_id, field="request_id") + _non_negative_int(self.chunk_index, field="chunk_index") + if self.prompt is not None and not isinstance(self.prompt, str): + raise ValueError("prompt must be a string or None.") + if not isinstance(self.reset, bool): + raise ValueError("reset must be a boolean.") + if not isinstance(self.close_session, bool): + raise ValueError("close_session must be a boolean.") + + event_ids = tuple( + _non_negative_int(event_id, field="applied_event_ids item") for event_id in self.applied_event_ids + ) + if event_ids != tuple(sorted(set(event_ids))): + raise ValueError("applied_event_ids must be unique and strictly increasing.") + controls = tuple(self.controls) + if any(not isinstance(control, ARDiffusionControlInput) for control in controls): + raise ValueError("controls must contain ARDiffusionControlInput values.") + object.__setattr__(self, "applied_event_ids", event_ids) + object.__setattr__(self, "controls", controls) + + def to_extra_args(self) -> dict[str, Any]: + """Serialize the tick under one namespaced ``extra_args`` key.""" + return { + AR_DIFFUSION_TICK_KEY: { + "session_id": self.session_id, + "request_id": self.request_id, + "chunk_index": self.chunk_index, + "applied_event_ids": list(self.applied_event_ids), + "prompt": self.prompt, + "controls": [control.to_dict() for control in self.controls], + "reset": self.reset, + "close_session": self.close_session, + } + } + + @classmethod + def from_extra_args( + cls, + extra_args: Mapping[str, Any] | None, + ) -> ARDiffusionTickRequest | None: + """Parse a typed tick, returning ``None`` for legacy AR requests.""" + if extra_args is None or AR_DIFFUSION_TICK_KEY not in extra_args: + return None + value = extra_args[AR_DIFFUSION_TICK_KEY] + if not isinstance(value, Mapping): + raise ValueError(f"{AR_DIFFUSION_TICK_KEY} must be a mapping.") + raw_event_ids = value.get("applied_event_ids", ()) + raw_controls = value.get("controls", ()) + if not isinstance(raw_event_ids, Sequence) or isinstance(raw_event_ids, (str, bytes)): + raise ValueError("applied_event_ids must be a sequence.") + if not isinstance(raw_controls, Sequence) or isinstance(raw_controls, (str, bytes)): + raise ValueError("controls must be a sequence.") + return cls( + session_id=value.get("session_id"), # type: ignore[arg-type] + request_id=value.get("request_id"), # type: ignore[arg-type] + chunk_index=value.get("chunk_index"), # type: ignore[arg-type] + applied_event_ids=tuple(raw_event_ids), + prompt=value.get("prompt"), # type: ignore[arg-type] + controls=tuple(ARDiffusionControlInput.from_dict(control) for control in raw_controls), + reset=value.get("reset", False), # type: ignore[arg-type] + close_session=value.get("close_session", False), # type: ignore[arg-type] + ) + + +@dataclass(frozen=True) +class ARDiffusionChunkMetadata: + """Identity metadata returned with one generated AR block.""" + + session_id: str + request_id: str + chunk_index: int + applied_event_ids: tuple[int, ...] = () + + @classmethod + def from_tick(cls, tick: ARDiffusionTickRequest) -> ARDiffusionChunkMetadata: + return cls( + session_id=tick.session_id, + request_id=tick.request_id, + chunk_index=tick.chunk_index, + applied_event_ids=tick.applied_event_ids, + ) + + def to_dict(self) -> dict[str, Any]: + return { + "session_id": self.session_id, + "request_id": self.request_id, + "chunk_index": self.chunk_index, + "applied_event_ids": list(self.applied_event_ids), + } From 9d2bb23ff676746e1cea69f2366a32f925c39f50 Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Sun, 16 Aug 2026 11:02:38 +0800 Subject: [PATCH 037/267] [Diffusion] Add loader-owned host-weight plans for DLO (TP=1) (#6213) Signed-off-by: Hongsheng Liu --- .../distributed_layerwise_offload.md | 177 +++- .../distributed_layerwise_offload.md | 61 +- ...{dlo_dp2_lifecycle.py => dlo_lifecycle.py} | 85 +- .../model_loader/test_diffusers_loader.py | 136 +++ .../test_minimax_h3_quantization.py | 43 + .../test_distributed_layerwise_backend.py | 595 ++++++++++-- .../diffusion/test_diffusion_model_runner.py | 9 +- .../checkpoint_adapters/__init__.py | 8 + .../checkpoint_adapters/direct_mmap.py | 108 +++ .../model_loader/diffusers_loader.py | 88 +- .../model_loader/host_weight_plan.py | 355 +++++++ .../models/minimax_h3/pipeline_minimax_h3.py | 7 - vllm_omni/diffusion/offloader/__init__.py | 28 +- vllm_omni/diffusion/offloader/base.py | 10 +- .../distributed_layerwise_backend.py | 891 ++++++++++-------- vllm_omni/diffusion/offloader/offload_plan.py | 17 - .../worker/diffusion_model_runner.py | 6 +- 17 files changed, 2009 insertions(+), 615 deletions(-) rename examples/offline_inference/minimax_h3/{dlo_dp2_lifecycle.py => dlo_lifecycle.py} (72%) create mode 100644 vllm_omni/diffusion/model_loader/checkpoint_adapters/direct_mmap.py create mode 100644 vllm_omni/diffusion/model_loader/host_weight_plan.py diff --git a/docs/design/feature/offloader/distributed_layerwise_offload.md b/docs/design/feature/offloader/distributed_layerwise_offload.md index ade39fa27..b38f0443d 100644 --- a/docs/design/feature/offloader/distributed_layerwise_offload.md +++ b/docs/design/feature/offloader/distributed_layerwise_offload.md @@ -4,7 +4,7 @@ This document describes distributed layerwise offload (DLO) for diffusion models. DLO keeps only a small number of DiT blocks on the accelerator and streams the remaining blocks from host memory. The distributed backend can either shard those host-side weights across an existing parallel group or keep -the standard loader's rank-local weights and avoid an additional collective. +complete rank-local block sources and avoid an additional collective. For user-facing commands, see the [distributed layerwise offloading guide](../../../user_guide/diffusion/offloader/distributed_layerwise_offload.md) @@ -14,10 +14,20 @@ and the [Cosmos3 recipe](https://github.com/vllm-project/vllm-omni/blob/main/rec DLO is implemented for multi-device diffusion execution. The default AllGather path is the primary path for DP and SP deployments. The -`--dlo-no-use-allgather` path is a rank-local compatibility mode: it is useful -for standard-loader sharding, workstation bring-up, and systems where an -additional DLO collective is undesirable, but it does not reduce host weight -storage across ranks. +`--dlo-no-use-allgather` path streams complete blocks independently and adds no +DLO weight collective. + +Host storage is selected separately from the transfer protocol. The loader can +produce a direct-checkpoint mmap plan for a proven-compatible runtime layout; +otherwise it uses the ordinary loader. Consequently, no-AllGather replicas on +the same node can share immutable checkpoint pages when direct mmap is +selected, while the ordinary-loader fallback still keeps a private runtime +copy per process. + +The Phase A shared-mmap support boundary is TP1. TP greater than one is an +ordinary-loader compatibility path: DLO can consume the resulting TP-local +tensors, but those configurations do not use checkpoint mmap and must not be +used to claim shared-mmap host-memory savings. The compatibility matrix below describes the current implementation. The unit-level guards are covered, but not every parallelism combination has a @@ -41,6 +51,36 @@ TP is deliberately not used as DLO's AllGather group. HSDP has its own parameter-sharding lifecycle and is not allowed to be sharded a second time by DLO's AllGather path. +### The loader owns host-weight planning + +Before it decides whether ordinary weight materialization can be skipped, the +diffusion loader builds one `HostWeightPlan`. A direct-checkpoint mmap plan is +accepted only when preflight proves all of the following: + +- every required DiT parameter and persistent buffer has exactly one source; +- runtime names, checkpoint keys, shapes, and dtypes match; +- the runtime topology is TP1 without HSDP or online quantization; and +- every custom loader operation is represented by a loader-owned checkpoint + adapter. + +The exact plan object is handed to DLO. The backend does not rescan checkpoint +files, repeat the capability decision, or reconstruct names from its block +topology. If preflight fails, the loader materializes weights normally and DLO +consumes those runtime tensors. + +The plan owns only dedicated DiT component sources. If a pipeline also exposes +ordinary sources for a text encoder or another non-DiT component, the loader +still consumes those sources and includes their loaded names in its strict +coverage check. Only the source prefixes covered by the plan skip ordinary +materialization. A source that mixes DiT and non-DiT weights fails closed to +the complete ordinary-loader path because it cannot be skipped safely as a +unit. + +This boundary keeps checkpoint semantics out of DLO and avoids model-pipeline +flags such as `_supports_mmap_loading` or parameter attributes for mmap-only +transforms. Model-specific direct-layout knowledge, when required, lives in a +checkpoint adapter beside the ordinary loader. + ### AllGather path With the default `dlo_use_allgather=True`, each rank stores approximately @@ -61,6 +101,15 @@ Buffers: [current slot] [prefetch slot] [current slot] The backend uses two shared device buffers, so accelerator weight residency is bounded by the largest streamed blocks rather than the complete model. +When direct checkpoint mmap is selected, the checkpoint mappings are only the +source used to prepare each rank's persistent shard. They can be closed after +shard preparation. Across the AllGather group, those private shards total +approximately one runtime model copy. + +An effective DLO group size of one performs no collective, even when +`dlo_use_allgather=True`; it follows the rank-local transfer path described +below. + When DP is greater than one, the engine can process one request per DP rank in the same denoising wave. Because AllGather is a collective, all participating requests must take the same execution path at every denoising step. @@ -68,9 +117,20 @@ requests must take the same execution path at every denoising step. ### Rank-local path without DLO AllGather With `--dlo-no-use-allgather`, DLO forces its internal offload shard size to -one. The regular model loader remains responsible for preparing each rank's -weights, including TP-local tensors or HSDP-managed parameters. DLO then -streams those rank-local tensors block by block using H2D copies only. +one and streams complete blocks using H2D copies only. The host backing may be +either a loader-approved checkpoint mapping or ordinary runtime tensors. + +For direct mmap, each process retains immutable safetensors views and uses two +bounded pinned host staging slots. Processes on the same node that map the same +files share physical checkpoint pages through the OS page cache. This removes +the persistent private full-model copy per pure-DP process, but each process +still packs and transfers every complete block. Sharing is node-local; each +node has its own page cache. + +When direct mmap preflight fails, the regular model loader remains responsible +for preparing each rank's weights, including TP-local tensors or HSDP-managed +parameters. In that fallback, each pure-DP process keeps a private full runtime +copy. This mode means: @@ -80,21 +140,18 @@ This mode means: not shard weights across SP ranks. - TP/HSDP/SP collectives, if configured, are not disabled by this flag; only DLO's additional weight AllGather is disabled. -- Pure DP deployments keep a full host-side model copy per rank, subject to - shared OS page-cache behavior. +- Pure DP deployments share one checkpoint-backed copy per node when direct + mmap is selected; the ordinary-loader fallback keeps one private runtime + copy per rank. - The scheduler does not require a synchronized DP request wave for DLO. -This path is intentionally not implemented by reusing the AllGather mmap -loader. It relies on the standard loader so model-specific TP/HSDP transforms -remain intact. - ## Parallelism compatibility | Parallelism | DLO + AllGather | DLO without AllGather | |---|---|---| -| **DP** | Supported primary path. DLO shards host weights across the DP group and can run DP multi-concurrency. | Supported rank-local path. DP replicas remain independent; no DLO weight sharding or cross-DP DLO collective. | +| **DP** | Supported primary path. DLO shards host weights across the DP group and can run DP multi-concurrency. | Supported rank-local path. Compatible TP1 replicas can share checkpoint pages on each node; fallback runtime tensors remain private. | | **SP** | Supported in the implementation. With DP=1, DLO uses the SP group for host-weight sharding; SP still shards sequence/activation work. | SP remains active, but DLO keeps standard-loader rank-local weights and adds no SP weight collective. | -| **TP > 1** | Unsupported in the DLO mmap path. TP-aware loader callbacks are bypassed, so the backend rejects this configuration when it enters that path. | Standard loading is retained and TP-local tensors can be streamed. This is the intended compatibility path, but it still needs broader model and hardware validation. | +| **TP > 1** | Outside the Phase A shared-mmap support scope. The loader falls back before mutation, preserves TP-local layouts, and DLO may apply DP/SP host sharding to those ordinary runtime tensors. | Outside the Phase A shared-mmap support scope. The ordinary TP-aware loader produces rank-local tensors, which DLO streams without an additional weight collective; DP replicas retain private runtime storage. | | **HSDP** | Rejected. HSDP has already sharded parameters, so DLO AllGather would double-shard them. | Accepted by configuration. HSDP owns parameter sharding and its own gathers; DLO only stages rank-local parameters. End-to-end coverage is limited. | ### Combined dimensions @@ -122,11 +179,15 @@ AllGather DP multi-concurrency requires: The no-AllGather path does not impose these DLO-specific synchronized-wave requirements. -The mmap loader is used only by the supported DLO+AllGather path when the -model has a compatible checkpoint layout. Online quantization is incompatible -with that sharded mmap path; use `--dlo-no-use-allgather` or disable online -quantization. Models that do not meet the mmap requirements use the regular -loader path. +Direct checkpoint mmap can back either transfer path. It is currently limited +to proven TP1, non-HSDP, non-online-quantized layouts. Other layouts use the +ordinary loader. Online quantization remains incompatible with DLO AllGather; +use `--dlo-no-use-allgather` or disable online quantization. + +A normalized runtime mmap cache, built through the ordinary loader, is the +proposed general mechanism for sharing transformed TP or quantized layouts. +That cache and its publication/lifecycle protocol are intentionally outside +this phase; see [RFC #6195](https://github.com/vllm-project/vllm-omni/issues/6195). ## Validation coverage @@ -134,15 +195,75 @@ Current source-level validation includes: - HSDP + DLO + AllGather rejection; - HSDP + DLO without AllGather acceptance at configuration level; -- TP rejection in the DLO+AllGather mmap path; +- loader preflight fallback for TP, HSDP, online quantization, unknown custom + loaders, missing keys, and shape/dtype mismatches; +- exact loader-to-backend plan transfer and ordinary-loader fallback; +- rank-local mmap source retention, bounded two-slot staging, and adapter + transforms without parameter-side flags; - resident-layer requests requiring no-AllGather; - DP request-wave validation for denoising-step compatibility; - sharding, double-buffer, AllGather-size, and heterogeneous-block regression tests. -The highest-value missing coverage is end-to-end numerical comparison against -ordinary layerwise offload for DP+SP, TP+no-AllGather, and HSDP+SP+no-AllGather -on the target CUDA/NCCL or CANN/HCCL hardware. +### B300 parallel-topology smoke matrix + +A four-GPU B300 smoke test covered MiniMax-H3 FL2VA with the same prompt, seed, +CUDNN attention backend, 256x256 output, two denoising steps, and +`dlo_resident_layers=0`. The TP2 rows used DiT DP2xTP2 with the text encoder and +VAEs at TP1. They validate the ordinary-loader fallback only, not direct mmap +or shared-mmap host-memory savings. + +| Configuration | Result | Warm E2E | Peak device memory | Host PSS | +|---|---:|---:|---:|---:| +| DP4xTP1 AllGather | Passed, 4 concurrent requests | 2.87 s / 4 requests | 13.84 GiB | 211.99 GiB | +| DP4xTP1 no-AllGather | Passed, 1 request | 15.02 s | 13.23 GiB | 187.77 GiB | +| DP2xTP2 AllGather | Passed, 2 concurrent requests | 4.16 s / 2 requests | 12.50 GiB | 211.97 GiB | +| DP2xTP2 no-AllGather | Passed, 1 request | 3.51 s | 11.88 GiB | 314.01 GiB | + +Within each topology, the AllGather and no-AllGather video and audio outputs +were byte-identical. All four runs completed without an `ERROR` or traceback +and released their device allocations. For DP4xTP1, no-AllGather direct mmap +reduced total PSS by 24.22 GiB (11.4%) and `Private_Dirty` from 211.33 to +125.32 GiB (40.7%) relative to AllGather. For DP2xTP2, preflight selected the +ordinary loader as designed; no-AllGather PSS was 314.01 GiB, about 48% above +AllGather, because DP replicas did not share checkpoint-backed runtime +weights. This is a functional and memory smoke test, not a production-quality +performance or output-quality benchmark. + +### Host-memory measurement + +A two-worker MiniMax-H3 FL2VA measurement on one L20X node compared the +ordinary-loader fallback with direct mmap. Both runs used +DP=2, TP=1, no DLO AllGather, BF16 weights, two denoising steps, and a +256x256 four-second request. The ordinary-loader workers were sampled after +initialization. The mmap workers were sampled after one completed request, so +the checkpoint working set had been faulted into the page cache; this is the +more conservative point for mmap. + +The values below come from `/proc//smaps_rollup` and include the whole +worker, not only the DiT. The stable rank-to-rank difference comes from other +pipeline components, so each worker should be compared with the same worker in +the other storage mode. + +| Worker | Ordinary RSS | mmap RSS | Ordinary PSS | mmap PSS | PSS reduction | +|---|---:|---:|---:|---:|---:| +| DP worker 0 | 168.27 GiB | 132.76 GiB | 167.84 GiB | 101.43 GiB | 66.40 GiB | +| DP worker 1 | 116.19 GiB | 79.97 GiB | 115.73 GiB | 48.64 GiB | 67.09 GiB | +| **Two-worker total** | — | — | **283.56 GiB** | **150.08 GiB** | **133.48 GiB (47.1%)** | + +The direct-mmap workers each reported 62.45 GiB `Shared_Clean` but only +31.20 GiB `Pss_File`, which is the proportional charge expected when the same +resident checkpoint pages are mapped by two workers. `Private_Dirty` also fell +from 167.53 to 70.24 GiB for worker 0 and from 115.40 to 17.44 GiB for worker +1, a reduction of about 97–98 GiB per worker. RSS understates this benefit +because it counts a shared physical page in every process that maps it; summed +PSS is the appropriate node-memory comparison. + +The highest-value missing coverage is broader end-to-end numerical and +lifecycle comparison against ordinary layerwise offload for DP+SP, +HSDP+SP+no-AllGather, and TP greater than one across additional models and +target CUDA/NCCL or CANN/HCCL hardware. That broader TP coverage does not +change the Phase A direct-mmap TP1 support boundary. ## Recommendations @@ -150,7 +271,9 @@ on the target CUDA/NCCL or CANN/HCCL hardware. scaling path. - Use **SP + DLO AllGather** for long-sequence workloads when DP concurrency is not the goal. -- Use **no-AllGather** to bring up TP or workstation PCIe configurations, with - the expectation of higher host-memory use and lower validation confidence. +- Use **no-AllGather** when independent replica execution is required. TP1 + direct-mmap deployments can share checkpoint pages per node; other layouts + retain the ordinary loader's private host memory behavior and are outside + the Phase A shared-mmap support scope. - Prefer **HSDP alone** for production HSDP deployments until the combined HSDP + DLO no-AllGather path has broader end-to-end coverage. diff --git a/docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md b/docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md index 15298c897..6157c866b 100644 --- a/docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md +++ b/docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md @@ -3,7 +3,9 @@ Distributed layerwise offloading (DLO) extends block streaming to multi-device deployments. With AllGather enabled, each rank stores roughly `1 / dp_size` of the host weights and reconstructs each layer at runtime. Without AllGather, -each rank streams its standard-loader rank-local weights independently. +each rank streams a complete block independently. Compatible TP1 deployments +can share checkpoint-backed host pages among processes on the same node; +otherwise DLO streams the ordinary loader's rank-local tensors. See the [DLO feature design](../../../design/feature/offloader/distributed_layerwise_offload.md) for the implementation contract and compatibility matrix. @@ -59,21 +61,48 @@ omni = Omni( | `--enable-distributed-layerwise-offload` | Enable DLO | `false` | | `--data-parallel-size N` | DP ranks and AllGather weight-sharding group | `1` | | `--dlo-use-allgather` | Shard host weights and reconstruct with AllGather | `true` | -| `--dlo-no-use-allgather` | Keep standard-loader rank-local weights | `false` | +| `--dlo-no-use-allgather` | Stream complete rank-local blocks without a DLO weight collective | `false` | | `--dlo-resident-layers N` | Keep N leading main-DiT blocks on device; requires no-AllGather and model-declared resident paths | `0` | -## mmap weight loading +## Host-weight loading -The DLO plus AllGather path: +The diffusion loader chooses host storage before DLO is enabled. It first +attempts to build a complete, validated direct-checkpoint mmap plan. If names, +coverage, shape, dtype, topology, or loader-callback compatibility cannot be +proven, it runs the ordinary model loader instead. DLO consumes that result and +does not make a second checkpoint-compatibility decision. + +The shared-mmap optimization in this phase is supported only with TP1. TP +greater than one falls back before model mutation to ordinary TP-aware loading. +DLO may still consume those TP-local tensors, but this is a compatibility path: +it does not share checkpoint-backed runtime weights across DP replicas and +provides no shared-mmap host-memory guarantee. + +The mmap plan skips only dedicated DiT weight sources. Other component sources, +such as a text encoder loaded through the shared diffusion loader, continue to +use their ordinary component loader. A checkpoint source that mixes DiT and +non-DiT weights falls back completely rather than leaving an unplanned +component uninitialized. + +With direct checkpoint mmap, the loader: 1. saves non-persistent buffers such as RoPE frequencies; 2. moves the normally created transformer to the meta device; 3. loads checkpoint tensors as mmap views backed by the shared OS page cache; -4. calls model-specific `post_load_weights()` conversions; and -5. restores the saved non-persistent buffers. +4. applies any loader-owned bounded layout adapters while packing blocks; +5. restores saved non-persistent buffers; and +6. preserves `post_load_weights()` and `validate_loaded_weights()` lifecycle + hooks. -This avoids one full checkpoint RSS copy per rank and does not require -model-specific loading code. +For AllGather with a group larger than one, each process copies only its +persistent shard and then releases the source mapping. For no-AllGather, each +process keeps the mapping open and packs complete blocks through two bounded +pinned staging slots. Processes mapping the same files on one node share the +immutable pages; no-AllGather still performs a complete-block H2D copy in each +process. + +When the effective DLO group size is one, `dlo_use_allgather=True` does not +perform a collective and uses the same rank-local transfer behavior. ## Declarative topology @@ -103,16 +132,22 @@ must enter each collective. ## Limitations -- Online FP8 quantization is rejected with the DLO plus AllGather mmap path. - Use `--dlo-no-use-allgather` or disable online quantization. -- Tensor parallel size greater than one is rejected in the mmap path because - it bypasses TP-aware loader callbacks. The no-AllGather path retains the - standard loader and remains experimental with TP. +- Direct checkpoint mmap currently requires TP1. TP greater than one is + outside the Phase A shared-mmap support scope and falls back before model + mutation to the ordinary TP-aware loader. DLO can stream that runtime layout, + but it provides no shared-mmap host-memory benefit or guarantee. - HSDP plus AllGather is rejected to avoid double sharding. HSDP without AllGather has limited end-to-end validation. +- Online quantization uses the ordinary loader with no-AllGather. It remains + incompatible with DLO AllGather. - Resident leading layers require `--dlo-no-use-allgather` and a model `OffloadPlan` that declares eligible `resident_dit_paths`. - DP concurrency requires an explicit, identical inference-step count. +Sharing transformed TP or quantized runtime layouts through a normalized mmap +cache is a follow-up design in +[RFC #6195](https://github.com/vllm-project/vllm-omni/issues/6195), not part of +the direct-checkpoint path. + See the [Cosmos3 DistOffload recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md) for an end-to-end example. diff --git a/examples/offline_inference/minimax_h3/dlo_dp2_lifecycle.py b/examples/offline_inference/minimax_h3/dlo_lifecycle.py similarity index 72% rename from examples/offline_inference/minimax_h3/dlo_dp2_lifecycle.py rename to examples/offline_inference/minimax_h3/dlo_lifecycle.py index f8117b0f9..5630777cb 100644 --- a/examples/offline_inference/minimax_h3/dlo_dp2_lifecycle.py +++ b/examples/offline_inference/minimax_h3/dlo_lifecycle.py @@ -1,21 +1,32 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Reproduce MiniMax-H3 request-mode and DLO/DP2 lifecycle behavior. +"""Validate MiniMax-H3 request-mode and DLO lifecycle behavior. + +The DLO modes accept any positive data-parallel size. With AllGather enabled, +the script submits one complete DP wave and verifies recovery after an invalid +wave. It also checks that every mode shuts down all worker processes cleanly. Examples: VLLM_WORKER_MULTIPROC_METHOD=spawn \ VLLM_OMNI_VIDEO_SYNC_TIMEOUT=14400 \ VLLM_OMNI_DLO_DP_WAVE_TIMEOUT=600 \ + CUDA_VISIBLE_DEVICES=0,1,2,3 \ + python examples/offline_inference/minimax_h3/dlo_lifecycle.py \ + --model /path/to/MiniMax-H3/FL2VA --mode dlo --dp-size 4 + + # Rank-local mmap storage, with no DLO collective or DP wave scheduling. + VLLM_WORKER_MULTIPROC_METHOD=spawn \ + VLLM_OMNI_VIDEO_SYNC_TIMEOUT=14400 \ CUDA_VISIBLE_DEVICES=0,1 \ - python examples/offline_inference/minimax_h3/dlo_dp2_lifecycle.py \ - --model /path/to/MiniMax-H3/FL2VA --mode dlo-dp2 + python examples/offline_inference/minimax_h3/dlo_lifecycle.py \ + --model /path/to/MiniMax-H3/FL2VA --mode dlo-no-allgather --dp-size 2 VLLM_WORKER_MULTIPROC_METHOD=spawn \ VLLM_OMNI_VIDEO_SYNC_TIMEOUT=14400 \ CUDA_VISIBLE_DEVICES=0,1 \ - python examples/offline_inference/minimax_h3/dlo_dp2_lifecycle.py \ - --model /path/to/MiniMax-H3/FL2VA --mode request + python examples/offline_inference/minimax_h3/dlo_lifecycle.py \ + --model /path/to/MiniMax-H3/FL2VA --mode request --tp-size 2 """ from __future__ import annotations @@ -40,14 +51,33 @@ DEFAULT_PROMPTS = ( ) +def positive_int(value: str) -> int: + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("must be a positive integer") + return parsed + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--model", required=True, help="Path to MiniMax-H3/FL2VA") parser.add_argument( "--mode", - choices=("dlo-dp2", "request"), + choices=("dlo", "dlo-no-allgather", "request"), required=True, - help="DLO AllGather DP2 or ordinary non-DLO TP2 request mode", + help="DLO (with or without AllGather) or ordinary non-DLO request mode", + ) + parser.add_argument( + "--dp-size", + type=positive_int, + default=2, + help="DLO data-parallel size (for example 1, 2, 4, or 8; default: 2)", + ) + parser.add_argument( + "--tp-size", + type=positive_int, + default=2, + help="Tensor-parallel size used only by request mode (default: 2)", ) parser.add_argument("--steps", type=int, default=2) parser.add_argument("--duration", type=float, default=5.0) @@ -60,10 +90,11 @@ def parse_args() -> argparse.Namespace: def engine_kwargs(args: argparse.Namespace) -> dict[str, Any]: + is_dlo = args.mode in {"dlo", "dlo-no-allgather"} common: dict[str, Any] = { "model": args.model, "trust_remote_code": True, - "num_gpus": 2, + "num_gpus": args.dp_size if is_dlo else args.tp_size, "usp": 1, "ring": 1, "vae_parallel_mode": "tile", @@ -74,22 +105,22 @@ def engine_kwargs(args: argparse.Namespace) -> dict[str, Any]: "stage_init_timeout": args.init_timeout, "init_timeout": args.init_timeout, } - if args.mode == "dlo-dp2": + if is_dlo: common.update( tensor_parallel_size=1, - data_parallel_size=2, + data_parallel_size=args.dp_size, text_encoder_tp_size=1, vae_patch_parallel_size=1, enable_distributed_layerwise_offload=True, - dlo_use_allgather=True, + dlo_use_allgather=args.mode == "dlo", dlo_resident_layers=0, ) else: common.update( - tensor_parallel_size=2, + tensor_parallel_size=args.tp_size, data_parallel_size=1, - text_encoder_tp_size=2, - vae_patch_parallel_size=2, + text_encoder_tp_size=args.tp_size, + vae_patch_parallel_size=args.tp_size, enable_distributed_layerwise_offload=False, ) return common @@ -172,7 +203,8 @@ async def run(args: argparse.Namespace) -> dict[str, Any]: "engine_kwargs": engine_kwargs(args), } try: - if args.mode == "dlo-dp2": + uses_dp_wave = args.mode == "dlo" and args.dp_size > 1 + if uses_dp_wave: started = time.perf_counter() asymmetric = await asyncio.gather( generate_one( @@ -182,12 +214,15 @@ async def run(args: argparse.Namespace) -> dict[str, Any]: prompt="", seed=1001, ), - generate_one( - engine, - args, - request_id="invalid-peer", - prompt=DEFAULT_PROMPTS[0], - seed=1002, + *( + generate_one( + engine, + args, + request_id=f"invalid-peer-{index}", + prompt=DEFAULT_PROMPTS[index % len(DEFAULT_PROMPTS)], + seed=1001 + index, + ) + for index in range(1, args.dp_size) ), return_exceptions=True, ) @@ -195,18 +230,18 @@ async def run(args: argparse.Namespace) -> dict[str, Any]: summary["asymmetric_errors"] = [ f"{type(result).__name__}: {result}" for result in asymmetric if isinstance(result, BaseException) ] - if len(summary["asymmetric_errors"]) != 2: - raise RuntimeError("Both requests in the asymmetric DP wave must fail before dispatch") + if len(summary["asymmetric_errors"]) != args.dp_size: + raise RuntimeError("Every request in the asymmetric DP wave must fail before dispatch") started = time.perf_counter() - request_count = 2 if args.mode == "dlo-dp2" else 1 + request_count = args.dp_size if uses_dp_wave else 1 outputs = await asyncio.gather( *( generate_one( engine, args, request_id=f"recovery-{index}", - prompt=DEFAULT_PROMPTS[index], + prompt=DEFAULT_PROMPTS[index % len(DEFAULT_PROMPTS)], seed=2000 + index, ) for index in range(request_count) diff --git a/tests/diffusion/model_loader/test_diffusers_loader.py b/tests/diffusion/model_loader/test_diffusers_loader.py index 69600d811..aa7651bb6 100644 --- a/tests/diffusion/model_loader/test_diffusers_loader.py +++ b/tests/diffusion/model_loader/test_diffusers_loader.py @@ -16,6 +16,11 @@ from vllm.config.load import LoadConfig from vllm_omni.diffusion.config import get_current_diffusion_config, get_current_diffusion_config_or_none from vllm_omni.diffusion.data import OmniDiffusionConfig from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader +from vllm_omni.diffusion.model_loader.host_weight_plan import ( + HostWeightPlan, + HostWeightPlanResult, + TensorBinding, +) from vllm_omni.diffusion.models.helios import HeliosPipeline from vllm_omni.diffusion.registry import initialize_model @@ -258,6 +263,137 @@ def test_load_model_custom_pipeline_sets_current_diffusion_config(monkeypatch): assert get_current_diffusion_config_or_none() is None +def test_dlo_transfers_loader_plan_and_skips_ordinary_weight_loading(monkeypatch): + import vllm_omni.diffusion.model_loader.diffusers_loader as loader_mod + + od_config = SimpleNamespace( + dtype=torch.float32, + parallel_config=SimpleNamespace(use_hsdp=False, tensor_parallel_size=1), + quantization_config=None, + enable_distributed_layerwise_offload=True, + dlo_use_allgather=False, + model="unused", + ) + loader = DiffusersPipelineLoader(LoadConfig(), od_config) + model = nn.Module() + model.transformer = nn.Linear(2, 2, bias=False) + plan = HostWeightPlan( + backing_kind="checkpoint_mmap", + bindings={}, + ) + loaded_ordinary_weights = False + + def load_weights(_model): + nonlocal loaded_ordinary_weights + loaded_ordinary_weights = True + + loader._init_from_load_format = lambda *_args, **_kwargs: model # type: ignore[method-assign] + loader.load_weights = load_weights # type: ignore[method-assign] + loader._apply_skip_softmax_calibration = lambda _model: None # type: ignore[method-assign] + monkeypatch.setattr( + loader_mod, + "build_checkpoint_mmap_plan", + lambda *_args, **_kwargs: HostWeightPlanResult(plan), + ) + + assert loader.load_model(load_device="cpu") is model + assert not loaded_ordinary_weights + assert loader.take_host_weight_plan() is plan + assert loader.take_host_weight_plan() is None + + +def test_dlo_plan_loads_component_sources_outside_planned_dit(monkeypatch): + import vllm_omni.diffusion.model_loader.diffusers_loader as loader_mod + + od_config = SimpleNamespace( + dtype=torch.float32, + parallel_config=SimpleNamespace(use_hsdp=False, tensor_parallel_size=1), + quantization_config=None, + enable_distributed_layerwise_offload=True, + dlo_use_allgather=False, + model="unused", + ) + loader = DiffusersPipelineLoader(LoadConfig(), od_config) + + class MixedSourceModel(nn.Module): + def __init__(self): + super().__init__() + self.transformer = nn.Linear(2, 2, bias=False) + self.text_encoder = nn.Linear(2, 2, bias=False) + self.weights_sources = ( + DiffusersPipelineLoader.ComponentSource("unused", None, None, prefix="transformer."), + DiffusersPipelineLoader.ComponentSource("unused", None, None, prefix="text_encoder."), + ) + self.loaded_weight_names: list[str] = [] + + def load_weights(self, weights): + self.loaded_weight_names = [name for name, _ in weights] + return set(self.loaded_weight_names) + + model = MixedSourceModel() + plan = HostWeightPlan( + backing_kind="checkpoint_mmap", + bindings={ + "transformer.weight": TensorBinding( + checkpoint_key="weight", + file_path="unused", + ) + }, + planned_source_prefixes=frozenset({"transformer."}), + ) + requested_prefixes: list[str] = [] + + def get_weights(source, model=None): + del model + requested_prefixes.append(source.prefix) + yield source.prefix + "weight", torch.ones(2, 2) + + loader._init_from_load_format = lambda *_args, **_kwargs: model # type: ignore[method-assign] + loader._get_weights_iterator = get_weights # type: ignore[method-assign] + loader._apply_skip_softmax_calibration = lambda _model: None # type: ignore[method-assign] + monkeypatch.setattr( + loader_mod, + "build_checkpoint_mmap_plan", + lambda *_args, **_kwargs: HostWeightPlanResult(plan), + ) + + assert loader.load_model(load_device="cpu") is model + assert requested_prefixes == ["text_encoder."] + assert model.loaded_weight_names == ["text_encoder.weight"] + assert loader.take_host_weight_plan() is plan + + +def test_dlo_plan_fallback_runs_ordinary_loader(monkeypatch): + import vllm_omni.diffusion.model_loader.diffusers_loader as loader_mod + + od_config = SimpleNamespace( + dtype=torch.float32, + parallel_config=SimpleNamespace(use_hsdp=False, tensor_parallel_size=1), + quantization_config=None, + enable_distributed_layerwise_offload=True, + dlo_use_allgather=False, + model="unused", + ) + loader = DiffusersPipelineLoader(LoadConfig(), od_config) + model = nn.Module() + model.transformer = nn.Linear(2, 2, bias=False) + calls: list[str] = [] + + loader._init_from_load_format = lambda *_args, **_kwargs: model # type: ignore[method-assign] + loader.load_weights = lambda _model: calls.append("load") # type: ignore[method-assign] + loader._process_weights_after_loading = lambda *_args: calls.append("process") # type: ignore[method-assign] + loader._apply_skip_softmax_calibration = lambda _model: None # type: ignore[method-assign] + monkeypatch.setattr( + loader_mod, + "build_checkpoint_mmap_plan", + lambda *_args, **_kwargs: HostWeightPlanResult(None, "not direct-compatible"), + ) + + assert loader.load_model(load_device="cpu") is model + assert calls == ["load", "process"] + assert loader.take_host_weight_plan() is None + + def test_hsdp_processes_quantized_weights_before_sharding(mocker): import vllm_omni.diffusion.model_loader.diffusers_loader as loader_mod from vllm_omni.diffusion.offloader.module_collector import PipelineModules diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py index 5f220d6ab..cc14d92e3 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py @@ -191,6 +191,49 @@ def test_model_load_weights_transforms_before_calling_vllm_loader(): ] +def test_loader_adapter_declares_equivalent_direct_mmap_transform(monkeypatch): + from vllm_omni.diffusion.model_loader.checkpoint_adapters import ( + get_direct_mmap_adapter, + ) + from vllm_omni.diffusion.models.minimax_h3 import minimax_h3_transformer as h3 + + monkeypatch.setattr(h3, "QKVParallelLinear", _FakeLinear) + monkeypatch.setattr(h3, "RowParallelLinear", _FakeLinear) + monkeypatch.setattr(h3, "Attention", _FakeAttention) + + arch = h3.MiniMaxH3DiTArchConfig( + hidden_size=1, + num_attention_heads=2, + attention_head_dim=1, + rope_inv_freq_len=1, + ) + attention = h3.MiniMaxH3Attention( + arch, + quant_config=None, + prefix="blocks.0.attn", + ) + transformer = object.__new__(h3.MiniMaxH3DiTModel) + nn.Module.__init__(transformer) + transformer.blocks = nn.ModuleList([nn.Module()]) + transformer.blocks[0].attn = attention + pipeline = nn.Module() + pipeline.transformer = transformer + + adapter = get_direct_mmap_adapter(pipeline) + assert adapter is not None + policy = adapter.policy_for( + "transformer.blocks.0.attn.qkv_proj.weight", + attention.qkv_proj.weight, + ) + assert policy is not None + assert policy.allow_custom_loader + assert policy.transform is not None + checkpoint_weight = torch.arange(6, dtype=torch.float32).reshape(6, 1) + + assert policy.transform(checkpoint_weight)[:, 0].tolist() == [0, 3, 1, 4, 2, 5] + assert not hasattr(attention.qkv_proj.weight, "mmap_weight_transform") + + def test_pipeline_resolves_transformer_component_quant_config(): from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import ( _resolve_component_quant_config, diff --git a/tests/diffusion/offloader/test_distributed_layerwise_backend.py b/tests/diffusion/offloader/test_distributed_layerwise_backend.py index 4c0cceb68..6c8248391 100644 --- a/tests/diffusion/offloader/test_distributed_layerwise_backend.py +++ b/tests/diffusion/offloader/test_distributed_layerwise_backend.py @@ -18,6 +18,11 @@ from torch import nn from torch.distributed.tensor import DeviceMesh, DTensor, Replicate import vllm_omni.diffusion.offloader.distributed_layerwise_backend as dist_backend_module +from vllm_omni.diffusion.model_loader.host_weight_plan import ( + HostWeightPlan, + TensorBinding, + build_checkpoint_mmap_plan, +) from vllm_omni.diffusion.offloader.base import OffloadConfig, OffloadStrategy from vllm_omni.diffusion.offloader.block_discovery import ( get_blocks_attr_names, @@ -30,7 +35,10 @@ from vllm_omni.diffusion.offloader.distributed_layerwise_backend import ( PinnedResidentLayerGroup, ) from vllm_omni.diffusion.offloader.module_residency import PinnedModuleStager -from vllm_omni.diffusion.offloader.offload_plan import OffloadPlan, get_offload_plan +from vllm_omni.diffusion.offloader.offload_plan import ( + OffloadPlan, + get_offload_plan, +) from vllm_omni.platforms import current_omni_platform pytestmark = [pytest.mark.diffusion, pytest.mark.cpu, pytest.mark.core_model] @@ -48,6 +56,9 @@ class DummyEvent: def record(self, _stream) -> None: return None + def synchronize(self) -> None: + return None + @contextmanager def dummy_stream(_stream): @@ -351,8 +362,9 @@ class TestDistributedLayerwiseOffloadHook: block.weight = nn.Parameter(torch.zeros(4, dtype=torch.float32)) next_block = nn.Module() next_block.weight = nn.Parameter(torch.arange(4, dtype=torch.float32)) - next_block.weight.mmap_weight_transform = lambda tensor: tensor.flip(0) - next_block.weight.mmap_weight_transform_pending = True + + def transform(tensor): + return tensor.flip(0) hook = DistributedLayerwiseOffloadHook( next_block=next_block, @@ -361,17 +373,17 @@ class TestDistributedLayerwiseOffloadHook: dp_size=2, rank=0, pin_memory=False, + tensor_transforms={id(next_block.weight): transform}, ) hook.initialize_hook(block) assert torch.equal(hook.cpu_shards[torch.float32], torch.tensor([3.0, 2.0])) - def test_regular_loader_weight_does_not_reapply_mmap_transform(self, dist_group, patched_offload_runtime): + def test_transform_is_not_applied_without_a_loader_plan(self, dist_group, patched_offload_runtime): block = nn.Module() block.weight = nn.Parameter(torch.zeros(4, dtype=torch.float32)) next_block = nn.Module() next_block.weight = nn.Parameter(torch.arange(4, dtype=torch.float32)) - next_block.weight.mmap_weight_transform = lambda tensor: tensor.flip(0) hook = DistributedLayerwiseOffloadHook( next_block=next_block, @@ -385,6 +397,82 @@ class TestDistributedLayerwiseOffloadHook: assert torch.equal(hook.cpu_shards[torch.float32], torch.tensor([0.0, 1.0])) + def test_rank_local_mmap_retains_source_and_uses_staging(self, dist_group, patched_offload_runtime): + current_block = nn.Linear(2, 2, bias=False) + next_block = nn.Linear(2, 2, bias=False) + next_block.weight.data.copy_(torch.arange(4, dtype=torch.float32).view(2, 2)) + source_storage = next_block.weight.untyped_storage().data_ptr() + + def transform(tensor): + return tensor.t() + + hook = DistributedLayerwiseOffloadHook( + next_block=next_block, + device=torch.device("cpu"), + dp_group=None, + dp_size=1, + rank=0, + pin_memory=False, + rank_local_mmap=True, + tensor_transforms={id(next_block.weight): transform}, + ) + hook.initialize_hook(current_block) + + source = hook.cpu_sources[torch.float32][0]["tensor"] + assert source.untyped_storage().data_ptr() == source_storage + assert hook.cpu_shards == {} + assert next_block.weight.numel() == 0 + + hook.cpu_staging_buffers = [ + {torch.float32: torch.empty(4)}, + {torch.float32: torch.empty(4)}, + ] + hook.prefetch_layer(slot=0, non_blocking=False) + + assert torch.equal( + next_block.weight, + torch.tensor([[0.0, 2.0], [1.0, 3.0]]), + ) + assert next_block.weight.stride() == (1, 2) + assert torch.equal(source, torch.arange(4, dtype=torch.float32).view(2, 2)) + + def test_rank_local_mmap_staging_is_bounded_by_largest_block(self, patched_offload_runtime): + hooks = [] + for size in (4, 9): + current_block = nn.Linear(1, 1, bias=False) + next_block = nn.Module() + next_block.weight = nn.Parameter(torch.arange(size, dtype=torch.float32)) + hook = DistributedLayerwiseOffloadHook( + next_block=next_block, + device=torch.device("cpu"), + dp_group=None, + dp_size=1, + rank=0, + pin_memory=False, + shared_buffers=[None, None], + rank_local_mmap=True, + ) + hook.initialize_hook(current_block) + hooks.append(hook) + + resident_block = nn.Module() + resident_block.weight = nn.Parameter(torch.arange(12, dtype=torch.float32)) + resident_group = PinnedResidentLayerGroup( + [resident_block], + device=torch.device("cpu"), + copy_stream=DummyStream(), + pin_memory=False, + rank_local_mmap=True, + ) + staging = DistributedLayerwiseOffloadBackend._allocate_shared_cpu_staging_buffers( + hooks, + resident_group, + ) + + assert len(staging) == 2 + assert staging[0][torch.float32].numel() == 12 + assert staging[1][torch.float32].numel() == 12 + class TestPinnedResidentLayerGroup: def test_load_offload_reuses_pinned_master_weights(self, patched_offload_runtime): @@ -439,23 +527,41 @@ class TestPinnedResidentLayerGroup: assert block.weight.stride() == expected_stride group.offload() - def test_mmap_loader_attrs_survive_to_empty_parameter_replacement(self): - module = nn.Linear(2, 2, bias=False) + def test_rank_local_mmap_resident_layers_retain_sources(self, patched_offload_runtime): + blocks = [nn.Linear(2, 2, bias=False) for _ in range(3)] + expected = [] + source_storage = [] + for index, block in enumerate(blocks): + block.weight.data.fill_(index + 1) + expected.append(block.weight.detach().clone()) + source_storage.append(block.weight.untyped_storage().data_ptr()) - def transform(tensor): - return tensor.flip(0) + group = PinnedResidentLayerGroup( + blocks, + device=torch.device("cpu"), + copy_stream=DummyStream(), + pin_memory=False, + rank_local_mmap=True, + ) - module.weight.mmap_weight_transform = transform + for index, state in enumerate(group._states): + source = state["cpu_sources"][torch.float32][0]["tensor"] + assert source.untyped_storage().data_ptr() == source_storage[index] + assert state["cpu_shards"] == {} + assert len(group._cpu_staging_buffers) == 2 - backend = object.__new__(DistributedLayerwiseOffloadBackend) - backend._remember_mmap_param_attrs(module) - module.to_empty(device="meta") + group.load() + for block, weight in zip(blocks, expected): + assert torch.equal(block.weight, weight) - replacement = nn.Parameter(torch.arange(4).reshape(2, 2).float()) - backend._attach_mmap_param_attrs("weight", replacement, module.weight) + blocks[0].weight.data.zero_() + group.offload() + group.load() + for block, weight in zip(blocks, expected): + assert torch.equal(block.weight, weight) - assert replacement.mmap_weight_transform is transform - assert replacement.mmap_weight_transform_pending is True + group.offload() + assert all(block.weight.numel() == 0 for block in blocks) def test_shared_allgather_output_is_narrowed_to_current_block( self, dist_group, patched_offload_runtime, monkeypatch @@ -598,7 +704,6 @@ class TestMmapWeightLoading: OffloadConfig( strategy=OffloadStrategy.DISTRIBUTED_LAYER_WISE, pin_cpu_memory=False, - model_path=str(tmp_path), ), torch.device("cpu"), ) @@ -606,13 +711,81 @@ class TestMmapWeightLoading: dits=[pipeline.transformer], dit_names=["transformer"], ) + plan = HostWeightPlan( + backing_kind="checkpoint_mmap", + bindings={ + name: TensorBinding( + checkpoint_key=name, + file_path=str(weight_file), + ) + for name in weights + }, + ) - backend._load_weights_via_mmap(pipeline, modules) + backend._load_weights_via_mmap(pipeline, modules, plan) assert pipeline.transformer.post_load_calls == 1 assert pipeline.transformer.time_embedder.weight.dtype == torch.float32 assert pipeline.transformer.blocks[0].weight.dtype == torch.bfloat16 + def test_allgather_flag_with_group_size_one_uses_rank_local_mmap( + self, + tmp_path, + patched_offload_runtime, + ): + class Transformer(nn.Module): + _layerwise_offload_blocks_attrs = ["blocks"] + + def __init__(self): + super().__init__() + self.blocks = nn.ModuleList( + [ + nn.Linear(2, 2, bias=False), + nn.Linear(2, 2, bias=False), + ] + ) + + class Pipeline(nn.Module): + def __init__(self): + super().__init__() + self.transformer = Transformer() + + @staticmethod + def _remap_ckpt_key(key): + return key + + pipeline = Pipeline() + weights = {name: torch.ones_like(param) for name, param in pipeline.named_parameters()} + checkpoint_file = tmp_path / "model.safetensors" + save_file(weights, str(checkpoint_file)) + result = build_checkpoint_mmap_plan( + pipeline, + dit_modules=(("transformer", pipeline.transformer),), + sources=(), + model_path=str(tmp_path), + tensor_parallel_size=1, + use_hsdp=False, + online_quantization=False, + ) + assert result.plan is not None + backend = DistributedLayerwiseOffloadBackend( + OffloadConfig( + strategy=OffloadStrategy.DISTRIBUTED_LAYER_WISE, + pin_cpu_memory=False, + dp_size=1, + dlo_use_allgather=True, + ), + torch.device("cpu"), + host_weight_plan=result.plan, + ) + + backend.enable(pipeline) + + assert backend._using_rank_local_mmap + assert backend.dp_group is None + assert all(hook.rank_local_mmap for group in backend._all_hook_groups for hook in group) + backend.disable() + class TestGetBlocksFromDit: def test_get_blocks_from_dit_single_block_attr(self): @@ -878,23 +1051,123 @@ class TestOffloadPlan: class TestMmapValidation: - """Tests for mmap loader validation: TP rejection, strict check, validate_loaded_weights.""" + """Tests for loader preflight and backend plan realization.""" - def test_tp_rejected_when_tp_world_size_gt_1(self, tmp_path, patched_offload_runtime, monkeypatch): - """DLO+AllGather should reject when TP world size > 1.""" - import json - from types import SimpleNamespace - - from safetensors.torch import save_file - - from vllm_omni.diffusion.offloader.base import OffloadConfig, OffloadStrategy - - # Mock TP world size = 2 - monkeypatch.setattr( - "vllm.distributed.parallel_state.get_tensor_model_parallel_world_size", - lambda: 2, + def test_component_source_prefix_builds_model_namespace(self, tmp_path): + source_root = tmp_path / "partition" + checkpoint_dir = source_root / "transformer" + checkpoint_dir.mkdir(parents=True) + checkpoint_file = checkpoint_dir / "model.safetensors" + save_file( + {"blocks.0.weight": torch.arange(4, dtype=torch.float32).reshape(1, 4)}, + str(checkpoint_file), ) + class Pipeline(nn.Module): + def __init__(self): + super().__init__() + self.transformer = nn.Module() + self.transformer.blocks = nn.ModuleList([nn.Linear(4, 1, bias=False)]) + self.weights_sources = [ + SimpleNamespace( + model_or_path=str(source_root), + subfolder="transformer", + revision=None, + prefix="transformer.", + ) + ] + + pipeline = Pipeline() + result = build_checkpoint_mmap_plan( + pipeline, + dit_modules=(("transformer", pipeline.transformer),), + sources=tuple(pipeline.weights_sources), + model_path=None, + tensor_parallel_size=1, + use_hsdp=False, + online_quantization=False, + ) + + assert result.fallback_reason is None + assert result.plan is not None + assert result.plan.bindings["transformer.blocks.0.weight"] == TensorBinding( + checkpoint_key="blocks.0.weight", + file_path=str(checkpoint_file), + ) + assert result.plan.planned_source_prefixes == frozenset({"transformer."}) + + def test_non_dedicated_component_source_falls_back_before_discovery(self): + pipeline = nn.Module() + pipeline.transformer = nn.Linear(2, 2, bias=False) + sources = ( + SimpleNamespace( + model_or_path="unused", + subfolder=None, + revision=None, + prefix="", + ), + ) + + result = build_checkpoint_mmap_plan( + pipeline, + dit_modules=(("transformer", pipeline.transformer),), + sources=sources, + model_path=None, + tensor_parallel_size=1, + use_hsdp=False, + online_quantization=False, + ) + + assert result.plan is None + assert "dedicated component weight source" in result.fallback_reason + + def test_preflight_falls_back_before_mutation_for_missing_model_tensors(self, tmp_path): + source_root = tmp_path / "partition" + checkpoint_dir = source_root / "transformer" + checkpoint_dir.mkdir(parents=True) + save_file( + {"blocks.0.weight": torch.ones((2, 2), dtype=torch.float32)}, + str(checkpoint_dir / "model.safetensors"), + ) + + class Transformer(nn.Module): + _layerwise_offload_blocks_attrs = ["blocks"] + + def __init__(self): + super().__init__() + self.blocks = nn.ModuleList([nn.Linear(2, 2, bias=False), nn.Linear(2, 2, bias=False)]) + + class Pipeline(nn.Module): + def __init__(self): + super().__init__() + self.transformer = Transformer() + self.weights_sources = [ + SimpleNamespace( + model_or_path=str(source_root), + subfolder="transformer", + revision=None, + prefix="transformer.", + ) + ] + + pipeline = Pipeline() + original_weight = pipeline.transformer.blocks[0].weight + result = build_checkpoint_mmap_plan( + pipeline, + dit_modules=(("transformer", pipeline.transformer),), + sources=tuple(pipeline.weights_sources), + model_path=None, + tensor_parallel_size=1, + use_hsdp=False, + online_quantization=False, + ) + + assert result.plan is None + assert "1 required DiT tensors" in result.fallback_reason + assert pipeline.transformer.blocks[0].weight is original_weight + assert not original_weight.is_meta + + def test_tp_falls_back_before_checkpoint_discovery(self): class SimpleModel(nn.Module): _layerwise_offload_blocks_attrs = ["blocks"] @@ -912,41 +1185,118 @@ class TestMmapValidation: return key pipeline = SimplePipeline() - weights = {name: torch.ones(p.shape, dtype=p.dtype) for name, p in pipeline.named_parameters() if not p.is_meta} - save_file(weights, str(tmp_path / "model.safetensors")) - (tmp_path / "model.safetensors.index.json").write_text( - json.dumps({"weight_map": {k: "model.safetensors" for k in weights}}) + result = build_checkpoint_mmap_plan( + pipeline, + dit_modules=(("transformer", pipeline.transformer),), + sources=(), + model_path="unused", + tensor_parallel_size=2, + use_hsdp=False, + online_quantization=False, ) - backend = DistributedLayerwiseOffloadBackend( - OffloadConfig( - strategy=OffloadStrategy.DISTRIBUTED_LAYER_WISE, - pin_cpu_memory=False, - model_path=str(tmp_path), - ), - torch.device("cpu"), - ) - modules = SimpleNamespace(dits=[pipeline.transformer], dit_names=["transformer"]) + assert result.plan is None + assert result.fallback_reason == "TP=2 requires the ordinary loader" - with pytest.raises(ValueError, match="Tensor Parallel"): - backend._load_weights_via_mmap(pipeline, modules) + @pytest.mark.parametrize( + ("use_hsdp", "online_quantization", "expected_reason"), + [ + (True, False, "HSDP requires the ordinary loader"), + (False, True, "online quantization requires the ordinary loader"), + ], + ) + def test_runtime_layouts_that_need_loader_callbacks_fall_back( + self, + use_hsdp, + online_quantization, + expected_reason, + ): + pipeline = nn.Module() + pipeline.transformer = nn.Linear(2, 2, bias=False) - def test_tp_allowed_when_tp_world_size_eq_1(self, tmp_path, patched_offload_runtime, monkeypatch): - """DLO+AllGather should work when TP world size = 1, even if params - have custom weight_loader attributes (e.g. QKVParallelLinear at TP=1).""" - import json - from types import SimpleNamespace - - from safetensors.torch import save_file - - from vllm_omni.diffusion.offloader.base import OffloadConfig, OffloadStrategy - - # Mock TP world size = 1 (default) - monkeypatch.setattr( - "vllm.distributed.parallel_state.get_tensor_model_parallel_world_size", - lambda: 1, + result = build_checkpoint_mmap_plan( + pipeline, + dit_modules=(("transformer", pipeline.transformer),), + sources=(), + model_path="unused", + tensor_parallel_size=1, + use_hsdp=use_hsdp, + online_quantization=online_quantization, ) + assert result.plan is None + assert result.fallback_reason == expected_reason + + @pytest.mark.parametrize( + ("checkpoint_tensor", "expected_reason"), + [ + (torch.ones((3, 2), dtype=torch.float32), "shape mismatch"), + (torch.ones((2, 2), dtype=torch.bfloat16), "dtype mismatch"), + ], + ) + def test_source_metadata_mismatch_falls_back(self, tmp_path, checkpoint_tensor, expected_reason): + checkpoint_file = tmp_path / "model.safetensors" + save_file({"transformer.weight": checkpoint_tensor}, str(checkpoint_file)) + + class Pipeline(nn.Module): + def __init__(self): + super().__init__() + self.transformer = nn.Linear(2, 2, bias=False) + + @staticmethod + def _remap_ckpt_key(key): + return key + + pipeline = Pipeline() + + result = build_checkpoint_mmap_plan( + pipeline, + dit_modules=(("transformer", pipeline.transformer),), + sources=(), + model_path=str(tmp_path), + tensor_parallel_size=1, + use_hsdp=False, + online_quantization=False, + ) + + assert result.plan is None + assert expected_reason in result.fallback_reason + + def test_non_persistent_buffers_do_not_need_checkpoint_bindings(self, tmp_path): + class Transformer(nn.Module): + def __init__(self): + super().__init__() + self.weight = nn.Parameter(torch.ones(2, 2)) + self.register_buffer("derived", torch.ones(2), persistent=False) + + checkpoint_file = tmp_path / "model.safetensors" + save_file({"transformer.weight": torch.ones(2, 2)}, str(checkpoint_file)) + + class Pipeline(nn.Module): + def __init__(self): + super().__init__() + self.transformer = Transformer() + + @staticmethod + def _remap_ckpt_key(key): + return key + + pipeline = Pipeline() + + result = build_checkpoint_mmap_plan( + pipeline, + dit_modules=(("transformer", pipeline.transformer),), + sources=(), + model_path=str(tmp_path), + tensor_parallel_size=1, + use_hsdp=False, + online_quantization=False, + ) + + assert result.plan is not None + assert set(result.plan.bindings) == {"transformer.weight"} + + def test_unadapted_custom_weight_loader_falls_back_at_tp1(self, tmp_path): def custom_weight_loader(param, weight): param.data.copy_(weight) @@ -976,27 +1326,67 @@ class TestMmapValidation: json.dumps({"weight_map": {k: "model.safetensors" for k in weights}}) ) - backend = DistributedLayerwiseOffloadBackend( - OffloadConfig( - strategy=OffloadStrategy.DISTRIBUTED_LAYER_WISE, - pin_cpu_memory=False, - model_path=str(tmp_path), - ), - torch.device("cpu"), + result = build_checkpoint_mmap_plan( + pipeline, + dit_modules=(("transformer", pipeline.transformer),), + sources=(), + model_path=str(tmp_path), + tensor_parallel_size=1, + use_hsdp=False, + online_quantization=False, ) - modules = SimpleNamespace(dits=[pipeline.transformer], dit_names=["transformer"]) - # Should NOT raise — TP=1 is allowed even with custom weight_loader - backend._load_weights_via_mmap(pipeline, modules) + assert result.plan is None + assert "custom weight_loader" in result.fallback_reason + + def test_cosmos3_adapter_preserves_existing_tp1_direct_mmap_contract(self, tmp_path): + from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import ( + Cosmos3OmniDiffusersPipeline, + ) + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import ( + Cosmos3VFMTransformer, + ) + + transformer = object.__new__(Cosmos3VFMTransformer) + nn.Module.__init__(transformer) + transformer.proj_in = nn.Linear(2, 2, bias=False) + transformer.proj_in.weight.weight_loader = lambda *_args: None + pipeline = object.__new__(Cosmos3OmniDiffusersPipeline) + nn.Module.__init__(pipeline) + pipeline.transformer = transformer + + source_root = tmp_path / "partition" + checkpoint_dir = source_root / "transformer" + checkpoint_dir.mkdir(parents=True) + checkpoint_file = checkpoint_dir / "model.safetensors" + save_file({"proj_in.weight": torch.ones(2, 2)}, str(checkpoint_file)) + sources = ( + SimpleNamespace( + model_or_path=str(source_root), + subfolder="transformer", + revision=None, + prefix="transformer.", + ), + ) + + result = build_checkpoint_mmap_plan( + pipeline, + dit_modules=(("transformer", transformer),), + sources=sources, + model_path=None, + tensor_parallel_size=1, + use_hsdp=False, + online_quantization=False, + ) + + assert result.plan is not None + assert result.plan.bindings["transformer.proj_in.weight"] == TensorBinding( + checkpoint_key="proj_in.weight", + file_path=str(checkpoint_file), + ) def test_validate_loaded_weights_called(self, tmp_path, patched_offload_runtime): """validate_loaded_weights should be called after mmap loading.""" - import json - from types import SimpleNamespace - - from safetensors.torch import save_file - - from vllm_omni.diffusion.offloader.base import OffloadConfig, OffloadStrategy class ModelWithValidate(nn.Module): _layerwise_offload_blocks_attrs = ["blocks"] @@ -1038,19 +1428,64 @@ class TestMmapValidation: backend = DistributedLayerwiseOffloadBackend( OffloadConfig( - strategy=OffloadStrategy.DISTRIBUTED_LAYER_WISE, pin_cpu_memory=False, model_path=str(tmp_path) + strategy=OffloadStrategy.DISTRIBUTED_LAYER_WISE, + pin_cpu_memory=False, ), torch.device("cpu"), ) modules = SimpleNamespace(dits=[pipeline.transformer], dit_names=["transformer"]) + result = build_checkpoint_mmap_plan( + pipeline, + dit_modules=(("transformer", pipeline.transformer),), + sources=(), + model_path=str(tmp_path), + tensor_parallel_size=1, + use_hsdp=False, + online_quantization=False, + ) - backend._load_weights_via_mmap(pipeline, modules) + assert result.plan is not None + backend._load_weights_via_mmap(pipeline, modules, result.plan) assert pipeline.transformer.validate_called, "validate_loaded_weights should be called" class TestConfigValidation: """Tests for configuration validation in OffloadConfig / execute_request.""" + def test_loader_plan_cannot_be_silently_dropped_on_unsupported_platform(self, monkeypatch): + import vllm_omni.diffusion.offloader as offloader_module + + od_config = SimpleNamespace( + enable_cpu_offload=False, + enable_layerwise_offload=False, + enable_distributed_layerwise_offload=True, + dlo_use_allgather=False, + dlo_resident_layers=0, + pin_cpu_memory=False, + model="unused", + parallel_config=SimpleNamespace( + use_hsdp=False, + data_parallel_size=1, + sequence_parallel_size=1, + ), + ) + plan = HostWeightPlan( + backing_kind="checkpoint_mmap", + bindings={}, + ) + monkeypatch.setattr( + offloader_module.current_omni_platform, + "supports_cpu_offload", + lambda: False, + ) + + with pytest.raises(RuntimeError, match="skipped ordinary weight materialization"): + offloader_module.get_offload_backend( + od_config, + device=torch.device("cpu"), + host_weight_plan=plan, + ) + def test_hsdp_with_allgather_rejected(self): """HSDP + DLO + AllGather should raise ValueError (double sharding).""" from vllm_omni.diffusion.offloader.base import OffloadConfig diff --git a/tests/diffusion/test_diffusion_model_runner.py b/tests/diffusion/test_diffusion_model_runner.py index 665196ad9..b60cec89c 100644 --- a/tests/diffusion/test_diffusion_model_runner.py +++ b/tests/diffusion/test_diffusion_model_runner.py @@ -740,6 +740,9 @@ def test_load_model_clears_cache_backend_for_unsupported_pipeline(monkeypatch): del kwargs return SimpleNamespace(transformer=torch.nn.Identity()) + def take_host_weight_plan(self): + return None + class _DummyMemoryProfiler: consumed_memory = 0 @@ -779,7 +782,11 @@ def test_load_model_clears_cache_backend_for_unsupported_pipeline(monkeypatch): monkeypatch.setattr(model_runner_module, "LoadConfig", lambda: object()) monkeypatch.setattr(model_runner_module, "DiffusersPipelineLoader", _DummyLoader) monkeypatch.setattr(model_runner_module, "DeviceMemoryProfiler", _DummyMemoryProfiler) - monkeypatch.setattr(model_runner_module, "get_offload_backend", lambda od_config, device: None) + monkeypatch.setattr( + model_runner_module, + "get_offload_backend", + lambda od_config, device, host_weight_plan: None, + ) monkeypatch.setattr( model_runner_module, "get_cache_backend", lambda cache_backend, cache_config: dummy_cache_backend ) diff --git a/vllm_omni/diffusion/model_loader/checkpoint_adapters/__init__.py b/vllm_omni/diffusion/model_loader/checkpoint_adapters/__init__.py index 584562193..804699bc7 100644 --- a/vllm_omni/diffusion/model_loader/checkpoint_adapters/__init__.py +++ b/vllm_omni/diffusion/model_loader/checkpoint_adapters/__init__.py @@ -9,6 +9,13 @@ from .modelopt import ( ) +def get_direct_mmap_adapter(model: nn.Module): + """Resolve direct-mmap adapters lazily to avoid model-loader cycles.""" + from .direct_mmap import get_direct_mmap_adapter as _get_direct_mmap_adapter + + return _get_direct_mmap_adapter(model) + + def get_checkpoint_adapter( model: nn.Module, source: object, @@ -29,4 +36,5 @@ __all__ = [ "ModelOptMixedPrecisionCheckpointAdapter", "ModelOptNvFp4CheckpointAdapter", "get_checkpoint_adapter", + "get_direct_mmap_adapter", ] diff --git a/vllm_omni/diffusion/model_loader/checkpoint_adapters/direct_mmap.py b/vllm_omni/diffusion/model_loader/checkpoint_adapters/direct_mmap.py new file mode 100644 index 000000000..86b23429b --- /dev/null +++ b/vllm_omni/diffusion/model_loader/checkpoint_adapters/direct_mmap.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Loader-side adapters for proven direct-checkpoint mmap layouts.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from functools import partial +from typing import Protocol + +import torch +from torch import nn + + +@dataclass(frozen=True) +class DirectMmapTensorPolicy: + """Loader-owned proof for one tensor's direct mmap behavior.""" + + allow_custom_loader: bool = False + transform: Callable[[torch.Tensor], torch.Tensor] | None = None + + +class DirectMmapAdapter(Protocol): + def policy_for( + self, + runtime_name: str, + target: torch.Tensor, + ) -> DirectMmapTensorPolicy | None: ... + + +class _MiniMaxH3DirectMmapAdapter: + """TP1 runtime-layout contract already enforced by MiniMax-H3's loader.""" + + def __init__(self, pipeline: nn.Module) -> None: + self.pipeline = pipeline + + def policy_for( + self, + runtime_name: str, + target: torch.Tensor, + ) -> DirectMmapTensorPolicy: + del target + transform = None + suffix = ".qkv_proj.weight" + if runtime_name.endswith(suffix): + from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import ( + MiniMaxH3Attention, + _reorder_grouped_qkv_to_qkv, + ) + + attention_path = runtime_name[: -len(suffix)] + attention = self.pipeline.get_submodule(attention_path) + if not isinstance(attention, MiniMaxH3Attention): + raise ValueError( + f"MiniMax-H3 direct-mmap adapter expected attention at {attention_path!r}, " + f"got {type(attention).__name__}" + ) + transform = partial( + _reorder_grouped_qkv_to_qkv, + num_query_groups=attention.total_num_heads, + heads_per_group=1, + head_dim=attention.head_dim, + ) + return DirectMmapTensorPolicy( + allow_custom_loader=True, + transform=transform, + ) + + +class _Cosmos3DirectMmapAdapter: + """Preserve Cosmos3's existing TP1 direct-mmap load contract.""" + + def policy_for( + self, + runtime_name: str, + target: torch.Tensor, + ) -> DirectMmapTensorPolicy: + del runtime_name, target + # Cosmos3 has no packed module mapping or TP1 semantic transform after + # its checkpoint-key remap. Its custom vLLM loaders reduce to exact + # copies at TP1; topology and metadata are checked by plan preflight. + return DirectMmapTensorPolicy(allow_custom_loader=True) + + +def get_direct_mmap_adapter(pipeline: nn.Module) -> DirectMmapAdapter | None: + """Return a loader-owned adapter without requiring a pipeline flag.""" + from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import ( + MiniMaxH3DiTModel, + ) + + if any(isinstance(module, MiniMaxH3DiTModel) for module in pipeline.modules()): + return _MiniMaxH3DirectMmapAdapter(pipeline) + + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import ( + Cosmos3VFMTransformer, + ) + + if any(isinstance(module, Cosmos3VFMTransformer) for module in pipeline.modules()): + return _Cosmos3DirectMmapAdapter() + return None + + +__all__ = [ + "DirectMmapAdapter", + "DirectMmapTensorPolicy", + "get_direct_mmap_adapter", +] diff --git a/vllm_omni/diffusion/model_loader/diffusers_loader.py b/vllm_omni/diffusion/model_loader/diffusers_loader.py index bde9a1543..5e05bd547 100644 --- a/vllm_omni/diffusion/model_loader/diffusers_loader.py +++ b/vllm_omni/diffusion/model_loader/diffusers_loader.py @@ -6,7 +6,7 @@ import glob import os import re import time -from collections.abc import Generator, Iterable +from collections.abc import Generator, Iterable, Sequence from pathlib import Path from typing import cast @@ -34,6 +34,11 @@ from vllm_omni.diffusion.distributed.hsdp import HSDPInferenceConfig, apply_hsdp from vllm_omni.diffusion.model_loader.checkpoint_adapters import ( get_checkpoint_adapter, ) +from vllm_omni.diffusion.model_loader.host_weight_plan import ( + HostWeightPlan, + build_checkpoint_mmap_plan, + has_online_quantization, +) from vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter import DiffusersAdapterPipeline from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery from vllm_omni.diffusion.registry import initialize_model @@ -135,6 +140,13 @@ class DiffusersPipelineLoader: self.od_config = od_config self.quant_config = od_config.quantization_config self.parallel_config = od_config.parallel_config + self.host_weight_plan: HostWeightPlan | None = None + + def take_host_weight_plan(self) -> HostWeightPlan | None: + """Transfer the loader-produced plan to the offload backend.""" + plan = self.host_weight_plan + self.host_weight_plan = None + return plan def _prepare_weights( self, @@ -293,8 +305,10 @@ class DiffusersPipelineLoader: def get_all_weights( self, model: nn.Module, + sources: Sequence["ComponentSource"] | None = None, ) -> Generator[tuple[str, torch.Tensor], None, None]: - sources = self._get_weight_sources(model) + if sources is None: + sources = self._get_weight_sources(model) for source in sources: yield from self._get_weights_iterator(source, model=model) @@ -386,6 +400,7 @@ class DiffusersPipelineLoader: device: torch.device | None = None, ) -> nn.Module: """Load a model with the given configurations.""" + self.host_weight_plan = None if load_format is None: load_format = "default" # CPU offload + quantization: for offline-quantized models (e.g., AutoRound MXFP8), @@ -417,22 +432,54 @@ class DiffusersPipelineLoader: else: model = self._init_from_load_format(load_format, target_device, custom_pipeline_name, is_hsdp=False) - # Skip load_weights only for DLO+AllGather when the model - # supports mmap loading and no online quantization is active. - # This condition MUST match the gate in - # DistributedLayerwiseOffloadBackend.enable() so that the - # loader skips ⟺ enable() uses mmap. - from vllm_omni.diffusion.offloader.offload_plan import supports_mmap_loading - _dist_offload = getattr(self.od_config, "enable_distributed_layerwise_offload", False) _use_ag = getattr(self.od_config, "dlo_use_allgather", True) _has_online_quant = self._has_online_quant(model) - _supports_mmap = supports_mmap_loading(model) + _tp_size = int(getattr(self.parallel_config, "tensor_parallel_size", 1)) + _use_hsdp = bool(getattr(self.parallel_config, "use_hsdp", False)) + _dp_size = int(getattr(self.parallel_config, "data_parallel_size", 1)) + _sp_size = int(getattr(self.parallel_config, "sequence_parallel_size", 1)) + _dlo_group_size = _dp_size if _dp_size > 1 else _sp_size - _skip_load = _dist_offload and _use_ag and _supports_mmap and not _has_online_quant + plan_result = None + weight_sources = self._get_weight_sources(model) + if _dist_offload: + modules = ModuleDiscovery.discover(model) + plan_result = build_checkpoint_mmap_plan( + model, + dit_modules=tuple(zip(modules.dit_names, modules.dits)), + sources=weight_sources, + model_path=str(getattr(self.od_config, "model", "")) or None, + tensor_parallel_size=_tp_size, + use_hsdp=_use_hsdp, + online_quantization=_has_online_quant, + ) + self.host_weight_plan = plan_result.plan + + _skip_load = self.host_weight_plan is not None if _skip_load: - logger.info("DLO+AllGather active: skipping load_weights (will load via mmap in enable())") + logger.info( + "DLO host-weight plan active (%s, %s): skipping ordinary materialization for %s", + "AllGather" if _use_ag and _dlo_group_size > 1 else "rank-local", + self.host_weight_plan.backing_kind, + sorted(self.host_weight_plan.planned_source_prefixes) or "legacy DiT sources", + ) + ordinary_sources = tuple( + source + for source in weight_sources + if source.prefix not in self.host_weight_plan.planned_source_prefixes + ) + if ordinary_sources: + logger.info( + "Loading %d component weight source(s) outside the DLO host-weight plan", + len(ordinary_sources), + ) + self.load_weights( + model, + sources=ordinary_sources, + planned_weights=self.host_weight_plan.bindings, + ) else: if _dist_offload and _use_ag and _has_online_quant: raise ValueError( @@ -442,9 +489,10 @@ class DiffusersPipelineLoader: "Please use --dlo-no-use-allgather or disable online " "quantization." ) - if _dist_offload and _use_ag and not _supports_mmap: + if _dist_offload and plan_result is not None: logger.info( - "DLO+AllGather active but model does not support mmap: loading weights via regular loader." + "DLO direct checkpoint mmap unavailable; using ordinary loader: %s", + plan_result.fallback_reason, ) logger.debug("Loading weights on %s ...", load_device) if offload_after_quant: @@ -498,11 +546,7 @@ class DiffusersPipelineLoader: """Whether any layer uses an online-quant method that defers weight materialization onto the ``meta`` device (upstream vLLM ``uses_meta_device=True``, e.g. online FP8).""" - for module in model.modules(): - quant_method = getattr(module, "quant_method", None) - if getattr(quant_method, "uses_meta_device", False): - return True - return False + return has_online_quantization(model) def _apply_skip_softmax_calibration(self, model: nn.Module) -> None: from vllm_omni.diffusion.attention.backends.trtllm_calibration import ( @@ -605,12 +649,16 @@ class DiffusersPipelineLoader: model: nn.Module, *, stream_online_quant_to_cpu: bool = False, + sources: Sequence["ComponentSource"] | None = None, + planned_weights: Iterable[str] = (), ) -> None: weights_to_load = self._get_expected_parameter_names(model) - weights = self.get_all_weights(model) + weights = self.get_all_weights(model) if sources is None else self.get_all_weights(model, sources=sources) if stream_online_quant_to_cpu: weights = self._stream_online_quant_weights_to_cpu(model, weights) loaded_weights = model.load_weights(weights) + if loaded_weights is not None: + loaded_weights = set(loaded_weights).union(planned_weights) self.counter_after_loading_weights = time.perf_counter() logger.info_once( diff --git a/vllm_omni/diffusion/model_loader/host_weight_plan.py b/vllm_omni/diffusion/model_loader/host_weight_plan.py new file mode 100644 index 000000000..a61d60cfb --- /dev/null +++ b/vllm_omni/diffusion/model_loader/host_weight_plan.py @@ -0,0 +1,355 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Loader-owned host-weight plans shared with diffusion offload backends.""" + +from __future__ import annotations + +import glob +import json +import os +from collections.abc import Callable, Sequence +from dataclasses import dataclass + +import torch +from safetensors import safe_open +from torch import nn +from vllm.logger import init_logger + +from vllm_omni.diffusion.model_loader.checkpoint_adapters import ( + get_direct_mmap_adapter, +) + +logger = init_logger(__name__) + +TensorTransform = Callable[[torch.Tensor], torch.Tensor] + + +@dataclass(frozen=True) +class TensorBinding: + """One runtime tensor backed by one safetensors entry.""" + + checkpoint_key: str + file_path: str + transform: TensorTransform | None = None + + +@dataclass(frozen=True) +class HostWeightPlan: + """Complete, prevalidated host backing consumed by an offload backend.""" + + backing_kind: str + bindings: dict[str, TensorBinding] + planned_source_prefixes: frozenset[str] = frozenset() + + +@dataclass(frozen=True) +class HostWeightPlanResult: + """A complete plan or a fail-closed reason to use the ordinary loader.""" + + plan: HostWeightPlan | None + fallback_reason: str | None = None + + +class _PlanIncompatibleError(RuntimeError): + pass + + +_SAFETENSORS_DTYPES: dict[str, torch.dtype] = { + "BOOL": torch.bool, + "U8": torch.uint8, + "I8": torch.int8, + "I16": torch.int16, + "I32": torch.int32, + "I64": torch.int64, + "F16": torch.float16, + "BF16": torch.bfloat16, + "F32": torch.float32, + "F64": torch.float64, +} +for _safe_name, _torch_name in ( + ("F8_E4M3", "float8_e4m3fn"), + ("F8_E5M2", "float8_e5m2"), +): + if (_dtype := getattr(torch, _torch_name, None)) is not None: + _SAFETENSORS_DTYPES[_safe_name] = _dtype + + +def has_online_quantization(model: nn.Module) -> bool: + """Whether any module defers online-quantized weights on ``meta``.""" + return any(getattr(getattr(module, "quant_method", None), "uses_meta_device", False) for module in model.modules()) + + +def _resolve_source_root(source: object) -> str: + source_root = os.fspath(getattr(source, "model_or_path")) + subfolder = getattr(source, "subfolder", None) + if not os.path.isdir(source_root): + from vllm.model_executor.model_loader.weight_utils import ( + download_weights_from_hf, + ) + + source_root = download_weights_from_hf( + model_name_or_path=source_root, + cache_dir=None, + allow_patterns=["*.safetensors", "*.safetensors.index.json"], + revision=getattr(source, "revision", None), + subfolder=subfolder, + ) + return os.path.join(source_root, subfolder) if subfolder is not None else source_root + + +def _record_binding( + model_to_ckpt: dict[str, tuple[str, str]], + model_name: str, + checkpoint_key: str, + file_path: str, +) -> None: + existing = model_to_ckpt.get(model_name) + candidate = (checkpoint_key, file_path) + if existing is not None and existing != candidate: + raise _PlanIncompatibleError( + f"multiple checkpoint entries map to runtime tensor {model_name!r}: {existing!r} and {candidate!r}" + ) + model_to_ckpt[model_name] = candidate + + +def _add_safetensors_source( + checkpoint_dir: str, + prefix: str, + remap_fn: Callable[[str], str | None] | None, + model_to_ckpt: dict[str, tuple[str, str]], +) -> int: + indexed_keys = 0 + index_files = sorted(glob.glob(os.path.join(checkpoint_dir, "*.safetensors.index.json"))) + if index_files: + for index_file in index_files: + with open(index_file, encoding="utf-8") as handle: + index = json.load(handle) + for checkpoint_key, filename in index.get("weight_map", {}).items(): + source_name = prefix + checkpoint_key + model_name = remap_fn(source_name) if remap_fn is not None else source_name + if model_name is not None: + _record_binding( + model_to_ckpt, + model_name, + checkpoint_key, + os.path.join(checkpoint_dir, filename), + ) + indexed_keys += 1 + return indexed_keys + + for filename in sorted(glob.glob(os.path.join(checkpoint_dir, "*.safetensors"))): + with safe_open(filename, framework="pt", device="cpu") as handle: + for checkpoint_key in handle.keys(): + source_name = prefix + checkpoint_key + model_name = remap_fn(source_name) if remap_fn is not None else source_name + if model_name is not None: + _record_binding(model_to_ckpt, model_name, checkpoint_key, filename) + indexed_keys += 1 + return indexed_keys + + +def _build_source_map( + *, + sources: Sequence[object], + model_path: str | None, + remap_fn: Callable[[str], str | None] | None, +) -> dict[str, tuple[str, str]]: + model_to_ckpt: dict[str, tuple[str, str]] = {} + indexed_keys = 0 + + for source in sources: + indexed_keys += _add_safetensors_source( + _resolve_source_root(source), + getattr(source, "prefix", ""), + remap_fn, + model_to_ckpt, + ) + + if not sources: + if remap_fn is None: + raise _PlanIncompatibleError( + "the loader has no component weight sources and the pipeline has no checkpoint-key remapper" + ) + if not model_path: + raise _PlanIncompatibleError("the loader has no model path for legacy checkpoint remapping") + + resolved_model_path = model_path + if not os.path.isdir(resolved_model_path): + from vllm.model_executor.model_loader.weight_utils import ( + download_weights_from_hf, + ) + + resolved_model_path = download_weights_from_hf( + model_name_or_path=resolved_model_path, + cache_dir=None, + allow_patterns=["*.safetensors", "*.safetensors.index.json"], + ) + checkpoint_dirs = { + os.path.dirname(path) + for pattern in ("*.safetensors.index.json", "*.safetensors") + for path in glob.glob(os.path.join(resolved_model_path, "**", pattern), recursive=True) + } + for checkpoint_dir in sorted(checkpoint_dirs): + indexed_keys += _add_safetensors_source( + checkpoint_dir, + "", + remap_fn, + model_to_ckpt, + ) + + if not model_to_ckpt: + raise _PlanIncompatibleError("no compatible safetensors entries were found in the loader's model sources") + logger.info( + "Indexed %d runtime tensor names from %d checkpoint keys for DLO preflight", + len(model_to_ckpt), + indexed_keys, + ) + return model_to_ckpt + + +def _is_persistent_buffer(module: nn.Module, local_name: str) -> bool: + parts = local_name.split(".") + owner = module.get_submodule(".".join(parts[:-1])) if len(parts) > 1 else module + return parts[-1] not in owner._non_persistent_buffers_set + + +def _collect_required_dit_tensors( + dit_modules: Sequence[tuple[str, nn.Module]], +) -> dict[str, torch.Tensor]: + required: dict[str, torch.Tensor] = {} + for dit_name, dit_module in dit_modules: + for local_name, tensor in dit_module.named_parameters(): + required[f"{dit_name}.{local_name}"] = tensor + for local_name, tensor in dit_module.named_buffers(): + if _is_persistent_buffer(dit_module, local_name): + required[f"{dit_name}.{local_name}"] = tensor + return required + + +def _planned_source_prefixes( + sources: Sequence[object], + dit_modules: Sequence[tuple[str, nn.Module]], +) -> frozenset[str]: + """Identify component sources that the DiT-only plan owns completely.""" + if not sources: + return frozenset() + + expected = frozenset(f"{dit_name}." for dit_name, _ in dit_modules) + available = {getattr(source, "prefix", "") for source in sources} + missing = sorted(expected - available) + if missing: + raise _PlanIncompatibleError( + f"direct mmap requires a dedicated component weight source for each DiT; missing prefixes: {missing}" + ) + return expected + + +def _validate_source_metadata( + required: dict[str, torch.Tensor], + bindings: dict[str, TensorBinding], +) -> None: + by_file: dict[str, list[tuple[str, torch.Tensor, TensorBinding]]] = {} + for runtime_name, target in required.items(): + by_file.setdefault(bindings[runtime_name].file_path, []).append((runtime_name, target, bindings[runtime_name])) + + for file_path, entries in by_file.items(): + if not os.path.isfile(file_path): + raise _PlanIncompatibleError(f"checkpoint file does not exist: {file_path}") + with safe_open(file_path, framework="pt", device="cpu") as handle: + available = set(handle.keys()) + for runtime_name, target, binding in entries: + if binding.checkpoint_key not in available: + raise _PlanIncompatibleError( + f"checkpoint key {binding.checkpoint_key!r} for {runtime_name!r} is missing from {file_path}" + ) + source = handle.get_slice(binding.checkpoint_key) + source_shape = tuple(source.get_shape()) + source_dtype = _SAFETENSORS_DTYPES.get(source.get_dtype()) + if source_shape != tuple(target.shape): + raise _PlanIncompatibleError( + f"shape mismatch for {runtime_name!r}: checkpoint={source_shape}, runtime={tuple(target.shape)}" + ) + if source_dtype is None or source_dtype != target.dtype: + raise _PlanIncompatibleError( + f"dtype mismatch for {runtime_name!r}: checkpoint={source.get_dtype()}, runtime={target.dtype}" + ) + + +def build_checkpoint_mmap_plan( + pipeline: nn.Module, + *, + dit_modules: Sequence[tuple[str, nn.Module]], + sources: Sequence[object], + model_path: str | None, + tensor_parallel_size: int, + use_hsdp: bool, + online_quantization: bool, +) -> HostWeightPlanResult: + """Build a complete direct-checkpoint plan or return a fallback reason.""" + if tensor_parallel_size != 1: + return HostWeightPlanResult(None, f"TP={tensor_parallel_size} requires the ordinary loader") + if use_hsdp: + return HostWeightPlanResult(None, "HSDP requires the ordinary loader") + if online_quantization: + return HostWeightPlanResult(None, "online quantization requires the ordinary loader") + + remap_fn = getattr(type(pipeline), "_remap_ckpt_key", None) + if not callable(remap_fn): + remap_fn = None + adapter = get_direct_mmap_adapter(pipeline) + + try: + planned_source_prefixes = _planned_source_prefixes(sources, dit_modules) + planned_sources = tuple( + source for source in sources if getattr(source, "prefix", "") in planned_source_prefixes + ) + model_to_ckpt = _build_source_map( + sources=planned_sources, + model_path=model_path, + remap_fn=remap_fn, + ) + required = _collect_required_dit_tensors(dit_modules) + if not required: + raise _PlanIncompatibleError("no DiT parameters or persistent buffers were discovered") + + missing = sorted(set(required) - set(model_to_ckpt)) + if missing: + raise _PlanIncompatibleError( + f"{len(missing)} required DiT tensors have no checkpoint binding (first 5: {missing[:5]})" + ) + + bindings: dict[str, TensorBinding] = {} + for runtime_name, target in required.items(): + policy = adapter.policy_for(runtime_name, target) if adapter is not None else None + has_custom_loader = callable(getattr(target, "weight_loader", None)) + if has_custom_loader and not (policy and policy.allow_custom_loader): + raise _PlanIncompatibleError( + f"{runtime_name!r} requires a custom weight_loader with no direct-mmap adapter" + ) + checkpoint_key, file_path = model_to_ckpt[runtime_name] + bindings[runtime_name] = TensorBinding( + checkpoint_key=checkpoint_key, + file_path=file_path, + transform=policy.transform if policy is not None else None, + ) + + _validate_source_metadata(required, bindings) + except (OSError, ValueError, _PlanIncompatibleError) as exc: + return HostWeightPlanResult(None, str(exc)) + + return HostWeightPlanResult( + HostWeightPlan( + backing_kind="checkpoint_mmap", + bindings=bindings, + planned_source_prefixes=planned_source_prefixes, + ) + ) + + +__all__ = [ + "HostWeightPlan", + "HostWeightPlanResult", + "TensorBinding", + "build_checkpoint_mmap_plan", + "has_online_quantization", +] diff --git a/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py b/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py index 5de203721..f639f3ebb 100644 --- a/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py +++ b/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py @@ -540,13 +540,6 @@ class MiniMaxH3Pipeline( encoder_block_attrs={"text_encoder": ("vision.blocks", "text_model.layers")}, on_demand_component_paths=frozenset({"text_encoder", "video_vae", "audio_vae"}), ) - # H3's regular loader performs checkpoint-layout conversions (grouped QKV - # and fused MLP). Keep it on that path until mmap can run the same loader - # callbacks for every affected parameter. - _supports_mmap_loading: ClassVar[bool] = False - # TODO(offload): Re-enable after the generic rank-local mmap path can run - # this model's grouped-QKV reorder and fused-MLP packing before TP sharding. - # Do not bypass the regular loader until that equivalence is tested. _PROFILER_TARGETS: ClassVar[list[str]] = [ "_prepare_reference_videos", "encode_prompt", diff --git a/vllm_omni/diffusion/offloader/__init__.py b/vllm_omni/diffusion/offloader/__init__.py index 9ff464494..2502fa76b 100644 --- a/vllm_omni/diffusion/offloader/__init__.py +++ b/vllm_omni/diffusion/offloader/__init__.py @@ -5,6 +5,7 @@ import torch from vllm.logger import init_logger from vllm_omni.diffusion.data import OmniDiffusionConfig +from vllm_omni.diffusion.model_loader.host_weight_plan import HostWeightPlan from vllm_omni.platforms import current_omni_platform from .base import OffloadBackend, OffloadConfig, OffloadStrategy @@ -17,7 +18,7 @@ from .distributed_layerwise_backend import ( ) from .layerwise_backend import LayerWiseOffloadBackend from .module_residency import PinnedModuleStager -from .offload_plan import OffloadPlan, get_offload_plan, supports_mmap_loading +from .offload_plan import OffloadPlan, get_offload_plan from .sequential_backend import ( ModelLevelOffloadBackend, apply_sequential_offload, @@ -49,7 +50,6 @@ __all__ = [ "remove_distributed_block_hook", "get_offload_backend", "get_offload_plan", - "supports_mmap_loading", "get_blocks_attr_names", "get_blocks_from_dit", "set_blocks_attr_names", @@ -64,12 +64,15 @@ __all__ = [ def get_offload_backend( od_config: OmniDiffusionConfig, device: torch.device | None = None, + host_weight_plan: HostWeightPlan | None = None, ) -> OffloadBackend | None: """Create appropriate offload backend based on configuration. Args: od_config: OmniDiffusionConfig with offload settings device: Target device (auto-detected if None) + host_weight_plan: Exact loader-produced backing plan, if ordinary + weight materialization was skipped. Returns: OffloadBackend instance or None if offloading disabled @@ -82,12 +85,22 @@ def get_offload_backend( # Extract and validate configuration config = OffloadConfig.from_od_config(od_config) + if host_weight_plan is not None and config.strategy != OffloadStrategy.DISTRIBUTED_LAYER_WISE: + raise RuntimeError( + "A loader-owned DLO host-weight plan was produced, but distributed layerwise offload is not selected" + ) + # Return None if no offloading requested if config.strategy == OffloadStrategy.NONE: return None # Validate platform (CUDA required for now) if not current_omni_platform.supports_cpu_offload() or current_omni_platform.get_device_count() < 1: + if host_weight_plan is not None: + raise RuntimeError( + "The loader skipped ordinary weight materialization for DLO, " + "but this platform cannot create the required offload backend" + ) logger.warning( "Current device: %s does not support CPU offloading. Skipping offloading.", current_omni_platform.get_device_name(), @@ -99,6 +112,11 @@ def get_offload_backend( try: device = current_omni_platform.get_torch_device() except (NotImplementedError, AttributeError) as exc: + if host_weight_plan is not None: + raise RuntimeError( + "The loader skipped ordinary weight materialization for DLO, " + "but the target device could not be resolved" + ) from exc logger.error("Failed to detect device: %s. Skipping offloading.", exc) return None @@ -108,7 +126,11 @@ def get_offload_backend( elif config.strategy == OffloadStrategy.LAYER_WISE: return LayerWiseOffloadBackend(config, device) elif config.strategy == OffloadStrategy.DISTRIBUTED_LAYER_WISE: - return DistributedLayerwiseOffloadBackend(config, device) + return DistributedLayerwiseOffloadBackend( + config, + device, + host_weight_plan=host_weight_plan, + ) else: logger.error("Unknown offload strategy: %s", config.strategy) return None diff --git a/vllm_omni/diffusion/offloader/base.py b/vllm_omni/diffusion/offloader/base.py index 2db80923d..ea74fbfb7 100644 --- a/vllm_omni/diffusion/offloader/base.py +++ b/vllm_omni/diffusion/offloader/base.py @@ -27,11 +27,10 @@ class OffloadConfig: pin_cpu_memory: bool = True use_hsdp: bool = False dp_size: int = 1 # derived from parallel_config, not user-configurable - # True: add DP sharding + AllGather. False: stream the standard loader's - # rank-local tensors (including TP-local shards) with H2D only. + # True: add DP sharding + AllGather. False: stream complete rank-local + # blocks from the loader-selected host backing with H2D only. dlo_use_allgather: bool = True dlo_resident_layers: int = 0 # leading DiT layers kept on device - model_path: str | None = None # checkpoint path for mmap weight loading @classmethod def from_od_config(cls, od_config: OmniDiffusionConfig) -> "OffloadConfig": @@ -58,7 +57,6 @@ class OffloadConfig: parallel_config = getattr(od_config, "parallel_config", None) use_hsdp = getattr(parallel_config, "use_hsdp", False) if parallel_config else False - # Derive dp_size from parallel_config — not user-configurable. # The offload adapts to whatever DP/SP is already configured. dp_size = 1 @@ -116,7 +114,8 @@ class OffloadConfig: dp_size = 1 logger.info( "Distributed layerwise offload: dlo_use_allgather=False, " - "streaming standard-loader rank-local weights (no DP shard or AllGather)" + "streaming complete rank-local blocks (no DLO shard or AllGather); " + "the backend will select mmap or standard-loader host storage" ) # HSDP already shards parameters into DTensors. Running distributed @@ -137,7 +136,6 @@ class OffloadConfig: dp_size=dp_size, dlo_use_allgather=dlo_use_allgather, dlo_resident_layers=dlo_resident_layers, - model_path=getattr(od_config, "model", None), ) diff --git a/vllm_omni/diffusion/offloader/distributed_layerwise_backend.py b/vllm_omni/diffusion/offloader/distributed_layerwise_backend.py index 2d6a34978..d184e71ea 100644 --- a/vllm_omni/diffusion/offloader/distributed_layerwise_backend.py +++ b/vllm_omni/diffusion/offloader/distributed_layerwise_backend.py @@ -1,22 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Distributed Layerwise Offload backend with H2D + AllGather overlap. +"""Distributed Layerwise Offload backend with double-buffered H2D. This module implements the RFC-1 "Distributed Layerwise Offload" mechanism that: -* Shards model weights across DP ranks and stores only the local shard - (1/DP_size of full model) in host pinned memory per rank. +* Optionally shards model weights across DP ranks and reconstructs each block + with AllGather, or streams a complete rank-local block without a collective. +* Can retain compatible checkpoint tensors as node-shared mmap sources instead + of creating a persistent private host copy in every rank. * Uses a fixed double-buffer scheme that keeps only two layers' worth of weights on each device at any time. -* Asynchronously pipelines both H2D transfers and AllGather communications - on dedicated streams, fully overlapping them with computation. +* Pipelines H2D transfers and, when enabled, AllGather communications on + dedicated streams, overlapping them with computation. * Is hardware-agnostic, supporting both NVIDIA GPU (CUDA) and Ascend NPU (CANN) platforms via vLLM-Omni's platform abstraction layer. """ from __future__ import annotations -import os from itertools import chain from typing import Any @@ -26,12 +27,18 @@ from torch import nn from vllm.logger import init_logger from vllm_omni.diffusion.hooks import HookRegistry, ModelHook +from vllm_omni.diffusion.model_loader.host_weight_plan import ( + HostWeightPlan, +) from vllm_omni.platforms import current_omni_platform from .base import OffloadBackend, OffloadConfig from .block_discovery import get_blocks_from_dit from .module_collector import ModuleDiscovery -from .offload_plan import OffloadPlan, get_offload_plan, supports_mmap_loading +from .offload_plan import ( + OffloadPlan, + get_offload_plan, +) from .tensor_utils import ( dtype_size as _dtype_size, ) @@ -74,6 +81,8 @@ class DistributedLayerwiseOffloadHook(ModelHook): comm_stream: Any | None = None, pin_memory: bool = True, shared_buffers: list[dict[torch.dtype, torch.Tensor] | None] | None = None, + rank_local_mmap: bool = False, + tensor_transforms: dict[int, Any] | None = None, ): assert isinstance(next_block, nn.Module), "transformer block must be type `torch.nn.Module`" @@ -83,6 +92,8 @@ class DistributedLayerwiseOffloadHook(ModelHook): self.dp_size = dp_size self.rank = rank self.pin_memory = pin_memory + self.rank_local_mmap = rank_local_mmap + self.tensor_transforms = tensor_transforms or {} self.copy_stream = copy_stream or current_omni_platform.Stream() self.comm_stream = comm_stream or current_omni_platform.Stream() @@ -98,8 +109,18 @@ class DistributedLayerwiseOffloadHook(ModelHook): # Sharded host weights for the next block, keyed by dtype self.cpu_shards: dict[torch.dtype, torch.Tensor] = {} + # File-backed source tensors for rank-local mmap. Unlike cpu_shards, + # these remain immutable views of the checkpoint and are never pinned + # or flattened into a model-sized private allocation. + self.cpu_sources: dict[torch.dtype, list[dict[str, Any]]] = {} self.metadata: dict[torch.dtype, list[dict[str, Any]]] = {} + # Rank-local mmap uses two host staging slots shared by every hook in + # this worker. They are assigned by the backend after all block sizes + # are known, mirroring the shared device-buffer allocation. + self.cpu_staging_buffers: list[dict[torch.dtype, torch.Tensor] | None] = [None, None] + self.cpu_staging_events: list[Any | None] = [None, None] + # Current slot index (0 or 1). Updated dynamically by the previous # hook's prefetch_layer call via _prefetched_slot. This ensures # correct slot tracking for ALL block counts (including odd N). @@ -154,14 +175,22 @@ class DistributedLayerwiseOffloadHook(ModelHook): self.next_block_parameters = dict(self.next_block.named_parameters()) self.next_block_buffers = dict(self.next_block.named_buffers()) - # Shard next block's weights and store local shard in pinned CPU memory - self.cpu_shards, self.metadata = self._shard_and_pin( - self.next_block_parameters, - self.next_block_buffers, - self.dp_size, - self.rank, - self.pin_memory, - ) + if self.rank_local_mmap: + self.cpu_sources, self.metadata = self._collect_mmap_sources( + self.next_block_parameters, + self.next_block_buffers, + self.tensor_transforms, + ) + else: + # Shard next block's weights and store local shard in pinned CPU memory. + self.cpu_shards, self.metadata = self._shard_and_pin( + self.next_block_parameters, + self.next_block_buffers, + self.dp_size, + self.rank, + self.pin_memory, + self.tensor_transforms, + ) # Allocate device buffers only if not using shared buffers from backend if self._owns_buffers: @@ -194,11 +223,78 @@ class DistributedLayerwiseOffloadHook(ModelHook): self._ag_output_sizes: dict[torch.dtype, int] = {} for dtype, metas in self.metadata.items(): total_numel = sum(m["numel"] for m in metas) - shard_numel = self.cpu_shards[dtype].numel() - self._ag_output_sizes[dtype] = shard_numel * self.dp_size if self.dp_size > 1 else total_numel + if self.rank_local_mmap: + self._ag_output_sizes[dtype] = total_numel + else: + shard_numel = self.cpu_shards[dtype].numel() + self._ag_output_sizes[dtype] = shard_numel * self.dp_size if self.dp_size > 1 else total_numel return module + @staticmethod + def _collect_mmap_sources( + params: dict[str, nn.Parameter], + bufs: dict[str, torch.Tensor], + tensor_transforms: dict[int, Any] | None = None, + ) -> tuple[dict[torch.dtype, list[dict[str, Any]]], dict[torch.dtype, list[dict[str, Any]]]]: + """Retain file-backed tensors and replace module storage with placeholders. + + The returned sources preserve safetensors mmap storage. Runtime-layout + adapters are applied only while packing a bounded staging slot, so no + full-model anonymous CPU copy is retained by a worker. + """ + cpu_sources: dict[torch.dtype, list[dict[str, Any]]] = {} + metadata: dict[torch.dtype, list[dict[str, Any]]] = {} + offsets: dict[torch.dtype, int] = {} + + for name, target in chain(params.items(), bufs.items()): + source = target.to_local() if hasattr(target, "to_local") else target + if source.device.type != "cpu": + raise ValueError( + f"Rank-local mmap storage requires CPU checkpoint views, but {name!r} is on {source.device}." + ) + + dtype = source.dtype + offset = offsets.get(dtype, 0) + transform = (tensor_transforms or {}).get(id(target)) + runtime_source = transform(source) if callable(transform) else source + if runtime_source.dtype != dtype or runtime_source.shape != source.shape: + raise ValueError( + "mmap weight transform changed tensor metadata for " + f"{name!r}: expected dtype={dtype}, shape={tuple(source.shape)}, " + f"got dtype={runtime_source.dtype}, shape={tuple(runtime_source.shape)}" + ) + stride = runtime_source.stride() + storage_numel = ( + 0 + if runtime_source.numel() == 0 + else 1 + sum((size - 1) * axis_stride for size, axis_stride in zip(runtime_source.shape, stride)) + ) + + cpu_sources.setdefault(dtype, []).append( + { + "name": name, + "tensor": source.detach(), + "transform": transform, + } + ) + metadata.setdefault(dtype, []).append( + { + "name": name, + "offset": offset, + "numel": storage_numel, + "shape": runtime_source.shape, + "stride": stride, + } + ) + offsets[dtype] = offset + storage_numel + + # The detached source above keeps the mmap storage alive while the + # module parameter/buffer is rebound to the rotating device slot. + set_tensor_storage(target, make_offload_placeholder(target)) + + return cpu_sources, metadata + @staticmethod def _shard_and_pin( params: dict[str, nn.Parameter], @@ -206,6 +302,7 @@ class DistributedLayerwiseOffloadHook(ModelHook): dp_size: int, rank: int, pin_memory: bool, + tensor_transforms: dict[int, Any] | None = None, ) -> tuple[dict[torch.dtype, torch.Tensor], dict[torch.dtype, list[dict[str, Any]]]]: """Flatten params+buffers by dtype, split into DP shards, store local shard. @@ -228,8 +325,8 @@ class DistributedLayerwiseOffloadHook(ModelHook): weights_with_local = [] for name, t in name2weights.items(): local_t = t.to_local() if hasattr(t, "to_local") else t - mmap_transform = getattr(t, "mmap_weight_transform", None) - if callable(mmap_transform) and getattr(t, "mmap_weight_transform_pending", False): + mmap_transform = (tensor_transforms or {}).get(id(t)) + if callable(mmap_transform): # Some checkpoints use a layout that is converted by the # regular weight loader (for example MiniMax-H3 grouped # QKV). Apply that conversion one block at a time while @@ -347,6 +444,61 @@ class DistributedLayerwiseOffloadHook(ModelHook): # Prefetch: H2D + AllGather (overlapped on dedicated streams) # # ------------------------------------------------------------------ # + @staticmethod + def _pack_mmap_sources( + cpu_sources: dict[torch.dtype, list[dict[str, Any]]], + metadata: dict[torch.dtype, list[dict[str, Any]]], + slot_buffers: dict[torch.dtype, torch.Tensor], + ) -> dict[torch.dtype, torch.Tensor]: + staged: dict[torch.dtype, torch.Tensor] = {} + for dtype, metas in metadata.items(): + total_numel = sum(meta["numel"] for meta in metas) + destination = slot_buffers[dtype][:total_numel] + sources = cpu_sources[dtype] + for source_info, meta in zip(sources, metas, strict=True): + source = source_info["tensor"] + transform = source_info["transform"] + if callable(transform): + source = transform(source) + if source.dtype != dtype or source.shape != meta["shape"] or source.stride() != meta["stride"]: + raise ValueError( + "mmap weight transform changed tensor layout for " + f"{source_info['name']!r}: expected dtype={dtype}, " + f"shape={tuple(meta['shape'])}, stride={meta['stride']}; " + f"got dtype={source.dtype}, shape={tuple(source.shape)}, " + f"stride={source.stride()}" + ) + start = meta["offset"] + physical_storage = destination[start : start + meta["numel"]] + if source.is_contiguous(): + physical_storage.copy_(source.flatten()) + else: + torch.as_strided( + physical_storage, + size=source.shape, + stride=source.stride(), + ).copy_(source) + staged[dtype] = destination + return staged + + def _stage_mmap_sources(self, slot: int) -> dict[torch.dtype, torch.Tensor]: + """Pack this block's mmap views into one bounded host staging slot.""" + previous_copy = self.cpu_staging_events[slot] + if previous_copy is not None: + synchronize = getattr(previous_copy, "synchronize", None) + if callable(synchronize): + synchronize() + else: + # Platform events normally expose synchronize(). Retain a + # correctness fallback for test and non-CUDA platform shims. + current_omni_platform.synchronize() + self.cpu_staging_events[slot] = None + + slot_buffers = self.cpu_staging_buffers[slot] + if slot_buffers is None: + raise RuntimeError(f"cpu_staging_buffers[{slot}] was not allocated") + return self._pack_mmap_sources(self.cpu_sources, self.metadata, slot_buffers) + @torch.compiler.disable def prefetch_layer(self, slot: int, non_blocking: bool = True) -> None: """Prepare next block's weights into the shared device buffer for *slot*. @@ -362,11 +514,17 @@ class DistributedLayerwiseOffloadHook(ModelHook): assert gpu_weights is not None, f"gpu_buffers[{slot}] not allocated" if self.dp_size <= 1 or self.dp_group is None: + cpu_weights = self._stage_mmap_sources(slot) if self.rank_local_mmap else self.cpu_shards with current_omni_platform.stream(self.copy_stream): - for dtype, cpu_shard in self.cpu_shards.items(): + for dtype, cpu_shard in cpu_weights.items(): gw = gpu_weights[dtype] - gw[: cpu_shard.numel()].copy_(cpu_shard, non_blocking=non_blocking) + async_copy = non_blocking and cpu_shard.is_pinned() + gw[: cpu_shard.numel()].copy_(cpu_shard, non_blocking=async_copy) evt.record(self.copy_stream) + if self.rank_local_mmap: + # The CPU slot may be overwritten only after this H2D copy has + # finished. The shared event protects reuse by another hook. + self.cpu_staging_events[slot] = evt else: gpu_shards: dict[torch.dtype, torch.Tensor] = {} shard_bufs = self.gpu_shard_buffers[slot] @@ -513,6 +671,8 @@ def apply_distributed_block_hook( comm_stream: Any | None = None, pin_memory: bool = True, shared_buffers: list[dict[torch.dtype, torch.Tensor] | None] | None = None, + rank_local_mmap: bool = False, + tensor_transforms: dict[int, Any] | None = None, ) -> DistributedLayerwiseOffloadHook: """Register a DistributedLayerwiseOffloadHook on *module*.""" registry = HookRegistry.get_or_create(module) @@ -526,6 +686,8 @@ def apply_distributed_block_hook( comm_stream=comm_stream, pin_memory=pin_memory, shared_buffers=shared_buffers, + rank_local_mmap=rank_local_mmap, + tensor_transforms=tensor_transforms, ) registry.register_hook(DistributedLayerwiseOffloadHook._HOOK_NAME, hook) return hook @@ -540,15 +702,16 @@ def remove_distributed_block_hook(module: nn.Module) -> None: class PinnedResidentLayerGroup: - """Keep selected layers in pinned host memory between requests. + """Keep selected layers available for stage-scoped device residency. TODO(offload): Extract this alongside PinnedModuleStager after the distributed shard-and-pin operation becomes a shared storage primitive. It currently remains here because it depends on DLO's local-shard layout. Unlike ``module.to(device)``/``module.to("cpu")``, this group retains a - pinned CPU master copy and never copies generated device weights back to - host. Entering the denoise stage performs one asynchronous H2D pass; + pinned CPU master copy (or mmap source plus bounded staging) and never + copies generated device weights back to host. Entering the denoise stage + performs one asynchronous H2D pass; leaving it only restores zero-sized placeholders and releases the device buffers. This lets the following VAE stage reuse the same HBM. @@ -563,10 +726,15 @@ class PinnedResidentLayerGroup: device: torch.device, copy_stream: Any, pin_memory: bool, + rank_local_mmap: bool = False, + defer_staging: bool = False, + tensor_transforms: dict[int, Any] | None = None, ) -> None: self.device = device self.copy_stream = copy_stream self.loaded = False + self.rank_local_mmap = rank_local_mmap + self.pin_memory = pin_memory self._states: list[dict[str, Any]] = [] self._gpu_buffers: list[dict[torch.dtype, torch.Tensor]] = [] @@ -574,21 +742,49 @@ class PinnedResidentLayerGroup: params = dict(block.named_parameters()) bufs = dict(block.named_buffers()) targets: dict[str, torch.Tensor] = {**params, **bufs} - cpu_shards, metadata = DistributedLayerwiseOffloadHook._shard_and_pin( - params, - bufs, - dp_size=1, - rank=0, - pin_memory=pin_memory, - ) + if rank_local_mmap: + cpu_sources, metadata = DistributedLayerwiseOffloadHook._collect_mmap_sources( + params, + bufs, + tensor_transforms, + ) + cpu_shards = {} + else: + cpu_shards, metadata = DistributedLayerwiseOffloadHook._shard_and_pin( + params, + bufs, + dp_size=1, + rank=0, + pin_memory=pin_memory, + tensor_transforms=tensor_transforms, + ) + cpu_sources = {} self._states.append( { "targets": targets, "cpu_shards": cpu_shards, + "cpu_sources": cpu_sources, "metadata": metadata, } ) + self._cpu_staging_buffers: list[dict[torch.dtype, torch.Tensor]] = [] + self._cpu_staging_events: list[Any | None] = [None, None] + if rank_local_mmap and not defer_staging: + max_sizes: dict[torch.dtype, int] = {} + for state in self._states: + for dtype, metas in state["metadata"].items(): + total = sum(meta["numel"] for meta in metas) + max_sizes[dtype] = max(max_sizes.get(dtype, 0), total) + for _ in range(2): + buffers = {} + for dtype, total in max_sizes.items(): + buffer = torch.empty(total, dtype=dtype, device="cpu") + if pin_memory: + buffer = buffer.pin_memory() + buffers[dtype] = buffer + self._cpu_staging_buffers.append(buffers) + def load(self) -> None: if self.loaded: return @@ -599,20 +795,41 @@ class PinnedResidentLayerGroup: gpu_buffers: list[dict[torch.dtype, torch.Tensor]] = [] for state in self._states: block_buffers: dict[torch.dtype, torch.Tensor] = {} - for dtype, cpu_shard in state["cpu_shards"].items(): - block_buffers[dtype] = torch.empty( - cpu_shard.shape, - dtype=dtype, - device=self.device, - ) + for dtype, metas in state["metadata"].items(): + total = sum(meta["numel"] for meta in metas) + block_buffers[dtype] = torch.empty(total, dtype=dtype, device=self.device) gpu_buffers.append(block_buffers) self.copy_stream.wait_stream(current_omni_platform.current_stream()) ready = current_omni_platform.Event() with current_omni_platform.stream(self.copy_stream): - for state, block_buffers in zip(self._states, gpu_buffers): - for dtype, cpu_shard in state["cpu_shards"].items(): - block_buffers[dtype].copy_(cpu_shard, non_blocking=True) + for index, (state, block_buffers) in enumerate(zip(self._states, gpu_buffers)): + if self.rank_local_mmap: + slot = index % 2 + previous_copy = self._cpu_staging_events[slot] + if previous_copy is not None: + synchronize = getattr(previous_copy, "synchronize", None) + if callable(synchronize): + synchronize() + else: + current_omni_platform.synchronize() + cpu_weights = DistributedLayerwiseOffloadHook._pack_mmap_sources( + state["cpu_sources"], + state["metadata"], + self._cpu_staging_buffers[slot], + ) + else: + cpu_weights = state["cpu_shards"] + + for dtype, cpu_weight in cpu_weights.items(): + block_buffers[dtype].copy_( + cpu_weight, + non_blocking=cpu_weight.is_pinned(), + ) + if self.rank_local_mmap: + slot_ready = current_omni_platform.Event() + slot_ready.record(self.copy_stream) + self._cpu_staging_events[slot] = slot_ready ready.record(self.copy_stream) for state, block_buffers in zip(self._states, gpu_buffers): @@ -665,9 +882,12 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): overlapped with computation. """ - _MMAP_PARAM_ATTRS = ("weight_loader", "mmap_weight_transform") - - def __init__(self, config: OffloadConfig, device: torch.device): + def __init__( + self, + config: OffloadConfig, + device: torch.device, + host_weight_plan: HostWeightPlan | None = None, + ): super().__init__(config, device) self.copy_stream = current_omni_platform.Stream() @@ -679,6 +899,10 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): self._all_hook_groups: list[list[DistributedLayerwiseOffloadHook]] = [] self._resident_blocks: list[nn.Module] = [] self._resident_layer_group: PinnedResidentLayerGroup | None = None + self._using_mmap = False + self._using_rank_local_mmap = False + self.host_weight_plan = host_weight_plan + self._mmap_transforms_by_tensor_id: dict[int, Any] = {} def load_resident_layers(self) -> None: """Load the model-declared leading blocks for the denoise stage.""" @@ -690,93 +914,30 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): if self._resident_layer_group is not None: self._resident_layer_group.offload() - def _remember_mmap_param_attrs(self, pipeline: nn.Module) -> None: - """Save loader metadata before ``to_empty`` replaces Parameters.""" - self._mmap_param_attrs = { - name: {attr: value for attr in self._MMAP_PARAM_ATTRS if (value := getattr(param, attr, None)) is not None} - for name, param in pipeline.named_parameters() - } - - def _attach_mmap_param_attrs( + def _load_weights_via_mmap( self, - name: str, - replacement: nn.Parameter, - source: nn.Parameter | None = None, + pipeline: nn.Module, + modules, + plan: HostWeightPlan, ) -> None: - attrs = getattr(self, "_mmap_param_attrs", {}).get(name, {}) - for attr in self._MMAP_PARAM_ATTRS: - value = attrs.get(attr) - if value is None and source is not None: - value = getattr(source, attr, None) - if value is not None: - setattr(replacement, attr, value) - if attr == "mmap_weight_transform": - replacement.mmap_weight_transform_pending = True - - def _load_weights_via_mmap(self, pipeline: nn.Module, modules) -> None: - """Load DiT weights from safetensors via mmap views (no RSS). + """Load DiT checkpoint tensors as file-backed safetensors views. When the transformer is created on meta device, this method replaces meta params with mmap views of the checkpoint files. - The views point to OS page cache (shared across ranks), so - no private copy is created. _shard_and_pin then copies only - the 1/dp_size shard from the mmap view to a private buffer. + The views point to OS page cache shared across ranks. AllGather mode + copies only the rank's 1/dp_size shard to a persistent private buffer; + rank-local mode retains the views and packs one block at a time into + bounded staging storage. Non-DiT modules (VAE, encoders) are NOT affected — they were created on CPU with real weights via from_pretrained. """ - import glob - import json - from safetensors import safe_open - model_path = self.config.model_path - if not model_path: - logger.warning("No model_path for mmap weight loading, skipping") - return - - # Resolve HF repo ID to local snapshot path. - # If model_path is a local directory, use it directly; otherwise - # download the safetensors index + weight files via HuggingFace Hub. - if not os.path.isdir(model_path): - from vllm.model_executor.model_loader.weight_utils import ( - download_weights_from_hf, - ) - - logger.info("model_path %s is not local, downloading from HF", model_path) - model_path = download_weights_from_hf( - model_name_or_path=model_path, - cache_dir=None, - allow_patterns=["*.safetensors", "*.safetensors.index.json"], - ) - - # Build {checkpoint_key: file_path} from safetensors index - weight_map: dict[str, str] = {} - for idx_file in glob.glob(os.path.join(model_path, "**", "*.safetensors.index.json"), recursive=True): - idx_dir = os.path.dirname(idx_file) - with open(idx_file) as f: - idx = json.load(f) - for key, filename in idx.get("weight_map", {}).items(): - weight_map[key] = os.path.join(idx_dir, filename) - - if not weight_map: - single_files = glob.glob(os.path.join(model_path, "**", "model.safetensors"), recursive=True) - for sf in single_files: - from safetensors import safe_open as _safe_open - - f = _safe_open(sf, framework="pt", device="cpu") - for key in f.keys(): - weight_map[key] = sf - logger.info("Found single-file safetensors: %s (%d keys)", sf, len(f.keys())) - - if not weight_map: - raise RuntimeError( - "Distributed layerwise offload could not find safetensors " - f"weights at {model_path}. Expected *.safetensors.index.json " - "or model.safetensors. Ensure the checkpoint path is correct." - ) - - logger.info("Loading DiT weights via mmap (meta → page cache, no RSS): %d keys", len(weight_map)) + logger.info( + "Loading DiT weights via mmap (meta -> shared page cache): %d tensors", + len(plan.bindings), + ) # --- Convert DiT modules to meta device --- # The transformer was created normally (with random weights and @@ -789,36 +950,10 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): # __init__. Save them before meta conversion and restore after # mmap loading, so we don't need model-specific buffer rebuild code. # - # Fail closed for unsupported configurations BEFORE to_empty. - # The mmap path bypasses AutoWeightsLoader, which means TP-aware - # weight_loader callbacks (fused QKV, row-parallel, etc.) are not - # invoked. When TP > 1, these callbacks shard weights across TP - # ranks — skipping them produces incorrect weights. - # Check the actual TP world size from parallel_state (initialized - # before enable()). Note: params may have custom weight_loader - # attributes even at TP=1 (e.g. QKVParallelLinear), so we cannot - # reject based on attribute presence alone. - try: - from vllm.distributed.parallel_state import ( - get_tensor_model_parallel_world_size, - ) - - tp_world = get_tensor_model_parallel_world_size() - except Exception: - tp_world = 1 - if tp_world > 1: - raise ValueError( - "Distributed layerwise offload with mmap loading does not " - "support Tensor Parallel (TP > 1). TP-aware weight_loader " - "callbacks are bypassed by the mmap path. Use DP or SP " - "instead of TP, or disable distributed layerwise offload." - ) - - # ``Module.to_empty`` creates new Parameter objects and drops custom - # attributes installed by vLLM weight loaders. Preserve them by full - # pipeline name so mmap replacements can reproduce checkpoint layout - # transforms such as MiniMax-H3 grouped QKV. - self._remember_mmap_param_attrs(pipeline) + # The loader proved topology, coverage, source metadata, and adapter + # compatibility before it skipped ordinary weight materialization. + # This method only realizes that exact plan; it does not select or + # rediscover a checkpoint layout. # # Important: when DiT modules are nested (e.g. transformer contains @@ -861,189 +996,63 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): len(saved_buffers.get(id(dit_module), {})), ) - # Build reverse mapping {model_param_name: (ckpt_key, file_path)} using - # the pipeline's _remap_ckpt_key (if available). This handles the - # model-specific checkpoint key remapping (e.g., Cosmos3's - # layers.0.mlp_moe_gen.gate_proj → transformer.gen_layers.0.mlp.gate_proj). - remap_fn = getattr(type(pipeline), "_remap_ckpt_key", None) - model_to_ckpt: dict[str, tuple[str, str]] = {} - if remap_fn is not None: - for ckpt_key, file_path in weight_map.items(): - model_name = remap_fn(ckpt_key) - if model_name is not None: - model_to_ckpt[model_name] = (ckpt_key, file_path) - logger.info( - "Built reverse remap: %d model params from %d checkpoint keys", len(model_to_ckpt), len(weight_map) - ) - else: - # No remap function — use checkpoint keys directly - for ckpt_key, file_path in weight_map.items(): - model_to_ckpt[ckpt_key] = (ckpt_key, file_path) - - # Collect all meta params in DiT modules - dit_ids = set() - for dit_module in modules.dits: - dit_ids.update(id(m) for m in dit_module.modules()) - # Cache open file handles file_cache: dict[str, Any] = {} - loaded = 0 - skipped = 0 loaded_names: set[str] = set() - # Discover blocks first so we can distinguish block params (deferred - # to _shard_and_pin) from non-block params (loaded here as mmap views). - block_module_ids: set[int] = set() - for dit_module in modules.dits: - blocks_attr_names, blocks = get_blocks_from_dit(dit_module) - for block in blocks: - block_module_ids.update(id(m) for m in block.modules()) + # Realize the loader's exact runtime-name bindings. In particular, + # do not reconstruct names from DLO block discovery: storage planning + # and transfer topology are separate contracts. + for runtime_name, binding in plan.bindings.items(): + parent_path, _, leaf_name = runtime_name.rpartition(".") + try: + parent = pipeline.get_submodule(parent_path) + except AttributeError as exc: + raise RuntimeError(f"Host-weight plan target module {parent_path!r} no longer exists") from exc - for name, param in pipeline.named_parameters(): - if not (hasattr(param, "is_meta") and param.is_meta): - continue + target = parent._parameters.get(leaf_name) + is_parameter = target is not None + if target is None: + target = parent._buffers.get(leaf_name) + if target is None: + raise RuntimeError(f"Host-weight plan target tensor {runtime_name!r} no longer exists") - # Check if this param is inside a block (will be handled by - # _shard_and_pin via mmap views in section 2 below). - parent = pipeline - parts = name.split(".") - is_block_param = False - for part in parts[:-1]: - try: - if part.isdigit(): - parent = parent[int(part)] - else: - parent = getattr(parent, part) - except (AttributeError, IndexError, TypeError): - break - if id(parent) in block_module_ids: - is_block_param = True - break - - if is_block_param: - # Block param — will be loaded as mmap view in section 2 - continue - - # Non-block param (merger, patch_embed, norm, proj_out, etc.) - # Look up in reverse remap mapping (handles Cosmos3 key remapping). - entry = model_to_ckpt.get(name) - if entry is None: - skipped += 1 - continue - - ckpt_key, file_path = entry - if file_path not in file_cache: - file_cache[file_path] = safe_open(file_path, framework="pt", device="cpu") - f = file_cache[file_path] - tensor = f.get_tensor(ckpt_key) - - # Replace meta param with mmap view (no copy!) - parent = pipeline - for part in parts[:-1]: - if part.isdigit(): - parent = parent[int(part)] - else: - parent = getattr(parent, part) - replacement = torch.nn.Parameter(tensor, requires_grad=param.requires_grad) - self._attach_mmap_param_attrs(name, replacement, param) - parent._parameters[parts[-1]] = replacement - loaded += 1 - loaded_names.add(name) - - logger.info("Mmap weight loading: %d non-block params loaded, %d skipped", loaded, skipped) - - # Now load DiT block params from safetensors using mmap views. - # Each block param is assigned an mmap view (no RSS). - # _shard_and_pin will later copy the shard portion to a private buffer. - block_loaded = 0 - for dit_idx, dit_module in enumerate(modules.dits): - dit_name = modules.dit_names[dit_idx] - blocks_attr_names, blocks = get_blocks_from_dit(dit_module) - if not blocks: - continue - - # Determine the blocks_attr for this DiT module (e.g. "layers" or "gen_layers") - blocks_attr = None - for attr_name in blocks_attr_names: - if hasattr(dit_module, attr_name): - candidate = getattr(dit_module, attr_name) - if isinstance(candidate, nn.ModuleList) and len(candidate) > 1: - blocks_attr = attr_name - break - - for block_idx, block in enumerate(blocks): - # Build the full model param prefix for this block - # e.g. "transformer.language_model.layers.0" or "transformer.gen_layers.0" - block_full_prefix = f"{dit_name}.{blocks_attr}.{block_idx}" - - for bname, bparam in block.named_parameters(): - if not (hasattr(bparam, "is_meta") and bparam.is_meta): - continue - - # Use reverse remap to find checkpoint key - full_param_name = f"{block_full_prefix}.{bname}" - entry = model_to_ckpt.get(full_param_name) - if entry is None: - skipped += 1 - continue - - ckpt_key, file_path = entry - if file_path not in file_cache: - file_cache[file_path] = safe_open(file_path, framework="pt", device="cpu") - f = file_cache[file_path] - tensor = f.get_tensor(ckpt_key) - - # Replace meta param with mmap view - parent = block - bparts = bname.split(".") - for part in bparts[:-1]: - if part.isdigit(): - parent = parent[int(part)] - else: - parent = getattr(parent, part) - replacement = torch.nn.Parameter(tensor, requires_grad=bparam.requires_grad) - self._attach_mmap_param_attrs(full_param_name, replacement, bparam) - parent._parameters[bparts[-1]] = replacement - block_loaded += 1 - loaded_names.add(full_param_name) - - logger.info("Mmap block loading: %d params loaded, %d skipped", block_loaded, skipped) - - # Strict validation: check that all meta params were loaded. - # This replaces the validation that AutoWeightsLoader.load_weights() - # performs in the regular load path. - remaining_meta = [ - name for name, param in pipeline.named_parameters() if hasattr(param, "is_meta") and param.is_meta - ] - if remaining_meta: - # Filter out params inside submodules that will be loaded later - # by _load_module_weights_from_mmap (non-block submodules) - dit_param_names = set() - for dit_module in modules.dits: - for pname, _ in dit_module.named_parameters(): - dit_param_names.add(pname) - truly_missing = [n for n in remaining_meta if n in dit_param_names] - if truly_missing: - logger.warning( - "Mmap loading: %d params still on meta device after " - "loading (first 5: %s). These may be loaded later by " - "_load_module_weights_from_mmap or are expected to be " - "meta (e.g. DTensor placeholders).", - len(truly_missing), - truly_missing[:5], + if binding.file_path not in file_cache: + file_cache[binding.file_path] = safe_open( + binding.file_path, + framework="pt", + device="cpu", ) + tensor = file_cache[binding.file_path].get_tensor(binding.checkpoint_key) + if is_parameter: + replacement = torch.nn.Parameter(tensor, requires_grad=target.requires_grad) + parent._parameters[leaf_name] = replacement + else: + replacement = tensor + parent._buffers[leaf_name] = replacement + loaded_names.add(runtime_name) + + logger.info("Realized %d loader-planned tensors as mmap views", len(loaded_names)) # Keep file handles open — _shard_and_pin will read from the mmap views. # They will be released after _shard_and_pin completes (when params are # replaced with offload placeholders). self._mmap_file_cache = file_cache - self._mmap_model_to_ckpt = model_to_ckpt - # The regular loader runs model-specific post-load transforms after # assigning checkpoint tensors. The mmap path bypasses that loader, so # preserve the same lifecycle for transforms such as Cosmos3's fp32 # timestep embedder. - for dit_module in modules.dits: + for dit_name, dit_module in zip(modules.dit_names, modules.dits): + # Restore non-persistent buffers before post-load hooks and strict + # validation. They are constructor-derived and intentionally have + # no checkpoint binding. + bufs = saved_buffers.get(id(dit_module), {}) + for name, buf in bufs.items(): + parent_path, _, leaf_name = name.rpartition(".") + parent = dit_module.get_submodule(parent_path) + restore_device = torch.device("cpu") if self._using_rank_local_mmap else self.device + parent._buffers[leaf_name] = buf.to(restore_device) + post_load_weights = getattr(dit_module, "post_load_weights", None) if callable(post_load_weights): post_load_weights() @@ -1054,73 +1063,45 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): # load path (e.g. Cosmos3 checks for missing audio/action weights). validate = getattr(dit_module, "validate_loaded_weights", None) if callable(validate): - validate(loaded_names) + local_loaded_names = { + name.removeprefix(f"{dit_name}.") for name in loaded_names if name.startswith(f"{dit_name}.") + } + validate(local_loaded_names) - # Restore non-persistent buffers that were saved before meta - # conversion. These buffers (e.g. RoPE inv_freq, timestep - # freqs) are computed from formulas in __init__ and are not - # present in the checkpoint. By saving and restoring them - # generically, we avoid model-specific buffer rebuild code. - bufs = saved_buffers.get(id(dit_module), {}) - for name, buf in bufs.items(): - parent = dit_module - parts = name.split(".") - for part in parts[:-1]: - if part.isdigit(): - parent = parent[int(part)] - else: - parent = getattr(parent, part) - # Place on the same device as the module's parameters - # Place on the offloader's target device (not the module's - # current device, which may still be meta at this point). - parent._buffers[parts[-1]] = buf.to(self.device) - - def _load_module_weights_from_mmap(self, module: nn.Module, dit_name: str, child_name: str) -> None: - """Load a non-block DiT submodule's weights from safetensors via mmap.""" - from safetensors import safe_open - - if not hasattr(self, "_mmap_file_cache") or not self._mmap_file_cache: - return - - file_cache = self._mmap_file_cache - model_to_ckpt = getattr(self, "_mmap_model_to_ckpt", {}) - loaded = 0 - - for pname, param in module.named_parameters(): - if not (hasattr(param, "is_meta") and param.is_meta): + # Post-load hooks may rebind parameters (for example while casting a + # submodule), so associate deferred bounded transforms with the final + # runtime tensor objects rather than the initial mmap replacements. + self._mmap_transforms_by_tensor_id.clear() + for runtime_name, binding in plan.bindings.items(): + if binding.transform is None: continue + parent_path, _, leaf_name = runtime_name.rpartition(".") + parent = pipeline.get_submodule(parent_path) + target = parent._parameters.get(leaf_name) + if target is None: + target = parent._buffers.get(leaf_name) + if target is None: + raise RuntimeError( + f"Host-weight transform target {runtime_name!r} no longer exists after post-load processing" + ) + self._mmap_transforms_by_tensor_id[id(target)] = binding.transform - # Build full model param name and look up in reverse remap - full_name = f"{dit_name}.{child_name}.{pname}" - entry = model_to_ckpt.get(full_name) - if entry is None: - # Try without dit_name prefix - full_name = f"{child_name}.{pname}" - entry = model_to_ckpt.get(full_name) - if entry is None: - continue - - ckpt_key, file_path = entry - if file_path not in file_cache: - file_cache[file_path] = safe_open(file_path, framework="pt", device="cpu") - f = file_cache[file_path] - tensor = f.get_tensor(ckpt_key) - - # Replace meta param with mmap view - parent = module - parts = pname.split(".") - for part in parts[:-1]: - if part.isdigit(): - parent = parent[int(part)] - else: - parent = getattr(parent, part) - replacement = torch.nn.Parameter(tensor, requires_grad=param.requires_grad) - self._attach_mmap_param_attrs(full_name, replacement, param) - parent._parameters[parts[-1]] = replacement - loaded += 1 - - if loaded > 0: - logger.info("Loaded %d params for submodule %s.%s via mmap", loaded, dit_name, child_name) + remaining_meta: list[str] = [] + for dit_name, dit_module in zip(modules.dit_names, modules.dits): + remaining_meta.extend( + f"{dit_name}.{name}" for name, tensor in dit_module.named_parameters() if tensor.is_meta + ) + for name, tensor in dit_module.named_buffers(): + parent_path, _, leaf_name = name.rpartition(".") + owner = dit_module.get_submodule(parent_path) + if leaf_name not in owner._non_persistent_buffers_set and tensor.is_meta: + remaining_meta.append(f"{dit_name}.{name}") + if remaining_meta: + raise RuntimeError( + "The prevalidated host-weight plan left " + f"{len(remaining_meta)} DiT tensors on the meta device " + f"(first 5: {remaining_meta[:5]})." + ) def _init_dp_group(self) -> None: """Reuse the process group initialized by parallel_state. @@ -1314,6 +1295,8 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): self.comm_stream, self.config.pin_cpu_memory, shared_buffers=[None, None], + rank_local_mmap=self._using_rank_local_mmap, + tensor_transforms=self._mmap_transforms_by_tensor_id, ) sub_hooks = [last_hook] for i, block in enumerate(blocks[:-1]): @@ -1329,6 +1312,8 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): self.comm_stream, self.config.pin_cpu_memory, shared_buffers=[None, None], + rank_local_mmap=self._using_rank_local_mmap, + tensor_transforms=self._mmap_transforms_by_tensor_id, ) sub_hooks.append(hook) @@ -1371,9 +1356,6 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): logger.debug("Skipped blocks module %s", name) continue - has_meta = any(getattr(param, "is_meta", False) for param in module.parameters()) - if has_meta: - self._load_module_weights_from_mmap(module, dit_name, name) module_mb = ( sum( param.nelement() * param.element_size() if not getattr(param, "is_meta", False) else 0 @@ -1394,7 +1376,6 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): try: module.to(self.device) except (NotImplementedError, RuntimeError): - self._load_module_weights_from_mmap(module, dit_name, name) # Non-persistent buffers such as RoPE frequencies do not # exist in the checkpoint and must be reconstructed. has_meta_buffer = any(getattr(buffer, "is_meta", False) for buffer in module.buffers(recurse=True)) @@ -1437,6 +1418,10 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): modules = ModuleDiscovery.discover(pipeline) if not modules.dits: + if self.host_weight_plan is not None: + raise RuntimeError( + "DLO received a loader-owned host-weight plan, but no DiT modules were discovered to consume it" + ) logger.warning("No DiT/transformer modules found, skipping distributed layer-wise offloading") return @@ -1451,33 +1436,40 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): self.config.dlo_resident_layers, ) - # Load weights via mmap for DLO+AllGather. - # TODO(offload): Add a rank-local mmap path for dlo_no_use_allgather. - # It must apply declarative checkpoint-to-runtime adapters before the - # standard TP loader shards each block, and must cover staged encoders - # and VAEs as well as DiT blocks. The current AllGather mmap path is - # deliberately not reused because it changes the weight layout. - # Gate condition MUST match diffusers_loader.py: - # supports_mmap_loading(pipeline) and not _has_online_quant - # When the gate is False, the loader has already loaded weights via - # regular load_weights() + _process_weights_after_loading(). - if self.config.dlo_use_allgather and self.dp_size > 1: - _has_online_quant = any( - getattr(getattr(module, "quant_method", None), "uses_meta_device", False) - for module in pipeline.modules() + # Storage selection belongs to the loader. DLO consumes the exact + # prevalidated plan that caused the loader to skip materialization; + # without a plan, all weights must already come from the ordinary + # loader. The transfer protocol is selected independently below. + self._using_mmap = self.host_weight_plan is not None + self._using_rank_local_mmap = self._using_mmap and self.dp_size <= 1 + if self._using_mmap: + if self.host_weight_plan.backing_kind != "checkpoint_mmap": + raise ValueError(f"Unsupported DLO host-weight backing: {self.host_weight_plan.backing_kind}") + self._load_weights_via_mmap( + pipeline, + modules, + self.host_weight_plan, ) - if _has_online_quant: - raise ValueError( - "Online quantization is incompatible with DLO+AllGather: " - "the sharding + AllGather mechanism flattens weights by " - "dtype, which breaks quantized weight/scale layouts. " - "Please use --dlo-no-use-allgather or disable online " - "quantization." + if self._using_rank_local_mmap: + logger.info( + "DLO rank-local mmap storage enabled: checkpoint pages are " + "node-shared; each worker owns only two bounded host staging slots" ) - if supports_mmap_loading(pipeline): - self._load_weights_via_mmap(pipeline, modules) - else: - logger.info("Weights loaded via regular loader — skipping mmap (model does not support mmap)") + else: + remaining_meta = [ + name + for dit_name, dit_module in zip(modules.dit_names, modules.dits) + for name, tensor in chain( + dit_module.named_parameters(), + dit_module.named_buffers(), + ) + if getattr(tensor, "is_meta", False) + ] + if remaining_meta: + raise RuntimeError( + f"DLO received meta tensors without a loader-owned host-weight plan (first 5: {remaining_meta[:5]})" + ) + logger.info("DLO is using host tensors materialized by the ordinary loader") # Keep VAE/encoders on CPU; move to GPU on-demand via hooks. # This saves ~4.3 GB HBM per card (VAE 1.3 + encoder 1.1 + sound 1.9) @@ -1577,6 +1569,8 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): self.comm_stream, self.config.pin_cpu_memory, shared_buffers=[None, None], + rank_local_mmap=self._using_rank_local_mmap, + tensor_transforms=self._mmap_transforms_by_tensor_id, ) block_hooks: list[DistributedLayerwiseOffloadHook] = [last_hook] @@ -1593,6 +1587,8 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): self.comm_stream, self.config.pin_cpu_memory, shared_buffers=[None, None], + rank_local_mmap=self._using_rank_local_mmap, + tensor_transforms=self._mmap_transforms_by_tensor_id, ) block_hooks.append(hook) @@ -1621,11 +1617,16 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): self.device, self.copy_stream, self.config.pin_cpu_memory, + rank_local_mmap=self._using_rank_local_mmap, + defer_staging=bool(self._all_hook_groups), + tensor_transforms=self._mmap_transforms_by_tensor_id, ) pipeline._dlo_residency_controller = self if not self._all_hook_groups: self.enabled = bool(self._resident_blocks) + if self._using_mmap and not self.enabled: + self._release_mmap_handles() return # Unified allocation: 2 shared output buffers + 2 shared shard buffers @@ -1639,6 +1640,21 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): unified_shard_buffers = None if self.dp_size > 1: unified_shard_buffers = self._allocate_shared_shard_buffers(all_hooks) + unified_cpu_staging = None + cpu_staging_events = None + if self._using_rank_local_mmap: + unified_cpu_staging = self._allocate_shared_cpu_staging_buffers( + all_hooks, + self._resident_layer_group, + ) + cpu_staging_events = [None, None] + if self._resident_layer_group is not None: + # Resident and streamed layers execute in the same stage and + # reuse the same host slots. Events serialize slot reuse. + self._resident_layer_group._cpu_staging_buffers = [ + buffers for buffers in unified_cpu_staging if buffers is not None + ] + self._resident_layer_group._cpu_staging_events = cpu_staging_events # Shared slot-group tracker: _shared_slot_group[slot] = group_id # that last wrote to that slot. Group-first hooks use this to @@ -1651,6 +1667,9 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): hook._owns_buffers = False if unified_shard_buffers is not None: hook.gpu_shard_buffers = unified_shard_buffers + if unified_cpu_staging is not None and cpu_staging_events is not None: + hook.cpu_staging_buffers = unified_cpu_staging + hook.cpu_staging_events = cpu_staging_events hook._group_id = group_idx hook._shared_slot_group = shared_slot_group @@ -1677,9 +1696,11 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): self.enabled = True - # Release mmap file handles — _shard_and_pin has copied all shards, - # params now point to offload placeholders (not mmap views). - self._release_mmap_handles() + if self._using_mmap and not self._using_rank_local_mmap: + # AllGather mode copied each rank's persistent shard, so the source + # mappings are no longer needed. Rank-local mode retains them as + # the node-shared host master until disable(). + self._release_mmap_handles() # Assign GPU buffers to sharded on-demand modules (VAE/encoders). # Each module gets a dedicated input (shard-sized) and output @@ -1702,11 +1723,10 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): def _release_mmap_handles(self) -> None: """Release safetensors mmap file handles.""" + self._mmap_transforms_by_tensor_id.clear() if hasattr(self, "_mmap_file_cache"): self._mmap_file_cache.clear() del self._mmap_file_cache - if hasattr(self, "_mmap_model_to_ckpt"): - del self._mmap_model_to_ckpt logger.info("Released safetensors mmap file handles") def _cleanup_after_loading(self) -> None: @@ -1723,9 +1743,12 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): pass def disable(self) -> None: - if not self.enabled: + if not self.enabled and not hasattr(self, "_mmap_file_cache"): return + if self._using_rank_local_mmap: + current_omni_platform.synchronize() + for blocks in self._blocks: for block in blocks: remove_distributed_block_hook(block) @@ -1740,6 +1763,9 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): self._all_hook_groups.clear() self._resident_blocks.clear() self._resident_layer_group = None + self._release_mmap_handles() + self._using_mmap = False + self._using_rank_local_mmap = False self.enabled = False logger.info("Distributed layer-wise offloading disabled") @@ -1779,6 +1805,41 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend): ) return shared_buffers + @staticmethod + def _allocate_shared_cpu_staging_buffers( + hooks: list[DistributedLayerwiseOffloadHook], + resident_group: PinnedResidentLayerGroup | None = None, + ) -> list[dict[torch.dtype, torch.Tensor] | None]: + """Allocate two bounded host slots for rank-local mmap -> device copies.""" + max_sizes: dict[torch.dtype, int] = {} + for hook in hooks: + for dtype, metas in hook.metadata.items(): + total = sum(meta["numel"] for meta in metas) + max_sizes[dtype] = max(max_sizes.get(dtype, 0), total) + if resident_group is not None: + for state in resident_group._states: + for dtype, metas in state["metadata"].items(): + total = sum(meta["numel"] for meta in metas) + max_sizes[dtype] = max(max_sizes.get(dtype, 0), total) + + pin_memory = hooks[0].pin_memory if hooks else bool(resident_group and resident_group.pin_memory) + shared_staging: list[dict[torch.dtype, torch.Tensor] | None] = [None, None] + for slot in range(2): + buffers: dict[torch.dtype, torch.Tensor] = {} + for dtype, total_numel in max_sizes.items(): + buffer = torch.empty(total_numel, dtype=dtype, device="cpu") + if pin_memory: + buffer = buffer.pin_memory() + buffers[dtype] = buffer + shared_staging[slot] = buffers + + logger.info( + "Allocated 2 shared host staging buffers for rank-local mmap (max block size: %s, pinned=%s)", + {str(k): f"{v * _dtype_size(k) / 1024 / 1024:.1f}MB" for k, v in max_sizes.items()}, + pin_memory, + ) + return shared_staging + @staticmethod def _allocate_shared_shard_buffers( hooks: list[DistributedLayerwiseOffloadHook], diff --git a/vllm_omni/diffusion/offloader/offload_plan.py b/vllm_omni/diffusion/offloader/offload_plan.py index f8943757e..2e1443df7 100644 --- a/vllm_omni/diffusion/offloader/offload_plan.py +++ b/vllm_omni/diffusion/offloader/offload_plan.py @@ -55,20 +55,3 @@ class OffloadPlan: def get_offload_plan(pipeline: nn.Module) -> OffloadPlan | None: """Retrieve the OffloadPlan declared by the pipeline, if any.""" return getattr(pipeline, "_offload_plan", None) - - -def supports_mmap_loading(pipeline: nn.Module) -> bool: - """Whether the pipeline supports mmap weight loading. - - A pipeline supports mmap if any module defines ``_remap_ckpt_key``, - which maps checkpoint keys to model parameter names. Models without - this method must use the regular ``load_weights()`` path. - - This check is shared between ``diffusers_loader.py`` (to decide whether - to skip ``load_weights``) and ``DistributedLayerwiseOffloadBackend.enable()`` - (to decide whether to call ``_load_weights_via_mmap``), ensuring both - paths use the same gate condition. - """ - return bool(getattr(pipeline, "_supports_mmap_loading", True)) and any( - callable(getattr(type(m), "_remap_ckpt_key", None)) for m in pipeline.modules() - ) diff --git a/vllm_omni/diffusion/worker/diffusion_model_runner.py b/vllm_omni/diffusion/worker/diffusion_model_runner.py index d118a9833..4a763d651 100644 --- a/vllm_omni/diffusion/worker/diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/diffusion_model_runner.py @@ -292,7 +292,11 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): ) # Apply CPU offloading - self.offload_backend = get_offload_backend(self.od_config, device=self.device) + self.offload_backend = get_offload_backend( + self.od_config, + device=self.device, + host_weight_plan=model_loader.take_host_weight_plan(), + ) if self.offload_backend is not None: logger.info(f" Enabling offloader backend: {self.offload_backend.__class__.__name__}") self.offload_backend.enable(self.pipeline) From baba7d1e2ef9d326486493146b43157f7cd4f8d0 Mon Sep 17 00:00:00 2001 From: Yukim1 <121286183+zwhzzz0821@users.noreply.github.com> Date: Sun, 16 Aug 2026 11:22:28 +0800 Subject: [PATCH 038/267] [Refactor] Remove legacy stage_args YAML loader (#6200) Signed-off-by: zwhzzz0821 <2831474076@qq.com> Co-authored-by: Hongsheng Liu --- .../offline_inference/aura_omni/end2end.py | 2 - examples/offline_inference/lance/end2end.py | 3 - .../nemotron_voicechat/end2end.py | 6 +- .../text_to_speech/fish_speech/end2end.py | 14 +--- .../qwen3_tts/word_timestamps.py | 8 +- tests/config/test_config_factory.py | 10 ++- .../test_hunyuanimage3_i2t_expansion.py | 9 +-- .../test_async_omni_diffusion_config.py | 18 +++++ tests/entrypoints/test_utils.py | 73 +++-------------- vllm_omni/config/stage_config.py | 5 ++ vllm_omni/engine/async_omni_engine.py | 10 +-- vllm_omni/entrypoints/utils.py | 79 ++++--------------- 12 files changed, 73 insertions(+), 164 deletions(-) diff --git a/examples/offline_inference/aura_omni/end2end.py b/examples/offline_inference/aura_omni/end2end.py index bac385d8e..29ccb3539 100644 --- a/examples/offline_inference/aura_omni/end2end.py +++ b/examples/offline_inference/aura_omni/end2end.py @@ -24,7 +24,6 @@ from vllm.assets.video import VideoAsset, video_to_ndarrays from vllm.multimodal.media.audio import load_audio from vllm.utils.argparse_utils import FlexibleArgumentParser -from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults from vllm_omni.entrypoints.omni import Omni from vllm_omni.model_executor.stage_input_processors.aura_omni import ( DEFAULT_QWEN3_TTS_REF_TEXT, @@ -228,7 +227,6 @@ def parse_args(): parser.add_argument("--init-timeout", type=int, default=2000, help="Overall initialization timeout.") parser.add_argument("--stage-init-timeout", type=int, default=2000, help="Per-stage initialization timeout.") parser.add_argument("--log-stats", action="store_true", default=False, help="Enable stats logging.") - nullify_stage_engine_defaults(parser) return parser.parse_args() diff --git a/examples/offline_inference/lance/end2end.py b/examples/offline_inference/lance/end2end.py index d26910161..2b70bd749 100644 --- a/examples/offline_inference/lance/end2end.py +++ b/examples/offline_inference/lance/end2end.py @@ -82,9 +82,6 @@ def parse_args(): # ``max_num_seqs=1`` …) are passed as flat kwargs to ``Omni`` below # and ``create_default_diffusion`` materializes the stage config. - from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults - - nullify_stage_engine_defaults(p) return p.parse_args() diff --git a/examples/offline_inference/nemotron_voicechat/end2end.py b/examples/offline_inference/nemotron_voicechat/end2end.py index b020e5bad..ac5fac376 100644 --- a/examples/offline_inference/nemotron_voicechat/end2end.py +++ b/examples/offline_inference/nemotron_voicechat/end2end.py @@ -67,7 +67,7 @@ def main() -> None: parser.add_argument("--output-dir", default="results/nemotron_voicechat") parser.add_argument("--system-prompt", default=DEFAULT_SYSTEM_PROMPT) parser.add_argument( - "--stage-configs-path", + "--deploy-config", default=None, help=( "Deploy yaml override. Pass vllm_omni/deploy/nemotron_labs_voicechat_streaming.yaml " @@ -134,8 +134,8 @@ def main() -> None: ] omni_kwargs: dict = {} - if args.stage_configs_path: - omni_kwargs["stage_configs_path"] = args.stage_configs_path + if args.deploy_config: + omni_kwargs["deploy_config"] = args.deploy_config omni = Omni(model=str(ckpt), trust_remote_code=True, **omni_kwargs) outputs = omni.generate(inputs, sampling_params_list) diff --git a/examples/offline_inference/text_to_speech/fish_speech/end2end.py b/examples/offline_inference/text_to_speech/fish_speech/end2end.py index 8ce4664b9..e2e5738fd 100644 --- a/examples/offline_inference/text_to_speech/fish_speech/end2end.py +++ b/examples/offline_inference/text_to_speech/fish_speech/end2end.py @@ -38,17 +38,6 @@ from vllm_omni.utils.tracking_parser import TrackingArgumentParser logger = logging.getLogger(__name__) DEFAULT_MODEL = "fishaudio/s2-pro" -DEFAULT_DEPLOY_CONFIG = os.path.join( - os.path.dirname(__file__), - "..", - "..", - "..", - "..", - "vllm_omni", - "model_executor", - "stage_configs", - "fish_speech_s2_pro.yaml", -) def build_prompt( @@ -152,7 +141,8 @@ def _build_omni_kwargs(args, model_name: str) -> dict: log_stats=args.log_stats, stage_init_timeout=args.stage_init_timeout, ) - omni_kwargs["deploy_config"] = args.deploy_config or DEFAULT_DEPLOY_CONFIG + if args.deploy_config: + omni_kwargs["deploy_config"] = args.deploy_config return omni_kwargs diff --git a/examples/offline_inference/text_to_speech/qwen3_tts/word_timestamps.py b/examples/offline_inference/text_to_speech/qwen3_tts/word_timestamps.py index 62f5e2caa..7b282c560 100644 --- a/examples/offline_inference/text_to_speech/qwen3_tts/word_timestamps.py +++ b/examples/offline_inference/text_to_speech/qwen3_tts/word_timestamps.py @@ -32,11 +32,10 @@ from end2end import _estimate_prompt_len from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm_omni import Omni -from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults from vllm_omni.utils.forced_aligner import align, build_forced_aligner_config -def _default_stage_config() -> str: +def _default_deploy_config() -> str: repo_root = Path(__file__).resolve().parents[4] return str(repo_root / "vllm_omni" / "deploy" / "qwen3_tts.yaml") @@ -109,7 +108,7 @@ def main(args: Any) -> None: "output_dir", ): omni_kwargs.pop(key, None) - omni_kwargs["stage_configs_path"] = args.stage_configs_path or _default_stage_config() + omni_kwargs["deploy_config"] = args.deploy_config or _default_deploy_config() omni_kwargs["log_stats"] = args.log_stats omni = Omni(**omni_kwargs) @@ -156,7 +155,7 @@ def parse_args() -> Any: default=None, help="Optional YAML file for forced aligner settings (incl. gpu_memory_utilization)", ) - parser.add_argument("--stage-configs-path", default=None, help="Qwen3-TTS deploy YAML") + parser.add_argument("--deploy-config", default=None, help="Qwen3-TTS deploy YAML") parser.add_argument("--text", default="Hello world.", help="Text to synthesize and align") parser.add_argument("--language", default="English", help="Qwen3-TTS language field") parser.add_argument("--speaker", default="Vivian", help="CustomVoice speaker name") @@ -164,7 +163,6 @@ def parse_args() -> Any: parser.add_argument("--max-new-tokens", type=int, default=2048, help="TTS max_new_tokens") parser.add_argument("--output-dir", default="output_audio", help="Directory for WAV and JSON sidecar") parser.add_argument("--log-stats", action="store_true", default=False, help="Enable vLLM-Omni stats logging") - nullify_stage_engine_defaults(parser) return parser.parse_args() diff --git a/tests/config/test_config_factory.py b/tests/config/test_config_factory.py index 5bfbb67e8..41fc15df5 100644 --- a/tests/config/test_config_factory.py +++ b/tests/config/test_config_factory.py @@ -1102,6 +1102,13 @@ class TestResolveScheduler: class TestDeployConfigLoading: + def test_rejects_legacy_stage_args_schema(self, tmp_path): + deploy_path = tmp_path / "legacy.yaml" + deploy_path.write_text("stage_args:\n - stage_id: 0\n", encoding="utf-8") + + with pytest.raises(ValueError, match=r"stage_args.*PipelineConfig.*stages"): + load_deploy_config(deploy_path) + def test_load_minicpmo_duplex_deploy_config(self): deploy_path = Path(get_deploy_config_path("minicpmo_4_5_duplex.yaml")) @@ -1377,7 +1384,8 @@ stages: assert async_stages[1].custom_process_input_func is None def test_no_bundled_legacy_stage_config_yamls(self): - stage_config_dir = Path(__file__).parent.parent / "vllm_omni" / "model_executor" / "stage_configs" + repo_root = Path(__file__).resolve().parents[2] + stage_config_dir = repo_root / "vllm_omni" / "model_executor" / "stage_configs" assert not list(stage_config_dir.glob("*.yaml")) def test_merge_pipeline_deploy(self): diff --git a/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py b/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py index 84a9291b0..c97de0069 100644 --- a/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py +++ b/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py @@ -3,19 +3,18 @@ """Smoke test for HunyuanImage-3.0 Image-to-Text (I2T) pipeline.""" from collections.abc import Generator -from pathlib import Path import pytest import torch from PIL import Image from tests.helpers.runtime import OmniRunner +from tests.helpers.stage_config import get_deploy_config_path from vllm_omni import Omni from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import build_prompt MODEL_NAME = "tencent/HunyuanImage-3.0-Instruct" -REPO_ROOT = Path(__file__).resolve().parents[3] -STAGE_CONFIG_PATH = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "hunyuan_image3_i2t.yaml" +DEPLOY_CONFIG_PATH = get_deploy_config_path("hunyuan_image3_ar.yaml") # First 20 generated token IDs from the HF greedy reference on this input. # vllm-omni AR output matches this prefix bitwise; the two implementations @@ -45,14 +44,14 @@ EXPECTED_PREFIX_TOKEN_IDS: list[int] = [ # Decoded form, kept only for human-readable assertion messages. EXPECTED_PREFIX_TEXT = "The image is a solid, uniform green color with no variations, objects, or details present. It" -pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] +pytestmark = [pytest.mark.full_model, pytest.mark.diffusion, pytest.mark.cuda] @pytest.fixture(scope="module") def omni() -> Generator[Omni, None, None]: with OmniRunner( MODEL_NAME, - deploy_config=str(STAGE_CONFIG_PATH), + deploy_config=DEPLOY_CONFIG_PATH, trust_remote_code=True, ) as runner: yield runner.omni diff --git a/tests/entrypoints/test_async_omni_diffusion_config.py b/tests/entrypoints/test_async_omni_diffusion_config.py index 75a9de2a7..1daca1e99 100644 --- a/tests/entrypoints/test_async_omni_diffusion_config.py +++ b/tests/entrypoints/test_async_omni_diffusion_config.py @@ -492,6 +492,24 @@ def test_resolve_stage_configs_injects_additional_config_into_diffusion_stage(mo assert stage_configs[1].engine_args.additional_config == {"torchair_graph_config": {"enabled": True}} +@pytest.mark.parametrize( + ("legacy_arg", "value"), + [ + ("stage_configs_path", "legacy.yaml"), + ("stage_configs", [{"stage_id": 0}]), + ], +) +def test_resolve_stage_configs_rejects_legacy_config_arguments(legacy_arg, value): + engine = AsyncOmniEngine.__new__(AsyncOmniEngine) + + with pytest.raises(ValueError, match=rf"`{legacy_arg}`.*`deploy_config`"): + engine._resolve_stage_configs( + "dummy-model", + {legacy_arg: value}, + trust_remote_code=False, + ) + + def test_default_stage_config_includes_quantization_config(): """Ensure structured quantization_config survives default diffusion-stage creation.""" quantization_config = { diff --git a/tests/entrypoints/test_utils.py b/tests/entrypoints/test_utils.py index 803097dcf..66c7d5c30 100644 --- a/tests/entrypoints/test_utils.py +++ b/tests/entrypoints/test_utils.py @@ -27,7 +27,7 @@ from vllm_omni.entrypoints.utils import ( filter_dataclass_kwargs, filter_stages, load_and_resolve_stage_configs, - load_stage_configs_from_yaml, + load_stage_configs_from_model, resolve_model_config_path, ) @@ -520,71 +520,20 @@ class TestLoadAndResolveStageConfigs: assert filtered[0].final_output_type is None -class TestLoadStageConfigsFromYaml: - """Regression tests for stage-config loading and merging.""" - - def test_deep_merges_stage_engine_args(self, mocker: MockerFixture): - yaml_config = create_config( - { - "async_chunk": True, - "stage_args": [ - { - "stage_id": 0, - "runtime": {"device": 0}, - "engine_args": { - "parallel_config": {"tensor_parallel_size": 4}, - }, - } - ], - } - ) +class TestLoadStageConfigsFromModel: + def test_unresolved_model_does_not_fall_back_to_yaml(self, mocker: MockerFixture, caplog): mocker.patch( - "vllm_omni.entrypoints.utils.load_yaml_config", - return_value=yaml_config, + "vllm_omni.entrypoints.utils.StageConfigFactory.create_legacy_stage_configs_from_model", + return_value=(None, None), ) + resolve_path = mocker.patch("vllm_omni.entrypoints.utils.resolve_model_config_path") - stages = load_stage_configs_from_yaml( - "fake.yaml", - base_engine_args={ - "parallel_config": { - "tensor_parallel_size": 1, - "pipeline_parallel_size": 2, - }, - "model": "base-model", - }, - ) + result = load_stage_configs_from_model("unregistered-model", trust_remote_code=False) - merged_engine_args = stages[0]["engine_args"] - assert merged_engine_args["parallel_config"]["tensor_parallel_size"] == 4 - assert merged_engine_args["parallel_config"]["pipeline_parallel_size"] == 2 - assert merged_engine_args["model"] == "base-model" - assert merged_engine_args["async_chunk"] is True - - def test_merges_nested_stage_engine_args(self, mocker: MockerFixture): - yaml_config = create_config( - { - "stage_args": [ - { - "stage_id": 0, - "engine_args": { - "nested": {"override": 2}, - }, - } - ], - } - ) - mocker.patch( - "vllm_omni.entrypoints.utils.load_yaml_config", - return_value=yaml_config, - ) - - stages = load_stage_configs_from_yaml( - "fake.yaml", - base_engine_args={"nested": {"base": 1}}, - ) - - assert stages[0]["engine_args"]["nested"]["base"] == 1 - assert stages[0]["engine_args"]["nested"]["override"] == 2 + assert result == ([], None) + resolve_path.assert_not_called() + assert "No registered PipelineConfig resolved" in caplog.text + assert "deploy_config" in caplog.text class TestCumulativeStreamingCoercion: diff --git a/vllm_omni/config/stage_config.py b/vllm_omni/config/stage_config.py index 86a6fed1b..cdfd35ee2 100644 --- a/vllm_omni/config/stage_config.py +++ b/vllm_omni/config/stage_config.py @@ -632,6 +632,11 @@ def resolve_deploy_yaml(path: str | Path) -> dict[str, Any]: def load_deploy_config(path: str | Path) -> DeployConfig: """Load a deploy YAML (with optional base_config inheritance).""" raw_dict = resolve_deploy_yaml(path) + if "stage_args" in raw_dict: + raise ValueError( + f"Deploy config {path} uses the removed `stage_args` schema; " + "define topology in PipelineConfig and deployment overrides under `stages`." + ) stages = [_parse_stage_deploy(s) for s in raw_dict.get("stages", [])] diff --git a/vllm_omni/engine/async_omni_engine.py b/vllm_omni/engine/async_omni_engine.py index c457b7052..38eea768e 100644 --- a/vllm_omni/engine/async_omni_engine.py +++ b/vllm_omni/engine/async_omni_engine.py @@ -1118,15 +1118,13 @@ class AsyncOmniEngine: ) -> tuple[str, list[Any]]: """Resolve stage configs and inject defaults shared by orchestrator/headless.""" + for legacy_arg in ("stage_configs_path", "stage_configs"): + if legacy_arg in kwargs: + raise ValueError(f"`{legacy_arg}` is no longer supported; use `deploy_config` instead.") + deploy_config_path = kwargs.pop("deploy_config", None) strategy_config_path = kwargs.pop("strategy_config", None) stage_overrides_json = kwargs.pop("stage_overrides", None) - explicit_stage_configs = kwargs.pop("stage_configs", None) - if explicit_stage_configs is not None: - logger.warning( - "`stage_configs` is not part of the public API. " - "Ignoring it and resolving stages from deploy_config/model factory." - ) # Parse --stage-overrides JSON string if provided stage_overrides = parse_stage_overrides(stage_overrides_json) diff --git a/vllm_omni/entrypoints/utils.py b/vllm_omni/entrypoints/utils.py index 9e884db48..e20dd7ec2 100644 --- a/vllm_omni/entrypoints/utils.py +++ b/vllm_omni/entrypoints/utils.py @@ -19,7 +19,7 @@ from vllm_omni.config.config_factory import ( ) from vllm_omni.config.pipeline_registry import OMNI_PIPELINES from vllm_omni.config.stage_config import _DEPLOY_DIR -from vllm_omni.config.yaml_util import create_config, load_yaml_config, merge_configs +from vllm_omni.config.yaml_util import create_config, load_yaml_config from vllm_omni.diffusion.utils.hf_utils import ( _looks_like_dreamzero, get_diffusion_model_index, @@ -369,11 +369,12 @@ def load_stage_configs_from_model( ) -> tuple[list, str | None]: """Load stage configurations from model's default config file. - For models registered in the pipeline registry (new path), uses + For models registered in the pipeline registry, uses ``StageConfigFactory.create_legacy_stage_configs_from_model()`` which merges PipelineConfig + DeployConfig + CLI overrides. - For other models (legacy path), loads stage configs from YAML. + Models that cannot be resolved through the registry return no stages so the + caller can apply its default stage configuration, when available. Args: model: Model name or path (used to determine model_type) @@ -383,7 +384,7 @@ def load_stage_configs_from_model( stage_overrides: Per-stage overrides from --stage-overrides. strategy_config_path: Optional path to a composable-parallel ``strategy.yaml`` whose derived sizing is overlaid onto the - registry-merged stages (opt-in; ignored on the legacy YAML path). + registry-merged stages. Returns: ``(stage_configs, omni_lb_policy)``: the list of stage configuration @@ -427,69 +428,17 @@ def load_stage_configs_from_model( # Convert StageConfig objects to OmegaConf for backward compat return [stage.to_omegaconf() for stage in stages], omni_lb_policy - # Legacy fallback: load from YAML. A composable-parallel strategy cannot be - # applied here (it overlays onto registry-merged stages), so warn rather than - # silently dropping the operator's --strategy-config. + strategy_note = "" if strategy_config_path is not None: - logger.warning( - "--strategy-config (%s) was provided but model %r resolves via the " - "legacy stage_configs YAML path, which does not support " - "composable-parallel strategies; the strategy is ignored. Use a " - "registry-based model to apply it.", - strategy_config_path, - model, - ) - stage_config_path = resolve_model_config_path(model) - if stage_config_path is None: - return [], None - stage_configs = load_stage_configs_from_yaml( - config_path=stage_config_path, - base_engine_args=base_engine_args, - prefer_stage_engine_args=True, + strategy_note = f" Strategy config {strategy_config_path!r} was not applied." + logger.warning( + "No registered PipelineConfig resolved for model %r. Legacy `stage_args` " + "YAMLs are no longer supported; register the pipeline and provide deployment " + "overrides through `deploy_config`.%s", + model, + strategy_note, ) - return stage_configs, None - - -def load_stage_configs_from_yaml( - config_path: str, - base_engine_args: dict | None = None, - prefer_stage_engine_args: bool = True, -) -> list: - """Load stage configurations from a YAML file (legacy OmegaConf path). - - TODO(@lishunyang12): remove once all models use PipelineConfig + DeployConfig. - - Args: - config_path: Path to the YAML configuration file - base_engine_args: Engine args supplied by the caller. - prefer_stage_engine_args: When True, YAML stage args override caller - engine args. When False, caller engine args override YAML defaults. - - Returns: - List of stage configuration dictionaries from the file's stage_args - """ - if base_engine_args is None: - base_engine_args = {} - config_data = load_yaml_config(config_path) - stage_args = config_data.stage_args - global_async_chunk = config_data.get("async_chunk", False) - # Convert any nested dataclass objects to dicts before creating DictConfig - base_engine_args = _convert_dataclasses_to_dict(base_engine_args) - base_engine_args = create_config(base_engine_args) - for stage_arg in stage_args: - base_engine_args_tmp = base_engine_args.copy() - # Update base_engine_args with stage-specific engine_args if they exist - if hasattr(stage_arg, "engine_args") and stage_arg.engine_args is not None: - if prefer_stage_engine_args: - merged_engine_args = merge_configs(base_engine_args_tmp, stage_arg.engine_args) - else: - merged_engine_args = merge_configs(stage_arg.engine_args, base_engine_args_tmp) - base_engine_args_tmp = create_config(merged_engine_args) - stage_type = getattr(stage_arg, "stage_type", "llm") - if hasattr(stage_arg, "runtime") and stage_arg.runtime is not None and stage_type != "diffusion": - base_engine_args_tmp.async_chunk = global_async_chunk - stage_arg.engine_args = base_engine_args_tmp - return stage_args + return [], None def filter_stages( From 611c083587ee6506d6648b294917ec2b98d7abdd Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Sun, 16 Aug 2026 19:51:17 -0700 Subject: [PATCH 039/267] [BugFix][TTS] R1.1-R1.4: turn silent async-chunk failures into visible ones (#6033) Signed-off-by: Yueqian Lin Co-authored-by: Hongsheng Liu --- .../sched/test_omni_scheduler_mixin_shared.py | 21 +- .../test_omni_scheduler_mixin_timeouts.py | 253 +++++++++++++++++- .../test_chunk_transfer_adapter.py | 109 ++++++++ .../test_orchestrator_error_handling.py | 45 ++++ .../test_orchestrator_stage_input_bridge.py | 28 +- vllm_omni/core/sched/omni_scheduler_mixin.py | 140 +++++++++- .../omni_connectors/transfer_adapter/base.py | 33 ++- .../chunk_transfer_adapter.py | 71 +++++ vllm_omni/engine/orchestrator.py | 70 ++++- .../omni_connector_model_runner_mixin.py | 12 +- 10 files changed, 750 insertions(+), 32 deletions(-) diff --git a/tests/core/sched/test_omni_scheduler_mixin_shared.py b/tests/core/sched/test_omni_scheduler_mixin_shared.py index 9aaa4c656..4f98fd092 100644 --- a/tests/core/sched/test_omni_scheduler_mixin_shared.py +++ b/tests/core/sched/test_omni_scheduler_mixin_shared.py @@ -4,7 +4,10 @@ from types import SimpleNamespace import pytest from vllm.v1.engine import FinishReason -from vllm_omni.core.sched.omni_scheduler_mixin import OmniSchedulerMixin +from vllm_omni.core.sched.omni_scheduler_mixin import ( + DEFAULT_INPUT_WAIT_TIMEOUT_S, + OmniSchedulerMixin, +) from vllm_omni.core.sched.output import OmniChunkRecvHandle pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -22,13 +25,25 @@ def test_schedule_lifecycle_helpers_process_and_restore_both_input_paths(): scheduler.requests = {"request": object()} scheduler._consume_pending_connector_output = lambda mode: calls.append(("consume", mode)) scheduler._process_pending_input_timeouts = lambda: calls.append(("timeouts",)) + + def _collect_timed_out(timeout_s): + calls.append(("chunk-timeouts", timeout_s)) + return set() + + def _collect_failed_sends(): + calls.append(("failed-sends",)) + return {} + scheduler.chunk_transfer_adapter = SimpleNamespace( + receives_chunks=True, process_pending_chunks=lambda waiting, running, scheduler_requests: calls.append( ("process", waiting, running, scheduler_requests) ), restore_queues=lambda waiting, running, scheduler_requests: calls.append( ("restore-chunks", waiting, running, scheduler_requests) ), + collect_timed_out_request_ids=_collect_timed_out, + collect_failed_send_request_ids=_collect_failed_sends, ) scheduler.input_coordinator = SimpleNamespace( restore_queues=lambda waiting: calls.append(("restore-full", waiting)) @@ -41,6 +56,10 @@ def test_schedule_lifecycle_helpers_process_and_restore_both_input_paths(): ("consume", "ar"), ("timeouts",), ("process", scheduler.waiting, scheduler.running, scheduler.requests), + # The chunk deadline runs after chunks are applied, so a chunk that + # arrived this cycle resets the clock before it is measured (R1.1). + ("chunk-timeouts", DEFAULT_INPUT_WAIT_TIMEOUT_S), + ("failed-sends",), ("restore-chunks", scheduler.waiting, scheduler.running, scheduler.requests), ("restore-full", scheduler.waiting), ] diff --git a/tests/core/sched/test_omni_scheduler_mixin_timeouts.py b/tests/core/sched/test_omni_scheduler_mixin_timeouts.py index 5fad4340c..1824d2809 100644 --- a/tests/core/sched/test_omni_scheduler_mixin_timeouts.py +++ b/tests/core/sched/test_omni_scheduler_mixin_timeouts.py @@ -1,6 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit coverage for _process_pending_input_timeouts. +"""Unit coverage for the two stage-input timeout hooks. + +``_process_pending_input_timeouts`` covers the full-payload transport; +``_process_pending_chunk_timeouts`` covers async-chunk. Both share +``VLLM_OMNI_INPUT_WAIT_TIMEOUT_S``. Verifies that the mixin correctly *delegates* timed-out requests to the base scheduler's ``finish_requests`` API with ``RequestStatus.FINISHED_ERROR``. @@ -12,6 +16,10 @@ this file only asserts the wiring from the mixin to that API. from __future__ import annotations +import importlib +import logging +import math +import os from types import SimpleNamespace import pytest @@ -102,5 +110,248 @@ def test_process_pending_input_timeouts_disabled_when_timeout_zero(monkeypatch): assert scheduler.finish_calls == [] +class _FakeChunkAdapter: + """Stands in for OmniChunkTransferAdapter's timeout-collection surface.""" + + def __init__(self, timed_out_ids, *, receives_chunks=True): + self._timed_out_ids = set(timed_out_ids) + self.receives_chunks = receives_chunks + self.calls = [] + + def collect_timed_out_request_ids(self, timeout_s): + self.calls.append(timeout_s) + return set(self._timed_out_ids) + + +class _FakeChunkScheduler(OmniSchedulerMixin): + def __init__(self, requests, adapter): + self.requests = requests + self.chunk_transfer_adapter = adapter + self.finish_calls = [] + + def finish_requests(self, req_ids, status): + self.finish_calls.append((set(req_ids), status)) + + +def test_process_pending_chunk_timeouts_delegates_to_finish_requests(): + """A request stalled in WAITING_FOR_CHUNK is failed, not left parked. + + This is the async-chunk half of the safety net (RFC #4855 R1.1); before it + existed, the chunk path had no deadline at all and issue #3833 could park a + request forever. + """ + req_id = "stalled-stream" + adapter = _FakeChunkAdapter(timed_out_ids={req_id}) + scheduler = _FakeChunkScheduler({req_id: SimpleNamespace(request_id=req_id)}, adapter) + + scheduler._process_pending_chunk_timeouts() + + assert len(adapter.calls) == 1 + assert adapter.calls[0] > 0 + assert len(scheduler.finish_calls) == 1 + finished_ids, status = scheduler.finish_calls[0] + assert finished_ids == {req_id} + assert getattr(status, "name", str(status)).endswith("FINISHED_ERROR") + + +def test_process_pending_chunk_timeouts_skips_already_freed_request(): + adapter = _FakeChunkAdapter(timed_out_ids={"already-freed"}) + scheduler = _FakeChunkScheduler(requests={}, adapter=adapter) + + scheduler._process_pending_chunk_timeouts() + + assert scheduler.finish_calls == [] + + +def test_process_pending_chunk_timeouts_noop_on_a_producer_stage(): + """Stage 0 does not receive chunks, so it has nothing to expire.""" + adapter = _FakeChunkAdapter(timed_out_ids={"r1"}, receives_chunks=False) + scheduler = _FakeChunkScheduler({"r1": SimpleNamespace(request_id="r1")}, adapter) + + scheduler._process_pending_chunk_timeouts() + + assert adapter.calls == [] + assert scheduler.finish_calls == [] + + +def test_process_pending_chunk_timeouts_noop_without_adapter(): + class _NoAdapter(OmniSchedulerMixin): + def __init__(self): + self.requests = {} + self.chunk_transfer_adapter = None + self.finish_calls = [] + + def finish_requests(self, req_ids, status): + self.finish_calls.append((set(req_ids), status)) + + scheduler = _NoAdapter() + scheduler._process_pending_chunk_timeouts() + assert scheduler.finish_calls == [] + + +def test_process_pending_chunk_timeouts_disabled_when_timeout_zero(monkeypatch): + """The shared knob disables both transports' nets together.""" + from vllm_omni.core.sched import omni_scheduler_mixin + + monkeypatch.setattr(omni_scheduler_mixin, "DEFAULT_INPUT_WAIT_TIMEOUT_S", 0.0) + + adapter = _FakeChunkAdapter(timed_out_ids={"r1"}) + scheduler = _FakeChunkScheduler({"r1": SimpleNamespace(request_id="r1")}, adapter) + scheduler._process_pending_chunk_timeouts() + + assert adapter.calls == [] + assert scheduler.finish_calls == [] + + +class _FakeSendFailureAdapter: + """Stands in for the adapter's drained send-failure map (R1.2).""" + + def __init__(self, failures): + self._failures = dict(failures) + self.collect_calls = 0 + + def collect_failed_send_request_ids(self): + self.collect_calls += 1 + failures, self._failures = self._failures, {} + return failures + + +class _FakeSendFailureScheduler(OmniSchedulerMixin): + def __init__(self, requests, adapter): + self.requests = requests + self.chunk_transfer_adapter = adapter + + +def test_log_failed_chunk_sends_reports_a_live_request(caplog): + adapter = _FakeSendFailureAdapter({"req-live": "OSError: No space left on device"}) + scheduler = _FakeSendFailureScheduler({"req-live": SimpleNamespace(request_id="req-live")}, adapter) + + with caplog.at_level(logging.ERROR): + scheduler._log_failed_chunk_sends() + + assert "req-live" in caplog.text + assert "No space left on device" in caplog.text + + +def test_log_failed_chunk_sends_still_reports_a_departed_request(caplog): + """``collect_*`` drains the map, so a failure for a request that already left + this scheduler is the one place a drop could still go unrecorded.""" + adapter = _FakeSendFailureAdapter({"req-gone": "OSError: No space left on device"}) + scheduler = _FakeSendFailureScheduler({}, adapter) + + with caplog.at_level(logging.WARNING): + scheduler._log_failed_chunk_sends() + + assert adapter.collect_calls == 1 + assert "req-gone" in caplog.text + assert "No space left on device" in caplog.text + assert "finished or aborted" in caplog.text + + +def test_log_failed_chunk_sends_noop_without_adapter(): + scheduler = _FakeSendFailureScheduler({}, None) + scheduler._log_failed_chunk_sends() + + +# --- R1.4: VLLM_OMNI_INPUT_WAIT_TIMEOUT_S is parsed at import time, so each of +# these reloads the module under a different environment. One knob now gates the +# deadline on both transports, which is why a silent 0 matters more than it did +# when only the full-payload path used it. + + +@pytest.fixture +def reload_with(monkeypatch): + """Reload the mixin module under a given env value, then put it back. + + ``importlib.reload`` re-executes into the live module dict, so whatever the + last test parsed is what every later test in the session sees -- ``-k + zero_warns`` alone would leave ``DEFAULT_INPUT_WAIT_TIMEOUT_S = 0.0`` and + silently disable both deadlines. The rejection cases are worse: the + ValueError fires *after* the rebind, so ``nan`` lingers in the module. + ``monkeypatch`` restores the environment but not the module, so reload once + more on the way out. + """ + import vllm_omni.core.sched.omni_scheduler_mixin as mod + + def _reload(raw): + if raw is None: + monkeypatch.delenv("VLLM_OMNI_INPUT_WAIT_TIMEOUT_S", raising=False) + else: + monkeypatch.setenv("VLLM_OMNI_INPUT_WAIT_TIMEOUT_S", raw) + return importlib.reload(mod) + + yield _reload + + monkeypatch.undo() + importlib.reload(mod) + + +@pytest.mark.parametrize( + "raw,expected", + [ + (None, 600.0), + ("30", 30.0), + ("0.5", 0.5), + ("0", 0.0), # explicit opt-out is still honoured + ], +) +def test_timeout_accepts_valid_values(reload_with, raw, expected): + mod = reload_with(raw) + assert mod.DEFAULT_INPUT_WAIT_TIMEOUT_S == expected + + +@pytest.mark.parametrize("raw", ["-1", "-600", "-0.5"]) +def test_negative_timeout_is_rejected(reload_with, raw): + """`-1` is a common "no limit" idiom; substituting 600 would give an operator + who meant "never time out" mysterious failures ten minutes in. Fail loudly.""" + with pytest.raises(ValueError, match="negative"): + reload_with(raw) + + +@pytest.mark.parametrize("raw", ["nan", "inf", "-inf", "1e999"]) +def test_non_finite_timeout_is_rejected(reload_with, raw): + """nan compares false against every deadline check and inf is unreachable; + both disable the net while looking like a number.""" + with pytest.raises(ValueError, match="not finite"): + reload_with(raw) + + +def test_zero_warns_and_names_both_transports(reload_with, caplog): + with caplog.at_level(logging.WARNING): + mod = reload_with("0") + assert mod.DEFAULT_INPUT_WAIT_TIMEOUT_S == 0.0 + text = caplog.text + assert "DISABLED" in text + assert "WAITING_FOR_INPUT" in text and "WAITING_FOR_CHUNK" in text + + +def test_unparsable_timeout_falls_back(reload_with): + mod = reload_with("not-a-number") + assert mod.DEFAULT_INPUT_WAIT_TIMEOUT_S == 600.0 + + +def test_a_rejected_value_does_not_outlive_its_test(monkeypatch): + """The reason ``reload_with`` reloads on the way out, done here by hand. + + ``nan`` is bound before the ValueError fires, so the reload that rejects it + still leaves it in the module -- and nan compares false against every + deadline check, which is exactly the silent-disable this item exists to + stop. Restoring the environment is not enough; the module has to be reloaded + too. + """ + import vllm_omni.core.sched.omni_scheduler_mixin as mod + + ambient = float(os.environ.get("VLLM_OMNI_INPUT_WAIT_TIMEOUT_S", "600")) + + monkeypatch.setenv("VLLM_OMNI_INPUT_WAIT_TIMEOUT_S", "nan") + with pytest.raises(ValueError, match="not finite"): + importlib.reload(mod) + assert math.isnan(mod.DEFAULT_INPUT_WAIT_TIMEOUT_S) + + monkeypatch.undo() + importlib.reload(mod) + assert mod.DEFAULT_INPUT_WAIT_TIMEOUT_S == ambient + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py index da4e81995..063751b01 100644 --- a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py +++ b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py @@ -1728,3 +1728,112 @@ def test_purge_is_noop_on_empty_deques(build_adapter): adapter.restore_queues(waiting_queue, running_queue, scheduler_requests={}) assert running_queue == [] assert waiting_queue == [] + + +# --------------------------------------------------------------------------- # +# Chunk-wait deadline (RFC #4855 R1.1, issue #3833) +# +# Before this, the async-chunk path had no deadline of any kind, so a dropped +# terminal chunk or an upstream stage that died mid-stream parked the request +# in WAITING_FOR_CHUNK forever. None of these scenarios were covered. +# --------------------------------------------------------------------------- # + + +def _park_in_chunk_wait(adapter, request, *, waiting=True): + """Drive a request through one process_pending_chunks round into the wait.""" + queue = DummyWaitingQueue([request]) if waiting else [request] + if waiting: + adapter.process_pending_chunks(queue, []) + else: + adapter.process_pending_chunks(DummyWaitingQueue(), queue) + return queue + + +def test_chunk_wait_clock_starts_when_a_request_parks(build_adapter): + adapter, _ = build_adapter(stage_id=1, model_mode="ar") + request = _req("r1", RequestStatus.WAITING) + + _park_in_chunk_wait(adapter, request) + + assert request.status == RequestStatus.WAITING_FOR_CHUNK + assert "r1" in adapter._waiting_since + + +def test_chunk_wait_does_not_expire_before_the_deadline(build_adapter): + adapter, _ = build_adapter(stage_id=1, model_mode="ar") + _park_in_chunk_wait(adapter, _req("r1", RequestStatus.WAITING)) + + assert adapter.collect_timed_out_request_ids(timeout_s=600.0) == set() + assert "r1" in adapter._waiting_since + + +def test_dropped_terminal_chunk_expires_the_request(build_adapter): + """The #3833 scenario: upstream stops sending and never marks the stream done.""" + adapter, _ = build_adapter(stage_id=1, model_mode="ar") + _park_in_chunk_wait(adapter, _req("r1", RequestStatus.WAITING)) + + adapter._waiting_since["r1"] -= 601.0 + + assert adapter.collect_timed_out_request_ids(timeout_s=600.0) == {"r1"} + # Cleared, so a second sweep does not re-report the same request. + assert adapter.collect_timed_out_request_ids(timeout_s=600.0) == set() + + +def test_arriving_chunk_resets_the_clock(build_adapter): + """A slow but healthy stream must never expire: the deadline measures stall + time between chunks, not the lifetime of the stream.""" + adapter, _ = build_adapter(stage_id=1, model_mode="ar") + request = _req("r1", RequestStatus.WAITING) + queue = _park_in_chunk_wait(adapter, request) + adapter.restore_queues(queue, [], scheduler_requests={"r1": request}) + + # Age the wait to just under the deadline, then deliver a chunk. + adapter._waiting_since["r1"] -= 599.0 + adapter._finished_load_reqs.add("r1") + adapter.process_pending_chunks(queue, []) + + assert "r1" not in adapter._waiting_since + assert adapter.collect_timed_out_request_ids(timeout_s=600.0) == set() + + +def test_a_disabled_deadline_never_expires(build_adapter): + adapter, _ = build_adapter(stage_id=1, model_mode="ar") + _park_in_chunk_wait(adapter, _req("r1", RequestStatus.WAITING)) + adapter._waiting_since["r1"] -= 10_000.0 + + assert adapter.collect_timed_out_request_ids(timeout_s=0.0) == set() + assert adapter.collect_timed_out_request_ids(timeout_s=-1.0) == set() + + +def test_finished_request_leaves_no_stale_timestamp(build_adapter): + """Otherwise a completed request would be reported as timed out later.""" + adapter, _ = build_adapter(stage_id=1, model_mode="ar") + request = _req("r1", RequestStatus.WAITING) + _park_in_chunk_wait(adapter, request) + + adapter.finish_requests(["r1"], RequestStatus.FINISHED_STOPPED, {"r1": request}) + + assert "r1" not in adapter._waiting_since + assert adapter.collect_timed_out_request_ids(timeout_s=0.001) == set() + + +def test_aborted_request_leaves_no_stale_timestamp(build_adapter): + adapter, _ = build_adapter(stage_id=1, model_mode="ar") + _park_in_chunk_wait(adapter, _req("r1", RequestStatus.WAITING)) + + adapter.cleanup_receiver("r1") + + assert "r1" not in adapter._waiting_since + + +def test_expiry_is_per_request(build_adapter): + """One stalled stream must not take down its healthy neighbours.""" + adapter, _ = build_adapter(stage_id=1, model_mode="ar", max_num_seqs=4) + stalled = _req("stalled", RequestStatus.WAITING) + healthy = _req("healthy", RequestStatus.WAITING) + adapter.process_pending_chunks(DummyWaitingQueue([stalled, healthy]), []) + + adapter._waiting_since["stalled"] -= 601.0 + + assert adapter.collect_timed_out_request_ids(timeout_s=600.0) == {"stalled"} + assert "healthy" in adapter._waiting_since diff --git a/tests/engine/test_orchestrator_error_handling.py b/tests/engine/test_orchestrator_error_handling.py index 780c5a17b..4176764bc 100644 --- a/tests/engine/test_orchestrator_error_handling.py +++ b/tests/engine/test_orchestrator_error_handling.py @@ -411,6 +411,51 @@ async def test_async_chunk_prewarm_failure_reports_failing_downstream_stage(orch orchestrator_fixture.thread.join(timeout=5) +@pytest.mark.asyncio +async def test_async_chunk_prewarm_without_prompt_token_ids_fails_only_that_request( + orchestrator_factory, +) -> None: + """R1.3 of #4855: an async-chunk request whose stage-0 prompt carries no + token ids cannot prewarm anything, so it must fail with a client error + instead of stalling. + + Non-fatal is the point: ``fatal=True`` reads as "the engine died" and the + entrypoints act on it (``OmniLLM.generate`` raises out of its batch loop, + the serving layer calls ``terminate_if_errored``). One client's embeds-only + prompt must not reach other requests. + """ + stage0 = FakeStageClient(stage_type="llm", final_output=False) + stage1 = FakeStageClient(stage_type="llm", final_output=True) + orchestrator_fixture = orchestrator_factory([stage0, stage1], async_chunk=True) + + try: + await _enqueue_add_request( + orchestrator_fixture, + request_id="req-embeds", + # No prompt_token_ids: what an embeds-only prompt looks like here. + prompt=SimpleNamespace(request_id="req-embeds"), + original_prompt={"prompt": "embeds only"}, + sampling_params_list=[_sampling_params(), _sampling_params()], + final_stage_id=1, + ) + + error_msg = await _wait_for_error_message(orchestrator_fixture, request_id="req-embeds") + assert error_msg.fatal is False + assert error_msg.status_code == 400 + assert error_msg.error_type == "BadRequestError" + assert error_msg.stage_id == 0 + assert "prompt_token_ids" in error_msg.error + + # Nothing was prewarmed downstream, and the request is gone. + assert stage1.add_request_calls == [] + await _wait_for(lambda: "req-embeds" not in orchestrator_fixture.orchestrator.request_states) + assert orchestrator_fixture.thread.is_alive() + finally: + if orchestrator_fixture.thread.is_alive(): + orchestrator_fixture.request_sync_q.put_nowait(ShutdownRequestMessage()) + orchestrator_fixture.thread.join(timeout=5) + + # ───────── Direct unit tests for the fault-isolation helpers ───────── diff --git a/tests/engine/test_orchestrator_stage_input_bridge.py b/tests/engine/test_orchestrator_stage_input_bridge.py index cf347e49f..25637913b 100644 --- a/tests/engine/test_orchestrator_stage_input_bridge.py +++ b/tests/engine/test_orchestrator_stage_input_bridge.py @@ -256,12 +256,13 @@ async def test_async_prewarm_skips_outgoing_only_stage() -> None: duplex_identity=SimpleNamespace(), ) - await orchestrator._prewarm_async_chunk_stages( + prewarmed = await orchestrator._prewarm_async_chunk_stages( "req-prewarm", SimpleNamespace(prompt_token_ids=[1, 2], resumable=True), req_state, ) + assert prewarmed is True assert stage1.submitted == [] assert len(stage2.submitted) == 1 assert 1 not in req_state.stage_submit_ts @@ -277,6 +278,7 @@ async def test_async_prewarm_skips_outgoing_only_stage() -> None: @pytest.mark.asyncio async def test_duplex_prewarm_runs_after_first_stage0_submission() -> None: port, stage_pools, request_states, prewarm, submission = _duplex_stage_port_submission() + prewarm.return_value = True result = await port.submit(submission) @@ -285,6 +287,30 @@ async def test_duplex_prewarm_runs_after_first_stage0_submission() -> None: prewarm.assert_awaited_once_with("req-duplex", ANY, request_states["req-duplex"]) +@pytest.mark.asyncio +async def test_duplex_submit_bails_out_when_prewarm_failed_the_request() -> None: + """A failed prewarm has already aborted the request and popped its state. + + Returning a success result here would hand the control plane a replica for a + request that no longer exists, and the trailing bookkeeping would re-register + a running counter the cleanup just released -- a leak that never decrements. + """ + port, stage_pools, request_states, prewarm, submission = _duplex_stage_port_submission() + counter = MagicMock() + port._running_counter = counter + prewarm.return_value = False + request_state = request_states["req-duplex"] + + with pytest.raises(RuntimeError, match="prewarm failed"): + await port.submit(submission) + + prewarm.assert_awaited_once() + assert request_state.duplex_stage_fences == {} + assert request_state.stage_submit_ts == {} + assert request_state.running_counter_registered is False + counter.increment.assert_not_called() + + @pytest.mark.asyncio async def test_async_route_forwards_to_outgoing_only_stage() -> None: orchestrator = object.__new__(Orchestrator) diff --git a/vllm_omni/core/sched/omni_scheduler_mixin.py b/vllm_omni/core/sched/omni_scheduler_mixin.py index e0df774c0..7ce5f4470 100644 --- a/vllm_omni/core/sched/omni_scheduler_mixin.py +++ b/vllm_omni/core/sched/omni_scheduler_mixin.py @@ -1,5 +1,6 @@ from __future__ import annotations +import math import os import time from collections.abc import Iterable @@ -42,25 +43,56 @@ logger = init_logger(__name__) _STATS_INTERVAL_S = 1.0 -# Upper bound on how long a request may sit in full-payload-input wait -# (the state ``OmniSchedulingCoordinator`` records via ``_waiting_since``) -# before the scheduler force-fails it. Defends against stuck consumer-side -# requests when the producer drops a full-payload, send fails, or recv -# never arrives. Override per-deployment via -# VLLM_OMNI_INPUT_WAIT_TIMEOUT_S; set <=0 to disable the safety net. +# Upper bound on how long a request may wait for stage input before the +# scheduler force-fails it. Defends against stuck consumer-side requests when +# the producer drops a payload, send fails, or recv never arrives. Override +# per-deployment via VLLM_OMNI_INPUT_WAIT_TIMEOUT_S; set exactly 0 to disable +# the safety net. Negative and non-finite values are rejected at startup. # -# Scope: this constant only covers the full-payload coordinator path -# (``input_coordinator``). The async-chunk path uses -# ``chunk_transfer_adapter`` and is not affected by this constant. +# Scope: both transports. The full-payload path measures time parked in +# WAITING_FOR_INPUT (``OmniSchedulingCoordinator._waiting_since``); the +# async-chunk path measures time stalled in WAITING_FOR_CHUNK +# (``OmniChunkTransferAdapter._waiting_since``, reset on each chunk arrival). +# One knob, because the question it answers -- "how long may a request wait for +# stage input?" -- does not depend on which transport carries it. +_INPUT_WAIT_TIMEOUT_DEFAULT = 600.0 _INPUT_WAIT_TIMEOUT_RAW = os.environ.get("VLLM_OMNI_INPUT_WAIT_TIMEOUT_S", "600") try: DEFAULT_INPUT_WAIT_TIMEOUT_S: float = float(_INPUT_WAIT_TIMEOUT_RAW) except ValueError: logger.warning( - "Invalid VLLM_OMNI_INPUT_WAIT_TIMEOUT_S=%r; falling back to 600 seconds.", + "Invalid VLLM_OMNI_INPUT_WAIT_TIMEOUT_S=%r; falling back to %s seconds.", _INPUT_WAIT_TIMEOUT_RAW, + _INPUT_WAIT_TIMEOUT_DEFAULT, + ) + DEFAULT_INPUT_WAIT_TIMEOUT_S = _INPUT_WAIT_TIMEOUT_DEFAULT + +if not math.isfinite(DEFAULT_INPUT_WAIT_TIMEOUT_S): + # nan compares false against everything, so every deadline check silently + # passes; inf makes the deadline unreachable. Both disable the net while + # looking like a number, which is the failure this item exists to stop. + raise ValueError( + f"VLLM_OMNI_INPUT_WAIT_TIMEOUT_S={_INPUT_WAIT_TIMEOUT_RAW!r} is not finite. " + "Use a positive number of seconds, or exactly 0 to disable the deadline." + ) +if DEFAULT_INPUT_WAIT_TIMEOUT_S < 0: + # Fail at startup rather than substituting a default. `-1` is a common idiom + # for "no limit", so silently turning it into 600 would give an operator who + # meant "never time out" mysterious failures ten minutes in. + raise ValueError( + f"VLLM_OMNI_INPUT_WAIT_TIMEOUT_S={_INPUT_WAIT_TIMEOUT_RAW!r} is negative. " + "Use a positive number of seconds, or exactly 0 to disable the deadline." + ) +elif DEFAULT_INPUT_WAIT_TIMEOUT_S == 0: + # Zero stays a supported opt-out, but it is no longer silent: it disables + # the deadline on BOTH transports, so a request whose stage input never + # arrives waits forever. Say which nets just went away. + logger.warning( + "VLLM_OMNI_INPUT_WAIT_TIMEOUT_S=0: stage-input deadlines are DISABLED for " + "both the full-payload path (WAITING_FOR_INPUT) and the async-chunk path " + "(WAITING_FOR_CHUNK). A request whose input never arrives will wait " + "indefinitely instead of failing." ) - DEFAULT_INPUT_WAIT_TIMEOUT_S = 600.0 class OmniSchedulerMixin: @@ -184,6 +216,8 @@ class OmniSchedulerMixin: self.running, scheduler_requests=self.requests, ) + self._process_pending_chunk_timeouts() + self._log_failed_chunk_sends() def _restore_omni_wait_queues(self) -> None: """Restore requests temporarily parked by Omni input gates.""" @@ -211,9 +245,9 @@ class OmniSchedulerMixin: Disabled when ``DEFAULT_INPUT_WAIT_TIMEOUT_S`` is <= 0. Scope: only covers ``input_coordinator`` (full-payload path). - Async-chunk requests park in ``chunk_transfer_adapter`` instead - and are not handled here -- if a similar safety net is needed - for the chunk path, it belongs in the chunk adapter. + Async-chunk requests park in ``chunk_transfer_adapter`` instead and + are handled by ``_process_pending_chunk_timeouts`` below, which shares + this timeout. """ if DEFAULT_INPUT_WAIT_TIMEOUT_S <= 0: return @@ -234,6 +268,84 @@ class OmniSchedulerMixin: ) self.finish_requests(present_ids, RequestStatus.FINISHED_ERROR) + def _log_failed_chunk_sends(self) -> None: + """Surface chunks the sender gave up on (R1.2 of #4855). + + ``save_loop`` pops a task with ``popleft()`` and never re-queues it, and + ``connector.put`` can report failure without raising, so a dropped chunk + is a give-up rather than a retry. Until this change the only trace was a + warning that printed ``None`` for the request id, because the task dict + has no ``request_id`` key. + + This only reports. Failing the request from here does not work: the + producer is not the ``final_output`` stage, so an abort synthesized by + ``finish_requests`` is not turned into a client-visible termination by + ``Orchestrator._route_output`` (see the ``final_output`` guards there). + Making a producer-side drop client-visible needs an orchestrator-side + path like ``_fail_request_dead_stage``; the consumer's R1.1 deadline is + what actually ends the request today. + """ + adapter = getattr(self, "chunk_transfer_adapter", None) + if adapter is None: + return + failures = adapter.collect_failed_send_request_ids() + for request_id, reason in failures.items(): + if request_id in self.requests: + logger.error( + "[OmniScheduler] req=%s: chunk send gave up (%s); the consumer will " + "fail on the stage-input deadline", + request_id, + reason, + ) + else: + # Drained but no longer scheduled here: the request finished or + # was aborted between the failed send and this sweep. Still say + # so -- collect_* empties the map, so staying quiet would make + # this the one drop path in a change whose point is visibility. + logger.warning( + "[OmniScheduler] req=%s: chunk send gave up (%s) after the request " + "left this scheduler (finished or aborted)", + request_id, + reason, + ) + + def _process_pending_chunk_timeouts(self) -> None: + """Force-fail requests stalled in ``WAITING_FOR_CHUNK`` too long. + + The async-chunk counterpart of ``_process_pending_input_timeouts``, + called right after ``chunk_transfer_adapter.process_pending_chunks``. + Until now the chunk path had no safety net at all: the scheduler-side + comment on the full-payload net said a similar guard "belongs in the + chunk adapter", while the worker mixin assumed the full-payload net + already covered it. Only the first was true, so a dropped terminal + chunk or a producer that exhausted its send retries parked the request + forever (vllm-project/vllm-omni#3833). Since ``async_chunk: true`` is + the default for the flagship TTS pipelines, that was the default + streaming path. + + Shares ``VLLM_OMNI_INPUT_WAIT_TIMEOUT_S`` with the full-payload net: + one knob for "how long may a request wait for stage input", regardless + of which transport carries it. Disabled when the value is <= 0. + """ + if DEFAULT_INPUT_WAIT_TIMEOUT_S <= 0: + return + adapter = getattr(self, "chunk_transfer_adapter", None) + if adapter is None or not getattr(adapter, "receives_chunks", False): + return + timed_out_ids = adapter.collect_timed_out_request_ids(timeout_s=DEFAULT_INPUT_WAIT_TIMEOUT_S) + if not timed_out_ids: + return + present_ids = {req_id for req_id in timed_out_ids if req_id in self.requests} + if not present_ids: + return + logger.warning( + "Marking %d request(s) as FINISHED_ERROR after stalling > %.0fs waiting for a chunk: %s", + len(present_ids), + DEFAULT_INPUT_WAIT_TIMEOUT_S, + sorted(present_ids), + ) + self.finish_requests(present_ids, RequestStatus.FINISHED_ERROR) + def _capture_omni_connector_output(self, model_runner_output: Any) -> None: """Stash the model runner's omni_connector_output for next schedule(). diff --git a/vllm_omni/distributed/omni_connectors/transfer_adapter/base.py b/vllm_omni/distributed/omni_connectors/transfer_adapter/base.py index bbaca48e0..578fbb233 100644 --- a/vllm_omni/distributed/omni_connectors/transfer_adapter/base.py +++ b/vllm_omni/distributed/omni_connectors/transfer_adapter/base.py @@ -35,6 +35,8 @@ class OmniTransferAdapterBase: self.stop_event = threading.Event() self._recv_cond = threading.Condition() self._save_cond = threading.Condition() + self._send_failures: dict[str, str] = {} + self._send_failure_lock = threading.Lock() self.recv_thread = threading.Thread(target=self.recv_loop, daemon=True) self.recv_thread.start() @@ -82,6 +84,24 @@ class OmniTransferAdapterBase: elif not any_success and not self.stop_event.is_set(): self._recv_cond.wait(timeout=0.001) + def record_send_failure(self, request_id: str | None, reason: str) -> None: + """Note that a chunk for *request_id* will never be delivered. + + Called from the sender thread; drained by the scheduler thread, hence + the lock. ``_waiting_since`` needs none because only the scheduler + touches it. + """ + if not request_id: + return + with self._send_failure_lock: + self._send_failures.setdefault(request_id, reason) + + def collect_failed_send_request_ids(self) -> dict[str, str]: + """Drain and return the requests whose send gave up.""" + with self._send_failure_lock: + failures, self._send_failures = self._send_failures, {} + return failures + def save_loop(self): """Loop to send outgoing data.""" while not self.stop_event.is_set(): @@ -90,7 +110,18 @@ class OmniTransferAdapterBase: try: self._send_single_request(task) except Exception as e: - logger.warning(f"Error saving data for {task.get('request_id')}: {e}") + # R1.2 of #4855. The task was popleft()-ed and is not re-queued, + # so this is a give-up, not a retry: the consumer will wait for a + # chunk that is never coming. Record it so the scheduler can fail + # the request now instead of leaving it to the wait deadline. + # The task dict carries the Request object, not a bare id -- + # `task.get("request_id")` is always None (it was that way in the + # original warning too, which logged "None" for every failure). + # Use the scheduler-side id: that is what `self.requests` is keyed + # by, and what `finish_requests` expects. + failed = getattr(task.get("request"), "request_id", None) + logger.error("Send gave up for %s: %s", failed, e) + self.record_send_failure(failed, f"{type(e).__name__}: {e}") with self._save_cond: if not self._pending_save_reqs and not self.stop_event.is_set(): diff --git a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py index d722adad2..ae1cb74b9 100644 --- a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py +++ b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import importlib +import time from collections import defaultdict, deque from collections.abc import Callable, Mapping from typing import Any @@ -94,6 +95,12 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): self._held_non_active: deque[Any] = deque() self.requests_num_chunks_sent: dict[str, int] = defaultdict(int) self._pending_streaming_prefills: dict[str, dict] = {} + # Monotonic timestamp of when each request last began waiting for a + # chunk, refreshed every time one arrives. Read by + # collect_timed_out_request_ids() so a stream that stops advancing + # becomes a client-visible error instead of parking forever. Mirrors + # OmniSchedulingCoordinator._waiting_since on the full-payload path. + self._waiting_since: dict[str, float] = {} @staticmethod def _is_truthy_scalar(value: Any) -> bool: @@ -383,6 +390,24 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): # has been sent successfully. This avoids cleanup->save races. if is_payload_finished: self.cleanup(request.request_id, external_req_id) + else: + # R1.2 of #4855. connector.put returning False is a silent drop: no + # exception, no retry, and the caller's `if success:` block simply + # does not run. /dev/shm exhaustion in SharedMemoryConnector.put is + # one way to get here. Record it so the request fails now rather + # than parking in WAITING_FOR_CHUNK until the deadline. + logger.error( + "Chunk send failed for %s (stage %s -> %s); giving up on this chunk", + external_req_id, + stage_id, + next_stage_id, + ) + # Key on the scheduler-side id. `external_req_id` is the user-facing id + # (InputProcessor renames request_id to an internal UUID and keeps the + # original in external_req_id, see async_omni_engine.py), while + # `self.requests` -- and therefore `finish_requests` -- is keyed by the + # internal one. + self.record_send_failure(request.request_id, "connector.put reported failure") if is_segment_finished: self.code_prompt_token_ids.pop(external_req_id, None) @@ -438,6 +463,7 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): self._cancelled_load_reqs.add(request_id) self._finished_load_reqs.discard(request_id) + self._waiting_since.pop(request_id, None) @staticmethod def _discard_from_chunk_deque(deque_list: deque[Any], request_id: str) -> None: @@ -579,6 +605,42 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): self._active_streams[request_id] = request return True + def collect_timed_out_request_ids(self, timeout_s: float) -> set[str]: + """Return IDs whose chunk wait has exceeded *timeout_s*. + + The async-chunk path had no deadline of any kind: a request parks in + ``WAITING_FOR_CHUNK`` and the receiver re-queues it on every failed + poll (``transfer_adapter/base.py``) with no attempt counter, so a + producer that crashed, gave up after its send retries, or simply never + emitted a terminal chunk left the request waiting forever + (vllm-project/vllm-omni#3833). The full-payload path has had a net + since ``OmniSchedulingCoordinator.collect_timed_out_request_ids``; this + is its async-chunk counterpart, and both are driven by the same + ``VLLM_OMNI_INPUT_WAIT_TIMEOUT_S``. + + The clock measures *stall* time, not stream lifetime: it starts when a + request begins waiting for a chunk and resets each time one arrives, so + a long but healthy stream is never failed. + + Clears ``_waiting_since`` for the expired IDs. The caller marks them + ``FINISHED_ERROR`` via the scheduler's ``finish_requests``, which routes + back through this adapter's own ``finish_requests`` and releases the + rest of the per-request state. + """ + if timeout_s <= 0 or not self._waiting_since: + return set() + now = time.monotonic() + timed_out_ids = {req_id for req_id, started in self._waiting_since.items() if now - started > timeout_s} + for req_id in timed_out_ids: + self._waiting_since.pop(req_id, None) + logger.warning( + "[Stage-%s] Request %s timed out waiting for a chunk (stalled > %.0fs)", + self.connector.stage_id, + req_id, + timeout_s, + ) + return timed_out_ids + @property def num_running_waiting_for_chunk(self) -> int: """Count running requests temporarily removed while awaiting a chunk.""" @@ -624,11 +686,15 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): # Requests that waiting for chunk self.load_async(request) request.status = RequestStatus.WAITING_FOR_CHUNK + self._waiting_since.setdefault(request.request_id, time.monotonic()) else: if request.request_id in finished_load_reqs: request.status = target_status finished_load_reqs.remove(request.request_id) self.requests_with_ready_chunks.add(request.request_id) + # A chunk landed: restart the clock for the next one, so the + # deadline measures stall time rather than total stream time. + self._waiting_since.pop(request.request_id, None) continue queue.remove(request) self.requests_origin_status[request.request_id] = target_status @@ -767,11 +833,15 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): # Requests that waiting for chunk self.load_async(request) request.status = RequestStatus.WAITING_FOR_CHUNK + self._waiting_since.setdefault(request.request_id, time.monotonic()) else: if request.request_id in finished_load_reqs: request.status = target_status finished_load_reqs.remove(request.request_id) self.requests_with_ready_chunks.add(request.request_id) + # A chunk landed: restart the clock for the next one, so the + # deadline measures stall time rather than total stream time. + self._waiting_since.pop(request.request_id, None) continue queue.remove(request) self.requests_origin_status[request.request_id] = target_status @@ -826,5 +896,6 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): self.upstream_exhausted_requests.discard(req_id) self._finished_load_reqs.discard(req_id) self._cancelled_load_reqs.add(req_id) + self._waiting_since.pop(req_id, None) return [] diff --git a/vllm_omni/engine/orchestrator.py b/vllm_omni/engine/orchestrator.py index 901bdb980..f112983e1 100644 --- a/vllm_omni/engine/orchestrator.py +++ b/vllm_omni/engine/orchestrator.py @@ -15,6 +15,7 @@ import asyncio import time as _time from collections.abc import Awaitable, Callable from dataclasses import dataclass, field +from http import HTTPStatus from typing import TYPE_CHECKING, Any import janus @@ -51,6 +52,7 @@ from vllm_omni.engine.messages import ( from vllm_omni.engine.orchestrator_monitor import create_orch_monitor, replica_key from vllm_omni.engine.serialization import serialize_additional_information from vllm_omni.engine.stage_pool import StagePool, StageUnavailableError +from vllm_omni.errors import DEFAULT_CLIENT_ERROR_TYPE from vllm_omni.metrics.prometheus import OmniRequestCounter from vllm_omni.metrics.stat_logger import OmniPrometheusStatLogger from vllm_omni.outputs import OmniRequestOutput @@ -221,7 +223,7 @@ class _OrchestratorDuplexStagePort: async_chunk: bool, prewarm_async_chunk_stages: Callable[ [str, Any, OrchestratorRequestState], - Awaitable[None], + Awaitable[bool], ], ) -> None: self._stage_pools = stage_pools @@ -307,11 +309,23 @@ class _OrchestratorDuplexStagePort: else: replica_id = await pool.submit_initial(context.request_id, request_state, request, prompt_text=None) if self._async_chunk and context.stage_id == 0: - await self._prewarm_async_chunk_stages( + prewarmed = await self._prewarm_async_chunk_stages( context.request_id, request, request_state, ) + if not prewarmed: + # The prewarm already failed the request, aborted it, and + # popped its state. Falling through would write fences and + # a submit timestamp onto that orphaned object and + # re-increment the running counter the cleanup just + # released, then hand the control plane a success result + # for a request that no longer exists. Raise instead: + # ``DuplexControlPlane.handle_append`` turns this into an + # error result for the append. + raise RuntimeError( + f"async-chunk prewarm failed for duplex request {context.request_id}; the request was aborted" + ) request_state.duplex_stage_fences[context.stage_id] = context.fence request_state.stage_submit_ts[context.stage_id] = _time.time() if not request_state.running_counter_registered and self._running_counter is not None: @@ -2217,18 +2231,54 @@ class Orchestrator: request_id: str, stage0_request: Any, req_state: OrchestratorRequestState, - ) -> None: - """Pre-submit downstream stages for async-chunk mode.""" + ) -> bool: + """Pre-submit downstream stages for async-chunk mode. + + Returns False when the request was failed and cleaned up in here, so a + caller still holding ``req_state`` stops instead of recording state on + an object the cleanup already popped from ``request_states``. + """ if req_state.final_stage_id <= 0: - return + return True prompt_token_ids = getattr(stage0_request, "prompt_token_ids", None) if prompt_token_ids is None: - logger.warning( - "[Orchestrator] async_chunk prewarm skipped for req=%s: stage0 prompt_token_ids missing", + # R1.3 of #4855. Skipping the prewarm leaves every downstream stage + # unsubmitted, so the request produces no output and no error -- it + # just stops. An embeds-only prompt in async-chunk mode is a config + # mistake, and the client should hear about it now rather than wait + # for the stage-input deadline to notice nothing ever arrived. + # + # Non-fatal, with a 4xx: `fatal=True` means "the engine is dead", and + # the entrypoints act on it -- `OmniBase._handle_output_message` + # raises `OmniEngineDeadError` out of `OmniLLM.generate`'s batch loop, + # failing every other request in that batch, and the serving layer + # calls `terminate_if_errored`. One client's bad prompt must not do + # either. This is the same shape `_handle_stage_error` uses for a + # request-scoped client error. + logger.error( + "[Orchestrator] req=%s: async_chunk prewarm needs stage0 prompt_token_ids " + "and none were provided; failing the request", request_id, ) - return + await self.output_async_queue.put( + ErrorMessage( + error=( + "async_chunk requires prompt_token_ids on the stage-0 request; " + "an embeds-only prompt cannot prewarm downstream stages" + ), + status_code=HTTPStatus.BAD_REQUEST.value, + error_type=DEFAULT_CLIENT_ERROR_TYPE, + request_id=request_id, + stage_id=0, + ) + ) + await self._cleanup_request_ids( + [request_id, *self._cfg_tracker.cleanup_parent(request_id)], + abort=True, + close_duplex_sessions=True, + ) + return False for next_stage_id in range(1, req_state.final_stage_id + 1): next_pool = self.stage_pools[next_stage_id] @@ -2303,7 +2353,7 @@ class Orchestrator: # Attribute the failure to the stage that failed, not stage 0; the # request is already cleaned up, so stop prewarming. if not submitted: - return + return False # The guard swallows submit_initial's return value; the pool binding # it recorded carries the same replica. @@ -2329,6 +2379,8 @@ class Orchestrator: tx_ms=_tx_ms, ) + return True + def _build_kv_sender_info( self, sender_stage_ids: list[int], diff --git a/vllm_omni/worker/omni_connector_model_runner_mixin.py b/vllm_omni/worker/omni_connector_model_runner_mixin.py index 07a0327a6..70e9dafd2 100644 --- a/vllm_omni/worker/omni_connector_model_runner_mixin.py +++ b/vllm_omni/worker/omni_connector_model_runner_mixin.py @@ -406,11 +406,13 @@ class OmniConnectorModelRunnerMixin: # thread by one execute_model() cycle, especially when the request was # added after the current scheduler_output snapshot. # - # Orphaned pending recv entries (e.g. from upstream stage crash) - # are handled by OmniSchedulingCoordinator.collect_timed_out_request_ids() - # which detects wait-time violations. The scheduler then removes the - # request from its queues, sets FINISHED_ERROR, and calls _free_request() - # which ultimately triggers cleanup_finished_request() here. + # Orphaned pending recv entries (e.g. from upstream stage crash) are + # handled by collect_timed_out_request_ids() -- on + # OmniSchedulingCoordinator for full-payload requests, and on + # OmniChunkTransferAdapter for async-chunk ones -- which detect + # wait-time violations. The scheduler then removes the request from + # its queues, sets FINISHED_ERROR, and calls _free_request() which + # ultimately triggers cleanup_finished_request() here. for attr_name in ( "_request_ids_mapping", "_get_req_chunk", From 4616823125500375828071558f9027cdd3cd3a01 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:21:28 +0800 Subject: [PATCH 040/267] [CI]demote slow nightly cases and trim Qwen-Image-Edit coverage (#5944) Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: [Your Name] Co-authored-by: Cursor --- .buildkite/cuda/test-nightly.yml | 206 +----------------- .buildkite/cuda/test-weekly.yml | 2 +- .buildkite/npu/test-npu-nightly.yml | 27 --- .../dfx/perf/tests/test_bagel_vllm_omni.json | 8 +- tests/dfx/perf/tests/test_higgs_audio_v3.json | 2 +- tests/dfx/perf/tests/test_ltx2_vllm_omni.json | 10 +- .../test_qwen_image_edit_2511_vllm_omni.json | 6 +- .../tests/test_qwen_image_edit_vllm_omni.json | 6 +- .../test_qwen_image_layered_vllm_omni.json | 2 +- tests/dfx/perf/tests/test_voxcpm2.json | 2 +- .../ltx/test_ltx25_official_similarity.py | 4 +- .../ltx/test_ltx_official_similarity.py | 2 +- tests/e2e/accuracy/test_qwen_image_edit.py | 4 +- tests/e2e/accuracy/test_qwen_image_layered.py | 2 +- .../test_video_streaming_output_similarity.py | 2 +- .../offline_inference/test_bagel_expansion.py | 2 +- ...est_glm_image_autoround_w4a16_expansion.py | 2 +- .../test_moss_tts_realtime.py | 4 +- .../online_serving/test_bagel_expansion.py | 2 +- .../test_bagel_multi_replicas.py | 6 +- tests/e2e/online_serving/test_boogu_image.py | 4 +- .../online_serving/test_boogu_image_edit.py | 4 +- .../test_boogu_image_edit_expansion.py | 4 +- .../test_boogu_image_expansion.py | 4 +- .../test_glm_image_expansion.py | 2 +- .../online_serving/test_lingbot_video_moe.py | 2 +- tests/e2e/online_serving/test_ltx.py | 2 +- tests/e2e/online_serving/test_ltx25.py | 2 +- .../test_ming_flash_omni_expansion.py | 2 +- .../online_serving/test_qwen_image_edit.py | 28 +-- .../test_qwen_image_edit_expansion.py | 48 +--- .../test_qwen_image_layered_expansion.py | 2 +- .../test_voxcpm2_tts_expansion.py | 2 +- .../test_wan_2_1_vace_expansion.py | 2 +- .../nightly/buildkite_testcase_statistics.py | 52 ++++- 35 files changed, 120 insertions(+), 341 deletions(-) diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index ed112b84b..0c5d94d81 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -159,19 +159,7 @@ steps: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - export VLLM_USE_DEEP_GEMM="0" - export VLLM_MOE_USE_DEEP_GEMM="0" - - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py - mirror_hardwares: l4_1 - - - label: ":full_moon: TTS · Accuracy Test · VoxCPM2" - timeout_in_minutes: 30 - commands: - - | - timeout 30m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 - pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts_expansion.py -m 'full_model and cuda and tts' --run-level 'full_model' - " + - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy mirror_hardwares: l4_1 - label: ":full_moon: TTS · Perf Test" @@ -188,35 +176,6 @@ steps: exit $$EXIT mirror_hardwares: h100_2 - - label: ":full_moon: TTS · VoxCPM2 · Perf Test" - key: nightly-tts-performance-voxcpm2 - timeout_in_minutes: 180 - commands: - - export BENCHMARK_DIR=tests/dfx/perf/results - - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_voxcpm2.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT - mirror_hardwares: h100_1 - - - label: ":full_moon: TTS · Higgs-Audio-V3 · Perf Test" - key: nightly-tts-performance-higgs-audio-v3 - timeout_in_minutes: 180 - commands: - - export BENCHMARK_DIR=tests/dfx/perf/results - - export VLLM_USE_DEEP_GEMM="0" - - export VLLM_MOE_USE_DEEP_GEMM="0" - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_higgs_audio_v3.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT - mirror_hardwares: h100_1 - - group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]" key: nightly-tiny-model-test-group depends_on: upload-nightly-pipeline @@ -246,8 +205,7 @@ steps: build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test" steps: - # Use explicit file shards so single-card cases do not reserve 2+ GPUs, - # and Bagel feature coverage is not duplicated via a broad tests/e2e sweep. + # Use explicit file shards so single-card cases do not reserve 2+ GPUs. - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU" timeout_in_minutes: 120 @@ -255,15 +213,6 @@ steps: - >- pytest -sv tests/e2e/online_serving/test_qwen_image_expansion.py - tests/e2e/online_serving/test_qwen_image_edit_expansion.py - tests/e2e/online_serving/test_qwen_image_layered_expansion.py - tests/e2e/online_serving/test_boogu_image.py - tests/e2e/online_serving/test_boogu_image_edit.py - tests/e2e/online_serving/test_boogu_image_expansion.py - tests/e2e/online_serving/test_boogu_image_edit_expansion.py - tests/e2e/online_serving/test_bagel_expansion.py - tests/e2e/online_serving/test_krea2_expansion.py - tests/e2e/offline_inference/test_krea2_expansion.py -m "full_model and diffusion and H100 and not distributed_cuda" --run-level "full_model" mirror_hardwares: h100_1 @@ -274,47 +223,14 @@ steps: - >- pytest -sv tests/e2e/online_serving/test_qwen_image_expansion.py - tests/e2e/online_serving/test_qwen_image_edit_expansion.py - tests/e2e/offline_inference/test_krea2_expansion.py -m "full_model and diffusion and H100 and distributed_cuda" --run-level "full_model" mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Layered/LongCat/FLUX" - timeout_in_minutes: 90 - commands: - - >- - pytest -sv - tests/e2e/online_serving/test_qwen_image_layered_expansion.py - -m "full_model and diffusion and H100 and distributed_cuda" - --run-level "full_model" - mirror_hardwares: h100_2 - - - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Bagel" - timeout_in_minutes: 120 - commands: - - >- - pytest -sv - tests/e2e/online_serving/test_bagel_expansion.py - -m "full_model and diffusion and H100 and distributed_cuda" - --run-level "full_model" - mirror_hardwares: h100_2 - - - label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100" - timeout_in_minutes: 120 - commands: - - >- - pytest -sv - tests/e2e/online_serving/test_glm_image_expansion.py - tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py - -m "full_model and diffusion and H100" - --run-level "full_model" - mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4" timeout_in_minutes: 120 commands: - - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy + - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy mirror_hardwares: l4_4 - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test" @@ -327,7 +243,7 @@ steps: timeout_in_minutes: 180 commands: - export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120 - - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model + - pytest -s -v tests/e2e/accuracy/test_qwen_image.py --run-level full_model - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model mirror_hardwares: h100_1 @@ -355,70 +271,6 @@ steps: exit $$EXIT mirror_hardwares: h100_4 - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit" - key: nightly-diffusion-x2iat-performance-qwen-image-edit - timeout_in_minutes: 180 - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - - export CACHE_DIT_VERSION=1.5.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT - mirror_hardwares: h100_4 - - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511" - key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511 - timeout_in_minutes: 180 - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - - export CACHE_DIT_VERSION=1.5.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT - mirror_hardwares: h100_4 - - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Layered" - key: nightly-diffusion-x2iat-performance-qwen-image-layered - timeout_in_minutes: 120 - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - - export CACHE_DIT_VERSION=1.5.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT - mirror_hardwares: h100_1 - - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · BAGEL" - key: nightly-diffusion-x2iat-performance-bagel - timeout_in_minutes: 180 - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - - export CACHE_DIT_VERSION=1.5.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT - mirror_hardwares: h100_1 - - group: ":card_index_dividers: Diffusion X2V Model Test" key: nightly-diffusion-x2v-group depends_on: upload-nightly-pipeline @@ -440,12 +292,6 @@ steps: - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model" mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2V · LingBot Video MoE Function Test" - timeout_in_minutes: 90 - commands: - - pytest -s -v tests/e2e/online_serving/test_lingbot_video_moe.py -k "video_generation" -m "full_model and cuda" --run-level "full_model" - mirror_hardwares: h100_1 - - label: ":full_moon: Diffusion X2V · Doc Test" timeout_in_minutes: 60 commands: @@ -456,15 +302,8 @@ steps: timeout_in_minutes: 90 commands: - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model" - - pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx.py -m "full_model and cuda" --run-level "full_model" mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2V · LTX-2.5 Online Test" - timeout_in_minutes: 60 - commands: - - pytest -s -v tests/e2e/online_serving/test_ltx25.py -m "full_model and cuda" --run-level "full_model" - mirror_hardwares: h100_1 - - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test" timeout_in_minutes: 180 commands: @@ -483,23 +322,8 @@ steps: commands: - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model - - pytest -s -v tests/e2e/accuracy/test_video_streaming_output_similarity.py -m full_model --run-level full_model mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2V · LTX Accuracy Test" - timeout_in_minutes: 180 - commands: - - pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_matches_official[ltx2_3_i2v]' -m full_model --run-level full_model - - pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_matches_official[ltx23_two_stage_layer_fused_i2v]' -m full_model --run-level full_model - mirror_hardwares: h100_1 - - - label: ":full_moon: Diffusion X2V · LTX-2.5 Official Parity Test" - timeout_in_minutes: 180 - commands: - - pytest -s -v tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py::test_ltx25_distilled_two_stage_matches_official -m full_model --run-level full_model - - pytest -s -v tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py::test_ltx25_full_matches_official -m full_model --run-level full_model - mirror_hardwares: h100_1 - - label: ":full_moon: Diffusion X2V · Perf Test" key: nightly-diffusion-x2v-performance timeout_in_minutes: 180 @@ -575,16 +399,6 @@ steps: - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model" mirror_hardwares: l4_1 - - label: ":bar_chart: Testcase Statistics" - key: nightly-testcase-statistics - timeout_in_minutes: 120 - depends_on: upload-nightly-pipeline - if: build.env("NIGHTLY") == "1" - commands: - - python tools/nightly/buildkite_testcase_statistics.py -o tests/dfx/perf/results/buildkite_testcase_statistics.html - - buildkite-agent artifact upload "tests/dfx/perf/results/*.html" - mirror_hardwares: h100_1 - # NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon. # Do not add new depends_on entries or extend this step. - label: ":email: Nightly Collection & Email" @@ -595,32 +409,20 @@ steps: - nightly-omni-performance-vllm-text - nightly-omni-performance-multi-replicas - nightly-tts-performance - - nightly-tts-performance-higgs-audio-v3 - nightly-diffusion-x2iat-performance-qwen-image - - nightly-diffusion-x2iat-performance-qwen-image-edit - - nightly-diffusion-x2iat-performance-qwen-image-edit-2511 - - nightly-diffusion-x2iat-performance-qwen-image-layered - - nightly-diffusion-x2iat-performance-bagel - nightly-diffusion-x2v-performance - - nightly-testcase-statistics if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1" commands: - pip install openpyxl - export DEFAULT_INPUT_DIR=tests/dfx/perf/results - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-higgs-audio-v3 - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2511 - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance - - buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics - python tools/nightly/generate_nightly_perf_excel.py - python tools/nightly/generate_nightly_perf_html.py - python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html" diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml index 0fd3a06ce..5ddbf22ce 100644 --- a/.buildkite/cuda/test-weekly.yml +++ b/.buildkite/cuda/test-weekly.yml @@ -149,4 +149,4 @@ steps: pytest -sv tests/e2e/ -m "slow and diffusion and H100 and distributed_cuda" --run-level "full_model" - mirror_hardwares: h100_2 + mirror_hardwares: h100_4 diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index 050cef4f7..2cdf0cd54 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -96,33 +96,6 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test" - key: nightly-diffusion-x2iat-group - depends_on: upload-nightly-pipeline - if: >- - build.env("NIGHTLY") == "1" || - build.pull_request.labels includes "nightly-test" || - build.pull_request.labels includes "diffusion-x2iat-test" - steps: - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511" - key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511 - timeout_in_minutes: 180 - mirror_hardwares: a3_npu_8 - env: - VLLM_WORKER_MULTIPROC_METHOD: spawn - VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" - HF_TOKEN: "${HF_TOKEN}" - DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export CACHE_DIT_VERSION=1.5.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT - group: ":card_index_dividers: Diffusion I2I Model Test" key: nightly-HunyuanImage3-x2i-test-group depends_on: upload-nightly-pipeline diff --git a/tests/dfx/perf/tests/test_bagel_vllm_omni.json b/tests/dfx/perf/tests/test_bagel_vllm_omni.json index 9f6ff0241..9dda3148d 100644 --- a/tests/dfx/perf/tests/test_bagel_vllm_omni.json +++ b/tests/dfx/perf/tests/test_bagel_vllm_omni.json @@ -10,7 +10,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Single-stage BAGEL (TP=1, CFG=1), t2i Task", @@ -61,7 +61,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Single-stage BAGEL (TP=1, CFG=1), i2i Task", @@ -113,7 +113,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, t2i Task", @@ -164,7 +164,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, i2i Task", diff --git a/tests/dfx/perf/tests/test_higgs_audio_v3.json b/tests/dfx/perf/tests/test_higgs_audio_v3.json index 75e5983b0..1197c88f1 100644 --- a/tests/dfx/perf/tests/test_higgs_audio_v3.json +++ b/tests/dfx/perf/tests/test_higgs_audio_v3.json @@ -10,7 +10,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "tts" ], "server_params": { diff --git a/tests/dfx/perf/tests/test_ltx2_vllm_omni.json b/tests/dfx/perf/tests/test_ltx2_vllm_omni.json index 1a0f31e98..9b0634a20 100644 --- a/tests/dfx/perf/tests/test_ltx2_vllm_omni.json +++ b/tests/dfx/perf/tests/test_ltx2_vllm_omni.json @@ -10,7 +10,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Single-device baseline with enforce-eager (no torch.compile)", @@ -64,7 +64,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Single-device with torch.compile (default, no enforce-eager)", @@ -117,7 +117,7 @@ "num_cards": 2 } }, - "full_model", + "local_model", "diffusion" ], "description": "CFG-parallel=2 with enforce-eager", @@ -172,7 +172,7 @@ "num_cards": 2 } }, - "full_model", + "local_model", "diffusion" ], "description": "CFG-parallel=2 with torch.compile", @@ -226,7 +226,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "CacheDiT with enforce-eager", diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json index dcd2188c5..fd8e437fd 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json @@ -10,7 +10,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Single-device baseline (two input images)", @@ -76,7 +76,7 @@ "num_cards": 4 } }, - "full_model", + "local_model", "diffusion" ], "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4", @@ -125,7 +125,7 @@ "num_cards": 4 } }, - "full_model", + "local_model", "diffusion" ], "description": "Ulysses SP=2 + CFG=2 + CacheDiT", diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json index 686bd6160..62ee70aa2 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json @@ -10,7 +10,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Single-device baseline", @@ -73,7 +73,7 @@ "num_cards": 4 } }, - "full_model", + "local_model", "diffusion" ], "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4", @@ -121,7 +121,7 @@ "num_cards": 4 } }, - "full_model", + "local_model", "diffusion" ], "description": "Ulysses SP=2 + CFG=2 + CacheDiT", diff --git a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json index bc4d93a24..b5fa2815b 100644 --- a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json @@ -10,7 +10,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Single-device baseline", diff --git a/tests/dfx/perf/tests/test_voxcpm2.json b/tests/dfx/perf/tests/test_voxcpm2.json index e5163e5bb..1756bc206 100644 --- a/tests/dfx/perf/tests/test_voxcpm2.json +++ b/tests/dfx/perf/tests/test_voxcpm2.json @@ -10,7 +10,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "tts" ], "server_params": { diff --git a/tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py b/tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py index 9f1623c8a..d72d14022 100644 --- a/tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py +++ b/tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py @@ -463,7 +463,7 @@ def _audio_metrics(reference: np.ndarray, prediction: np.ndarray) -> dict[str, f } -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.benchmark @pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=1) @@ -591,7 +591,7 @@ def test_ltx25_distilled_two_stage_matches_official(accuracy_artifact_root: Path assert audio_metrics["relative_l2"] <= LTX25_AUDIO_RELATIVE_L2_THRESHOLD -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.benchmark @pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=1) diff --git a/tests/e2e/accuracy/ltx/test_ltx_official_similarity.py b/tests/e2e/accuracy/ltx/test_ltx_official_similarity.py index 8a3be4a99..59a8540aa 100644 --- a/tests/e2e/accuracy/ltx/test_ltx_official_similarity.py +++ b/tests/e2e/accuracy/ltx/test_ltx_official_similarity.py @@ -544,7 +544,7 @@ def _audio_metrics(reference: np.ndarray, prediction: np.ndarray) -> dict[str, f @pytest.mark.parametrize("case", CASES, ids=lambda case: case.name) -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.benchmark @pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=1) diff --git a/tests/e2e/accuracy/test_qwen_image_edit.py b/tests/e2e/accuracy/test_qwen_image_edit.py index 9ee9c2c87..935a032d8 100644 --- a/tests/e2e/accuracy/test_qwen_image_edit.py +++ b/tests/e2e/accuracy/test_qwen_image_edit.py @@ -17,7 +17,7 @@ from tests.helpers.mark import hardware_test from tests.helpers.media import get_asset_path from tests.helpers.runtime import OmniServer -pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] +pytestmark = [pytest.mark.diffusion] SINGLE_MODEL = "Qwen/Qwen-Image-Edit" @@ -253,6 +253,7 @@ def _diffusers_output_single_image(accuracy_artifact_root: Path, qwen_bear_image ) +@pytest.mark.slow @pytest.mark.benchmark @hardware_test(res={"cuda": "H100"}, num_cards=1) def test_qwen_image_edit_single_matches_diffusers( @@ -278,6 +279,7 @@ def test_qwen_image_edit_single_matches_diffusers( ) +@pytest.mark.slow @pytest.mark.benchmark @hardware_test(res={"cuda": "H100"}, num_cards=1) def test_qwen_image_edit_2511_matches_diffusers_pixelwise(accuracy_artifact_root: Path) -> None: diff --git a/tests/e2e/accuracy/test_qwen_image_layered.py b/tests/e2e/accuracy/test_qwen_image_layered.py index f2bc1b6aa..3ad8d3328 100644 --- a/tests/e2e/accuracy/test_qwen_image_layered.py +++ b/tests/e2e/accuracy/test_qwen_image_layered.py @@ -17,7 +17,7 @@ from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServer -pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] +pytestmark = [pytest.mark.slow, pytest.mark.diffusion] MODEL_ID = "Qwen/Qwen-Image-Layered" diff --git a/tests/e2e/accuracy/test_video_streaming_output_similarity.py b/tests/e2e/accuracy/test_video_streaming_output_similarity.py index 799e6e125..1aa7cdfc2 100644 --- a/tests/e2e/accuracy/test_video_streaming_output_similarity.py +++ b/tests/e2e/accuracy/test_video_streaming_output_similarity.py @@ -11,7 +11,7 @@ from tests.e2e.accuracy.helpers import assert_video_similarity_metrics, probe_vi from tests.helpers.mark import hardware_marks from tests.helpers.runtime import DiffusionResponse, OmniServer, OpenAIClientHandler -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = "BestWishYsh/Helios-Distilled" PROMPT = "A serene lakeside sunrise with mist over the water." diff --git a/tests/e2e/offline_inference/test_bagel_expansion.py b/tests/e2e/offline_inference/test_bagel_expansion.py index e8ac6c19f..4c31be722 100644 --- a/tests/e2e/offline_inference/test_bagel_expansion.py +++ b/tests/e2e/offline_inference/test_bagel_expansion.py @@ -138,7 +138,7 @@ def _make_file_lora_request(adapter_dir: Path) -> LoRARequest: pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.diffusion, pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), ] diff --git a/tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py b/tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py index 28eaab9b3..86b8cee8a 100644 --- a/tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py +++ b/tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py @@ -109,7 +109,7 @@ def _diffusion_sampling_params( pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.diffusion, ] diff --git a/tests/e2e/offline_inference/test_moss_tts_realtime.py b/tests/e2e/offline_inference/test_moss_tts_realtime.py index 892a395ba..34d36118d 100644 --- a/tests/e2e/offline_inference/test_moss_tts_realtime.py +++ b/tests/e2e/offline_inference/test_moss_tts_realtime.py @@ -96,7 +96,7 @@ def _get_test_config() -> str: pytestmark = [ pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4700"), - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.tts, pytest.mark.parametrize( "omni_runner", @@ -271,7 +271,6 @@ def _collect_audio(omni_runner: OmniRunner, request: dict) -> tuple[torch.Tensor # --------------------------------------------------------------------------- -@pytest.mark.advanced_model @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_moss_tts_realtime_english(omni_runner: OmniRunner, ref_audio_path: str) -> None: """MossTTSRealtime: English voice_clone produces non-empty 24 kHz audio.""" @@ -283,7 +282,6 @@ def test_moss_tts_realtime_english(omni_runner: OmniRunner, ref_audio_path: str) assert not torch.all(audio == 0), "Audio is silence" -@pytest.mark.advanced_model @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_moss_tts_realtime_chinese(omni_runner: OmniRunner, ref_audio_path: str) -> None: """MossTTSRealtime: Chinese input produces non-empty audio.""" diff --git a/tests/e2e/online_serving/test_bagel_expansion.py b/tests/e2e/online_serving/test_bagel_expansion.py index fa17791e5..3308efdcb 100644 --- a/tests/e2e/online_serving/test_bagel_expansion.py +++ b/tests/e2e/online_serving/test_bagel_expansion.py @@ -22,7 +22,7 @@ from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] PROMPT = "A futuristic city skyline at twilight, cyberpunk style, ultra-detailed, high resolution." NEGATIVE_PROMPT = "low quality, blurry, distorted, deformed, watermark" diff --git a/tests/e2e/online_serving/test_bagel_multi_replicas.py b/tests/e2e/online_serving/test_bagel_multi_replicas.py index cb7aeeb94..6409ddd58 100644 --- a/tests/e2e/online_serving/test_bagel_multi_replicas.py +++ b/tests/e2e/online_serving/test_bagel_multi_replicas.py @@ -1,6 +1,8 @@ -"""Nightly-only e2e coverage for BAGEL diffusion multi-replica serving. +"""Weekly e2e coverage for BAGEL diffusion multi-replica serving. This test needs 4 H100 GPUs; keep it out of generic test_bagel_* jobs. +It is covered by the weekly Diffusion · H100 · Multi-GPU lane +(``.buildkite/cuda/test-weekly.yml``, ``mirror_hardwares: h100_4``). """ import os @@ -17,7 +19,7 @@ MODEL = "ByteDance-Seed/BAGEL-7B-MoT" MULTI_REPLICA_DEPLOY = get_deploy_config_path("ci/bagel_multi_replicas_4gpu.yaml") ROUTE_STRESS_REQUESTS = 6 -pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] +pytestmark = [pytest.mark.slow, pytest.mark.diffusion] test_params = [ OmniServerParams( diff --git a/tests/e2e/online_serving/test_boogu_image.py b/tests/e2e/online_serving/test_boogu_image.py index 5a5494d7d..440e031f1 100644 --- a/tests/e2e/online_serving/test_boogu_image.py +++ b/tests/e2e/online_serving/test_boogu_image.py @@ -16,7 +16,7 @@ parity vs. the Diffusers baseline is covered separately (see the support plan, s From ``tests/``:: pytest -s -v e2e/online_serving/test_boogu_image.py \ - -m "full_model and diffusion and H100" --run-level=full_model + -m "slow and diffusion and H100" """ import os @@ -28,7 +28,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = "Boogu/Boogu-Image-0.1-Base" T2I_PROMPT = "A mountain lake at sunset, photorealistic, cinematic lighting" diff --git a/tests/e2e/online_serving/test_boogu_image_edit.py b/tests/e2e/online_serving/test_boogu_image_edit.py index 45f999d26..d7b6d1dd1 100644 --- a/tests/e2e/online_serving/test_boogu_image_edit.py +++ b/tests/e2e/online_serving/test_boogu_image_edit.py @@ -19,7 +19,7 @@ reference image is supported. From ``tests/``:: pytest -s -v e2e/online_serving/test_boogu_image_edit.py \ - -m "full_model and diffusion and H100" --run-level=full_model + -m "slow and diffusion and H100" """ import base64 @@ -35,7 +35,7 @@ from tests.helpers.runtime import ( dummy_messages_from_mix_data, ) -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] EDIT_PROMPT = "Change the style to a colored pencil drawing." SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) diff --git a/tests/e2e/online_serving/test_boogu_image_edit_expansion.py b/tests/e2e/online_serving/test_boogu_image_edit_expansion.py index efc968eb6..1825aaafc 100644 --- a/tests/e2e/online_serving/test_boogu_image_edit_expansion.py +++ b/tests/e2e/online_serving/test_boogu_image_edit_expansion.py @@ -25,7 +25,7 @@ model and are intentionally omitted. From ``tests/``:: pytest -s -v e2e/online_serving/test_boogu_image_edit_expansion.py \ - -m "full_model and diffusion and H100" --run-level=full_model + -m "slow and diffusion and H100" """ import os @@ -38,7 +38,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = "Boogu/Boogu-Image-0.1-Edit" EDIT_PROMPT = "Change the style to a colored pencil drawing." diff --git a/tests/e2e/online_serving/test_boogu_image_expansion.py b/tests/e2e/online_serving/test_boogu_image_expansion.py index ef6d3a62e..a56e5a338 100644 --- a/tests/e2e/online_serving/test_boogu_image_expansion.py +++ b/tests/e2e/online_serving/test_boogu_image_expansion.py @@ -30,7 +30,7 @@ omitted (add rows here when those features land). From ``tests/``:: pytest -s -v e2e/online_serving/test_boogu_image_expansion.py \ - -m "full_model and diffusion and H100" --run-level=full_model + -m "slow and diffusion and H100" """ import os @@ -42,7 +42,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = "Boogu/Boogu-Image-0.1-Base" T2I_PROMPT = "A mountain lake at sunset, photorealistic, cinematic lighting" diff --git a/tests/e2e/online_serving/test_glm_image_expansion.py b/tests/e2e/online_serving/test_glm_image_expansion.py index d2a08f633..42e8d444a 100644 --- a/tests/e2e/online_serving/test_glm_image_expansion.py +++ b/tests/e2e/online_serving/test_glm_image_expansion.py @@ -30,7 +30,7 @@ from tests.helpers.runtime import ( ) from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = os.environ.get("GLM_IMAGE_MODEL_PATH", "zai-org/GLM-Image") diff --git a/tests/e2e/online_serving/test_lingbot_video_moe.py b/tests/e2e/online_serving/test_lingbot_video_moe.py index 777880732..7e4d16568 100644 --- a/tests/e2e/online_serving/test_lingbot_video_moe.py +++ b/tests/e2e/online_serving/test_lingbot_video_moe.py @@ -63,7 +63,7 @@ def test_text_to_image_moe(omni_server: OmniServer, openai_client: OpenAIClientH ) -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize("omni_server", _get_server_cases(MODEL), indirect=True) def test_video_generation_modes_moe(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: diff --git a/tests/e2e/online_serving/test_ltx.py b/tests/e2e/online_serving/test_ltx.py index 43df5df2d..e4d8f51a8 100644 --- a/tests/e2e/online_serving/test_ltx.py +++ b/tests/e2e/online_serving/test_ltx.py @@ -10,7 +10,7 @@ from tests.helpers.mark import hardware_marks from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] DISTILLED_MODEL = os.getenv("VLLM_TEST_LTX23_DISTILLED_MODEL", "diffusers/LTX-2.3-Distilled-Diffusers") BASE_MODEL = os.getenv("VLLM_TEST_LTX23_MODEL", "diffusers/LTX-2.3-Diffusers") diff --git a/tests/e2e/online_serving/test_ltx25.py b/tests/e2e/online_serving/test_ltx25.py index 2fad9f16a..da5c53851 100644 --- a/tests/e2e/online_serving/test_ltx25.py +++ b/tests/e2e/online_serving/test_ltx25.py @@ -20,7 +20,7 @@ MODEL = os.getenv("VLLM_TEST_LTX25_MODEL", DEFAULT_MODEL) MODEL_REVISION = os.getenv("VLLM_TEST_LTX25_MODEL_REVISION", DEFAULT_REVISION if MODEL == DEFAULT_MODEL else "") PROMPT = "A red fox walks through a snowy forest while the camera remains fixed." -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"}) diff --git a/tests/e2e/online_serving/test_ming_flash_omni_expansion.py b/tests/e2e/online_serving/test_ming_flash_omni_expansion.py index 8ab327022..6f6881d68 100644 --- a/tests/e2e/online_serving/test_ming_flash_omni_expansion.py +++ b/tests/e2e/online_serving/test_ming_flash_omni_expansion.py @@ -20,7 +20,7 @@ from tests.helpers.stage_config import get_deploy_config_path os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -pytestmark = [pytest.mark.omni, pytest.mark.full_model] +pytestmark = [pytest.mark.omni, pytest.mark.slow] _SKIP_NEED_4_H100_NOT_CI = pytest.mark.skip( reason="Requires 4x H100 GPUs; skipped in CI for now.", diff --git a/tests/e2e/online_serving/test_qwen_image_edit.py b/tests/e2e/online_serving/test_qwen_image_edit.py index a02bc1b57..14d27318c 100644 --- a/tests/e2e/online_serving/test_qwen_image_edit.py +++ b/tests/e2e/online_serving/test_qwen_image_edit.py @@ -2,20 +2,19 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ -Online serving tests for the Qwen-Image-Edit family (image-to-image via chat completions). +Online serving tests for ``Qwen/Qwen-Image-Edit-2511`` (image-to-image via chat completions). -- ``test_single_image_to_image_001``: ``Qwen/Qwen-Image-Edit`` — one reference image, fixed 512×512. -- ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2511`` — two reference images, fixed 512×512. -- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2511`` only, ``slow`` — mixed input - resolutions; ``extra_body`` uses per-output ``width``/``height`` lists; the test client sends one - scalar-size request per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``). +- ``test_single_image_to_image_001``: one reference image, fixed 512×512. +- ``test_multi_images_to_image_001``: two reference images, fixed 512×512. +- ``test_different_sizes_001``: ``slow`` — mixed input resolutions; ``extra_body`` uses + per-output ``width``/``height`` lists; the test client sends one scalar-size request + per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``). -``_get_diffusion_feature_cases`` registers a single ``default`` ``OmniServerParams`` row per model. +``_get_diffusion_feature_cases`` registers a single ``default`` ``OmniServerParams`` row. From ``tests/``:: - pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "core_model and diffusion" --run-level=core_model - pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "advanced_model and diffusion" --run-level=advanced_model + pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "slow and diffusion" --run-level=full_model """ import pytest @@ -29,6 +28,7 @@ from tests.helpers.runtime import ( dummy_messages_from_mix_data, ) +MODEL = "Qwen/Qwen-Image-Edit-2511" EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting." MULTI_EDIT_PROMPT = ( "Transform the first image into a Dadaism collage art. " @@ -39,7 +39,7 @@ NEGATIVE_PROMPT = "blurry, low quality, modern, geometrist" SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) -def _get_diffusion_feature_cases(model: str): +def _get_diffusion_feature_cases(model: str = MODEL): """Return one ``default`` ``OmniServerParams`` row for ``model`` (no extra ``server_args``).""" return [ pytest.param( @@ -56,11 +56,11 @@ def _get_diffusion_feature_cases(model: str): @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", - _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit"), + _get_diffusion_feature_cases(), indirect=True, ) def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler): - """Single-reference edit smoke for ``Qwen/Qwen-Image-Edit`` (CFG when negative prompt + ``true_cfg_scale`` > 1).""" + """Single-reference edit smoke for ``Qwen/Qwen-Image-Edit-2511`` (CFG when negative prompt + ``true_cfg_scale`` > 1).""" image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT) @@ -86,7 +86,7 @@ def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenA @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", - _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"), + _get_diffusion_feature_cases(), indirect=True, ) def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler): @@ -117,7 +117,7 @@ def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenA @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", - _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"), + _get_diffusion_feature_cases(), indirect=True, ) def test_different_sizes_001(omni_server: OmniServer, openai_client: OpenAIClientHandler): diff --git a/tests/e2e/online_serving/test_qwen_image_edit_expansion.py b/tests/e2e/online_serving/test_qwen_image_edit_expansion.py index b3d6f8b19..e5e8c6c2b 100644 --- a/tests/e2e/online_serving/test_qwen_image_edit_expansion.py +++ b/tests/e2e/online_serving/test_qwen_image_edit_expansion.py @@ -1,8 +1,6 @@ """ Comprehensive tests of diffusion features that are available in online serving mode -and are supported by the following models: -- Qwen-Image-Edit: single image input -- Qwen-Image-Edit-2511: single image input and two image inputs +and are supported by Qwen-Image-Edit-2511 (single image input and two image inputs). """ import pytest @@ -11,10 +9,10 @@ from tests.helpers.mark import hardware_marks from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] -EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting." -SINGLE_EDIT_PROMPT_2511 = "Restyle this image into a Vincent van Gogh style impressionist painting." +MODEL = "Qwen/Qwen-Image-Edit-2511" +SINGLE_EDIT_PROMPT = "Restyle this image into a Vincent van Gogh style impressionist painting." MULTI_EDIT_PROMPT = ( "Transform the first image into a Dadaism collage art. " "Transform the second image into a Vincent van Gogh style painting. " @@ -25,9 +23,7 @@ SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2) -# This test file targets two models, so I write a helper function. -# If a similar test only involves one model, one can just define a global list variable. -def _get_diffusion_feature_cases(model: str): +def _get_diffusion_feature_cases(model: str = MODEL): return [ pytest.param( OmniServerParams( @@ -123,35 +119,7 @@ def _get_diffusion_feature_cases(model: str): @pytest.mark.parametrize( "omni_server", - _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit"), - indirect=True, -) -def test_qwen_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHandler): - """Test all diffusion features with Qwen-Image-Edit in regular end-user scenarios.""" - image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" - - messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT) - - # CFG parallel is only activated when a negative prompt and true_cfg_scale > 1.0 are both present - request_config = { - "model": omni_server.model, - "messages": messages, - "extra_body": { - "height": 512, - "width": 512, - "num_inference_steps": 2, - "negative_prompt": NEGATIVE_PROMPT, - "true_cfg_scale": 4.0, - "seed": 42, - }, - } - - openai_client.send_diffusion_request(request_config) - - -@pytest.mark.parametrize( - "omni_server", - _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"), + _get_diffusion_feature_cases(), indirect=True, ) def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_client: OpenAIClientHandler): @@ -166,7 +134,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien """ image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" - messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT_2511) + messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT) request_config = { "model": omni_server.model, @@ -186,7 +154,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien @pytest.mark.parametrize( "omni_server", - _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"), + _get_diffusion_feature_cases(), indirect=True, ) def test_qwen_image_edit_2511_two_images(omni_server: OmniServer, openai_client: OpenAIClientHandler): diff --git a/tests/e2e/online_serving/test_qwen_image_layered_expansion.py b/tests/e2e/online_serving/test_qwen_image_layered_expansion.py index 315fba6ef..8c5001a4a 100644 --- a/tests/e2e/online_serving/test_qwen_image_layered_expansion.py +++ b/tests/e2e/online_serving/test_qwen_image_layered_expansion.py @@ -18,7 +18,7 @@ from tests.helpers.mark import hardware_marks from tests.helpers.media import decode_b64_image, generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = "Qwen/Qwen-Image-Layered" EDIT_PROMPT = "Decompose this image into layers." diff --git a/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py b/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py index 996585a1d..de73ae6da 100644 --- a/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py +++ b/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py @@ -16,7 +16,7 @@ from tests.helpers.media import load_test_audio_data_url from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path -pytestmark = [pytest.mark.full_model, pytest.mark.tts] +pytestmark = [pytest.mark.slow, pytest.mark.tts] os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" diff --git a/tests/e2e/online_serving/test_wan_2_1_vace_expansion.py b/tests/e2e/online_serving/test_wan_2_1_vace_expansion.py index d58bb9e32..4c6380314 100644 --- a/tests/e2e/online_serving/test_wan_2_1_vace_expansion.py +++ b/tests/e2e/online_serving/test_wan_2_1_vace_expansion.py @@ -26,7 +26,7 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = "Wan-AI/Wan2.1-VACE-1.3B-diffusers" PROMPT = "A cat walking slowly across a sunlit garden path" diff --git a/tools/nightly/buildkite_testcase_statistics.py b/tools/nightly/buildkite_testcase_statistics.py index 740f896f5..06f09c791 100644 --- a/tools/nightly/buildkite_testcase_statistics.py +++ b/tools/nightly/buildkite_testcase_statistics.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 """ -Parse pytest commands from Buildkite test-ready.yml, test-merge.yml, test-nightly.yml, -and test-weekly.yml; collect test cases (including parametrized) via pytest --collect-only -q -and produce an HTML report. +Parse pytest commands from Buildkite .buildkite/cuda/test-{ready,merge,nightly,weekly}.yml; +collect test cases (including parametrized) via pytest --collect-only -q and produce an HTML report. Leaf steps may live under ``group:`` blocks (nested ``steps``); those are flattened with the group title carried into the report. @@ -30,7 +29,7 @@ import yaml # Repo root (parent of the directory containing this script) REPO_ROOT = Path(__file__).resolve().parent.parent.parent -BUILDKITE_DIR = REPO_ROOT / ".buildkite" +BUILDKITE_DIR = REPO_ROOT / ".buildkite" / "cuda" PIPELINE_FILES = ["test-ready.yml", "test-merge.yml", "test-nightly.yml", "test-weekly.yml"] @@ -256,13 +255,34 @@ def _resolve_pytest_target(target: str, repo_root: Path, extra: list[str], raw_l rel_dir = Path(target.rstrip("/")) dir_path = (repo_root / rel_dir).resolve() if dir_path.exists() and dir_path.is_dir(): - return [str(dir_path)], target.replace("\\", "/").rstrip("/"), 120 + return [str(dir_path)], target.replace("\\", "/").rstrip("/"), 300 if not extra: raise RuntimeError(f"Failed to parse pytest sidecar args from line: {raw_line!r}") if not (repo_root / "tests").exists(): raise FileNotFoundError("tests/ directory not found under repo root") - return ["tests/"], "tests/", 120 + # Broad ``tests/`` collection loads many conftests; allow more time on busy CI agents. + return ["tests/"], "tests/", 300 + + +def _format_pytest_collect_failure( + path_args: list[str], + extra: list[str], + result: subprocess.CompletedProcess[str], +) -> str: + """pytest often puts collection errors on stdout; include both streams + exit code.""" + stdout = (result.stdout or "").strip() + stderr = (result.stderr or "").strip() + parts = [ + f"pytest --collect-only failed for {path_args} with args {extra} (exit={result.returncode})", + ] + if stderr: + parts.append(f"stderr:\n{stderr}") + if stdout: + parts.append(f"stdout:\n{stdout}") + if not stderr and not stdout: + parts.append("(no stdout/stderr captured)") + return "\n".join(parts) def collect_test_names(target: str, repo_root: Path, raw_line: str) -> list[str]: @@ -273,6 +293,8 @@ def collect_test_names(target: str, repo_root: Path, raw_line: str) -> list[str] flags ``--ignore``, ``--test-config-file``, ``-m``, ``-k``, ``--run-level`` are taken from the full ``raw_line`` and appended to the same invocation so collection matches CI (perf scripts parametrize from ``--test-config-file`` at import time). + + Pytest exit code 5 (no tests collected) is treated as an empty result, not a hard failure. """ extra = _pytest_collect_sidecar_args(raw_line) path_args, _fallback, timeout_quiet = _resolve_pytest_target(target, repo_root, extra, raw_line) @@ -286,8 +308,15 @@ def collect_test_names(target: str, repo_root: Path, raw_line: str) -> list[str] text=True, timeout=timeout_quiet, ) + # 5 == no tests collected under the given -m/-k filters. + if result.returncode == 5: + print( + f"pytest --collect-only: no tests matched for {path_args} with args {extra}", + file=sys.stderr, + ) + return [] if result.returncode != 0: - raise RuntimeError(f"pytest --collect-only failed for {path_args} with args {extra}: {result.stderr.strip()}") + raise RuntimeError(_format_pytest_collect_failure(path_args, extra, result)) print(f"pytest --collect-only success for {path_args} with args {extra}: {result.stdout.strip()}") return _parse_collect_only_stdout(result.stdout or "", stderr=result.stderr or "") @@ -962,7 +991,7 @@ def main() -> None: "--buildkite-dir", type=Path, default=BUILDKITE_DIR, - help="Path to .buildkite directory", + help="Path to Buildkite CUDA pipeline directory (default: .buildkite/cuda)", ) default_out = REPO_ROOT / "buildkite_testcase_statistics.html" parser.add_argument( @@ -1023,7 +1052,12 @@ def main() -> None: names = [] skipped_ids: set[str] = set() for target, raw in targets: - collected = collect_test_names(target, REPO_ROOT, raw) + try: + collected = collect_test_names(target, REPO_ROOT, raw) + except (RuntimeError, FileNotFoundError, subprocess.TimeoutExpired) as exc: + # Keep generating the report for other steps; surface the root cause. + print(f"WARNING: collect failed for {module_key} target={target!r}: {exc}", file=sys.stderr) + collected = [] skipped_ids.update(get_skip_status(target, raw, REPO_ROOT)) names.extend(collected) count = len(names) - len(skipped_ids) From 65b3d41b3ad9525bdfa2cbe4bd0bd099282d55b2 Mon Sep 17 00:00:00 2001 From: boatman <1930807094@qq.com> Date: Mon, 17 Aug 2026 12:14:26 +0800 Subject: [PATCH 041/267] [Migrate] Move sampling parameter overrides from legacy dispatches to TTS adapters (#5272) Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- .../openai_api/test_audex_serving_guards.py | 33 +- .../openai_api/test_serving_speech.py | 114 ++++++- .../openai_api/test_tts_detection.py | 17 +- tests/tools/test_check_tts_adapter.py | 10 - tests/worker/test_gpu_ar_model_runner.py | 10 + tools/pre_commit/check_tts_adapter.py | 44 +-- .../entrypoints/openai/serving_speech.py | 281 +----------------- .../openai/tts_adapters/__init__.py | 16 +- .../entrypoints/openai/tts_adapters/audex.py | 70 ++++- .../openai/tts_adapters/audex_tta.py | 84 +++++- .../entrypoints/openai/tts_adapters/base.py | 48 ++- .../openai/tts_adapters/cosyvoice3.py | 68 ++++- .../openai/tts_adapters/fish_speech.py | 18 +- .../openai/tts_adapters/glm_tts.py | 67 ++++- .../openai/tts_adapters/higgs_audio_v2.py | 13 +- .../openai/tts_adapters/higgs_audio_v3.py | 13 +- .../openai/tts_adapters/indextts2.py | 11 +- .../openai/tts_adapters/ming_tts.py | 31 +- .../openai/tts_adapters/qwen3_tts.py | 13 +- .../openai/tts_adapters/voxcpm2.py | 13 +- .../openai/tts_adapters/voxtral.py | 13 +- 22 files changed, 565 insertions(+), 424 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 491123225..535bc1452 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -66,7 +66,7 @@ repos: entry: python tools/pre_commit/check_tts_adapter.py language: python pass_filenames: false - files: ^(vllm_omni/entrypoints/openai/(serving_speech\.py|tts_adapters/__init__\.py)|tools/pre_commit/check_tts_adapter\.py)$ + files: ^(vllm_omni/entrypoints/openai/serving_speech\.py|tools/pre_commit/check_tts_adapter\.py)$ # The budget-is-now-slack reminder is printed on a PASSING run, and # pre-commit hides output from passing hooks unless the hook is verbose. verbose: true diff --git a/tests/entrypoints/openai_api/test_audex_serving_guards.py b/tests/entrypoints/openai_api/test_audex_serving_guards.py index 68661865f..27ed11356 100644 --- a/tests/entrypoints/openai_api/test_audex_serving_guards.py +++ b/tests/entrypoints/openai_api/test_audex_serving_guards.py @@ -26,8 +26,13 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] def _serving(model_type: str = "audex") -> OmniOpenAIServingSpeech: + from vllm_omni.entrypoints.openai.tts_adapters import resolve_adapter + from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext + serving = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech) serving._tts_model_type = model_type + adapter_cls = resolve_adapter(model_type) + serving._adapter = adapter_cls(SpeechServingContext(server=serving)) if adapter_cls is not None else None serving._mark_ref_audio_artifact_ready_for_request = lambda request_id: None serving._discard_ref_audio_artifact_warmup = lambda request_id: None return serving @@ -154,7 +159,7 @@ class TestAudexCfgValidation: assert _adapter().validate(_speech_request(cfg_scale=None)) is None serving = TestAudexCfgInjection()._serving_with_fake_tokenizer() params = SimpleNamespace(extra_args={"cfg_scale": None}) - out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": "p"}, params) + out = serving._adapter._inject_audex_cfg_pair_args("req-1", {"prompt": "p"}, params) assert out.extra_args == {} @pytest.mark.parametrize("key", ["cfg_role", "cfg_pair_id", "cfg_null_prompt"]) @@ -177,21 +182,21 @@ class TestAudexCfgInjection: def test_no_extra_args_is_untouched(self): serving = self._serving_with_fake_tokenizer() params = SimpleNamespace(extra_args=None) - out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": self._cond_prompt()}, params) + out = serving._adapter._inject_audex_cfg_pair_args("req-1", {"prompt": self._cond_prompt()}, params) assert out.extra_args is None assert params.extra_args is None def test_scale_one_leaves_params_byte_identical(self): serving = self._serving_with_fake_tokenizer() params = SimpleNamespace(extra_args={"cfg_scale": 1.0}) - out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": self._cond_prompt()}, params) + out = serving._adapter._inject_audex_cfg_pair_args("req-1", {"prompt": self._cond_prompt()}, params) assert out.extra_args == {} def test_guided_request_gains_pair_contract(self): serving = self._serving_with_fake_tokenizer() params = SimpleNamespace(extra_args={"cfg_scale": 1.5}) cond = self._cond_prompt("Some transcription text here.") - out = serving._inject_audex_cfg_pair_args("req-42", {"prompt": cond}, params) + out = serving._adapter._inject_audex_cfg_pair_args("req-42", {"prompt": cond}, params) extra = out.extra_args assert extra["cfg_scale"] == 1.5 @@ -207,14 +212,14 @@ class TestAudexCfgInjection: serving = self._serving_with_fake_tokenizer() params = SimpleNamespace(extra_args={"cfg_scale": 1.5}) with pytest.raises(ValueError, match="adapter-built text prompt"): - serving._inject_audex_cfg_pair_args("req-1", {"prompt_token_ids": [1]}, params) + serving._adapter._inject_audex_cfg_pair_args("req-1", {"prompt_token_ids": [1]}, params) def test_injection_never_mutates_input_params(self): """The input may be the engine's SHARED default params (review P2): pair state must land only on the returned clone.""" serving = self._serving_with_fake_tokenizer() shared = SimpleNamespace(extra_args={"cfg_scale": 1.5}, temperature=0.1) - out = serving._inject_audex_cfg_pair_args("req-7", {"prompt": self._cond_prompt()}, shared) + out = serving._adapter._inject_audex_cfg_pair_args("req-7", {"prompt": self._cond_prompt()}, shared) assert out is not shared assert shared.extra_args == {"cfg_scale": 1.5} assert shared.temperature == 0.1 @@ -252,11 +257,11 @@ def test_guided_injection_sets_guided_temperature(): serving = TestAudexCfgInjection()._serving_with_fake_tokenizer() cond = TestAudexCfgInjection()._cond_prompt() params = SimpleNamespace(extra_args={"cfg_scale": 1.5}, temperature=0.1) - out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": cond}, params) + out = serving._adapter._inject_audex_cfg_pair_args("req-1", {"prompt": cond}, params) assert out.temperature == 0.05 unguided = SimpleNamespace(extra_args={"cfg_scale": 1.0}, temperature=0.1) - out = serving._inject_audex_cfg_pair_args("req-2", {"prompt": cond}, unguided) + out = serving._adapter._inject_audex_cfg_pair_args("req-2", {"prompt": cond}, unguided) assert out.temperature == 0.1 @@ -340,7 +345,7 @@ class TestAudexTTAInjection: def test_default_injection_applies_rvq_and_cfg3(self): serving = self._serving() params = SimpleNamespace(extra_args=None) - out = serving._inject_audex_tta_args("req-9", self._prompt(), params) + out = serving._adapter._inject_audex_tta_args("req-9", self._prompt(), params) extra = out.extra_args rvq = extra["tta_rvq"] @@ -355,15 +360,15 @@ class TestAudexTTAInjection: def test_explicit_scale_one_disables_cfg_but_keeps_rvq(self): serving = self._serving() params = SimpleNamespace(extra_args={"cfg_scale": 1.0}) - out = serving._inject_audex_tta_args("req-1", self._prompt(), params) + out = serving._adapter._inject_audex_tta_args("req-1", self._prompt(), params) assert "tta_rvq" in out.extra_args assert "cfg_scale" not in out.extra_args assert "cfg_role" not in out.extra_args def test_rvq_contract_cached_per_process(self): serving = self._serving() - out_a = serving._inject_audex_tta_args("a", self._prompt(), SimpleNamespace(extra_args=None)) - out_b = serving._inject_audex_tta_args("b", self._prompt(), SimpleNamespace(extra_args=None)) + out_a = serving._adapter._inject_audex_tta_args("a", self._prompt(), SimpleNamespace(extra_args=None)) + out_b = serving._adapter._inject_audex_tta_args("b", self._prompt(), SimpleNamespace(extra_args=None)) assert out_a.extra_args["tta_rvq"] is out_b.extra_args["tta_rvq"] def test_injection_never_mutates_input_params(self): @@ -372,8 +377,8 @@ class TestAudexTTAInjection: per-request pair state must land only on the returned clone.""" serving = self._serving() shared = SimpleNamespace(extra_args=None) - out_1 = serving._inject_audex_tta_args("req-1", self._prompt(), shared) - out_2 = serving._inject_audex_tta_args("req-2", self._prompt(), shared) + out_1 = serving._adapter._inject_audex_tta_args("req-1", self._prompt(), shared) + out_2 = serving._adapter._inject_audex_tta_args("req-2", self._prompt(), shared) assert shared.extra_args is None assert out_1 is not shared and out_2 is not shared assert out_1.extra_args["cfg_pair_id"] == "req-1" diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 94d84d041..934c0ac64 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -52,6 +52,7 @@ from vllm_omni.model_executor.models.ming_tts.constants import ( KEY_SPEAKER_WAVEFORM, KEY_SPEAKER_WAVEFORM_LENGTHS, SPEAKER_EMBEDDING_DIM, + TEXT_EOS_TOKEN_ID, ) from vllm_omni.outputs import OmniRequestOutput @@ -3618,7 +3619,21 @@ class TestCosyVoice3Serving: assert error is not None assert "max_new_tokens" in error - def test_prepare_speech_generation_cosyvoice3(self, cosyvoice3_server, mocker: MockerFixture): + @pytest.mark.parametrize( + ("max_new_tokens", "expected_min_tokens", "expected_max_tokens"), + [ + (None, 10, 2000), + (5, 5, 5), + ], + ) + def test_prepare_speech_generation_cosyvoice3( + self, + cosyvoice3_server, + mocker: MockerFixture, + max_new_tokens: int | None, + expected_min_tokens: int, + expected_max_tokens: int, + ): cosyvoice3_server._build_cosyvoice3_prompt = mocker.AsyncMock( return_value={ "prompt": "Hello", @@ -3626,15 +3641,23 @@ class TestCosyVoice3Serving: "mm_processor_kwargs": {"prompt_text": "ref text", "sample_rate": 24000}, } ) - cosyvoice3_server._apply_cosyvoice3_dynamic_tokens = mocker.MagicMock(side_effect=lambda spl, req: spl) + cosyvoice3_server.model_config.hf_config = SimpleNamespace( + min_token_text_ratio=1, max_token_text_ratio=200, allowed_special=True + ) + cosyvoice3_server._cosyvoice3_tokenizer = mocker.MagicMock() + mocker.patch("vllm_omni.model_executor.models.cosyvoice3.utils.extract_text_token", return_value=[None, 10]) request = OpenAICreateSpeechRequest( input="Hello", ref_audio="data:audio/wav;base64,abc", ref_text="Reference text", + max_new_tokens=max_new_tokens, ) request_id, generator, tts_params = asyncio.run(cosyvoice3_server._prepare_speech_generation(request)) + sampling_params = cosyvoice3_server.engine_client.generate.call_args.kwargs["sampling_params_list"][0] + assert sampling_params.min_tokens == expected_min_tokens + assert sampling_params.max_tokens == expected_max_tokens assert request_id.startswith("speech-") assert generator == "generator" assert tts_params == {} @@ -3718,6 +3741,90 @@ class TestGLMTTSServing: assert text_token_len == 3 load_tokenizer.assert_called_once() + def test_prepare_speech_generation_glm_tts(self, glm_tts_server, mocker: MockerFixture): + glm_tts_server._build_glm_tts_prompt = mocker.AsyncMock( + return_value={ + "prompt": "Hello", + "multi_modal_data": {"audio": (np.zeros(24000), 24000)}, + "mm_processor_kwargs": {"prompt_text": "ref text", "sample_rate": 24000}, + } + ) + request = OpenAICreateSpeechRequest( + input="Hello", + ref_audio="data:audio/wav;base64,abc", + ref_text="Reference text", + ) + glm_tts_server.model_config.hf_config = SimpleNamespace(min_token_text_ratio=1, max_token_text_ratio=200) + glm_tts_server._estimate_glm_tts_text_token_len = mocker.MagicMock(return_value=10) + request_id, generator, tts_params = asyncio.run(glm_tts_server._prepare_speech_generation(request)) + sampling_params = glm_tts_server.engine_client.generate.call_args.kwargs["sampling_params_list"][0] + + assert sampling_params.min_tokens == 10 + assert sampling_params.max_tokens == 2000 + assert request_id.startswith("speech-") + assert generator == "generator" + assert tts_params == {} + glm_tts_server._build_glm_tts_prompt.assert_awaited_once() + + +@pytest.fixture +def ming_tts_server(mocker: MockerFixture): + mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set()) + mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None) + + mock_engine_client = mocker.MagicMock() + mock_engine_client.errored = False + mock_engine_client.model_config = mocker.MagicMock( + model="inclusionAI/Ming-omni-tts-0.5B", + hf_config=SimpleNamespace(min_token_text_ratio=2, max_token_text_ratio=20), + ) + mock_engine_client.default_sampling_params_list = [ + SimpleNamespace(max_tokens=2048, min_tokens=None, extra_args=None) + ] + mock_engine_client.tts_batch_max_items = 32 + mock_engine_client.generate = mocker.MagicMock(return_value="generator") + mock_engine_client.stage_configs = [ + SimpleNamespace( + engine_args=SimpleNamespace( + model_stage="llm", model_arch="MingTTSForConditionalGeneration", worker_type="ar" + ), + tts_args={}, + ) + ] + + mock_models = mocker.MagicMock() + mock_models.is_base_model.return_value = True + return OmniOpenAIServingSpeech( + engine_client=mock_engine_client, + models=mock_models, + request_logger=mocker.MagicMock(), + ) + + +class TestMingTTSServing: + def test_prepare_speech_generation_ming(self, ming_tts_server, mocker: MockerFixture): + ming_tts_server._adapter.build = mocker.AsyncMock( + return_value=PreparedRequest( + prompt={"prompt": "hello"}, + tts_params={}, + model_type="ming_tts", + ) + ) + + request = OpenAICreateSpeechRequest( + input="Hello", + ref_audio="data:audio/wav;base64,abc", + ref_text="Reference text", + max_new_tokens=7, + ) + asyncio.run(ming_tts_server._prepare_speech_generation(request)) + + assert ming_tts_server._tts_model_type == "ming_tts" + ming_tts_server._adapter.build.assert_awaited_once() + sampling_params = ming_tts_server.engine_client.generate.call_args.kwargs["sampling_params_list"][0] + assert sampling_params.stop_token_ids == [TEXT_EOS_TOKEN_ID] + assert sampling_params.max_tokens == 8 + @pytest.fixture def ming_flash_omni_tts_server(mocker: MockerFixture): @@ -3901,6 +4008,9 @@ class TestTTSAsyncOffloading: model_type=adapter_model_type, ) + def apply_sampling_overrides(self, sampling_params_list, request, prompt, request_id=None): + return sampling_params_list + voxtral_server._tts_model_type = legacy_tts_model_type mocker.patch.object(voxtral_server, "_get_tts_adapter", return_value=FakeAdapter()) log_info = mocker.patch("vllm_omni.entrypoints.openai.serving_speech.logger.info") diff --git a/tests/entrypoints/openai_api/test_tts_detection.py b/tests/entrypoints/openai_api/test_tts_detection.py index c63e5c8e4..2ac27bb18 100644 --- a/tests/entrypoints/openai_api/test_tts_detection.py +++ b/tests/entrypoints/openai_api/test_tts_detection.py @@ -14,7 +14,6 @@ import itertools import pytest from vllm_omni.entrypoints.openai.tts_adapters import ( - LEGACY_TTS_DETECTORS, TTS_ADAPTER_REGISTRY, all_tts_stage_keys, detect_tts_model_type, @@ -282,27 +281,17 @@ def test_entry_stage_archs_is_ming_only(): assert tts_entry_stage_archs() == frozenset({"MingTTSForConditionalGeneration"}) -def test_every_detected_type_is_adapter_backed_or_declared_legacy(): - """Detection may only name a model that has an adapter or an explicit - :data:`LEGACY_TTS_DETECTORS` entry — never an undeclared string.""" - legacy_names = {d.name for d in LEGACY_TTS_DETECTORS} +def test_every_detected_type_is_adapter_backed(): + """Detection may only name a model that has a registered adapter.""" for stage, arch in itertools.product(_STAGES, _ARCHS): # full cross product on purpose detected = detect_tts_model_type(stage, arch) if detected is None: continue - assert detected in TTS_ADAPTER_REGISTRY or detected in legacy_names, ( + assert detected in TTS_ADAPTER_REGISTRY, ( f"detection produced undeclared model type {detected!r} for stage={stage!r} arch={arch!r}" ) -def test_legacy_detectors_have_no_adapter(): - """A legacy entry that gained an adapter must be deleted from the list.""" - for detector in LEGACY_TTS_DETECTORS: - assert resolve_adapter(detector.name) is None, ( - f"{detector.name!r} now has an adapter; remove it from LEGACY_TTS_DETECTORS" - ) - - def test_no_ambiguous_detectors_at_equal_priority(): """Two detectors that can match the same deployment must be ordered. diff --git a/tests/tools/test_check_tts_adapter.py b/tests/tools/test_check_tts_adapter.py index 486a629d5..47c6ac89f 100644 --- a/tests/tools/test_check_tts_adapter.py +++ b/tests/tools/test_check_tts_adapter.py @@ -17,9 +17,7 @@ import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools" / "pre_commit")) from check_tts_adapter import ( # noqa: E402 - MAX_LEGACY_DETECTORS, MAX_MODEL_TYPE_BRANCHES, - _legacy_detectors, _model_type_branches, main, ) @@ -60,12 +58,6 @@ def test_dict_dispatch_is_a_known_gap(): assert _count('TABLE = {"foo": 1}\nvalue = TABLE[self._tts_model_type]') == 0 -def test_legacy_detector_counting(): - src = 'X = (LegacyDetector(name="a", stage_keys=frozenset({"s"})), LegacyDetector(name="b"))' - found = _legacy_detectors(ast.parse(src)) - assert [name for _, name in found] == ["a", "b"] - - def test_gate_passes_on_current_tree(): """The recorded budgets must match reality, or the ratchet is decorative.""" assert main([]) == 0 @@ -74,6 +66,4 @@ def test_gate_passes_on_current_tree(): def test_budgets_are_the_real_counts(): root = Path(__file__).resolve().parents[2] serving = root / "vllm_omni" / "entrypoints" / "openai" / "serving_speech.py" - adapters = root / "vllm_omni" / "entrypoints" / "openai" / "tts_adapters" / "__init__.py" assert len(_model_type_branches(ast.parse(serving.read_text()))) == MAX_MODEL_TYPE_BRANCHES - assert len(_legacy_detectors(ast.parse(adapters.read_text()))) == MAX_LEGACY_DETECTORS diff --git a/tests/worker/test_gpu_ar_model_runner.py b/tests/worker/test_gpu_ar_model_runner.py index 116b5ee4d..967407169 100644 --- a/tests/worker/test_gpu_ar_model_runner.py +++ b/tests/worker/test_gpu_ar_model_runner.py @@ -2,6 +2,7 @@ import asyncio import copy from collections.abc import Sequence from types import SimpleNamespace +from typing import Any import numpy as np import pytest @@ -200,6 +201,15 @@ def test_speech_extra_params_reach_model_sampler_as_sampling_metadata(monkeypatc async def build(self, request, sampling_params_list, has_inline_ref_audio): return PreparedRequest(prompt={"prompt": request.input}, tts_params={}, model_type="higgs_audio_v3") + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return sampling_params_list + engine = SimpleNamespace( errored=False, default_sampling_params_list=[SamplingParams(temperature=1.0, top_p=0.95, top_k=50)], diff --git a/tools/pre_commit/check_tts_adapter.py b/tools/pre_commit/check_tts_adapter.py index 626323b4f..bdd62f823 100644 --- a/tools/pre_commit/check_tts_adapter.py +++ b/tools/pre_commit/check_tts_adapter.py @@ -6,10 +6,8 @@ Per-model behaviour is being moved out of ``serving_speech.py`` and into nothing stopped a new model from adding one more ``self._tts_model_type == ...`` branch faster than the migration removed them. -This gate makes the two debt counters monotonically non-increasing: - -1. ``self._tts_model_type`` comparisons in ``serving_speech.py`` -2. entries in ``tts_adapters.LEGACY_TTS_DETECTORS`` (models with no adapter) +This gate makes the number of ``self._tts_model_type`` comparisons in +``serving_speech.py`` monotonically non-increasing. Lowering a budget after removing branches is expected and encouraged — the check insists on it, so the recorded numbers cannot silently drift upward. Raising one @@ -28,11 +26,9 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] SERVING_SPEECH = Path("vllm_omni/entrypoints/openai/serving_speech.py") -ADAPTERS_INIT = Path("vllm_omni/entrypoints/openai/tts_adapters/__init__.py") # Budgets. These may only go DOWN. See the module docstring. -MAX_MODEL_TYPE_BRANCHES = 27 -MAX_LEGACY_DETECTORS = 0 +MAX_MODEL_TYPE_BRANCHES = 20 _GUIDANCE = """ Move the per-model behaviour into its adapter under @@ -109,20 +105,6 @@ def _model_type_branches(tree: ast.AST) -> list[tuple[int, str]]: return found -def _legacy_detectors(tree: ast.AST) -> list[tuple[int, str]]: - """Every ``LegacyDetector(...)`` construction.""" - found: list[tuple[int, str]] = [] - for node in ast.walk(tree): - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "LegacyDetector": - name = "?" - for kw in node.keywords: - if kw.arg == "name" and isinstance(kw.value, ast.Constant): - name = str(kw.value.value) - found.append((node.lineno, name)) - found.sort() - return found - - def _check( label: str, path: Path, @@ -151,21 +133,18 @@ def _check( def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) - # pre-commit passes the staged filenames; we always audit the same two files, + # pre-commit passes the staged filenames; we always audit the same file, # so they are accepted and ignored. parser.add_argument("filenames", nargs="*") parser.parse_args(argv) serving_path = REPO_ROOT / SERVING_SPEECH - adapters_path = REPO_ROOT / ADAPTERS_INIT - for path in (serving_path, adapters_path): - if not path.is_file(): - print(f"check_tts_adapter: expected file not found: {path}", file=sys.stderr) - return 1 + if not serving_path.is_file(): + print(f"check_tts_adapter: expected file not found: {serving_path}", file=sys.stderr) + return 1 notices: list[str] = [] branches = _model_type_branches(ast.parse(serving_path.read_text())) - detectors = _legacy_detectors(ast.parse(adapters_path.read_text())) errors = _check( "`_tts_model_type` comparisons", @@ -176,15 +155,6 @@ def main(argv: list[str] | None = None) -> int: "MAX_MODEL_TYPE_BRANCHES", notices, ) - errors += _check( - "unmigrated model types in LEGACY_TTS_DETECTORS", - ADAPTERS_INIT, - len(detectors), - MAX_LEGACY_DETECTORS, - [f"line {lineno}: {name}" for lineno, name in detectors], - "MAX_LEGACY_DETECTORS", - notices, - ) for notice in notices: print(f"check_tts_adapter: {notice}", file=sys.stderr) diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 3612a2ff1..1b44d397d 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -92,19 +92,7 @@ logger = init_logger(__name__) # reference (issue #4644). Matches the offline example/test and upstream demo. _COSYVOICE3_PROMPT_DELIMITER = "<|endofprompt|>" _COSYVOICE3_PROMPT_PREFIX = f"You are a helpful assistant.{_COSYVOICE3_PROMPT_DELIMITER}" -_SAMPLING_MAX_TOKENS_TTS_MODEL_TYPES = { - "fish_tts", - "qwen3_tts", - "voxtral_tts", - "cosyvoice3", - "voxcpm2", - "higgs_audio_v2", - "higgs_audio_v3", - "indextts2", - "indextts2_5", - "audex", - "audex_tta", -} + # Audex contract: zero-codec / invalid generations arrive as empty terminal # payloads and must fail the request, never serialize as a successful empty # WAV. Covers both the TTS ("audex") and TTA ("audex_tta") pipelines. @@ -444,7 +432,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Cached per process: the CosyVoice3 Qwen tokenizer + resolved model # path used for dynamic-token sizing. Without this, every request # re-ran snapshot_download + reloaded the tokenizer (~100 ms on the - # TTFP critical path) in _apply_cosyvoice3_dynamic_tokens. + # TTFP critical path) in the CosyVoice3 adapter's sampling override. self._cosyvoice3_tokenizer = None # Determine TTS model type or None @@ -502,9 +490,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Resolve the per-model serving adapter (RFC #4327), keyed on the # detected model-type. Every dedicated TTS model has an adapter; the - # adapter owns request validation + prompt/param building. Sampling - # overrides and the model-type label remain in the orchestrator tail - # (keyed on ``_tts_model_type``) during this incremental migration. + # adapter owns request validation, prompt/param building, and sampling + # overrides; the model-type label remains available for compatibility. self._adapter = None if self._tts_stage is not None: adapter_cls = resolve_adapter(self._tts_model_type) @@ -2825,154 +2812,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self._audex_tokenizer = AutoTokenizer.from_pretrained(os.path.join(root, folder)) return self._audex_tokenizer - def _inject_audex_tta_args(self, request_id: str, prompt: Any, stage0_params: Any) -> Any: - """Complete the Audex TTA contract on a CLONE of the stage-0 params. - - Always attaches the RVQ phase-mask contract (required for token - validity) and the CFG pair contract at the official default scale - 3.0 unless the caller passed ``cfg_scale`` (1.0 disables guidance). - - Returns the cloned params; the input is never mutated. On the - no-extra_params online path the input is an element of the engine's - SHARED ``default_sampling_params_list`` — writing per-request pair - state (``cfg_pair_id``/``cfg_null_prompt``) into it would race - concurrent requests and leak stale CFG metadata into later ones. - """ - import copy - - from vllm_omni.model_executor.models.audex.prompt import build_tta_null_prompt - from vllm_omni.model_executor.models.audex.tta import build_tta_phase_token_ids - - cond_prompt = prompt.get("prompt") if isinstance(prompt, dict) else prompt - if not isinstance(cond_prompt, str) or not cond_prompt: - raise ValueError("Audex TTA requires the adapter-built caption prompt") - - stage0_params = copy.deepcopy(stage0_params) - tokenizer = self._get_audex_tokenizer() - if self._audex_tta_rvq is None: - phase_token_ids, start_tid, end_tid = build_tta_phase_token_ids(tokenizer) - self._audex_tta_rvq = { - "phase_token_ids": phase_token_ids, - "start_tid": start_tid, - "end_tid": end_tid, - # Official generation cap (decode truncates at 500 frames). - "codec_cap": 4000, - # The TTA prompt ends with . - "start_in_prompt": True, - } - - extra_args = getattr(stage0_params, "extra_args", None) - if extra_args is None: - extra_args = {} - stage0_params.extra_args = extra_args - extra_args["tta_rvq"] = self._audex_tta_rvq - - cfg_scale = extra_args.get("cfg_scale") - cfg_scale = 3.0 if cfg_scale is None else float(cfg_scale) - if cfg_scale <= 1.0: - extra_args.pop("cfg_scale", None) - return stage0_params - extra_args.update( - { - "cfg_scale": cfg_scale, - "cfg_role": "cond", - "cfg_pair_id": request_id, - "cfg_null_prompt": build_tta_null_prompt(cond_prompt, tokenizer), - } - ) - return stage0_params - - def _inject_audex_cfg_pair_args(self, request_id: str, prompt: Any, stage0_params: Any) -> Any: - """Complete the Audex CFG contract on a CLONE of the stage-0 params. - - The adapter validated ``cfg_scale``; here (where the final request id - exists) the cond role, pair id, and the length-matched null prompt are - attached. ``cfg_scale`` absent/1.0 returns a clone value-identical to - the non-CFG flow (the key is dropped rather than forwarded). - - Returns the cloned params; the input is never mutated (it may be an - element of the engine's shared ``default_sampling_params_list`` — - see ``_inject_audex_tta_args``). - """ - import copy - - stage0_params = copy.deepcopy(stage0_params) - extra_args = getattr(stage0_params, "extra_args", None) - if not extra_args or extra_args.get("cfg_scale") is None: - if extra_args: - extra_args.pop("cfg_scale", None) - return stage0_params - cfg_scale = float(extra_args["cfg_scale"]) - if cfg_scale <= 1.0: - extra_args.pop("cfg_scale", None) - return stage0_params - - from vllm_omni.model_executor.models.audex.prompt import build_null_prompt - - cond_prompt = prompt.get("prompt") if isinstance(prompt, dict) else prompt - if not isinstance(cond_prompt, str) or not cond_prompt: - raise ValueError("Audex CFG requires the adapter-built text prompt") - null_prompt = build_null_prompt(cond_prompt, self._get_audex_tokenizer()) - extra_args.update( - { - "cfg_scale": cfg_scale, - "cfg_role": "cond", - "cfg_pair_id": request_id, - "cfg_null_prompt": null_prompt, - } - ) - # Guided decoding sharpens the distribution; the unguided default - # temperature (0.1) adds excess sampling noise under CFG. Measured - # on the en-24 gate corpus at cfg 1.5: temp 0.1 -> CER 7.31%, - # temp 0.05 -> CER 6.87% (unguided baseline 7.24%). - stage0_params.temperature = 0.05 - return stage0_params - - def _apply_cosyvoice3_dynamic_tokens( - self, - sampling_params_list: list, - request: OpenAICreateSpeechRequest, - ) -> list: - """Set min/max tokens from tokenized text length (ratios target tokens, not chars).""" - import copy - - from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer - from vllm_omni.model_executor.models.cosyvoice3.utils import extract_text_token - - sampling_params_list = copy.deepcopy(sampling_params_list) - hf_cfg = self.model_config.hf_config - # Build the Qwen tokenizer once per process (resolving the model dir via - # snapshot_download at most once) and reuse it across requests. - tokenizer = self._cosyvoice3_tokenizer - if tokenizer is None: - model_path = self.engine_client.model_config.model - if not os.path.isdir(model_path): - from huggingface_hub import snapshot_download - - model_path = snapshot_download(model_path) - tokenizer = get_qwen_tokenizer( - token_path=os.path.join(model_path, hf_cfg.qwen_pretrain_path), - skip_special_tokens=hf_cfg.skip_special_tokens, - version=hf_cfg.version, - ) - self._cosyvoice3_tokenizer = tokenizer - _, text_token_len = extract_text_token( - request.input, - tokenizer, - hf_cfg.allowed_special, - ) - min_ratio = getattr(hf_cfg, "min_token_text_ratio", 2) - max_ratio = getattr(hf_cfg, "max_token_text_ratio", 20) - sampling_params_list[0].min_tokens = max(1, int(text_token_len * min_ratio)) - sampling_params_list[0].max_tokens = min(2048, int(text_token_len * max_ratio)) - logger.info( - "CosyVoice3 dynamic tokens: text_tokens=%d, min_tokens=%d, max_tokens=%d", - text_token_len, - sampling_params_list[0].min_tokens, - sampling_params_list[0].max_tokens, - ) - return sampling_params_list - # ---- GLM-TTS helpers ---- async def _build_glm_tts_prompt( @@ -3172,9 +3011,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Build prompt + tts_params via the per-model adapter (RFC #4327). Every # dedicated TTS model resolves to an adapter that owns its validation, - # uploaded-speaker handling, and prompt/param building. Sampling - # overrides and the model-type label remain in the orchestrator tail - # below (keyed on ``_tts_model_type``) during this incremental migration. + # uploaded-speaker handling, prompt/param building, and sampling + # overrides. The model-type label remains available for compatibility. # Non-TTS deployments (no adapter) fall through to the rejection below. # Capture inline-ref-audio status BEFORE validate(): several adapters # apply uploaded speakers inside validate(), which sets request.ref_audio @@ -3225,63 +3063,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): model_type, ) - # CosyVoice3: set dynamic min/max tokens based on text length. - # The official model requires min_token_text_ratio to prevent early - # EOS and max_token_text_ratio to cap generation length. - if self._tts_model_type == "cosyvoice3" and sampling_params_list: - sampling_params_list = self._apply_cosyvoice3_dynamic_tokens(sampling_params_list, request) - - # GLM-TTS: set dynamic min/max tokens based on text length. - if self._tts_model_type == "glm_tts" and sampling_params_list: - import copy - - sampling_params_list = copy.deepcopy(sampling_params_list) - glm_metadata = prompt.get("additional_information") if isinstance(prompt, dict) else None - text_len_value = None - if isinstance(glm_metadata, dict): - text_len_value = glm_metadata.get("glm_tts_text_token_len") - if isinstance(text_len_value, list) and text_len_value: - text_len_value = text_len_value[0] - text_token_len = ( - int(text_len_value) - if text_len_value is not None - else self._estimate_glm_tts_text_token_len(request.input) - ) - hf_cfg = self.model_config.hf_config - min_ratio = getattr(hf_cfg, "min_token_text_ratio", 2) - max_ratio = getattr(hf_cfg, "max_token_text_ratio", 20) - stage_min_tokens = getattr(sampling_params_list[0], "min_tokens", None) - stage_max_tokens = getattr(sampling_params_list[0], "max_tokens", None) - cap_candidates = [int(cap) for cap in (stage_max_tokens, request.max_new_tokens) if cap is not None] - hard_cap = min(cap_candidates) if cap_candidates else None - - min_tokens = max(1, int(text_token_len * min_ratio)) - if stage_min_tokens is not None: - min_tokens = max(min_tokens, int(stage_min_tokens)) - if hard_cap is not None: - min_tokens = min(min_tokens, hard_cap) - - max_tokens = max(min_tokens, int(text_token_len * max_ratio)) - if hard_cap is not None: - max_tokens = min(max_tokens, hard_cap) - sampling_params_list[0].min_tokens = min_tokens - sampling_params_list[0].max_tokens = max_tokens - seed = getattr(request, "seed", None) - if seed is not None: - sampling_params_list[0].seed = seed - logger.info( - "GLM-TTS dynamic tokens: text_tokens=%d, min_ratio=%s, max_ratio=%s, " - "stage_min=%s, stage_max=%s, request_max=%s, min_tokens=%d, max_tokens=%d", - text_token_len, - min_ratio, - max_ratio, - stage_min_tokens, - stage_max_tokens, - request.max_new_tokens, - min_tokens, - max_tokens, - ) - # Apply model-specific extra parameters if request.extra_params is not None and sampling_params_list: if not isinstance(request.extra_params, dict): @@ -3300,52 +3081,10 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): sampling_params_list[0].extra_args.update(request.extra_params) logger.info("Applied extra_params: %s", request.extra_params) - # Audex CFG: turn an adapter-validated cfg_scale into the engine-side - # pair contract. This must run here (not in the adapter) because the - # pair id is the final request_id, which the adapter never sees. - # The injectors return a CLONE: without extra_params above, - # sampling_params_list still holds the engine's shared default params, - # which must never carry per-request CFG state. - if self._tts_model_type == "audex" and sampling_params_list: - sampling_params_list[0] = self._inject_audex_cfg_pair_args(request_id, prompt, sampling_params_list[0]) - elif self._tts_model_type == "audex_tta" and sampling_params_list: - sampling_params_list[0] = self._inject_audex_tta_args(request_id, prompt, sampling_params_list[0]) - - # Some TTS model defaults come from deploy YAML. Their AR - # generation length is controlled by SamplingParams.max_tokens, so only - # override it when the caller explicitly requests max_new_tokens. - if ( - self._tts_model_type in _SAMPLING_MAX_TOKENS_TTS_MODEL_TYPES - and request.max_new_tokens is not None - and sampling_params_list - ): - import copy - - sampling_params_list = copy.deepcopy(sampling_params_list) - sampling_params_list[0].max_tokens = request.max_new_tokens - if self._tts_model_type == "cosyvoice3": - sampling_params_list[0].min_tokens = min( - getattr(sampling_params_list[0], "min_tokens", 0), - request.max_new_tokens, - ) - elif self._tts_model_type == "ming_tts" and sampling_params_list: - import copy - - from vllm_omni.model_executor.models.ming_tts.config_ming_tts import ( - MOE_TEXT_EOS_TOKEN_ID, - TEXT_EOS_TOKEN_ID, - ) - - hf_config = self.engine_client.model_config.hf_config - is_moe = getattr(hf_config, "model_type", "") == "bailingmm" - stop_token_id = MOE_TEXT_EOS_TOKEN_ID if is_moe else TEXT_EOS_TOKEN_ID - - sampling_params_list = copy.deepcopy(sampling_params_list) - sampling_params_list[0].stop_token_ids = [int(stop_token_id)] - if request.max_new_tokens is not None: - # Ming emits TEXT_EOS after the latent decode budget is exhausted, so - # Stage-0 needs one extra token beyond ming_max_decode_steps. - sampling_params_list[0].max_tokens = int(request.max_new_tokens) + 1 + # Apply adapter-owned sampling overrides, including request-level token + # limits and model-specific dynamic token or stop-token configuration. + if sampling_params_list and (adapter := self._get_tts_adapter()) is not None: + sampling_params_list = adapter.apply_sampling_overrides(sampling_params_list, request, prompt, request_id) if request.seed is not None and sampling_params_list: import copy diff --git a/vllm_omni/entrypoints/openai/tts_adapters/__init__.py b/vllm_omni/entrypoints/openai/tts_adapters/__init__.py index cd44f0ff5..74597a2d1 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/__init__.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/__init__.py @@ -9,8 +9,6 @@ key alone cannot distinguish. That metadata is the single source of truth. :func:`detect_tts_model_type` and :func:`all_tts_stage_keys` derive the whole stage -> model-type mapping from it, so a new TTS model is one adapter file and needs no edit to ``serving_speech.py``. -Models still awaiting an adapter are listed explicitly in -:data:`LEGACY_TTS_DETECTORS`. """ from vllm.logger import init_logger @@ -18,7 +16,6 @@ from vllm.logger import init_logger from vllm_omni.entrypoints.openai.tts_adapters.base import ( ARTTSAdapter, DiffusionTTSAdapter, - LegacyDetector, OutputPolicy, PreparedRequest, SpeechServingContext, @@ -29,10 +26,6 @@ logger = init_logger(__name__) TTS_ADAPTER_REGISTRY: dict[str, type[TTSModelAdapter]] = {} -#: Model types the serving layer detects that have no adapter yet. See -#: :class:`LegacyDetector`. This list must only ever shrink. -LEGACY_TTS_DETECTORS: tuple[LegacyDetector, ...] = () - def register_tts_adapter(cls: type[TTSModelAdapter]) -> type[TTSModelAdapter]: """Class decorator: index ``cls`` under its ``name`` (model-type).""" @@ -57,16 +50,13 @@ def resolve_adapter(model_type: str | None) -> type[TTSModelAdapter] | None: return TTS_ADAPTER_REGISTRY.get(model_type) -def iter_tts_detectors() -> list[type[TTSModelAdapter] | LegacyDetector]: +def iter_tts_detectors() -> list[type[TTSModelAdapter]]: """Every detector, in resolution order. Sorted by ``detect_priority`` then ``name``, so the order is total and does not depend on adapter import order. """ - detectors: list[type[TTSModelAdapter] | LegacyDetector] = [ - *TTS_ADAPTER_REGISTRY.values(), - *LEGACY_TTS_DETECTORS, - ] + detectors = list(TTS_ADAPTER_REGISTRY.values()) detectors.sort(key=lambda d: (d.detect_priority, d.name)) return detectors @@ -132,12 +122,10 @@ from vllm_omni.entrypoints.openai.tts_adapters import ( # noqa: E402,F401 __all__ = [ "ARTTSAdapter", "DiffusionTTSAdapter", - "LegacyDetector", "OutputPolicy", "PreparedRequest", "SpeechServingContext", "TTSModelAdapter", - "LEGACY_TTS_DETECTORS", "TTS_ADAPTER_REGISTRY", "all_tts_model_types", "all_tts_stage_keys", diff --git a/vllm_omni/entrypoints/openai/tts_adapters/audex.py b/vllm_omni/entrypoints/openai/tts_adapters/audex.py index 4533a420b..ab7c7c25e 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/audex.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/audex.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 """Audex (Nemotron-Labs-Audex-2B) TTS serving adapter.""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens from vllm_omni.model_executor.models.audex.prompt import build_cond_prompt if TYPE_CHECKING: @@ -17,8 +17,8 @@ if TYPE_CHECKING: AUDEX_CFG_SCALE_MIN = 1.0 AUDEX_CFG_SCALE_MAX = 10.0 -# Internal CFG pair-plumbing keys; injected by the serving layer once the -# request id exists, never accepted from callers. +# Internal CFG pair-plumbing keys; injected by this adapter once it receives +# the final request id, never accepted from callers. _AUDEX_INTERNAL_CFG_KEYS = ("cfg_role", "cfg_pair_id", "cfg_null_prompt") @@ -74,3 +74,65 @@ class AudexAdapter(ARTTSAdapter): ) -> PreparedRequest: prompt = {"prompt": build_cond_prompt(request.input)} return PreparedRequest(prompt=prompt, tts_params={}, model_type="audex") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + if request_id is None: + raise ValueError("Audex CFG requires a request id") + stage0_params = sampling_params_list[0] + sampling_params_list[0] = self._inject_audex_cfg_pair_args( + request_id=request_id, prompt=prompt, stage0_params=stage0_params + ) + return apply_max_new_tokens(sampling_params_list, request) + + def _inject_audex_cfg_pair_args(self, request_id: str, prompt: Any, stage0_params: Any) -> Any: + """Complete the Audex CFG contract on a CLONE of the stage-0 params. + + The adapter validated ``cfg_scale``; here (where the final request id + exists) the cond role, pair id, and the length-matched null prompt are + attached. ``cfg_scale`` absent/1.0 returns a clone value-identical to + the non-CFG flow (the key is dropped rather than forwarded). + + Returns the cloned params; the input is never mutated (it may be an + element of the engine's shared ``default_sampling_params_list`` — + see ``_inject_audex_tta_args``). + """ + server = self.ctx.server + import copy + + stage0_params = copy.deepcopy(stage0_params) + extra_args = getattr(stage0_params, "extra_args", None) + if not extra_args or extra_args.get("cfg_scale") is None: + if extra_args: + extra_args.pop("cfg_scale", None) + return stage0_params + cfg_scale = float(extra_args["cfg_scale"]) + if cfg_scale <= 1.0: + extra_args.pop("cfg_scale", None) + return stage0_params + + from vllm_omni.model_executor.models.audex.prompt import build_null_prompt + + cond_prompt = prompt.get("prompt") if isinstance(prompt, dict) else prompt + if not isinstance(cond_prompt, str) or not cond_prompt: + raise ValueError("Audex CFG requires the adapter-built text prompt") + null_prompt = build_null_prompt(cond_prompt, server._get_audex_tokenizer()) + extra_args.update( + { + "cfg_scale": cfg_scale, + "cfg_role": "cond", + "cfg_pair_id": request_id, + "cfg_null_prompt": null_prompt, + } + ) + # Guided decoding sharpens the distribution; the unguided default + # temperature (0.1) adds excess sampling noise under CFG. Measured + # on the en-24 gate corpus at cfg 1.5: temp 0.1 -> CER 7.31%, + # temp 0.05 -> CER 6.87% (unguided baseline 7.24%). + stage0_params.temperature = 0.05 + return stage0_params diff --git a/vllm_omni/entrypoints/openai/tts_adapters/audex_tta.py b/vllm_omni/entrypoints/openai/tts_adapters/audex_tta.py index f754f0158..33dd2294a 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/audex_tta.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/audex_tta.py @@ -2,16 +2,16 @@ """Audex (Nemotron-Labs-Audex-2B) text-to-audio serving adapter. Caption → general audio through the ``audex_tta`` pipeline. -The adapter builds the TTA prompt; the serving layer completes the request -after the final request id exists: the RVQ phase-mask contract +The adapter builds the TTA prompt and, after receiving the final request id, +completes the sampling parameters with the RVQ phase-mask contract (``extra_args["tta_rvq"]``) and the CFG pair contract (official default scale 3.0 — guidance is effectively mandatory for TTA quality). """ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens from vllm_omni.model_executor.models.audex.prompt import build_tta_cond_prompt if TYPE_CHECKING: @@ -21,8 +21,7 @@ AUDEX_TTA_DEFAULT_CFG_SCALE = 3.0 AUDEX_TTA_CFG_SCALE_MIN = 1.0 AUDEX_TTA_CFG_SCALE_MAX = 10.0 -# Internal plumbing keys; injected by the serving layer, never accepted -# from callers. +# Internal plumbing keys; injected by this adapter, never accepted from callers. _INTERNAL_KEYS = ("cfg_role", "cfg_pair_id", "cfg_null_prompt", "tta_rvq") @@ -67,3 +66,76 @@ class AudexTTAAdapter(ARTTSAdapter): ) -> PreparedRequest: prompt = {"prompt": build_tta_cond_prompt(request.input)} return PreparedRequest(prompt=prompt, tts_params={}, model_type="audex_tta") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + if request_id is None: + raise ValueError("Audex TTA requires a request id") + stage0_params = sampling_params_list[0] + sampling_params_list[0] = self._inject_audex_tta_args( + request_id=request_id, prompt=prompt, stage0_params=stage0_params + ) + return apply_max_new_tokens(sampling_params_list, request) + + def _inject_audex_tta_args(self, request_id: str, prompt: Any, stage0_params: Any) -> Any: + """Complete the Audex TTA contract on a CLONE of the stage-0 params. + + Always attaches the RVQ phase-mask contract (required for token + validity) and the CFG pair contract at the official default scale + 3.0 unless the caller passed ``cfg_scale`` (1.0 disables guidance). + + Returns the cloned params; the input is never mutated. On the + no-extra_params online path the input is an element of the engine's + SHARED ``default_sampling_params_list`` — writing per-request pair + state (``cfg_pair_id``/``cfg_null_prompt``) into it would race + concurrent requests and leak stale CFG metadata into later ones. + """ + server = self.ctx.server + import copy + + from vllm_omni.model_executor.models.audex.prompt import build_tta_null_prompt + from vllm_omni.model_executor.models.audex.tta import build_tta_phase_token_ids + + cond_prompt = prompt.get("prompt") if isinstance(prompt, dict) else prompt + if not isinstance(cond_prompt, str) or not cond_prompt: + raise ValueError("Audex TTA requires the adapter-built caption prompt") + + stage0_params = copy.deepcopy(stage0_params) + tokenizer = server._get_audex_tokenizer() + if server._audex_tta_rvq is None: + phase_token_ids, start_tid, end_tid = build_tta_phase_token_ids(tokenizer) + server._audex_tta_rvq = { + "phase_token_ids": phase_token_ids, + "start_tid": start_tid, + "end_tid": end_tid, + # Official generation cap (decode truncates at 500 frames). + "codec_cap": 4000, + # The TTA prompt ends with . + "start_in_prompt": True, + } + + extra_args = getattr(stage0_params, "extra_args", None) + if extra_args is None: + extra_args = {} + stage0_params.extra_args = extra_args + extra_args["tta_rvq"] = server._audex_tta_rvq + + cfg_scale = extra_args.get("cfg_scale") + cfg_scale = 3.0 if cfg_scale is None else float(cfg_scale) + if cfg_scale <= 1.0: + extra_args.pop("cfg_scale", None) + return stage0_params + extra_args.update( + { + "cfg_scale": cfg_scale, + "cfg_role": "cond", + "cfg_pair_id": request_id, + "cfg_null_prompt": build_tta_null_prompt(cond_prompt, tokenizer), + } + ) + return stage0_params diff --git a/vllm_omni/entrypoints/openai/tts_adapters/base.py b/vllm_omni/entrypoints/openai/tts_adapters/base.py index b8679041c..643d71ad6 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/base.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/base.py @@ -7,9 +7,7 @@ model's request normalization, validation, prompt/param building, sampling overrides, and output policy, so adding a model means writing one adapter file instead of editing the shared serving module in ~10 scattered places. -See the RFC for the full design (issue #4327). This is the foundation landed in -the first migration PR; Qwen3-TTS is the first model routed through it while the -remaining models stay on the legacy path until individually migrated. +See the RFC for the full design (issue #4327). """ from abc import ABC, abstractmethod @@ -39,6 +37,21 @@ def conditioning_cache_salt(request: "OpenAICreateSpeechRequest", tts_params: di return _conditioning_cache_salt_fn(request, tts_params) +def apply_max_new_tokens( + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", +) -> list: + """Apply a request-level ``max_new_tokens`` limit.""" + if request.max_new_tokens is None: + return sampling_params_list + + import copy + + sampling_params_list = copy.deepcopy(sampling_params_list) + sampling_params_list[0].max_tokens = request.max_new_tokens + return sampling_params_list + + @dataclass class OutputPolicy: """How the orchestrator aggregates engine output for a model. @@ -184,6 +197,8 @@ class TTSModelAdapter(ABC): self, sampling_params_list: list, request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, ) -> list: """Apply model-specific sampling mutations. @@ -228,37 +243,10 @@ class DiffusionTTSAdapter(TTSModelAdapter): return frozenset(params) if params is not None else frozenset() -@dataclass(frozen=True) -class LegacyDetector: - """A model type the serving layer detects but has no adapter for yet. - - Detection has to keep naming these models so ``serving_speech.py`` can route - them down its legacy path, but they own none of the adapter contract. Each - entry is a migration debt: delete it when the model gets an adapter. The - pre-commit gate (``tools/pre_commit/check_tts_adapter.py``) fails if the list - grows. - - Exposes the subset of the :class:`TTSModelAdapter` class surface that - detection reads, so both kinds live in one sorted sequence. - """ - - name: str - stage_keys: frozenset[str] = frozenset() - model_archs: frozenset[str] = frozenset() - arch_identifies_entry_stage: bool = False - detect_priority: int = 100 - - def matches(self, model_stage: str | None, model_arch: str | None) -> bool: - if model_arch is not None and model_arch in self.model_archs: - return True - return model_stage is not None and model_stage in self.stage_keys - - # Re-exported here to avoid import cycles at call sites. __all__ = [ "ARTTSAdapter", "DiffusionTTSAdapter", - "LegacyDetector", "OutputPolicy", "PreparedRequest", "SpeechServingContext", diff --git a/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py b/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py index 85648442f..ec1c145c7 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py @@ -1,7 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 """CosyVoice3 TTS serving adapter.""" -from typing import TYPE_CHECKING +import os +from typing import TYPE_CHECKING, Any + +from vllm.logger import init_logger from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest @@ -9,6 +12,8 @@ from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, Prepare if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest +logger = init_logger(__name__) + @register_tts_adapter class CosyVoice3Adapter(ARTTSAdapter): @@ -47,6 +52,63 @@ class CosyVoice3Adapter(ARTTSAdapter): self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: prompt = await self.ctx.server._build_cosyvoice3_prompt(request, has_inline_ref_audio=has_inline_ref_audio) - # NOTE: CosyVoice3 dynamic-token sampling stays in the orchestrator tail - # (keyed on _tts_model_type) during this incremental migration. return PreparedRequest(prompt=prompt, tts_params={}, model_type="cosyvoice3") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + """Set dynamic min/max tokens based on tokenized text length. + + The official model uses ``min_token_text_ratio`` to prevent early EOS + and ``max_token_text_ratio`` to cap the generated token count. + """ + import copy + + from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer + from vllm_omni.model_executor.models.cosyvoice3.utils import extract_text_token + + server = self.ctx.server + sampling_params_list = copy.deepcopy(sampling_params_list) + hf_cfg = server.model_config.hf_config + # Build the Qwen tokenizer once per process (resolving the model dir via + # snapshot_download at most once) and reuse it across requests. + tokenizer = server._cosyvoice3_tokenizer + if tokenizer is None: + model_path = server.engine_client.model_config.model + if not os.path.isdir(model_path): + from huggingface_hub import snapshot_download + + model_path = snapshot_download(model_path) + tokenizer = get_qwen_tokenizer( + token_path=os.path.join(model_path, hf_cfg.qwen_pretrain_path), + skip_special_tokens=hf_cfg.skip_special_tokens, + version=hf_cfg.version, + ) + server._cosyvoice3_tokenizer = tokenizer + _, text_token_len = extract_text_token( + request.input, + tokenizer, + hf_cfg.allowed_special, + ) + min_ratio = getattr(hf_cfg, "min_token_text_ratio", 2) + max_ratio = getattr(hf_cfg, "max_token_text_ratio", 20) + sampling_params_list[0].min_tokens = max(1, int(text_token_len * min_ratio)) + sampling_params_list[0].max_tokens = min(2048, int(text_token_len * max_ratio)) + + if request.max_new_tokens is not None: + sampling_params_list[0].max_tokens = request.max_new_tokens + sampling_params_list[0].min_tokens = min( + getattr(sampling_params_list[0], "min_tokens", 0), + request.max_new_tokens, + ) + logger.info( + "CosyVoice3 dynamic tokens: text_tokens=%d, min_tokens=%d, max_tokens=%d", + text_token_len, + sampling_params_list[0].min_tokens, + sampling_params_list[0].max_tokens, + ) + return sampling_params_list diff --git a/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py b/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py index e8080a427..48d5886c4 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py @@ -1,10 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 """Fish Speech serving adapter (retires the legacy ``_is_fish_speech`` flag).""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, conditioning_cache_salt +from vllm_omni.entrypoints.openai.tts_adapters.base import ( + ARTTSAdapter, + PreparedRequest, + apply_max_new_tokens, + conditioning_cache_salt, +) if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -53,3 +58,12 @@ class FishSpeechAdapter(ARTTSAdapter): tts_params = {} prompt["cache_salt"] = conditioning_cache_salt(request, tts_params) return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="fish_speech") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py index ddfd194e2..9ef342666 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 """GLM-TTS serving adapter.""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any + +from vllm.logger import init_logger from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest @@ -9,6 +11,8 @@ from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, Prepare if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest +logger = init_logger(__name__) + @register_tts_adapter class GlmTTSAdapter(ARTTSAdapter): @@ -43,6 +47,63 @@ class GlmTTSAdapter(ARTTSAdapter): self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: prompt = await self.ctx.server._build_glm_tts_prompt(request, has_inline_ref_audio=has_inline_ref_audio) - # GLM-TTS dynamic-token sampling stays in the orchestrator tail - # (keyed on _tts_model_type) during this incremental migration. return PreparedRequest(prompt=prompt, tts_params={}, model_type="glm_tts") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + # GLM-TTS: set dynamic min/max tokens based on text length. + import copy + + server = self.ctx.server + sampling_params_list = copy.deepcopy(sampling_params_list) + glm_metadata = prompt.get("additional_information") if isinstance(prompt, dict) else None + text_len_value = None + if isinstance(glm_metadata, dict): + text_len_value = glm_metadata.get("glm_tts_text_token_len") + if isinstance(text_len_value, list) and text_len_value: + text_len_value = text_len_value[0] + text_token_len = ( + int(text_len_value) + if text_len_value is not None + else server._estimate_glm_tts_text_token_len(request.input) + ) + hf_cfg = server.model_config.hf_config + min_ratio = getattr(hf_cfg, "min_token_text_ratio", 2) + max_ratio = getattr(hf_cfg, "max_token_text_ratio", 20) + stage_min_tokens = getattr(sampling_params_list[0], "min_tokens", None) + stage_max_tokens = getattr(sampling_params_list[0], "max_tokens", None) + cap_candidates = [int(cap) for cap in (stage_max_tokens, request.max_new_tokens) if cap is not None] + hard_cap = min(cap_candidates) if cap_candidates else None + + min_tokens = max(1, int(text_token_len * min_ratio)) + if stage_min_tokens is not None: + min_tokens = max(min_tokens, int(stage_min_tokens)) + if hard_cap is not None: + min_tokens = min(min_tokens, hard_cap) + + max_tokens = max(min_tokens, int(text_token_len * max_ratio)) + if hard_cap is not None: + max_tokens = min(max_tokens, hard_cap) + sampling_params_list[0].min_tokens = min_tokens + sampling_params_list[0].max_tokens = max_tokens + seed = getattr(request, "seed", None) + if seed is not None: + sampling_params_list[0].seed = seed + logger.info( + "GLM-TTS dynamic tokens: text_tokens=%d, min_ratio=%s, max_ratio=%s, " + "stage_min=%s, stage_max=%s, request_max=%s, min_tokens=%d, max_tokens=%d", + text_token_len, + min_ratio, + max_ratio, + stage_min_tokens, + stage_max_tokens, + request.max_new_tokens, + min_tokens, + max_tokens, + ) + return sampling_params_list diff --git a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py index 99949d8b3..74ec591cc 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 """Higgs-Audio v2 serving adapter.""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -99,3 +99,12 @@ class HiggsAudioV2Adapter(ARTTSAdapter): additional["voice_name"] = voice_lower additional["voice_created_at"] = server._voice_created_at(voice_lower) return PreparedRequest(prompt=prompt, tts_params={}, model_type="higgs_audio_v2") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py index 9cae9aa2d..411d97f31 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 """Higgs-Audio v3 serving adapter.""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -36,3 +36,12 @@ class HiggsAudioV3Adapter(ARTTSAdapter): ) -> PreparedRequest: prompt = await self.ctx.server._build_higgs_audio_v3_params(request) return PreparedRequest(prompt=prompt, tts_params={}, model_type="higgs_audio_v3") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py b/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py index d1cb2a469..de70cd5b8 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py @@ -14,7 +14,7 @@ from vllm.inputs import tokens_input from vllm.utils import random_uuid from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens from vllm_omni.model_executor.models.indextts2.configuration_indextts2 import ( INDEXTTS25_MAX_DURATION_FACTOR, INDEXTTS25_MIN_DURATION_FACTOR, @@ -241,6 +241,15 @@ class IndexTTS2Adapter(ARTTSAdapter): prompt["cache_salt"] = indextts2_conditioning_cache_salt(request, tts_params) return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type=self.name) + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: OpenAICreateSpeechRequest, + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) + async def _build_params(self, request: OpenAICreateSpeechRequest) -> dict[str, Any]: server = self.ctx.server params: dict[str, Any] = {"text": [request.input]} diff --git a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py index dd8b8b621..cf76cdafe 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 """Ming-TTS (dense) serving adapter.""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm.logger import init_logger @@ -73,9 +73,36 @@ class MingTTSAdapter(ARTTSAdapter): voice_created_at=uploaded_audio_created_at, ) tts_params = prompt.get("additional_information", {}) - # Ming stop-token / max_tokens sampling stays in the orchestrator tail. return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="ming_tts") + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + import copy + + server = self.ctx.server + + from vllm_omni.model_executor.models.ming_tts.config_ming_tts import ( + MOE_TEXT_EOS_TOKEN_ID, + TEXT_EOS_TOKEN_ID, + ) + + hf_config = server.engine_client.model_config.hf_config + is_moe = getattr(hf_config, "model_type", "") == "bailingmm" + stop_token_id = MOE_TEXT_EOS_TOKEN_ID if is_moe else TEXT_EOS_TOKEN_ID + + sampling_params_list = copy.deepcopy(sampling_params_list) + sampling_params_list[0].stop_token_ids = [int(stop_token_id)] + if request.max_new_tokens is not None: + # Ming emits TEXT_EOS after the latent decode budget is exhausted, so + # Stage-0 needs one extra token beyond ming_max_decode_steps. + sampling_params_list[0].max_tokens = int(request.max_new_tokens) + 1 + return sampling_params_list + def _validate_ming_tts_single_speaker_request(self, request: "OpenAICreateSpeechRequest") -> str | None: server = self.ctx.server if request.ref_audio is not None: diff --git a/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py index c0be8c685..c3dcf6a54 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py @@ -2,12 +2,12 @@ """Qwen3-TTS serving adapter.""" from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm.logger import init_logger from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -163,3 +163,12 @@ class Qwen3TTSAdapter(ARTTSAdapter): model_type=tts_params.get("task_type", ["unknown"])[0], warmup_artifact_key=warmup_key, ) + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py b/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py index b0f188fbb..92b84774b 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 """VoxCPM2 serving adapter (AR base-LM + diffusion side-computation).""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -70,3 +70,12 @@ class VoxCPM2Adapter(ARTTSAdapter): additional["voice_name"] = voice_lower additional["voice_created_at"] = server._voice_created_at(voice_lower) return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="voxcpm2") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py b/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py index 0b2d400b1..09de585d8 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 """Voxtral TTS serving adapter.""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -48,3 +48,12 @@ class VoxtralTTSAdapter(ARTTSAdapter): ) -> PreparedRequest: prompt = await self.ctx.server._build_voxtral_prompt_async(request) return PreparedRequest(prompt=prompt, tts_params={}, model_type="voxtral_tts") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) From 309d9c3ea8083e73685562dbbc97c8b40ab6e0b1 Mon Sep 17 00:00:00 2001 From: Zhenghan Song Date: Mon, 17 Aug 2026 13:25:09 +0800 Subject: [PATCH 042/267] [Bugfix] Respect media redirect policy for image references (#6122) Signed-off-by: HaningZS Co-authored-by: Hongsheng Liu --- docs/serving/videos_api.md | 8 ++ .../openai_api/test_video_api_utils.py | 106 ++++++++++++++++++ .../openai_api/test_video_server.py | 43 ++++++- .../entrypoints/openai/video_api_utils.py | 15 ++- 4 files changed, 169 insertions(+), 3 deletions(-) diff --git a/docs/serving/videos_api.md b/docs/serving/videos_api.md index 6a94f9dab..5995e0911 100644 --- a/docs/serving/videos_api.md +++ b/docs/serving/videos_api.md @@ -167,6 +167,14 @@ transfer options such as `control_path`, `control_guidance`, `num_conditional_frames`, `show_control_condition`, and `show_input`; see the Cosmos3 recipe for complete examples. +HTTP redirects for `image_reference.image_url` follow vLLM's +`VLLM_MEDIA_URL_ALLOW_REDIRECTS` setting. Before starting the server, set it to +`1` (the default) to allow redirects or `0` to reject them. A redirect target +can differ from the original host, and vLLM-Omni does not yet fully implement +upstream vLLM's media URL allowlist protection. Deployments should therefore +treat remote media URLs as untrusted and choose this setting as part of their +URL access policy. + ### Speech-to-Video For models that support audio-driven generation (e.g., Wan2.2-S2V), pass both diff --git a/tests/entrypoints/openai_api/test_video_api_utils.py b/tests/entrypoints/openai_api/test_video_api_utils.py index 08d223c68..bb47356c6 100644 --- a/tests/entrypoints/openai_api/test_video_api_utils.py +++ b/tests/entrypoints/openai_api/test_video_api_utils.py @@ -2,16 +2,122 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Unit tests for OpenAI-compatible video API encoding helpers.""" +import base64 +from io import BytesIO + +import httpx import numpy as np import pytest import torch +from PIL import Image +from vllm import envs from vllm_omni.diffusion.postprocess import rife_interpolator from vllm_omni.entrypoints.openai import video_api_utils +from vllm_omni.entrypoints.openai.errors import InvalidInputReferenceError pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +def _png_bytes() -> bytes: + buffer = BytesIO() + Image.new("RGB", (2, 1), color=(12, 34, 56)).save(buffer, format="PNG") + return buffer.getvalue() + + +def _install_http_transport(monkeypatch, handler, client_kwargs): + async_client = httpx.AsyncClient + + def _client_factory(*args, **kwargs): + client_kwargs.append(kwargs.copy()) + return async_client(*args, transport=httpx.MockTransport(handler), **kwargs) + + monkeypatch.setattr(video_api_utils.httpx, "AsyncClient", _client_factory) + + +@pytest.mark.asyncio +async def test_decode_image_url_follows_redirects_when_allowed(monkeypatch): + requested_paths = [] + client_kwargs = [] + + def _handler(request): + requested_paths.append(request.url.path) + if request.url.path == "/redirect.png": + return httpx.Response(302, headers={"location": "/image.png"}) + return httpx.Response(200, content=_png_bytes(), headers={"content-type": "image/png"}) + + monkeypatch.setattr(envs, "VLLM_MEDIA_URL_ALLOW_REDIRECTS", True) + _install_http_transport(monkeypatch, _handler, client_kwargs) + + image = await video_api_utils.decode_image_url("https://example.com/redirect.png") + + assert image.size == (2, 1) + assert image.mode == "RGB" + assert requested_paths == ["/redirect.png", "/image.png"] + assert client_kwargs == [{"timeout": 60, "follow_redirects": True}] + + +@pytest.mark.asyncio +async def test_decode_image_url_rejects_redirects_when_disabled(monkeypatch): + requested_paths = [] + client_kwargs = [] + + def _handler(request): + requested_paths.append(request.url.path) + return httpx.Response(302, headers={"location": "/image.png"}) + + monkeypatch.setattr(envs, "VLLM_MEDIA_URL_ALLOW_REDIRECTS", False) + _install_http_transport(monkeypatch, _handler, client_kwargs) + + with pytest.raises(InvalidInputReferenceError, match="redirect.*VLLM_MEDIA_URL_ALLOW_REDIRECTS"): + await video_api_utils.decode_image_url("https://example.com/redirect.png") + + assert requested_paths == ["/redirect.png"] + assert client_kwargs == [{"timeout": 60, "follow_redirects": False}] + + +@pytest.mark.asyncio +async def test_decode_image_url_reports_http_status(monkeypatch): + client_kwargs = [] + + def _handler(request): + return httpx.Response(404) + + monkeypatch.setattr(envs, "VLLM_MEDIA_URL_ALLOW_REDIRECTS", True) + _install_http_transport(monkeypatch, _handler, client_kwargs) + + with pytest.raises(InvalidInputReferenceError, match="server returned HTTP 404"): + await video_api_utils.decode_image_url("https://example.com/missing.png") + + +@pytest.mark.asyncio +async def test_decode_image_url_reports_connection_failure(monkeypatch): + client_kwargs = [] + + def _handler(request): + raise httpx.ConnectError("connection refused", request=request) + + monkeypatch.setattr(envs, "VLLM_MEDIA_URL_ALLOW_REDIRECTS", True) + _install_http_transport(monkeypatch, _handler, client_kwargs) + + with pytest.raises(InvalidInputReferenceError, match="failed to download image"): + await video_api_utils.decode_image_url("https://example.com/unreachable.png") + + +@pytest.mark.asyncio +async def test_decode_image_url_keeps_data_urls_local(monkeypatch): + def _unexpected_http_client(*args, **kwargs): + pytest.fail("data URLs must not create an HTTP client") + + monkeypatch.setattr(video_api_utils.httpx, "AsyncClient", _unexpected_http_client) + encoded_image = base64.b64encode(_png_bytes()).decode() + + image = await video_api_utils.decode_image_url(f"data:image/png;base64,{encoded_image}") + + assert image.size == (2, 1) + assert image.mode == "RGB" + + def _install_fake_video_mux(monkeypatch, mux_calls): def _fake_mux_video_audio_bytes(frames, audio, fps, audio_sample_rate, video_codec_options=None): mux_calls.append( diff --git a/tests/entrypoints/openai_api/test_video_server.py b/tests/entrypoints/openai_api/test_video_server.py index b7527e233..b9b7df4df 100644 --- a/tests/entrypoints/openai_api/test_video_server.py +++ b/tests/entrypoints/openai_api/test_video_server.py @@ -16,15 +16,17 @@ from pathlib import Path from types import SimpleNamespace import av +import httpx import numpy as np import pytest from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient from PIL import Image from pytest_mock import MockerFixture +from vllm import envs from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes -from vllm_omni.entrypoints.openai import api_server +from vllm_omni.entrypoints.openai import api_server, video_api_utils from vllm_omni.entrypoints.openai.api_server import router from vllm_omni.entrypoints.openai.protocol.videos import ( VideoGenerationRequest, @@ -599,6 +601,45 @@ def test_i2v_video_generation_with_image_reference_form(test_client, mocker: Moc assert input_image.size == (40, 24) +def test_i2v_video_generation_follows_allowed_image_redirect(test_client, mocker: MockerFixture, monkeypatch): + requested_paths = [] + async_client = httpx.AsyncClient + + def _handler(request): + requested_paths.append(request.url.path) + if request.url.path == "/redirect.png": + return httpx.Response(302, headers={"location": "/image.png"}) + return httpx.Response(200, content=_make_test_image_bytes((40, 24))) + + def _client_factory(*args, **kwargs): + return async_client(*args, transport=httpx.MockTransport(_handler), **kwargs) + + monkeypatch.setattr(envs, "VLLM_MEDIA_URL_ALLOW_REDIRECTS", True) + monkeypatch.setattr(video_api_utils.httpx, "AsyncClient", _client_factory) + mocker.patch( + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", + ) + + response = test_client.post( + "/v1/videos", + data={ + "prompt": "A fox running through snow.", + "image_reference": json.dumps({"image_url": "https://example.com/redirect.png"}), + }, + ) + + assert response.status_code == 200 + video_id = response.json()["id"] + _wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value) + assert requested_paths == ["/redirect.png", "/image.png"] + + engine = test_client.app.state.openai_serving_video._engine_client + input_image = engine.captured_prompt["multi_modal_data"]["image"] + assert isinstance(input_image, Image.Image) + assert input_image.size == (40, 24) + + def test_v2v_video_generation_form(test_client, mocker: MockerFixture): video_bytes = _make_test_video_bytes((32, 24), num_frames=3) diff --git a/vllm_omni/entrypoints/openai/video_api_utils.py b/vllm_omni/entrypoints/openai/video_api_utils.py index 5a4f9f826..d954bd821 100644 --- a/vllm_omni/entrypoints/openai/video_api_utils.py +++ b/vllm_omni/entrypoints/openai/video_api_utils.py @@ -17,6 +17,7 @@ import httpx import numpy as np import torch from PIL import Image, UnidentifiedImageError +from vllm import envs from vllm.multimodal.video import ( VIDEO_LOADER_REGISTRY, VideoBackend, @@ -172,11 +173,21 @@ async def decode_image_url(image_url: str) -> Image.Image: return _decode_base64_image(image_url, source="image_reference.image_url") if image_url.startswith(("http://", "https://")): - async with httpx.AsyncClient(timeout=60) as client: + allow_redirects = envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS + async with httpx.AsyncClient(timeout=60, follow_redirects=allow_redirects) as client: try: response = await client.get(image_url) response.raise_for_status() - except httpx.HTTPError as exc: + except httpx.HTTPStatusError as exc: + if exc.response.has_redirect_location and not allow_redirects: + raise InvalidInputReferenceError( + "Invalid image_reference.image_url: redirect response was rejected because " + "VLLM_MEDIA_URL_ALLOW_REDIRECTS is disabled." + ) from exc + raise InvalidInputReferenceError( + f"Invalid image_reference.image_url: server returned HTTP {exc.response.status_code}." + ) from exc + except httpx.RequestError as exc: raise InvalidInputReferenceError( "Invalid image_reference.image_url: failed to download image." ) from exc From e96d3fe2a6b15c1820bf7cb79681927f5c4c899b Mon Sep 17 00:00:00 2001 From: akshatvishu Date: Mon, 17 Aug 2026 11:28:51 +0530 Subject: [PATCH 043/267] [ROCm][Recipe] Add MI300X coverage for verified recipes (#6207) Signed-off-by: akshatvishu Signed-off-by: Hongsheng Liu Co-authored-by: Hongsheng Liu --- recipes/MammothModa2/MammothModa2.md | 40 +++++++++- recipes/Qwen/Qwen3-TTS.md | 50 ++++++++++++ recipes/README.md | 7 +- recipes/SenseNova/SenseNova-U1.md | 42 ++++++++++ recipes/StabilityAI/Stable-Audio-Open.md | 47 +++++++++++- recipes/inclusionAI/Ming-omni-tts.md | 29 +++++++ recipes/k2-fsa/OmniVoice.md | 98 ++++++++++++++++++++++++ 7 files changed, 307 insertions(+), 6 deletions(-) create mode 100644 recipes/k2-fsa/OmniVoice.md diff --git a/recipes/MammothModa2/MammothModa2.md b/recipes/MammothModa2/MammothModa2.md index 7056a6f8f..6dc5f12e9 100644 --- a/recipes/MammothModa2/MammothModa2.md +++ b/recipes/MammothModa2/MammothModa2.md @@ -81,7 +81,7 @@ MammothModa2 generation parameters as a JSON object through `--extra-body`: ```bash python examples/offline_inference/text_to_image/text_to_image.py \ --model ./MammothModa2-Preview \ - --stage-configs-path vllm_omni/deploy/mammoth_moda2.yaml \ + --deploy-config vllm_omni/deploy/mammoth_moda2.yaml \ --prompt "A stylish woman riding a motorcycle in NYC, movie poster style" \ --height 1024 \ --width 1024 \ @@ -132,6 +132,44 @@ ls -lh mammoth_t2i.png python -c "from PIL import Image; print(Image.open('mammoth_t2i.png').size)" ``` +### 1x AMD MI300X, MammothModa2 Preview + +#### Environment + +- OS: Linux 6.8.0-134-generic, x86_64 +- Container: official ROCm image built from `docker/Dockerfile.rocm` +- Python: 3.12.13 +- PyTorch: 2.11.0+gitd0c8b1f +- Driver / runtime: AMD 6.19.14.31400000 / ROCm 7.2.53211 +- GPU: one AMD Instinct MI300X, `gfx942:sramecc+:xnack-`, 191.69 GiB visible HBM +- vLLM version: 0.27.0+rocm723 +- vLLM Omni version or commit: `73e1368c7bb940efe1a025859c9d6c8eeeb2e3f0` +- Installed vLLM Omni package metadata: `0.27.0rc2.dev44+g55abdade9.rocm` + +#### Offline Commands + +The checked run used the committed stage split, with `gpu_memory_utilization` set to 0.5 for AR and 0.3 for DiT: + +```bash +python3 examples/offline_inference/text_to_image/text_to_image.py \ + --model bytedance-research/MammothModa2-Preview \ + --deploy-config vllm_omni/deploy/mammoth_moda2.yaml \ + --prompt "A stylish woman riding a motorcycle in NYC, movie poster style" \ + --height 1024 \ + --width 1024 \ + --seed 42 \ + --extra-body '{"text_guidance_scale": 4.0, "cfg_range": [0.0, 1.0], "num_inference_steps": 50}' \ + --enable-diffusion-pipeline-profiler \ + --log-stats \ + --output mammoth_t2i.png +``` + +#### Verification + +The first request took 85.224 seconds. The AR stage generated 4,161 visual tokens in 72.996 seconds, and the DiT stage took 12.163 seconds. AR weight loading used 21.4 GiB and took 8.250 seconds. DiT weight loading used 5.49 GiB and took 1.824 seconds. The largest one second whole device memory sample was 106.57 GiB, including the AR KV cache reserved by the 0.5 memory setting. + +The output was a valid 1024 by 1024 RGB PNG. + ## MammothModa2-Dev unified inference MammothModa2-Dev uses a Qwen3-VL AR backbone, while MammothModa2-Preview uses diff --git a/recipes/Qwen/Qwen3-TTS.md b/recipes/Qwen/Qwen3-TTS.md index b6d2c9293..7365aafae 100644 --- a/recipes/Qwen/Qwen3-TTS.md +++ b/recipes/Qwen/Qwen3-TTS.md @@ -252,3 +252,53 @@ curl -X POST http://localhost:8091/v1/audio/speech \ - stage_id: 1 gpu_memory_utilization: 0.15 ``` + +### 1x AMD MI300X, 1.7B checkpoints + +#### Environment + +- OS: Linux 6.8.0-134-generic, x86_64 +- Container: official ROCm image built from `docker/Dockerfile.rocm` +- Python: 3.12.13 +- PyTorch: 2.11.0+gitd0c8b1f +- Driver / runtime: AMD 6.19.14.31400000 / ROCm 7.2.53211 +- GPU: one AMD Instinct MI300X, `gfx942:sramecc+:xnack-`, 191.69 GiB visible HBM +- vLLM version: 0.27.0+rocm723 +- vLLM Omni version or commit: `3fecb6953ca8dc51210cc0421ef24552267a41ef` +- Installed vLLM Omni package metadata: `0.27.0rc2.dev44+g55abdade9.rocm` +- ONNX Runtime: onnxruntime-rocm 1.22.2.post3 with `ROCMExecutionProvider` +- transformers: 5.15.0 + +#### Command + +The MI300X tests also worked without `enforce_eager`. To use that setup, remove `enforce_eager: true` from stage 1 under `platforms.rocm` in `vllm_omni/deploy/qwen3_tts.yaml`. + +Set this environment variable before you start Qwen3-TTS: + +```bash +export MIOPEN_FIND_MODE=FAST +``` + +#### Verification + +Each row records one offline request with one prompt and batch size one. Request time is the logged `e2e_wall_time_ms`, and stage 1 time is the logged `e2e_stage_1_wall_time_ms`. + +| Checkpoint | Code2Wav configuration | `MIOPEN_FIND_MODE` | Request time | Stage 1 time | Output duration | Real time factor | Maximum sampled device memory | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| CustomVoice | Eager | Unset | 1.683 s | 1.613 s | 5.68 s | 0.30 | 60.91 GiB | +| CustomVoice | Graphs allowed | `FAST` | 1.326 s | 1.259 s | 5.68 s | 0.23 | 62.19 GiB | +| VoiceDesign | Eager | Unset | 1.195 s | 1.180 s | 4.40 s | 0.27 | 60.91 GiB | +| VoiceDesign | Graphs allowed | `FAST` | 1.127 s | 1.054 s | 4.40 s | 0.26 | 62.21 GiB | +| Base | Eager | Unset | 13.330 s | 13.263 s | 4.32 s | 3.09 | 61.74 GiB | +| Base | Graphs allowed | `FAST` | 14.314 s | 14.245 s | 4.80 s | 2.98 | 62.84 GiB | + +All six processes finished successfully, and each output passed the 24 kHz mono WAV checks. + +#### Notes + +- The current ROCm config keeps Code2Wav in eager mode. +- The tested alternative allows graphs and sets [`MIOPEN_FIND_MODE=FAST`](https://rocm.docs.amd.com/projects/MIOpen/en/develop/reference/env_variables.html). MIOpen uses a saved kernel choice when available and uses its immediate fallback otherwise. +- CustomVoice used a graph for one of four logged Code2Wav batches. VoiceDesign used a graph for two of four batches. Base used no graphs for its four logged batches. +- The recorded request time with graphs allowed was lower for CustomVoice and VoiceDesign and higher for Base. The Base runs generated different amounts of audio, so their request times are not a direct comparison. +- Each setup was tested with one request, so the timing is not a performance benchmark. +- GPU memory was sampled once per second with `rocm-smi`. diff --git a/recipes/README.md b/recipes/README.md index 5fee4fdae..32c68e8c1 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -49,6 +49,7 @@ recipes/ | [`Helios/Helios.md`](./Helios/Helios.md) | Text-to-video, image-to-video, and video-to-video generation | 1x NVIDIA H20 | | [`inclusionAI/Ming-flash-omni-2.0.md`](./inclusionAI/Ming-flash-omni-2.0.md) | Online serving for multimodal chat + standalone TTS | 4x H100 / 1x H100 80GB | | [`inclusionAI/Ming-omni-tts.md`](./inclusionAI/Ming-omni-tts.md) | Offline + online dense Ming TTS/audio generation | 1x H100 80GB / 1x AMD MI300X (ROCm 7.2) | +| [`k2-fsa/OmniVoice.md`](./k2-fsa/OmniVoice.md) | Offline and online multilingual text-to-speech | 1x AMD MI300X (ROCm 7.2) | | [`IndexTeam/IndexTTS-2.md`](./IndexTeam/IndexTTS-2.md) | Online serving for voice-cloned TTS with optional emotion control | 1x L4 24GB or larger CUDA GPU | | [`IndexTeam/IndexTTS-2_5.md`](./IndexTeam/IndexTTS-2_5.md) | Online + offline multilingual voice-cloned TTS with native speed and emotion control | 1x NVIDIA H20 96GB | | [`MiniMaxAI/MiniMax-H3.md`](./MiniMaxAI/MiniMax-H3.md) | T2VA, FL2VA, image+audio Ref2VA, and multi-video Ref2VA serving | 1x GPU with CPU offload / 4x B300 | @@ -73,9 +74,9 @@ recipes/ | [`Qwen/Qwen-Image-2512.md`](./Qwen/Qwen-Image-2512.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt FP8 / mixed FP8/NVFP4 | 1x A800 80GB / 2x B200 | | [`Qwen/Qwen-Image-Edit.md`](./Qwen/Qwen-Image-Edit.md) | Text-guided single-image editing | 1x or 2x H200 141GB | | [`Qwen/Qwen3-Omni.md`](./Qwen/Qwen3-Omni.md) | Online serving for multimodal chat | 1x A100 80GB | -| [`Qwen/Qwen3-TTS.md`](./Qwen/Qwen3-TTS.md) | Text-to-speech serving (CustomVoice / VoiceDesign / Base) | 1x H100/A100 80GB | -| [`SenseNova/SenseNova-U1.md`](./SenseNova/SenseNova-U1.md) | Unified image generation and understanding | 1x H200 (144GB) | -| [`StabilityAI/Stable-Audio-Open.md`](./StabilityAI/Stable-Audio-Open.md) | Offline + online text-to-audio generation (Stable Audio Open) | 1x RTX 4090 24GB | +| [`Qwen/Qwen3-TTS.md`](./Qwen/Qwen3-TTS.md) | Text-to-speech serving (CustomVoice / VoiceDesign / Base) | 1x H100/A100 80GB / 1x AMD MI300X | +| [`SenseNova/SenseNova-U1.md`](./SenseNova/SenseNova-U1.md) | Unified image generation and understanding | 1x or 2x H200 / 1x AMD MI300X | +| [`StabilityAI/Stable-Audio-Open.md`](./StabilityAI/Stable-Audio-Open.md) | Offline + online text-to-audio generation (Stable Audio Open) | 1x RTX 4090 24GB / 1x AMD MI300X | | [`Tencent/Covo-Audio-Chat.md`](./Tencent/Covo-Audio-Chat.md) | Online serving for audio chat | 1x A100 80GB | | [`Tencent/HunyuanImage-3.0-Instruct.md`](./Tencent/HunyuanImage-3.0-Instruct.md) | DiT-only text-to-image serving and benchmark, including ModelOpt mixed FP8/NVFP4 | 4x H100/H800 80GB / 2x B200 | | [`Wan-AI/Wan2.2-T2V.md`](./Wan-AI/Wan2.2-T2V.md) | Text-to-video serving (Wan2.2 14B) with Skip-Softmax sparse attention | 1x datacenter Blackwell (B200/B300) | diff --git a/recipes/SenseNova/SenseNova-U1.md b/recipes/SenseNova/SenseNova-U1.md index be25df570..8e9091d63 100644 --- a/recipes/SenseNova/SenseNova-U1.md +++ b/recipes/SenseNova/SenseNova-U1.md @@ -178,6 +178,48 @@ with the expected 1536×2720 resolution. - The LLM transformer uses `QKVParallelLinear` and `MergedColumnParallelLinear` for fused QKV and gate/up projections with TP support. +### 1x AMD MI300X 192GB + +#### Environment + +- OS: Linux 6.8.0-134-generic, x86_64 +- Container: official ROCm image built from `docker/Dockerfile.rocm` +- Python: 3.12.13 +- PyTorch: 2.11.0+gitd0c8b1f +- Driver / runtime: AMD 6.19.14.31400000 / ROCm 7.2.53211 +- GPU: one AMD Instinct MI300X, `gfx942:sramecc+:xnack-`, 191.69 GiB visible HBM +- vLLM version: 0.27.0+rocm723 +- vLLM Omni version or commit: `a704c8759c96e123c0d7c89b11f120b1c0f120cf` +- Installed vLLM Omni package metadata: `0.27.0rc2.dev44+g55abdade9.rocm` + +#### Command + +```bash +python3 examples/offline_inference/text_to_image/text_to_image.py \ + --model SenseNova/SenseNova-U1-8B-MoT \ + --prompt "Close portrait of an elderly woman by a farmhouse window, textured skin, gentle smile, warm natural light, emotional documentary look." \ + --width 1536 \ + --height 2720 \ + --seed 42 \ + --num-inference-steps 50 \ + --cfg-scale 4.0 \ + --extra-body '{"think": true, "cfg_norm": "none", "timestep_shift": 3.0, "t_eps": 0.02}' \ + --enable-diffusion-pipeline-profiler \ + --log-stats \ + --output sensenova_u1_mi300x.png +``` + +#### Verification + +The command completed and wrote a valid 1536 by 2720 RGB PNG. + +#### Notes + +- Generation with 50 inference steps took 34.011 seconds. +- Model loading used 32.774 GiB and took 13.987 seconds. +- The internal profiler recorded 35.67 GB reserved and 35.10 GB allocated for the request. +- The highest one second whole device memory sample was 37.93 GiB. + ## Online Serving SenseNova-U1 supports all four modalities via the OpenAI-compatible diff --git a/recipes/StabilityAI/Stable-Audio-Open.md b/recipes/StabilityAI/Stable-Audio-Open.md index 97c39f7ee..4b5fd59c2 100644 --- a/recipes/StabilityAI/Stable-Audio-Open.md +++ b/recipes/StabilityAI/Stable-Audio-Open.md @@ -164,7 +164,50 @@ PY `torchaudio==2.11.0+cu130`. - The `NIXL is not available`, `GLOO_SOCKET_IFNAME`, and `torchsde` boundary warnings observed during validation did not prevent successful generation. -- This recipe was validated on one RTX 4090 24GB GPU only. Other GPU counts, - ROCm, XPU, and NPU setups are not covered here. +- The RTX 4090 entry was validated on one 24 GB GPU. The MI300X entry below covers one 192 GB GPU. - Long generations, higher inference-step counts, and non-WAV response formats were not benchmarked in this recipe. + +#### 1x AMD MI300X 192GB + +##### Environment + +- OS: Linux 6.8.0-134-generic, x86_64 +- Container: official ROCm image built from `docker/Dockerfile.rocm` +- Python: 3.12.13 +- PyTorch: 2.11.0+gitd0c8b1f +- Driver / runtime: AMD 6.19.14.31400000 / ROCm 7.2.53211 +- GPU: one AMD Instinct MI300X, `gfx942:sramecc+:xnack-`, 191.69 GiB visible HBM +- vLLM version: 0.27.0+rocm723 +- vLLM Omni version or commit: `73e1368c7bb940efe1a025859c9d6c8eeeb2e3f0` +- Installed vLLM Omni package metadata: `0.27.0rc2.dev44+g55abdade9.rocm` + +##### Commands + +```bash +python3 examples/offline_inference/text_to_audio/text_to_audio.py \ + --model stabilityai/stable-audio-open-1.0 \ + --prompt "A gentle piano melody with soft room ambience" \ + --negative-prompt "Low quality, distorted, noisy" \ + --seed 42 \ + --guidance-scale 7.0 \ + --audio-length 10.0 \ + --num-inference-steps 50 \ + --cache-backend tea_cache \ + --enable-diffusion-pipeline-profiler \ + --output stable_audio_10s.wav +``` + +##### Verification + +The command completed and wrote a valid 44.1 kHz stereo WAV with 10.00 seconds of audio. + +##### Notes + +- TeaCache ran with `rel_l1_thresh=0.2`. +- Model loading used 2.7891 GiB and took 3.706 seconds. +- Generation took 4.750 seconds, which gives a real time factor of 0.475 for the 10.00 second output. +- The internal profiler recorded 15.65 GB reserved and 9.68 GB allocated for the request. +- The highest one second whole device memory sample was 19.61 GiB. +- The output RMS was 0.0887, and its peak absolute amplitude was 0.5761. +- The full process took 384 seconds, including startup and compilation. diff --git a/recipes/inclusionAI/Ming-omni-tts.md b/recipes/inclusionAI/Ming-omni-tts.md index 0f0034721..0b3584691 100644 --- a/recipes/inclusionAI/Ming-omni-tts.md +++ b/recipes/inclusionAI/Ming-omni-tts.md @@ -170,6 +170,35 @@ python examples/online_serving/text_to_speech/ming_tts/openai_speech_client.py \ `--ref-audio` matches upstream `use_spk_emb=True`; do not add `--ref-text` for the dialect case. +**Offline smoke result on ROCm 7.2:** + +A later offline smoke run used the same MI300X model with the following software: + +- GPU: one AMD Instinct MI300X with 191.69 GiB visible HBM and gfx942 +- Kernel: Linux 6.8.0-134-generic, x86_64 +- Python: 3.12.13 +- PyTorch: 2.11.0+gitd0c8b1f +- ROCm or HIP: 7.2.53211 +- vLLM: 0.27.0+rocm723 +- vLLM Omni checkout: `73e1368c7bb940efe1a025859c9d6c8eeeb2e3f0` + +```bash +python3 examples/offline_inference/text_to_speech/ming_tts/end2end.py \ + --model inclusionAI/Ming-omni-tts-0.5B \ + --deploy-config vllm_omni/deploy/ming_tts.yaml \ + --case basic \ + --text "你好，这是 AMD MI300X ROCm 配方验证。" \ + --output-dir ./ming_tts_output \ + --output-name ming_omni_tts_mi300x.wav \ + --enforce-eager \ + --log-stats \ + --metadata-json ./ming_tts_manifest.json +``` + +The checked run produced a valid 44.1 kHz mono WAV with 4.80 seconds of audio, RMS 0.1449, and peak absolute amplitude 0.8621. The first request took 20.406 seconds, which gives a real time factor of 4.25. The stage weight loads used 1.31 GiB and 1.47 GiB and took 1.756 seconds and 1.264 seconds. The largest one second whole device memory sample was 91.05 GiB, including the reserved KV cache. Both stages used eager mode, and the AR stage selected `TRITON_ATTN`. + +The run used the official ROCm image built from `docker/Dockerfile.rocm`. + ## Notes - The official ROCm image includes the platform dependencies. diff --git a/recipes/k2-fsa/OmniVoice.md b/recipes/k2-fsa/OmniVoice.md new file mode 100644 index 000000000..74e1b12f5 --- /dev/null +++ b/recipes/k2-fsa/OmniVoice.md @@ -0,0 +1,98 @@ +# OmniVoice + +> Offline and online text to speech with automatic voice, voice design, and reference voice cloning + +## Summary + +- Vendor: k2-fsa +- Model: `k2-fsa/OmniVoice` +- Task: Multilingual text to speech +- Mode: Offline inference for all modes and online serving for automatic voice +- Maintainer: Community + +## References + +- [Hugging Face model](https://huggingface.co/k2-fsa/OmniVoice) +- [Offline text to speech examples](../../docs/user_guide/examples/offline_inference/text_to_speech.md#omnivoice) +- [Online text to speech examples](../../docs/user_guide/examples/online_serving/text_to_speech.md#omnivoice) +- [Deploy config](../../vllm_omni/deploy/omnivoice.yaml) + +## Requirements + +Download the checkpoint before the first run: + +```bash +huggingface-cli download k2-fsa/OmniVoice +``` + +Automatic voice and voice design require `transformers>=4.57.0`. Reference voice cloning requires `transformers>=5.3.0`. + +The vLLM Omni deploy config runs the OmniVoice generator and decoder inside one float32 diffusion stage. + +## ROCm + +### 1x AMD MI300X + +#### Environment + +- GPU: one AMD Instinct MI300X with 191.69 GiB visible HBM and gfx942 +- Kernel: Linux 6.8.0-134-generic, x86_64 +- Python: 3.12.13 +- PyTorch: 2.11.0+gitd0c8b1f +- ROCm or HIP: 7.2.53211 +- vLLM: 0.27.0+rocm723 +- vLLM Omni checkout: `73e1368c7bb940efe1a025859c9d6c8eeeb2e3f0` +- transformers: 5.15.0 + +The run used the official ROCm image built from `docker/Dockerfile.rocm`. + +#### Automatic voice + +Run the offline example from the repository root: + +```bash +python3 examples/offline_inference/text_to_speech/omnivoice/end2end.py \ + --model k2-fsa/OmniVoice \ + --deploy-config vllm_omni/deploy/omnivoice.yaml \ + --text "Hello, this is OmniVoice running on one AMD MI300X." \ + --seed 42 \ + --output omnivoice_mi300x.wav +``` + +The checked run produced a valid 24 kHz mono WAV with 3.60 seconds of audio, RMS 0.1194, and peak absolute amplitude 0.9129. The first request took 0.685 seconds, which gives a real time factor of 0.19. Model loading used 3.5739 GiB and took 9.564 seconds. The request used 4.56 GB reserved and 3.64 GB allocated according to the internal profiler, while the largest one second whole device sample was 8.33 GiB. The log states that the OmniVoice Triton kernels loaded. + +#### Reference voice cloning + +The offline interface also accepts a reference WAV and its transcript: + +```bash +python3 examples/offline_inference/text_to_speech/omnivoice/end2end.py \ + --model k2-fsa/OmniVoice \ + --text "Hello, this is a cloned voice." \ + --ref-audio ref.wav \ + --ref-text "This is the reference transcription." \ + --output cloned.wav +``` + +#### Voice design + +```bash +python3 examples/offline_inference/text_to_speech/omnivoice/end2end.py \ + --model k2-fsa/OmniVoice \ + --text "Hello, this is a designed voice." \ + --instruct "female, low pitch, British accent" \ + --output designed.wav +``` + +## Online serving + +Online serving currently supports automatic voice: + +```bash +vllm serve k2-fsa/OmniVoice --omni --port 8091 --trust-remote-code +``` + +```bash +cd examples/online_serving/text_to_speech/omnivoice +python speech_client.py --text "Hello, how are you?" +``` From 5664e69dbf05486468e84f81ff1245458f3d1847 Mon Sep 17 00:00:00 2001 From: Shenglei Fu <117230642+ShengleiFu@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:02:50 -0700 Subject: [PATCH 044/267] [CI/Build] Gate one-word pronunciation on a success rate, not per request (#5681) Signed-off-by: Shenglei Fu Co-authored-by: amy-why-3459 --- .../test_qwen3_omni_expansion.py | 42 +++++-- tests/helpers/assertions.py | 113 +++++++++++++++++- tests/helpers/runtime.py | 61 +++++++--- .../tests/test_success_rate_helpers.py | 110 +++++++++++++++++ 4 files changed, 291 insertions(+), 35 deletions(-) create mode 100644 tests/helpers/tests/test_success_rate_helpers.py diff --git a/tests/e2e/online_serving/test_qwen3_omni_expansion.py b/tests/e2e/online_serving/test_qwen3_omni_expansion.py index 1ba94c011..99066468d 100644 --- a/tests/e2e/online_serving/test_qwen3_omni_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_omni_expansion.py @@ -452,11 +452,31 @@ def test_audio_in_video_default_loader_sampling_regression(omni_server, openai_c @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_one_word_prompt_001(omni_server, openai_client) -> None: - """ + """Catastrophic-regression gate on one-word pronunciation. + + Whether a one-word answer is pronounced intelligibly is sampled, not guaranteed: the + talker runs at temperature 0.9 (the upstream default, see qwen3_omni_moe.yaml stage 1), + so asserting it per request gates on a random variable, which is what the old 10-retry + loop was compensating for. + + Sends a fixed 40 requests and requires at least 29 to transcribe as the expected + word. 85.6% is a predeclared minimum acceptable success probability, taken from a + measured HF reference run (77/90); vLLM-Omni itself measured 92.2%. Assuming 40 + independent Bernoulli trials, >=29/40 has a nominal rejection probability of 0.9% + at p=.856, with 92.9% power at p=.60 and 55.9% at p=.70. + + Those are conditional on p, not a measured false-failure rate for this CI: 77/90 is + one point estimate (Wilson 76.8%-91.4%), and at the low end of that interval the gate + would reject about 20% of runs. Pinning down a real rate would take repeated + report-only runs on the target hardware. + + A drop to 70% is caught only about half the time, so this is a smoke gate for gross + pipeline breakage, not a quality tracker -- resolving moderate drift needs ~80+ + samples and belongs in an accuracy benchmark. See #5395. + Input Modal: text only (one-word answer constraint). Output Modal: text, audio (default ``modalities``); ``key_words`` only assert on text. Input Setting: stream=True - Datasets: single request """ messages = dummy_messages_from_mix_data( system_prompt=get_system_prompt(), @@ -474,17 +494,13 @@ def test_one_word_prompt_001(omni_server, openai_client) -> None: "transcript_language": "en", } - # Retry only when assert_omni_response fails on text/audio cosine similarity (see tests/helpers/assertions.py). - _similarity_assert_msg = "The audio content is not same as the text" - _max_retries = 10 - for attempt in range(_max_retries): - try: - openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) - break - except AssertionError as e: - if _similarity_assert_msg not in str(e) or attempt == _max_retries - 1: - raise - print(f"Similarity assertion failed, retrying {attempt + 2}/{_max_retries}: {e!r}") + # The judge runs per request so mismatches can be counted, which also transcribes + # concurrently where the previous request_num=5 path judged serially -- worth watching + # on tighter cards, since each judge loads whisper on the same device as the talker and + # code2wav stages. + openai_client.send_omni_request( + request_config, request_num=40, min_successes=29, max_concurrency=get_max_batch_size() + ) @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) diff --git a/tests/helpers/assertions.py b/tests/helpers/assertions.py index 3b6fbe137..70c772435 100644 --- a/tests/helpers/assertions.py +++ b/tests/helpers/assertions.py @@ -2,12 +2,16 @@ import io import json +import math import tempfile import threading import wave +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass from io import BytesIO from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Literal, TypeVar if TYPE_CHECKING: from tests.helpers.runtime import DiffusionResponse @@ -24,8 +28,109 @@ from tests.helpers.media import ( preprocess_text, ) +_ResponseT = TypeVar("_ResponseT") + _GENDER_PIPELINE = None _GENDER_PIPELINE_LOCK = threading.Lock() + +# Assertions that sample a random variable rather than testing whether serving works: +# whether TTS audio is intelligible enough to transcribe back, and whether a preset voice's +# timbre estimates as the expected gender. Both are decided by the model and the estimator, +# so a caller measuring a success rate has to tell them apart from a serving failure. The +# messages are constants because a caller matching the wording by hand would silently stop +# matching if it were reworded, reclassifying every quality failure as a hard failure. +AUDIO_MISMATCH_MESSAGE = "The audio content is not same as the text" +GENDER_MISMATCH_MESSAGE = "estimated gender is" +_QUALITY_FAILURE_MESSAGES = (AUDIO_MISMATCH_MESSAGE, GENDER_MISMATCH_MESSAGE) + + +@dataclass(frozen=True) +class SuccessRateGate: + """Gate a sampled quality check on how many requests pass, not on each one passing. + + Built by the send helpers from their own arguments, so a test states a policy and never + touches this class. See ``collect_at_success_rate``. + + Attributes: + min_successes: Fail below this many successes out of ``request_num``. Predeclare it + from a measured baseline; tuning it until CI passes defeats the gate. + concurrency: In-flight requests, in batches until the sample count is reached. + Defaults to sending all of them at once. + """ + + min_successes: int + concurrency: int | None = None + + +def is_quality_failure(exc: BaseException) -> bool: + """True when ``exc`` is one of the sampled quality assertions above. + + A caller absorbing a few of these into a success-rate budget must not absorb an HTTP + failure, missing audio or a text-keyword miss as well, or a broken server passes. + """ + if not isinstance(exc, AssertionError): + return False + return any(message in str(exc) for message in _QUALITY_FAILURE_MESSAGES) + + +def collect_at_success_rate( + sample: Callable[[], _ResponseT], + gate: SuccessRateGate, + *, + request_num: int, + report: Callable[[str], None] = print, +) -> list[_ResponseT]: + """Send ``request_num`` samples and gate on how many passed, returning the successes. + + ``sample`` sends one request and asserts it. Only the sampled quality assertions are + counted; anything else propagates immediately. Does not retry and does not exit early, + since both would bias the rate being measured. + """ + if gate.min_successes > request_num: + raise ValueError(f"min_successes={gate.min_successes} exceeds request_num={request_num}, gate can never pass") + concurrency = min(gate.concurrency or request_num, request_num) + responses: list[_ResponseT] = [] + failures: list[str] = [] + lock = threading.Lock() + + def _one_sample(_: int) -> None: + try: + response = sample() + except AssertionError as exc: + if not is_quality_failure(exc): + raise + with lock: + failures.append(str(exc)) + return + with lock: + responses.append(response) + + for start in range(0, request_num, concurrency): + size = min(concurrency, request_num - start) + with ThreadPoolExecutor(max_workers=size) as executor: + for _ in executor.map(_one_sample, range(size)): + pass + + successes = request_num - len(failures) + report(f"success rate: {successes}/{request_num} succeeded") + assert successes >= gate.min_successes, ( + f"success rate {successes}/{request_num} is below the " + f"{gate.min_successes}/{request_num} gate " + f"(95% CI {wilson_interval(successes, request_num)}). Failures:\n" + + "\n".join(f" - {failure}" for failure in failures) + ) + return responses + + +def wilson_interval(successes: int, total: int, z: float = 1.96) -> str: + """Format a 95% Wilson score interval, so a rate failure shows how marginal it is.""" + p = successes / total + denom = 1 + z * z / total + centre = (p + z * z / (2 * total)) / denom + half = z / denom * math.sqrt(p * (1 - p) / total + z * z / (4 * total * total)) + return f"{max(0.0, centre - half):.1%}-{min(1.0, centre + half):.1%}" + + # Transcript gates default to whisper ``small`` for speed. ``small`` mishears a # short TTS clip ~0.5% of the time (e.g. "Hello"->"fellow", or hallucinating a # leading SFX token), which flakes the deterministic similarity gate. Short @@ -474,7 +579,7 @@ def _assert_preset_voice_gender_from_audio( print(f"Preset voice gender check: preset={key!r}, estimated={estimated_gender!r}, expected={expected_gender!r}") if estimated_gender != "unknown": assert estimated_gender == expected_gender, ( - f"{voice_name!r} is expected {expected_gender}, but estimated gender is {estimated_gender!r}" + f"{voice_name!r} is expected {expected_gender}, but {GENDER_MISMATCH_MESSAGE} {estimated_gender!r}" ) @@ -655,7 +760,7 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve shorter_clean = _re.sub(r"[^\w\s]", "", shorter).strip() longer_clean = _re.sub(r"[^\w\s]", "", longer).strip() assert shorter_clean and (shorter_clean in longer_clean), ( - f"The audio content is not same as the text " + f"{AUDIO_MISMATCH_MESSAGE} " f"(short-text containment check failed: " f"text={text_output!r}, transcript={transcript!r})" ) @@ -666,7 +771,7 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve text_output.lower(), ) print(f"similarity is: {similarity}") - assert similarity > similarity_threshold, "The audio content is not same as the text" + assert similarity > similarity_threshold, AUDIO_MISMATCH_MESSAGE if audio_ref_text: assert transcript is not None, "No audio transcript for reference-text validation" audio_similarity = cosine_similarity_text( diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index cfc8cc75d..c48411c9b 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -38,11 +38,13 @@ from vllm.distributed.parallel_state import ( from vllm.logger import init_logger from tests.helpers.assertions import ( + SuccessRateGate, assert_audio_speech_response, assert_diffusion_response, assert_http_error, assert_images_generations_response, assert_omni_response, + collect_at_success_rate, ) from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup from tests.helpers.media import ( @@ -1891,8 +1893,20 @@ class OpenAIClientHandler: ) ] - def send_omni_request(self, request_config: dict[str, Any], request_num: int = 1) -> list[OmniResponse]: - """Chat completions via the OpenAI Python SDK (not raw HTTP).""" + def send_omni_request( + self, + request_config: dict[str, Any], + request_num: int = 1, + *, + min_successes: int | None = None, + max_concurrency: int | None = None, + ) -> list[OmniResponse]: + """Chat completions via the OpenAI Python SDK (not raw HTTP). + + ``min_successes`` gates the batch on how many of the ``request_num`` requests pass + rather than requiring each one to, and returns only the successes; ``max_concurrency`` + caps how many are in flight at once. See ``SuccessRateGate``. + """ responses: list[OmniResponse] = [] stream = request_config.get("stream", False) modalities = request_config.get("modalities", ["text", "audio"]) @@ -1923,22 +1937,6 @@ class OpenAIClientHandler: if extra_body: create_kwargs["extra_body"] = extra_body - if request_num == 1: - wall_start = time.perf_counter() - chat_completion = self.client.chat.completions.create(**create_kwargs) - resp = ( - self._process_stream_omni_response(chat_completion, wall_start=wall_start) - if stream - else self._process_non_stream_omni_response(chat_completion, wall_start=wall_start) - ) - assert_omni_response(resp, request_config, run_level=self.run_level) - if resp.e2e_latency is not None: - self._print_client_stat(f"[omni] request#1 success in {resp.e2e_latency:.3f}s") - else: - self._print_client_stat("[omni] request#1 completed") - responses.append(resp) - return responses - def _one(): wall_start = time.perf_counter() chat_completion = self.client.chat.completions.create(**create_kwargs) @@ -1948,6 +1946,33 @@ class OpenAIClientHandler: else self._process_non_stream_omni_response(chat_completion, wall_start=wall_start) ) + if min_successes is not None: + gate = SuccessRateGate(min_successes=min_successes, concurrency=max_concurrency) + + def _sample() -> OmniResponse: + resp = _one() + assert_omni_response(resp, request_config, run_level=self.run_level) + return resp + + # Not _print_client_stat: the count is the gate's verdict, so it has to survive + # log_stats=False. + return collect_at_success_rate( + _sample, + gate, + request_num=request_num, + report=lambda line: print(f"[omni] {line}", flush=True), + ) + + if request_num == 1: + resp = _one() + assert_omni_response(resp, request_config, run_level=self.run_level) + if resp.e2e_latency is not None: + self._print_client_stat(f"[omni] request#1 success in {resp.e2e_latency:.3f}s") + else: + self._print_client_stat("[omni] request#1 completed") + responses.append(resp) + return responses + with concurrent.futures.ThreadPoolExecutor(max_workers=request_num) as executor: futures = {executor.submit(_one): i + 1 for i in range(request_num)} for future in concurrent.futures.as_completed(futures): diff --git a/tests/helpers/tests/test_success_rate_helpers.py b/tests/helpers/tests/test_success_rate_helpers.py new file mode 100644 index 000000000..c1b8f4417 --- /dev/null +++ b/tests/helpers/tests/test_success_rate_helpers.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import threading +from types import SimpleNamespace + +import pytest + +from tests.helpers import assertions +from tests.helpers.assertions import ( + AUDIO_MISMATCH_MESSAGE, + SuccessRateGate, + assert_omni_response, + collect_at_success_rate, + is_quality_failure, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_GATE = SuccessRateGate(min_successes=7, concurrency=2) + + +def _fake_response(text: str) -> SimpleNamespace: + return SimpleNamespace(success=True, text_content=text, audio_content=None, audio_bytes=b"fake-wav") + + +def _omni_response_failure(monkeypatch, *, text: str, transcript: str) -> AssertionError: + """Drive the real assert_omni_response to failure and hand back what it raised.""" + monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", lambda *a, **kw: transcript) + request_config = {"modalities": ["text", "audio"], "key_words": {"text": ["london"]}} + with pytest.raises(AssertionError) as excinfo: + assert_omni_response(_fake_response(text), request_config, run_level="full_model") + return excinfo.value + + +def test_real_audio_mismatch_is_tolerated(monkeypatch): + # Ties the classifier to the actual producer: if assert_omni_response's wording drifts, + # this fails instead of the gate silently turning every quality failure into a hard one. + assert is_quality_failure(_omni_response_failure(monkeypatch, text="London", transcript="런던")) + + +def test_real_keyword_failure_is_not_tolerated(monkeypatch): + assert not is_quality_failure(_omni_response_failure(monkeypatch, text="Paris", transcript="London")) + + +@pytest.mark.parametrize("exc", [RuntimeError(AUDIO_MISMATCH_MESSAGE), ValueError("boom")]) +def test_non_assertion_failures_are_not_tolerated(exc): + # A transport error carrying the same text is still a serving failure, not a sample. + assert not is_quality_failure(exc) + + +class _ScriptedSampler: + """One scripted outcome per call: raised if it is an exception, else returned.""" + + def __init__(self, outcomes: list) -> None: + self._outcomes = outcomes + self._lock = threading.Lock() + self.calls = 0 + + def __call__(self): + with self._lock: + outcome = self._outcomes[self.calls] + self.calls += 1 + if isinstance(outcome, BaseException): + raise outcome + return outcome + + +def _mismatch(detail: str) -> AssertionError: + return AssertionError(f"{AUDIO_MISMATCH_MESSAGE} (transcript={detail!r})") + + +def test_tolerated_failures_are_counted_and_every_sample_is_sent(): + sampler = _ScriptedSampler(["ok"] * 7 + [_mismatch("a"), _mismatch("b"), _mismatch("c")]) + + responses = collect_at_success_rate(sampler, _GATE, request_num=10, report=lambda _: None) + + # No early exit once the gate is already satisfied: it would bias the measured rate. + assert sampler.calls == 10 + assert responses == ["ok"] * 7 + + +def test_gate_failure_reports_the_count_and_every_failing_sample(): + outcomes = ["ok"] * 6 + [_mismatch("first"), _mismatch("second"), _mismatch("third"), _mismatch("fourth")] + + with pytest.raises(AssertionError) as excinfo: + collect_at_success_rate(_ScriptedSampler(outcomes), _GATE, request_num=10, report=lambda _: None) + + message = str(excinfo.value) + assert "6/10" in message and "7/10 gate" in message + assert all(detail in message for detail in ("first", "second", "third", "fourth")) + + +def test_serving_failure_is_not_absorbed_into_the_budget(): + sampler = _ScriptedSampler(["ok", AssertionError("No audio output is generated"), *["ok"] * 8]) + + with pytest.raises(AssertionError, match="No audio output"): + collect_at_success_rate(sampler, _GATE, request_num=10, report=lambda _: None) + + # Stopped on the serving failure rather than spending the remaining samples. + assert sampler.calls < 10 + + +def test_unreachable_gate_is_rejected_before_sending(): + sampler = _ScriptedSampler(["ok"] * 3) + + with pytest.raises(ValueError, match="can never pass"): + collect_at_success_rate(sampler, _GATE, request_num=3, report=lambda _: None) + + assert sampler.calls == 0 From 2a9f6aaa8e470eda306a7f2890c40025e69df9bc Mon Sep 17 00:00:00 2001 From: Yichen Wang Date: Sun, 16 Aug 2026 23:29:17 -0700 Subject: [PATCH 045/267] =?UTF-8?q?Add=20=CF=800=20VLA=20model=20support?= =?UTF-8?q?=20(#4222)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Yichen Wang Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com> Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com> --- docs/models/supported_models.md | 1 + examples/online_serving/pi0/README.md | 104 ++ examples/online_serving/pi0/openpi_client.py | 119 +++ recipes/lerobot/Pi0.md | 108 ++ tests/diffusion/models/pi0/__init__.py | 2 + tests/diffusion/models/pi0/test_pi0_parity.py | 366 +++++++ tests/diffusion/models/pi0/test_pi0_units.py | 585 +++++++++++ .../e2e/online_serving/test_pi0_expansion.py | 85 ++ tests/helpers/runtime.py | 144 +++ tests/model_tests/diffusion/test_alignment.py | 1 + vllm_omni/config/pipeline_registry.py | 2 + vllm_omni/deploy/pi0.yaml | 52 + vllm_omni/diffusion/data.py | 6 + vllm_omni/diffusion/models/pi0/__init__.py | 48 + vllm_omni/diffusion/models/pi0/config.py | 125 +++ .../diffusion/models/pi0/modeling_pi0.py | 944 ++++++++++++++++++ .../diffusion/models/pi0/pipeline_pi0.py | 277 +++++ .../diffusion/models/pi0/processor_pi0.py | 222 ++++ vllm_omni/diffusion/registry.py | 11 + 19 files changed, 3202 insertions(+) create mode 100644 examples/online_serving/pi0/README.md create mode 100644 examples/online_serving/pi0/openpi_client.py create mode 100644 recipes/lerobot/Pi0.md create mode 100644 tests/diffusion/models/pi0/__init__.py create mode 100644 tests/diffusion/models/pi0/test_pi0_parity.py create mode 100644 tests/diffusion/models/pi0/test_pi0_units.py create mode 100644 tests/e2e/online_serving/test_pi0_expansion.py create mode 100644 vllm_omni/deploy/pi0.yaml create mode 100644 vllm_omni/diffusion/models/pi0/__init__.py create mode 100644 vllm_omni/diffusion/models/pi0/config.py create mode 100644 vllm_omni/diffusion/models/pi0/modeling_pi0.py create mode 100644 vllm_omni/diffusion/models/pi0/pipeline_pi0.py create mode 100644 vllm_omni/diffusion/models/pi0/processor_pi0.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 62645aae6..d9fdb09a4 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -53,6 +53,7 @@ th { | `LingBotVideoPipeline` | LingBot-Video dense and MoE T2I, T2V, TI2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | | — | | `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | ✅︎ | | | [Published](https://recipes.vllm.ai/MiniMaxAI/MiniMax-H3) | | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | +| `Pi0Pipeline` | π0 (Pi-Zero) VLA | `lerobot/pi0_base` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi0.md) | | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | | — | | `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | | — | | `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ | — | diff --git a/examples/online_serving/pi0/README.md b/examples/online_serving/pi0/README.md new file mode 100644 index 000000000..1f424a88b --- /dev/null +++ b/examples/online_serving/pi0/README.md @@ -0,0 +1,104 @@ +# π0 (Pi-Zero) VLA — OpenPI realtime serving + +[π0](https://www.physicalintelligence.company/blog/pi0) is a Vision-Language-Action +model from Physical Intelligence: multi-camera images + a language instruction + +robot proprioceptive state → a continuous action chunk via flow-matching denoising +(it does **not** emit text tokens). This example serves π0 over the OpenPI realtime +websocket protocol at `/v1/realtime/robot/openpi`. + +## Install extras + +The core `pip install -e .` does not include the OpenPI client used here: + +- `openpi-client` (from the [openpi](https://github.com/Physical-Intelligence/openpi) + repo: `pip install -e packages/openpi-client`), `websockets`, `msgpack`, `msgpack-numpy` + +## Weights + +π0 uses the LeRobot `lerobot/pi0_base` checkpoint (HF, ~13 GB). Either let the server +download it (`MODEL=lerobot/pi0_base`) or point at a local copy +(`MODEL=/path/to/pi0_base`). + +## Run the server + +```bash +vllm serve lerobot/pi0_base --omni --port 8000 \ + --served-model-name pi0 \ + --deploy-config vllm_omni/deploy/pi0.yaml \ + --enforce-eager --disable-log-stats +``` + +The deploy config (`vllm_omni/deploy/pi0.yaml`) declares a single diffusion stage +(`Pi0Pipeline`), float32, `max_num_seqs: 1`, and the `policy_server_config` handshake +metadata (3 cameras, `joint_position`, action horizon 50, action dim 32). + +### Commonly adjusted `model_config` keys + +These live under `stages[0].model_config` in `vllm_omni/deploy/pi0.yaml`: + +| Key | Default | Meaning | +|---|---|---| +| `chunk_size` | `50` | Action-chunk length (timesteps) the model predicts per inference. | +| `num_inference_steps` | `10` | Flow-matching Euler denoising steps. | +| `max_action_dim` | `32` | Action dimensionality (state/action are padded to this). | +| `max_state_dim` | `32` | Proprioceptive-state dimensionality (zero-padded to this). | +| `image_resolution` | `[224, 224]` | Per-camera input size (square; SigLIP). | +| `tokenizer_max_length` | `48` | Max PaliGemma prompt tokens. | +| `max_cameras` | `3` | Camera slots the model attends to (real + `-1`-padded). | +| `image_feature_keys` | 3 `observation.images.*` keys | Camera order the model attends to. | +| `image_key_map` | `{}` | Map raw obs camera keys → `image_feature_keys` (empty = verbatim). | + +The `policy_server_config` block below them is the OpenPI handshake metadata +advertised to the client; keep its `action_horizon` / `action_dim` / +`image_resolution` in sync with the `model_config` values above. The e2e test +`tests/e2e/online_serving/test_pi0_expansion.py::test_pi0_openpi_online` connects to a +live server and asserts the advertised metadata matches these `pi0.yaml` values. + +## Run the client + +```bash +python examples/online_serving/pi0/openpi_client.py --host 127.0.0.1 --port 8000 \ + --prompt "pick up the red block and place it in the bin" +``` + +It connects, prints the server metadata, sends robot observations, and prints the +returned `[action_horizon, action_dim] = [50, 32]` action chunks. Replace the blank +cameras / zero state in `_make_dummy_obs` with real frames (HWC uint8) and +proprioceptive state to drive a robot. + +### Observation format + +The client sends a flat dict per inference: + +```python +{ + "observation.images.base_0_rgb": np.uint8[H, W, 3], + "observation.images.left_wrist_0_rgb": np.uint8[H, W, 3], + "observation.images.right_wrist_0_rgb":np.uint8[H, W, 3], + "state": np.float32[state_dim], # zero-padded to max_state_dim=32 server-side + "prompt": "pick up the red block", + "session_id": "", # accepted but ignored (π0 is stateless) +} +``` + +Camera keys must match the server's `image_feature_keys` (the checkpoint's +`input_features` order). If your robot uses different camera names, set +`model_config.image_key_map` in `pi0.yaml` to map raw obs keys → feature keys. + +## Correctness + +π0's flow-matching kernel is bit-for-bit matched to the LeRobot `PI0Policy` reference +(`max|Δ| = 7.15e-07`, CPU/float32, fixed noise; see +`tests/diffusion/models/pi0/test_pi0_parity.py::test_pi0_vllm_omni_vs_lerobot`) and is +version-stable across transformers releases (the version-stability checks in +`tests/diffusion/models/pi0/test_pi0_units.py`). An OpenPI websocket online-serving e2e +lives in `tests/e2e/online_serving/test_pi0_expansion.py::test_pi0_openpi_online`. + +## Limitations + +- **Normalization stats**: `lerobot/pi0_base` uses identity normalization, which is + fully supported (state passes through, actions are returned in the model's space). + Per-dataset `norm_stats` declared in a checkpoint's `config.json` are honored, but + stats stored only in LeRobot's `policy_preprocessor.json` companion safetensors are + not yet loaded — a fine-tuned checkpoint that relies on those would need that bridge + added before its actions are in real-world units. diff --git a/examples/online_serving/pi0/openpi_client.py b/examples/online_serving/pi0/openpi_client.py new file mode 100644 index 000000000..f2154b33d --- /dev/null +++ b/examples/online_serving/pi0/openpi_client.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Minimal OpenPI client for the π0 (Pi-Zero) VLA policy server. + +Connects to a running π0 server (see README.md), sends a single robot +observation (3 cameras + state + language instruction), and prints the returned +continuous action chunk ``[action_horizon, action_dim]``. + +The server speaks the OpenPI websocket protocol: + connect -> server sends msgpack(policy_server_config metadata) + infer -> client sends msgpack(obs), server replies msgpack(actions ndarray) + +Usage: + python openpi_client.py --host 127.0.0.1 --port 8000 \ + --prompt "pick up the red block" +""" + +from __future__ import annotations + +import argparse +import logging +import sys +import uuid +from pathlib import Path +from typing import Any + +import numpy as np + +try: + # Avoid shadowing the installed ``openpi_client`` package with this dir. + example_dir = str(Path(__file__).resolve().parent) + removed = sys.path and sys.path[0] == example_dir + if removed: + sys.path.pop(0) + try: + from openpi_client.websocket_client_policy import WebsocketClientPolicy + finally: + if removed: + sys.path.insert(0, example_dir) +except ImportError as exc: # pragma: no cover - runtime dependency guard + raise ImportError("π0 OpenPI example requires `openpi-client`.") from exc + +logger = logging.getLogger("pi0_openpi_client") + +DEFAULT_HOST = "127.0.0.1" +DEFAULT_PORT = 8000 +DEFAULT_PATH = "/v1/realtime/robot/openpi" +DEFAULT_PROMPT = "pick up the red block and place it in the bin" + +# π0 / pi0_base camera identities (must match the server's image_feature_keys). +CAMERA_KEYS = ( + "observation.images.base_0_rgb", + "observation.images.left_wrist_0_rgb", + "observation.images.right_wrist_0_rgb", +) +STATE_DIM = 32 + + +def _make_dummy_obs(prompt: str, session_id: str, image_size: int = 224) -> dict[str, Any]: + """Build a single π0 observation with blank cameras + zero state. + + Replace the zeros with real camera frames (HWC uint8) and proprioceptive + state to drive an actual robot. + """ + obs: dict[str, Any] = {cam: np.zeros((image_size, image_size, 3), dtype=np.uint8) for cam in CAMERA_KEYS} + obs["state"] = np.zeros(STATE_DIM, dtype=np.float32) + obs["prompt"] = prompt + obs["session_id"] = session_id + return obs + + +def _ws_uri(host: str, port: int, path: str) -> str: + """Build the full websocket URI. WebsocketClientPolicy takes the whole URI as + ``host`` (it has no separate path arg).""" + if host.startswith(("ws://", "wss://")): + return host + return f"ws://{host}:{port}{path}" + + +def _as_action_array(response: Any) -> np.ndarray: + if isinstance(response, dict) and response.get("type") == "error": + raise RuntimeError(f"Inference failed: {response.get('message', response)}") + return np.asarray(response, dtype=np.float32) + + +def main() -> int: + parser = argparse.ArgumentParser(description="π0 OpenPI client") + parser.add_argument("--host", default=DEFAULT_HOST) + parser.add_argument("--port", type=int, default=DEFAULT_PORT) + parser.add_argument("--path", default=DEFAULT_PATH) + parser.add_argument("--prompt", default=DEFAULT_PROMPT) + parser.add_argument("--num-steps", type=int, default=2, help="Inference observations to send.") + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(message)s") + + policy = WebsocketClientPolicy(host=_ws_uri(args.host, args.port, args.path)) + metadata = dict(policy.get_server_metadata()) + logger.info("Server metadata: %s", metadata) + + session_id = str(uuid.uuid4()) + for step in range(args.num_steps): + obs = _make_dummy_obs(args.prompt, session_id) + actions = _as_action_array(policy.infer(obs)) + logger.info( + "[step %d] actions shape=%s mean=%.4f std=%.4f", + step, + actions.shape, + float(actions.mean()), + float(actions.std()), + ) + if not np.isfinite(actions).all(): + raise RuntimeError("Server returned non-finite actions.") + logger.info("Done — π0 OpenPI round-trip OK.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/recipes/lerobot/Pi0.md b/recipes/lerobot/Pi0.md new file mode 100644 index 000000000..6341e57a0 --- /dev/null +++ b/recipes/lerobot/Pi0.md @@ -0,0 +1,108 @@ +# Pi0 (π0, Pi-Zero) VLA + +> Vision-Language-Action policy serving over the OpenPI realtime websocket + +## Summary + +- Vendor: Physical Intelligence (checkpoint redistributed by LeRobot) +- Model: `lerobot/pi0_base` +- Task: Vision-Language-Action — multi-camera images + a language instruction + + robot proprioceptive state → a continuous action chunk + `[action_horizon, action_dim] = [50, 32]` via flow-matching denoising. π0 does + **not** emit text tokens. +- Mode: Online serving via the OpenPI realtime robot API + (`/v1/realtime/robot/openpi` websocket) +- Maintainer: Community + +## When to use this recipe + +Use this recipe to deploy `lerobot/pi0_base` as a robot policy server: a robot +client connects over the OpenPI websocket and, per control step, sends an +observation (camera frames + state + instruction) and receives an action chunk. + +A single pipeline class (`Pi0Pipeline`) serves the policy. π0 is stateless across +calls (no KV reuse), so the protocol's `session_id` / `reset` are accepted but +ignored. + +## Weights + +π0 uses the LeRobot `lerobot/pi0_base` checkpoint (HF, ~13 GB). Either let the +server download it (`MODEL=lerobot/pi0_base`) or point at a local copy. + +## Hardware Support + +### 1x H200 141GB (Online serving) + +#### Environment + +- OS: Ubuntu 22.04+ +- Python: 3.12+ +- Driver / runtime: NVIDIA CUDA environment +- vLLM / vLLM-Omni: use the commit you are deploying from +- OpenPI client extras (for the example client): `openpi-client` + (`pip install -e packages/openpi-client` from the + [openpi](https://github.com/Physical-Intelligence/openpi) repo), `websockets`, + `msgpack`, `msgpack-numpy` + +#### Command + +```bash +vllm serve lerobot/pi0_base \ + --omni \ + --host 0.0.0.0 --port 8000 \ + --served-model-name pi0 \ + --deploy-config vllm_omni/deploy/pi0.yaml \ + --enforce-eager --disable-log-stats +``` + +The deploy config (`vllm_omni/deploy/pi0.yaml`) declares a single diffusion stage +(`Pi0Pipeline`), `float32`, `max_num_seqs: 1`, and the `policy_server_config` +handshake metadata (3 cameras, `joint_position`, action horizon 50, action dim +32) that enables the OpenPI realtime serving layer. + +#### Verification + +Run the example client (sends a synthetic observation and prints the returned +action chunk): + +```bash +python examples/online_serving/pi0/openpi_client.py \ + --host 127.0.0.1 --port 8000 \ + --prompt "pick up the red block and place it in the bin" +``` + +It connects, prints the server metadata, sends robot observations, and prints the +returned `[action_horizon, action_dim] = [50, 32]` action chunks. The observation +is a flat dict per inference: + +```python +{ + "observation.images.base_0_rgb": np.uint8[H, W, 3], + "observation.images.left_wrist_0_rgb": np.uint8[H, W, 3], + "observation.images.right_wrist_0_rgb": np.uint8[H, W, 3], + "state": np.float32[state_dim], # zero-padded to max_state_dim=32 server-side + "prompt": "pick up the red block", + "session_id": "", # accepted but ignored (π0 is stateless) +} +``` + +Camera keys must match the server's `image_feature_keys` (the checkpoint's +`input_features` order). If your robot uses different camera names, set +`model_config.image_key_map` in `pi0.yaml` to map raw obs keys → feature keys. + +## Correctness + +π0's flow-matching kernel is bit-for-bit matched to the LeRobot `PI0Policy` +reference (`max|Δ| = 7.15e-07`, CPU/float32, fixed noise; see +`tests/diffusion/models/pi0/test_pi0_parity.py::test_pi0_vllm_omni_vs_lerobot`). + +## References + +- Model blog: +- Checkpoint: +- Pipeline: [`vllm_omni/diffusion/models/pi0/pipeline_pi0.py`](../../vllm_omni/diffusion/models/pi0/pipeline_pi0.py) +- Deploy config: [`vllm_omni/deploy/pi0.yaml`](../../vllm_omni/deploy/pi0.yaml) +- Example client: [`examples/online_serving/pi0/`](../../examples/online_serving/pi0/) +- Tests: CPU units + LeRobot parity in [`tests/diffusion/models/pi0/`](../../tests/diffusion/models/pi0/), + OpenPI websocket e2e in + [`tests/e2e/online_serving/test_pi0_expansion.py`](../../tests/e2e/online_serving/test_pi0_expansion.py) diff --git a/tests/diffusion/models/pi0/__init__.py b/tests/diffusion/models/pi0/__init__.py new file mode 100644 index 000000000..208f01a7c --- /dev/null +++ b/tests/diffusion/models/pi0/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/diffusion/models/pi0/test_pi0_parity.py b/tests/diffusion/models/pi0/test_pi0_parity.py new file mode 100644 index 000000000..3b6358b50 --- /dev/null +++ b/tests/diffusion/models/pi0/test_pi0_parity.py @@ -0,0 +1,366 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""π0 LeRobot parity (in-process): the bit-for-bit correctness oracle. + +Verifies that vllm-omni's ``Pi0ForActionPrediction`` produces bit-for-bit +matching action chunks with LeRobot's ``PI0Policy`` when fed: + * the same weights (``lerobot/pi0_base``) + * the same pre-processed inputs (images, masks, tokens, state) + * the same initial noise tensor + +π0 is a flow-matching model (Euler-integrated ODE from t=1 → t=0 with a fixed +``num_steps``), so the output is deterministic once the noise is fixed and +``torch.allclose`` on the final action chunk is a valid oracle. This is the +authoritative correctness oracle for the vllm-omni π0 port (max|Δ| < 1e-4). + +The OpenPI websocket online-serving e2e lives separately in +``tests/e2e/online_serving/test_pi0_expansion.py``. + +Run in a SEPARATE ``lerobot[pi]`` venv (avoids dep conflict with the vllm-omni +env), with the vllm-omni ``pi0`` package importable:: + + python -m pytest tests/diffusion/models/pi0/test_pi0_parity.py -v -s + +Skipped automatically when LeRobot is not installed (e.g. the vllm-omni env). +The run is CPU/float32 with fixed defaults; the only override is +``PI0_PARITY_MODEL_PATH`` (a local pi0_base dir in LeRobot format, to skip the +HF download; defaults to ``lerobot/pi0_base``). +""" + +from __future__ import annotations + +import copy +import importlib.util +import os + +import pytest +import torch + +# local_model: needs real weights + a lerobot venv (transformers 5.3.0), so it +# runs locally rather than in the ready-CI (see docs/contributing/ci/CI_5levels.md). +# Additionally gated on lerobot being importable. +_HAS_LEROBOT = importlib.util.find_spec("lerobot") is not None + +pytestmark = [pytest.mark.local_model, pytest.mark.diffusion] + + +# ─── Config (fixed; matches LeRobot defaults for ``lerobot/pi0_base``) ── +DEVICE = "cpu" +DTYPE_STR = "float32" +ATOL = 1e-4 +NUM_STEPS = 10 +BATCH_SIZE = 2 +ACTION_DIM = 32 +STATE_DIM = 32 +ACTION_HORIZON = 50 +MAX_TOKEN_LEN = 48 + +# The only knob: point at a local pi0_base dir (LeRobot format) to skip the HF +# download; defaults to the HF repo id. +MODEL_PATH = os.environ.get("PI0_PARITY_MODEL_PATH", "lerobot/pi0_base") + + +def _resolve_checkpoint_dir() -> str: + """Return a local dir containing the pi0_base checkpoint (download if needed).""" + if os.path.isdir(MODEL_PATH): + return MODEL_PATH + from huggingface_hub import snapshot_download + + return snapshot_download(repo_id=MODEL_PATH, repo_type="model") + + +# ─── Dummy dataset stats (identity) ────────────────────────────────── +def _dummy_dataset_stats() -> dict: + return { + "observation.state": { + "mean": torch.zeros(STATE_DIM), + "std": torch.ones(STATE_DIM), + "q01": torch.zeros(STATE_DIM), + "q99": torch.ones(STATE_DIM), + }, + "action": { + "mean": torch.zeros(ACTION_DIM), + "std": torch.ones(ACTION_DIM), + "q01": torch.zeros(ACTION_DIM), + "q99": torch.ones(ACTION_DIM), + }, + "images": { + cam: { + "mean": torch.zeros(3, 224, 224), + "std": torch.ones(3, 224, 224), + "q01": torch.zeros(3, 224, 224), + "q99": torch.ones(3, 224, 224), + } + for cam in ("base_0_rgb", "left_wrist_0_rgb", "right_wrist_0_rgb") + }, + } + + +def _create_dummy_batch(batch_size: int = BATCH_SIZE, device: str = DEVICE) -> dict: + """Reproducible dummy inputs — identical across both implementations.""" + g = torch.Generator(device="cpu").manual_seed(0) + prompt = "Pick up the red block and place it in the bin" + return { + "observation.state": torch.randn(batch_size, STATE_DIM, generator=g, dtype=torch.float32).to(device), + "action": torch.randn(batch_size, ACTION_HORIZON, ACTION_DIM, generator=g, dtype=torch.float32).to(device), + "observation.images.base_0_rgb": torch.rand(batch_size, 3, 224, 224, generator=g, dtype=torch.float32).to( + device + ), + "observation.images.left_wrist_0_rgb": torch.rand(batch_size, 3, 224, 224, generator=g, dtype=torch.float32).to( + device + ), + "observation.images.right_wrist_0_rgb": torch.rand( + batch_size, 3, 224, 224, generator=g, dtype=torch.float32 + ).to(device), + "task": [prompt for _ in range(batch_size)], + } + + +# ─── LeRobot instantiation ──────────────────────────────────────────── +def _instantiate_lerobot(): + from lerobot.policies.pi0 import PI0Policy + from lerobot.policies.pi0.processor_pi0 import make_pi0_pre_post_processors + + policy = PI0Policy.from_pretrained(MODEL_PATH, strict=True) + policy.to(DEVICE) + policy.config.device = DEVICE + policy.eval() + + pre, post = make_pi0_pre_post_processors(config=policy.config, dataset_stats=_dummy_dataset_stats()) + return policy, pre, post + + +# ─── vllm-omni instantiation ────────────────────────────────────────── +def _instantiate_vllm_omni(): + """Build the vllm-omni π0 model in isolation (no pipeline, no engine).""" + from vllm_omni.diffusion.models.pi0 import Pi0Config, Pi0ForActionPrediction + + cfg = Pi0Config( + max_action_dim=ACTION_DIM, + max_state_dim=STATE_DIM, + chunk_size=ACTION_HORIZON, + num_inference_steps=NUM_STEPS, + dtype=DTYPE_STR, + ) + model = Pi0ForActionPrediction(cfg) + model.to(DEVICE).eval() + _load_lerobot_weights(model) + return model + + +def _load_lerobot_weights(model): + """Feed the ``lerobot/pi0_base`` safetensors into the vllm-omni model. + + ``Pi0ForActionPrediction.load_weights`` handles the leading ``model.`` + prefix and the flat→nested / lm_head→embed_tokens remaps, so we can pass + the raw checkpoint dict. + """ + import safetensors.torch + + cache_dir = _resolve_checkpoint_dir() + path = os.path.join(cache_dir, "model.safetensors") + state = safetensors.torch.load_file(path) + model.load_weights(list(state.items())) + + +# ─── Helpers to extract LeRobot's pre-processed inputs ──────────────── +def _extract_lerobot_model_inputs(lerobot_policy, processed_batch): + """Mimic what ``PI0Policy.predict_action_chunk`` feeds into + ``self.model.sample_actions``. We use these *exact* tensors for vllm-omni + so any divergence must come from the core model, not preprocessing. + """ + images, img_masks = lerobot_policy._preprocess_images(processed_batch) + from lerobot.utils.constants import ( + OBS_LANGUAGE_ATTENTION_MASK, + OBS_LANGUAGE_TOKENS, + ) + + lang_tokens = processed_batch[OBS_LANGUAGE_TOKENS] + lang_masks = processed_batch[OBS_LANGUAGE_ATTENTION_MASK] + state = lerobot_policy.prepare_state(processed_batch) + return images, img_masks, lang_tokens, lang_masks, state + + +# ─── Shared fixed-noise sampler ─────────────────────────────────────── +def _make_fixed_noise(batch_size: int, device: str) -> torch.Tensor: + g = torch.Generator(device="cpu").manual_seed(42) + return torch.randn(batch_size, ACTION_HORIZON, ACTION_DIM, generator=g, dtype=torch.float32).to(device) + + +# ─── Main test ──────────────────────────────────────────────────────── +@pytest.mark.skipif(not _HAS_LEROBOT, reason="lerobot not installed (run in a lerobot venv).") +def test_pi0_vllm_omni_vs_lerobot(): + print("\n[parity] Instantiating LeRobot…") + lerobot_policy, lerobot_pre, _ = _instantiate_lerobot() + + print("[parity] Instantiating vllm-omni…") + omni_model = _instantiate_vllm_omni() + + print("[parity] Preparing shared inputs…") + raw_batch = _create_dummy_batch() + processed_batch = lerobot_pre(copy.deepcopy(raw_batch)) + images, img_masks, lang_tokens, lang_masks, state = _extract_lerobot_model_inputs(lerobot_policy, processed_batch) + noise = _make_fixed_noise(raw_batch["observation.state"].shape[0], DEVICE) + + print(f"[parity] state.shape={state.shape} lang_tokens.shape={lang_tokens.shape}") + print(f"[parity] images[0].shape={images[0].shape} (num_cams={len(images)})") + print(f"[parity] noise.shape={noise.shape} dtype={noise.dtype}") + + # ── LeRobot forward ── + print("[parity] Running LeRobot sample_actions…") + with torch.no_grad(): + lerobot_actions = lerobot_policy.model.sample_actions( + images, + img_masks, + lang_tokens, + lang_masks, + state, + noise=noise, + num_steps=NUM_STEPS, + ) + print( + f"[parity] LeRobot actions: shape={lerobot_actions.shape} " + f"mean={lerobot_actions.mean().item():.6f} std={lerobot_actions.std().item():.6f}" + ) + + # ── vllm-omni forward ── + print("[parity] Running vllm-omni sample_actions…") + with torch.no_grad(): + omni_actions = omni_model.sample_actions( + images=images, + image_masks=img_masks, + lang_tokens=lang_tokens, + lang_masks=lang_masks, + state=state, + noise=noise, + num_steps=NUM_STEPS, + ) + print( + f"[parity] vllm-omni actions: shape={omni_actions.shape} " + f"mean={omni_actions.mean().item():.6f} std={omni_actions.std().item():.6f}" + ) + + # ── Compare ── + diff = (lerobot_actions.float() - omni_actions.float()).abs() + max_diff = diff.max().item() + mean_diff = diff.mean().item() + print(f"[parity] |Δ| max={max_diff:.2e} mean={mean_diff:.2e} atol={ATOL:.1e}") + close = torch.allclose(lerobot_actions.float(), omni_actions.float(), atol=ATOL) + print(f"[parity] torch.allclose(atol={ATOL}): {close}") + + if not close: + print("\n[parity] ⚠️ Outputs diverge — running per-stage diagnostics…") + _diagnose_divergence( + lerobot_policy.model, + omni_model, + images, + img_masks, + lang_tokens, + lang_masks, + state, + noise, + ) + + assert close, ( + f"vllm-omni vs LeRobot actions differ beyond atol={ATOL}. max_diff={max_diff:.2e} mean_diff={mean_diff:.2e}" + ) + + +# ─── Per-stage divergence diagnostics ───────────────────────────────── +@torch.no_grad() +def _diagnose_divergence( + lerobot_flow_model, + omni_model, + images, + img_masks, + lang_tokens, + lang_masks, + state, + noise, +): + """Localize a numerical mismatch to a specific pipeline stage: + 1. Prefix embeddings (SigLIP / embed_tokens / projector) — image vs lang. + 2. Prefix KV cache layer 0 — PaliGemma LM attention. + 3. A single denoise_step velocity at t=1.0 — action expert forward. + """ + from vllm_omni.diffusion.models.pi0.modeling_pi0 import ( + make_att_2d_masks, + prepare_attention_masks_4d, + ) + + # ── Stage 1: prefix embeddings ── + lr_prefix_embs, lr_prefix_pad, lr_prefix_att = lerobot_flow_model.embed_prefix( + images, img_masks, lang_tokens, lang_masks + ) + sg_prefix_embs, sg_prefix_pad, sg_prefix_att = omni_model.embed_prefix(images, img_masks, lang_tokens, lang_masks) + total_diff = (lr_prefix_embs.float() - sg_prefix_embs.float()).abs().max().item() + print(f"[diag] prefix_embs max |Δ| = {total_diff:.2e} (shape={tuple(sg_prefix_embs.shape)})") + print(f"[diag] prefix_pad_masks equal: {torch.equal(lr_prefix_pad, sg_prefix_pad)}") + print(f"[diag] prefix_att_masks equal: {torch.equal(lr_prefix_att.bool(), sg_prefix_att.bool())}") + + num_cams = len(images) + img_len = 256 * num_cams + lang_len = lr_prefix_embs.shape[1] - img_len + img_diff = (lr_prefix_embs[:, :img_len].float() - sg_prefix_embs[:, :img_len].float()).abs().max().item() + lang_diff = (lr_prefix_embs[:, img_len:].float() - sg_prefix_embs[:, img_len:].float()).abs().max().item() + print(f"[diag] image slice [:{img_len}] max|Δ| = {img_diff:.2e} (num_cams={num_cams})") + print(f"[diag] lang slice [{img_len}:] max|Δ| = {lang_diff:.2e} (len={lang_len})") + + def _stats(name, t): + t = t.float() + print( + f"[diag] {name} prefix_embs: mean={t.mean().item():+.4f} " + f"std={t.std().item():.4f} min={t.min().item():+.2f} " + f"max={t.max().item():+.2f}" + ) + + _stats("LeRobot ", lr_prefix_embs) + _stats("vllm-omni", sg_prefix_embs) + + # ── Stage 2: prefix KV cache ── + prefix_att_2d = make_att_2d_masks(sg_prefix_pad, sg_prefix_att) + prefix_pos = torch.cumsum(sg_prefix_pad, dim=1) - 1 + prefix_att_4d = prepare_attention_masks_4d(prefix_att_2d) + + lerobot_flow_model.paligemma_with_expert.paligemma.language_model.config._attn_implementation = "eager" + _, lr_kv = lerobot_flow_model.paligemma_with_expert.forward( + attention_mask=prefix_att_4d, + position_ids=prefix_pos, + past_key_values=None, + inputs_embeds=[lr_prefix_embs, None], + use_cache=True, + ) + _, sg_kv = omni_model.paligemma_with_expert.forward( + attention_mask=prefix_att_4d, + position_ids=prefix_pos, + past_key_values=None, + inputs_embeds=[sg_prefix_embs, None], + use_cache=True, + ) + + def _layer0_kv(cache): + if isinstance(cache, list): + return cache[0] + layer0 = cache.layers[0] + return layer0.keys, layer0.values + + try: + lr_k, lr_v = _layer0_kv(lr_kv) + sg_k, sg_v = _layer0_kv(sg_kv) + dk = (lr_k.float() - sg_k.float()).abs().max().item() + dv = (lr_v.float() - sg_v.float()).abs().max().item() + print(f"[diag] prefix KV layer0 K max|Δ|={dk:.2e} V max|Δ|={dv:.2e}") + except Exception as e: # noqa: BLE001 + print(f"[diag] could not extract prefix KV for comparison: {e}") + + # ── Stage 3: a single denoise_step at t=1.0 ── + bsize = state.shape[0] + t = torch.ones(bsize, dtype=torch.float32, device=state.device) + lr_vt = lerobot_flow_model.denoise_step(state, sg_prefix_pad, lr_kv, noise, t) + sg_vt = omni_model.denoise_step(state, sg_prefix_pad, sg_kv, noise, t) + print(f"[diag] denoise_step(t=1) v_t max|Δ| = {(lr_vt.float() - sg_vt.float()).abs().max().item():.2e}") + + +if __name__ == "__main__": + test_pi0_vllm_omni_vs_lerobot() diff --git a/tests/diffusion/models/pi0/test_pi0_units.py b/tests/diffusion/models/pi0/test_pi0_units.py new file mode 100644 index 000000000..a1fa18f1a --- /dev/null +++ b/tests/diffusion/models/pi0/test_pi0_units.py @@ -0,0 +1,585 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CPU unit tests for the π0 (Pi-Zero) VLA model. + +Deliberately lightweight: pure utility code (config plumbing, mask building, +sinusoidal embeddings, normalization, image preprocessing, weight-load remap) +that runs on CPU with no weights and no lerobot. The full LeRobot parity check +lives in ``tests/diffusion/models/pi0/test_pi0_parity.py`` and requires a model download. + + pytest tests/diffusion/models/pi0/test_pi0_units.py -v +""" + +from __future__ import annotations + +import logging +import math + +import pytest +import torch + +from vllm_omni.diffusion.models.pi0.config import Pi0Config +from vllm_omni.diffusion.models.pi0.modeling_pi0 import ( + OPENPI_ATTENTION_MASK_VALUE, + Pi0ForActionPrediction, + _apply_norm, + _build_norm_buffers, + create_sinusoidal_pos_embedding, + get_gemma_config, + make_att_2d_masks, + prepare_attention_masks_4d, +) +from vllm_omni.diffusion.models.pi0.processor_pi0 import ( + Pi0ImageProcessor, + build_model_inputs, + pil_image_to_tensor, + resize_with_pad, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +# ---------------------------------------------------------------------------- +# Pi0Config dataclass + checkpoint resolver +# ---------------------------------------------------------------------------- +_LEROBOT_CFG = { + "type": "pi0", + "paligemma_variant": "gemma_2b", + "action_expert_variant": "gemma_300m", + "chunk_size": 50, + "max_action_dim": 32, + "max_state_dim": 32, + "num_inference_steps": 10, + "image_resolution": [224, 224], + "tokenizer_max_length": 48, + "dtype": "float32", + # Training-only keys that must be filtered out of the dataclass. + "optimizer_lr": 2.5e-5, + "scheduler_warmup_steps": 1000, + "input_features": { + "observation.images.base_0_rgb": {"type": "VISUAL", "shape": [3, 224, 224]}, + "observation.images.left_wrist_0_rgb": {"type": "VISUAL", "shape": [3, 224, 224]}, + "observation.images.right_wrist_0_rgb": {"type": "VISUAL", "shape": [3, 224, 224]}, + "observation.state": {"type": "STATE", "shape": [32]}, + }, +} + +_EXPECTED_CAMERA_ORDER = [ + "observation.images.base_0_rgb", + "observation.images.left_wrist_0_rgb", + "observation.images.right_wrist_0_rgb", +] + + +def test_config_camera_order_from_input_features(): + c = Pi0Config.from_model_config(_LEROBOT_CFG) + assert c.image_feature_keys == _EXPECTED_CAMERA_ORDER + + +def test_config_filters_training_only_keys(): + c = Pi0Config.from_model_config(_LEROBOT_CFG) + assert not hasattr(c, "optimizer_lr") + assert not hasattr(c, "scheduler_warmup_steps") + + +def test_config_coerces_image_resolution_to_tuple(): + c = Pi0Config.from_model_config(_LEROBOT_CFG) + assert c.image_resolution == (224, 224) + assert isinstance(c.image_resolution, tuple) + + +def test_config_non_square_resolution_raises(): + with pytest.raises(ValueError): + Pi0Config(image_resolution=(224, 256)) + + +@pytest.mark.parametrize("bad", [(224,), (224, 224, 3), 224]) +def test_config_malformed_resolution_raises(bad): + with pytest.raises(ValueError): + Pi0Config(image_resolution=bad) + + +def test_config_defaults_and_empty(): + c = Pi0Config.from_model_config(None) + assert c.chunk_size == 50 + assert c.max_action_dim == 32 + assert c.max_state_dim == 32 + assert c.num_inference_steps == 10 + assert c.image_resolution == (224, 224) + assert c.tokenizer_max_length == 48 + assert c.max_cameras == 3 + assert c.norm_stats is None + assert c.image_key_map == {} + + +def test_config_deploy_yaml_override(): + """Deploy-yaml model_config is authoritative (e.g. force dtype/variant).""" + c = Pi0Config.from_model_config({**_LEROBOT_CFG, "dtype": "bfloat16", "num_inference_steps": 4}) + assert c.dtype == "bfloat16" + assert c.num_inference_steps == 4 + + +# ---------------------------------------------------------------------------- +# Kernel utilities: sinusoidal timestep embedding +# ---------------------------------------------------------------------------- +def test_sinusoidal_embedding_shape(): + emb = create_sinusoidal_pos_embedding(torch.tensor([0.0, 0.5, 1.0]), dimension=64) + assert emb.shape == (3, 64) + + +def test_sinusoidal_embedding_values_at_zero(): + emb = create_sinusoidal_pos_embedding(torch.tensor([0.0]), dimension=4) + assert torch.allclose(emb[0, :2].float(), torch.zeros(2), atol=1e-6) + assert torch.allclose(emb[0, 2:].float(), torch.ones(2), atol=1e-6) + + +def test_sinusoidal_embedding_odd_dim_raises(): + with pytest.raises(ValueError): + create_sinusoidal_pos_embedding(torch.tensor([1.0]), dimension=3) + + +def test_sinusoidal_embedding_non_1d_raises(): + with pytest.raises(ValueError): + create_sinusoidal_pos_embedding(torch.tensor([[0.0, 0.5]]), dimension=4) + + +def test_sinusoidal_embedding_distinct_timesteps_differ(): + emb = create_sinusoidal_pos_embedding(torch.tensor([0.1, 0.2, 0.5, 0.9]), dimension=16) + for i in range(emb.shape[0]): + for j in range(i + 1, emb.shape[0]): + assert not torch.allclose(emb[i], emb[j]) + + +# ---------------------------------------------------------------------------- +# Kernel utilities: attention mask construction +# ---------------------------------------------------------------------------- +def test_make_att_2d_masks_bidirectional(): + result = make_att_2d_masks(torch.ones(1, 4, dtype=torch.bool), torch.zeros(1, 4, dtype=torch.long)) + assert result.all() + + +def test_make_att_2d_masks_causal(): + result = make_att_2d_masks(torch.ones(1, 4, dtype=torch.bool), torch.ones(1, 4, dtype=torch.long)) + expected = torch.tril(torch.ones(4, 4, dtype=torch.bool)) + assert torch.equal(result[0], expected) + + +def test_make_att_2d_masks_prefix_lm(): + result = make_att_2d_masks(torch.ones(1, 5, dtype=torch.bool), torch.tensor([[0, 0, 0, 1, 1]], dtype=torch.long)) + assert result[0, 0, 0] + assert not result[0, 0, 3] + assert result[0, 3, 0] + assert not result[0, 3, 4] + + +def test_make_att_2d_masks_respects_padding(): + pad = torch.tensor([[1, 1, 1, 0]], dtype=torch.bool) # last token is padding + att = torch.zeros(1, 4, dtype=torch.long) # full bidirectional + result = make_att_2d_masks(pad, att) + assert not result[0, :, 3].any(), "Nothing should attend to padded key" + assert not result[0, 3, :].any(), "Padded query should attend to nothing" + + +def test_prepare_attention_masks_4d(): + mask_2d = torch.tensor([[[True, False], [True, True]]]) + mask_4d = prepare_attention_masks_4d(mask_2d) + assert mask_4d.shape == (1, 1, 2, 2) + assert mask_4d[0, 0, 0, 0].item() == 0.0 + assert mask_4d[0, 0, 1, 1].item() == 0.0 + expected = torch.tensor(OPENPI_ATTENTION_MASK_VALUE, dtype=mask_4d.dtype) + assert torch.equal(mask_4d[0, 0, 0, 1], expected) + + +def test_attention_mask_value_finite_and_negative(): + assert math.isfinite(OPENPI_ATTENTION_MASK_VALUE) + assert OPENPI_ATTENTION_MASK_VALUE < -1e30 + + +# ---------------------------------------------------------------------------- +# Normalization round-trips +# ---------------------------------------------------------------------------- +def test_build_norm_buffers_missing(): + assert _build_norm_buffers(None, "state") is None + assert _build_norm_buffers({}, "state") is None + assert _build_norm_buffers({"state": None}, "state") is None + + +def test_build_norm_buffers_mean_std(): + stats = {"state": {"mode": "mean_std", "mean": [1.0, 2.0], "std": [0.5, 0.5]}} + out = _build_norm_buffers(stats, "state") + assert out["mode"] == "mean_std" + assert torch.equal(out["mean"], torch.tensor([1.0, 2.0])) + assert torch.equal(out["std"], torch.tensor([0.5, 0.5])) + + +def test_build_norm_buffers_min_max(): + stats = {"action": {"mode": "min_max", "min": [-1.0, 0.0], "max": [1.0, 2.0]}} + out = _build_norm_buffers(stats, "action") + assert out["mode"] == "min_max" + assert torch.equal(out["min"], torch.tensor([-1.0, 0.0])) + assert torch.equal(out["max"], torch.tensor([1.0, 2.0])) + + +def test_apply_norm_mean_std_forward_inverse(): + stats = _build_norm_buffers({"s": {"mode": "mean_std", "mean": [0.5, -0.2], "std": [2.0, 0.1]}}, "s") + x = torch.tensor([[1.0, 0.3]]) + z = _apply_norm(x, stats, inverse=False) + assert torch.allclose(z, torch.tensor([[0.25, 5.0]]), atol=1e-5) + x_back = _apply_norm(z, stats, inverse=True) + assert torch.allclose(x_back, x, atol=1e-5) + + +def test_apply_norm_min_max_forward_inverse(): + stats = _build_norm_buffers({"s": {"mode": "min_max", "min": [0.0, -1.0], "max": [1.0, 1.0]}}, "s") + x = torch.tensor([[0.25, 0.5]]) + z = _apply_norm(x, stats, inverse=False) + assert torch.allclose(z, torch.tensor([[-0.5, 0.5]]), atol=1e-5) + x_back = _apply_norm(z, stats, inverse=True) + assert torch.allclose(x_back, x, atol=1e-5) + + +def test_apply_norm_preserves_padded_tail(): + stats = _build_norm_buffers({"s": {"mode": "mean_std", "mean": [0.0, 0.0], "std": [1.0, 1.0]}}, "s") + x = torch.tensor([[1.0, 2.0, 99.0, -99.0]]) + z = _apply_norm(x, stats, inverse=False) + assert torch.allclose(z[:, :2], x[:, :2]) + assert torch.allclose(z[:, 2:], x[:, 2:]) + + +def test_apply_norm_noop_when_stats_none(): + x = torch.randn(2, 5) + assert torch.equal(_apply_norm(x, None, inverse=False), x) + assert torch.equal(_apply_norm(x, None, inverse=True), x) + + +# ---------------------------------------------------------------------------- +# Gemma variant configs +# ---------------------------------------------------------------------------- +def test_gemma_2b_dims(): + c = get_gemma_config("gemma_2b") + assert c.width == 2048 + assert c.depth == 18 + + +def test_gemma_300m_dims(): + c = get_gemma_config("gemma_300m") + assert c.width == 1024 + assert c.depth == 18 + + +def test_gemma_unknown_variant_raises(): + with pytest.raises(ValueError): + get_gemma_config("gemma_7b") + + +# ---------------------------------------------------------------------------- +# load_weights remap rules (tiny model; needs torch + transformers) +# ---------------------------------------------------------------------------- +def _tiny_pi0_model(): + """Construct a tiny π0 model for remap tests (full Gemma depth but small + action/state dims — the PaliGemma backbone is always full-size).""" + cfg = Pi0Config(max_action_dim=8, max_state_dim=8, chunk_size=2) + return Pi0ForActionPrediction(cfg) + + +@pytest.fixture(autouse=True) +def _silence_pi0_loader(caplog): + # The remap tests feed a single synthetic weight; the loader's + # "missing params" warning is expected noise here. + logging.getLogger("vllm_omni.diffusion.models.pi0.modeling_pi0").setLevel(logging.ERROR) + yield + + +@pytest.mark.slow +def test_remap_paligemma_submodules_to_nested_layout(): + model = _tiny_pi0_model() + params = dict(model.named_parameters()) + target = next(k for k in params if k.startswith("paligemma_with_expert.paligemma.model.vision_tower.")) + flat_key = "model." + target.replace( + "paligemma_with_expert.paligemma.model.vision_tower.", + "paligemma_with_expert.paligemma.vision_tower.", + ) + payload = torch.full(params[target].shape, 0.12345) + before = params[target].detach().clone() + model.load_weights([(flat_key, payload)]) + after = dict(model.named_parameters())[target] + assert not torch.equal(before, after), "flat→nested remap broken" + assert torch.allclose(after, payload) + + +@pytest.mark.slow +def test_lm_head_remap_to_embed_tokens(): + model = _tiny_pi0_model() + params = dict(model.named_parameters()) + target = "paligemma_with_expert.paligemma.model.language_model.embed_tokens.weight" + assert target in params + payload = torch.full(params[target].shape, 0.5) + before = params[target].detach().clone() + model.load_weights([("model.paligemma_with_expert.paligemma.lm_head.weight", payload)]) + after = dict(model.named_parameters())[target] + assert not torch.equal(before, after), "lm_head→embed_tokens remap broken" + assert torch.allclose(after, payload) + + +@pytest.mark.slow +def test_vision_tower_version_robust_remap(): + """transformers >=5.4 flattens SigLIP from vision_tower.vision_model.* to + vision_tower.*; the lerobot/pi0_base checkpoint uses the vision_model infix. + load_weights must adapt so vision weights aren't silently dropped (which + would leave the vision encoder at random init).""" + model = _tiny_pi0_model() + params = dict(model.named_parameters()) + # A representative vision-tower param as it exists on the model. + target = next( + k + for k in params + if k.startswith("paligemma_with_expert.paligemma.model.vision_tower.") and "patch_embedding.weight" in k + ) + vt = "paligemma_with_expert.paligemma.model.vision_tower." + rest = target[len(vt) :] + # Build the checkpoint key in whichever layout differs from the model's. + if rest.startswith("vision_model."): + ckpt_key = vt + rest[len("vision_model.") :] + else: + ckpt_key = vt + "vision_model." + rest + payload = torch.full(params[target].shape, 0.073) + before = params[target].detach().clone() + model.load_weights([("model." + ckpt_key, payload)]) + after = dict(model.named_parameters())[target] + assert not torch.equal(before, after), "vision_tower version-robust remap broken" + assert torch.allclose(after, payload) + + +# ---------------------------------------------------------------------------- +# Image processor +# ---------------------------------------------------------------------------- +def test_resize_with_pad_square(): + out = resize_with_pad(torch.ones(1, 3, 224, 224), 224, 224) + assert out.shape == (1, 3, 224, 224) + + +def test_resize_with_pad_wide(): + """Wide source (H < W): padding applied to top/bottom (rows).""" + out = resize_with_pad(torch.zeros(1, 3, 100, 200), 224, 224) + assert out.shape == (1, 3, 224, 224) + assert out[0, 0, 0, 112].item() == -1.0 # top row is padded + + +def test_resize_with_pad_tall(): + """Tall source (H > W): padding applied to left/right (cols).""" + out = resize_with_pad(torch.zeros(1, 3, 400, 100), 224, 224) + assert out.shape == (1, 3, 224, 224) + assert out[0, 0, 112, 0].item() == -1.0 # left-most column padded + + +def test_resize_with_pad_rejects_non_4d(): + with pytest.raises(ValueError): + resize_with_pad(torch.zeros(3, 100, 200), 224, 224) + + +def test_pil_to_tensor(): + import numpy as np + from PIL import Image + + img = Image.fromarray(np.full((10, 10, 3), 128, dtype=np.uint8)) + t = pil_image_to_tensor(img) + assert t.shape == (1, 3, 10, 10) + expected = 128.0 / 255.0 * 2.0 - 1.0 + assert abs(t[0, 0, 5, 5].item() - expected) < 1e-4 + + +def test_pil_to_tensor_rgba_to_rgb(): + import numpy as np + from PIL import Image + + rgba = np.full((8, 8, 4), 200, dtype=np.uint8) + img = Image.fromarray(rgba, mode="RGBA") + t = pil_image_to_tensor(img) + assert t.shape == (1, 3, 8, 8) + + +def test_make_empty_image(): + proc = Pi0ImageProcessor(image_size=224) + empty = proc.make_empty_image() + assert empty.shape == (1, 3, 224, 224) + assert (empty == -1.0).all() + + +def test_preprocess_single_resizes_ndarray(): + import numpy as np + + proc = Pi0ImageProcessor(image_size=224) + arr = np.full((120, 120, 3), 100, dtype=np.uint8) + t = proc.preprocess_single(arr) + assert t.shape == (1, 3, 224, 224) + + +# ---------------------------------------------------------------------------- +# build_model_inputs: camera ordering, padding, masks, state +# ---------------------------------------------------------------------------- +class _FakeTokenizer: + """Minimal stand-in for the PaliGemma tokenizer (no download).""" + + def __call__( + self, text, padding=None, max_length=None, truncation=None, add_special_tokens=None, return_tensors=None + ): + ids = [2] + [ord(c) % 100 for c in text][: max_length - 1] + ids = ids[:max_length] + attn = [1] * len(ids) + while len(ids) < max_length: + ids.append(0) + attn.append(0) + return {"input_ids": ids, "attention_mask": attn} + + +def test_build_model_inputs_camera_order_and_padding(): + import numpy as np + + config = Pi0Config.from_model_config(_LEROBOT_CFG) # 3-camera config + device = torch.device("cpu") + # Provide only 2 of the 3 declared cameras. + robot_obs = { + "images": { + "observation.images.base_0_rgb": np.full((224, 224, 3), 50, dtype=np.uint8), + "observation.images.left_wrist_0_rgb": np.full((224, 224, 3), 80, dtype=np.uint8), + }, + "state": [0.1] * 14, # raw state shorter than max_state_dim=32 + "prompt": "pick up the block", + } + images, image_masks, lang_tokens, lang_masks, state = build_model_inputs( + robot_obs, config, _FakeTokenizer(), device + ) + assert len(images) == 3 + assert all(img.shape == (1, 3, 224, 224) for img in images) + masks = [bool(m.item()) for m in image_masks] + assert masks == [True, True, False] + # The padded (3rd) camera must be all -1. + assert (images[2] == -1.0).all() + assert lang_tokens.shape == (1, 48) + assert lang_masks.shape == (1, 48) + assert state.shape == (1, 32) + # State padded with zeros beyond the 14 real entries. + assert torch.allclose(state[0, 14:], torch.zeros(18)) + + +def test_build_model_inputs_state_truncated(): + config = Pi0Config.from_model_config(_LEROBOT_CFG) + robot_obs = {"images": {}, "state": [0.5] * 40, "prompt": "x"} + _, _, _, _, state = build_model_inputs(robot_obs, config, _FakeTokenizer(), torch.device("cpu")) + assert state.shape == (1, 32) + + +def test_build_model_inputs_image_key_map(): + import numpy as np + + cfg = Pi0Config.from_model_config({**_LEROBOT_CFG, "image_key_map": {"cam_high": "observation.images.base_0_rgb"}}) + robot_obs = { + "images": {"cam_high": np.full((224, 224, 3), 60, dtype=np.uint8)}, + "prompt": "go", + } + _, image_masks, _, _, _ = build_model_inputs(robot_obs, cfg, _FakeTokenizer(), torch.device("cpu")) + masks = [bool(m.item()) for m in image_masks] + assert masks == [True, False, False] # only base_0_rgb (mapped) is present + + +# ---------------------------------------------------------------------------- +# Version-stability / self-consistency (gated on a real pi0_base checkpoint) +# +# The LeRobot parity gate (test_pi0_parity.py) only validates correctness at +# transformers 5.3.0 (where LeRobot loads pi0_base cleanly). Production vllm-omni +# ships transformers 5.6.2, and LeRobot is not a usable reference there (it +# incorrectly loads the SigLIP vision tower at >=5.4). So we pin the kernel's deterministic +# output to golden values established at 5.3.0 and re-check them here: running on +# the production transformers proves the port is version-stable and guards the +# vision_tower / embed_scale version-robust paths against regressions. +# +# PI0_SELFCONSIST_CKPT=/path/to/pi0_base \ +# pytest tests/diffusion/models/pi0/test_pi0_units.py -k version_stable -v -s +# ---------------------------------------------------------------------------- +import os # noqa: E402 + +_SELFCONSIST_CKPT = os.environ.get("PI0_SELFCONSIST_CKPT") + +# Golden values established at transformers 5.3.0 (where parity = 7.15e-07). +_GOLDEN_SUM = -67.808113 +_GOLDEN_STD = 0.252789 +_GOLDEN_MEAN = -0.021190 +_GOLDEN_EMBED_IMAGE_STD = 2.17603 + + +def _selfconsist_inputs(device): + # NB: draw order is load-bearing — the golden values were established with + # state drawn first, then the 3 images, then lang (shared RNG, seed 0). + g = torch.Generator(device="cpu").manual_seed(0) + b = 2 + state = torch.randn(b, 32, generator=g) + images = [torch.rand(b, 3, 224, 224, generator=g) for _ in range(3)] + masks = [torch.ones(b, dtype=torch.bool) for _ in range(3)] + lang = torch.randint(0, 1000, (b, 48), generator=g) + lmask = torch.ones(b, 48, dtype=torch.bool) + ng = torch.Generator(device="cpu").manual_seed(42) + noise = torch.randn(b, 50, 32, generator=ng) + return ( + [im.to(device) for im in images], + [m.to(device) for m in masks], + lang.to(device), + lmask.to(device), + state.to(device), + noise.to(device), + ) + + +def _selfconsist_model(device): + import safetensors.torch + + model = Pi0ForActionPrediction( + Pi0Config(max_action_dim=32, max_state_dim=32, chunk_size=50, num_inference_steps=10, dtype="float32") + ) + state = safetensors.torch.load_file(os.path.join(_SELFCONSIST_CKPT, "model.safetensors")) + filled = model.load_weights(list(state.items())) + expected = {n for n, _ in model.named_parameters() if "rotary_emb" not in n and not n.endswith(".inv_freq")} + missing = expected - (filled or set()) + assert not missing, f"{len(missing)} params received no weight (e.g. {sorted(missing)[:3]})" + return model.to(device).eval() + + +@pytest.mark.slow +@pytest.mark.skipif(not _SELFCONSIST_CKPT, reason="Set PI0_SELFCONSIST_CKPT to a pi0_base dir.") +@torch.no_grad() +def test_version_stable_actions(): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = _selfconsist_model(device) + images, masks, lang, lmask, state, noise = _selfconsist_inputs(device) + actions = ( + model.sample_actions( + images=images, + image_masks=masks, + lang_tokens=lang, + lang_masks=lmask, + state=state, + noise=noise, + num_steps=10, + ) + .float() + .cpu() + ) + s, m, std = actions.sum().item(), actions.mean().item(), actions.std().item() + assert abs(s - _GOLDEN_SUM) < 0.5, f"action sum drifted: {s} vs {_GOLDEN_SUM}" + assert abs(std - _GOLDEN_STD) < 0.01, f"action std drifted: {std} vs {_GOLDEN_STD}" + assert abs(m - _GOLDEN_MEAN) < 0.01, f"action mean drifted: {m} vs {_GOLDEN_MEAN}" + + +@pytest.mark.slow +@pytest.mark.skipif(not _SELFCONSIST_CKPT, reason="Set PI0_SELFCONSIST_CKPT to a pi0_base dir.") +@torch.no_grad() +def test_version_stable_embed_image(): + """Directly guards the SigLIP vision-tower load across the >=5.4 flattening.""" + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = _selfconsist_model(device) + g = torch.Generator(device="cpu").manual_seed(7) + img = torch.rand(1, 3, 224, 224, generator=g, dtype=torch.float32).to(device) + std = model.paligemma_with_expert.embed_image(img).float().std().item() + assert abs(std - _GOLDEN_EMBED_IMAGE_STD) < 0.05, ( + f"embed_image std drifted: {std} vs {_GOLDEN_EMBED_IMAGE_STD} — " + "likely a vision_tower weight-load regression under this transformers version." + ) diff --git a/tests/e2e/online_serving/test_pi0_expansion.py b/tests/e2e/online_serving/test_pi0_expansion.py new file mode 100644 index 000000000..3a9c3b177 --- /dev/null +++ b/tests/e2e/online_serving/test_pi0_expansion.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""E2E online serving test for π0 (Pi-Zero) OpenPI websocket serving. + +Boots ``vllm serve --omni --deploy-config pi0.yaml`` and drives the real OpenPI +websocket (``/v1/realtime/robot/openpi``) — the same wire path a robot uses +(handshake metadata → send observation → receive action chunk). Mirrors +``tests/e2e/online_serving/test_dreamzero_expansion.py``. Needs a GPU + a +pi0_base checkpoint; skipped in CI unless explicitly run. + +The in-process LeRobot parity oracle lives separately in +``tests/diffusion/models/pi0/test_pi0_parity.py``. +""" + +from __future__ import annotations + +import os + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import ( + OmniServerParams, + get_open_port, + pi0_openpi_require_dependencies, + pi0_openpi_run_policy_session, + pi0_openpi_validate_session_result, +) + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" +os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" + +MODEL = "lerobot/pi0_base" + + +test_params = [ + OmniServerParams( + model=MODEL, + port=8092, + server_args=[ + "--deploy-config", + "vllm_omni/deploy/pi0.yaml", + "--served-model-name", + "pi0", + "--enforce-eager", + "--disable-log-stats", + ], + env_dict={ + "ATTENTION_BACKEND": "torch", + "DIFFUSION_ATTENTION_BACKEND": "TORCH_SDPA", + "VLLM_DISABLE_COMPILE_CACHE": "1", + "MASTER_PORT": str(get_open_port()), + }, + ) +] + + +@pytest.mark.full_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "H100"}) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_pi0_openpi_online(omni_server): + try: + pi0_openpi_require_dependencies() + except ModuleNotFoundError as exc: + pytest.skip(str(exc)) + + result = pi0_openpi_run_policy_session( + host=omni_server.host, + port=omni_server.port, + prompt="pick up the red block and place it in the bin", + session_id="pi0-online-e2e", + num_steps=2, + ) + + # Asserts every returned chunk is [50, 32] + finite, and the handshake + # metadata matches pi0.yaml's policy_server_config. + pi0_openpi_validate_session_result(result) + + metadata = result["metadata"] + assert tuple(metadata["image_resolution"]) == (224, 224) + assert metadata["needs_wrist_camera"] is True + assert metadata["needs_session_id"] is False + assert metadata["action_space"] == "joint_position" diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index c48411c9b..4d06eb86a 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -3265,6 +3265,146 @@ def iter_omni_runner( print("OmniRunner stopped") +# ───────────────────────────────────────────────────────────────────── +# π0 (Pi-Zero) OpenPI websocket policy client +# +# Minimal client for the OpenPI realtime robot protocol +# (``/v1/realtime/robot/openpi``): connect → read handshake metadata → send +# observation → receive an action chunk. Used by the π0 online-serving e2e +# (tests/e2e/online_serving/test_pi0_expansion.py). π0's observation is 3 RGB +# cameras + proprioceptive state + a language prompt, built as synthetic numpy +# frames here (no video assets). +# ───────────────────────────────────────────────────────────────────── +PI0_OPENPI_PATH = "/v1/realtime/robot/openpi" +PI0_OPENPI_DEFAULT_PROMPT = "pick up the red block and place it in the bin" +# π0 / pi0_base camera identities (must match the server's image_feature_keys, +# i.e. vllm_omni/deploy/pi0.yaml, whose image_key_map is empty). +PI0_CAMERA_KEYS = ( + "observation.images.base_0_rgb", + "observation.images.left_wrist_0_rgb", + "observation.images.right_wrist_0_rgb", +) +PI0_ACTION_HORIZON = 50 +PI0_ACTION_DIM = 32 +PI0_STATE_DIM = 32 +PI0_IMAGE_SIZE = 224 + + +def pi0_openpi_require_dependencies() -> None: + """Raise ModuleNotFoundError if the OpenPI websocket client deps are missing.""" + missing = [] + try: + import websockets.sync.client # noqa: F401 + except ImportError: + missing.append("websockets") + try: + from openpi_client import msgpack_numpy # noqa: F401 + except ImportError: + missing.append("openpi-client") + if missing: + raise ModuleNotFoundError(f"π0 OpenPI test dependencies are missing: {', '.join(missing)}") + + +def _pi0_decode_action_response(response: bytes | str) -> np.ndarray: + from openpi_client import msgpack_numpy + + if isinstance(response, str): + raise RuntimeError(f"Inference failed: {response}") + decoded = msgpack_numpy.unpackb(response) + if isinstance(decoded, dict) and decoded.get("type") == "error": + raise RuntimeError(f"Inference failed: {decoded.get('message', decoded)}") + return np.asarray(decoded, dtype=np.float32) + + +def pi0_make_dummy_obs(*, prompt: str, session_id: str, image_size: int = PI0_IMAGE_SIZE) -> dict[str, Any]: + """A single π0 observation: 3 blank cameras (HWC uint8) + zero state + prompt.""" + obs: dict[str, Any] = {cam: np.zeros((image_size, image_size, 3), dtype=np.uint8) for cam in PI0_CAMERA_KEYS} + obs["state"] = np.zeros(PI0_STATE_DIM, dtype=np.float32) + obs["prompt"] = prompt + obs["session_id"] = session_id + return obs + + +def pi0_openpi_run_policy_session( + *, + host: str = "127.0.0.1", + port: int = 8000, + path: str = PI0_OPENPI_PATH, + prompt: str = PI0_OPENPI_DEFAULT_PROMPT, + session_id: str | None = None, + num_steps: int = 2, +) -> dict[str, Any]: + """Connect, read handshake metadata, send ``num_steps`` observations.""" + import uuid + + import websockets.sync.client as websockets_client + from openpi_client import msgpack_numpy + + session_id = session_id or str(uuid.uuid4()) + uri = f"ws://{host}:{port}{path}" + packer = msgpack_numpy.Packer() + conn = websockets_client.connect(uri, compression=None, max_size=None, ping_interval=300, ping_timeout=3600) + try: + metadata = msgpack_numpy.unpackb(conn.recv()) + if not isinstance(metadata, dict): + raise TypeError(f"Expected dict metadata from server, got {type(metadata)!r}") + actions = [] + for _ in range(num_steps): + payload = pi0_make_dummy_obs(prompt=prompt, session_id=session_id) + payload["endpoint"] = "infer" + conn.send(packer.pack(payload)) + actions.append(_pi0_decode_action_response(conn.recv())) + return {"metadata": dict(metadata), "actions": actions, "session_id": session_id} + finally: + conn.close() + + +def pi0_openpi_validate_session_result( + result: dict[str, Any], + *, + expected_action_horizon: int = PI0_ACTION_HORIZON, + expected_action_dim: int = PI0_ACTION_DIM, +) -> None: + """Assert the handshake metadata + every returned action chunk for π0.""" + metadata = result["metadata"] + required_keys = ( + "image_resolution", + "needs_wrist_camera", + "needs_session_id", + "action_space", + "action_horizon", + "action_dim", + ) + missing = [key for key in required_keys if key not in metadata] + if missing: + raise AssertionError(f"Missing π0 metadata keys: {missing}") + + if tuple(metadata["image_resolution"]) != (PI0_IMAGE_SIZE, PI0_IMAGE_SIZE): + raise AssertionError(f"Unexpected image_resolution: {metadata['image_resolution']!r}") + if not metadata["needs_wrist_camera"]: + raise AssertionError("π0 test expects needs_wrist_camera=True") + if metadata["needs_session_id"]: + raise AssertionError("π0 is stateless; needs_session_id must be False") + if metadata["action_space"] != "joint_position": + raise AssertionError(f"Unexpected action_space: {metadata['action_space']!r}") + if int(metadata["action_horizon"]) != expected_action_horizon: + raise AssertionError(f"Unexpected action_horizon: {metadata['action_horizon']!r}") + if int(metadata["action_dim"]) != expected_action_dim: + raise AssertionError(f"Unexpected action_dim: {metadata['action_dim']!r}") + + actions = result["actions"] + if not actions: + raise AssertionError("No actions returned from the server") + for index, action in enumerate(actions): + if action.shape != (expected_action_horizon, expected_action_dim): + raise AssertionError( + f"Action {index} shape mismatch: expected " + f"{(expected_action_horizon, expected_action_dim)}, got {action.shape}" + ) + if not np.isfinite(action).all(): + raise AssertionError(f"Action {index} contains non-finite values") + + __all__ = [ "DiffusionResponse", "HttpResponse", @@ -3286,4 +3426,8 @@ __all__ = [ "OpenAIClientHandler", "get_open_port", "dummy_messages_from_mix_data", + "pi0_openpi_require_dependencies", + "pi0_openpi_run_policy_session", + "pi0_openpi_validate_session_result", + "pi0_make_dummy_obs", ] diff --git a/tests/model_tests/diffusion/test_alignment.py b/tests/model_tests/diffusion/test_alignment.py index 423ccc7a8..7faf175f8 100644 --- a/tests/model_tests/diffusion/test_alignment.py +++ b/tests/model_tests/diffusion/test_alignment.py @@ -77,6 +77,7 @@ EXCLUDED_MODELS = [ "DreamZeroPipeline", "StableDiffusionXLPipeline", "Gr00tN1d7Pipeline", + "Pi0Pipeline", "SoulXSingerPipeline", "SoulXSingerSVCPipeline", ] diff --git a/vllm_omni/config/pipeline_registry.py b/vllm_omni/config/pipeline_registry.py index 69e3ebe11..e8e3448b2 100644 --- a/vllm_omni/config/pipeline_registry.py +++ b/vllm_omni/config/pipeline_registry.py @@ -33,6 +33,7 @@ from vllm.logger import init_logger from vllm_omni.config.stage_config import ( PipelineConfig, ) +from vllm_omni.diffusion.models.pi0.pipeline_pi0 import PI0_PIPELINE from vllm_omni.model_executor.models.audex.pipeline import ( AUDEX_S2S_PIPELINE, AUDEX_THINKER_ONLY_PIPELINE, @@ -139,6 +140,7 @@ OMNI_PIPELINES: dict[str, PipelineConfig | PipelineResolverFunc] = { "lance": LANCE_PIPELINE, "dreamzero": DREAMZERO_PIPELINE, "Gr00tN1d7": GR00T_N1D7_PIPELINE, + "pi0": PI0_PIPELINE, "gepard": GEPARD_PIPELINE, "glm_image": GLM_IMAGE_PIPELINE, "hunyuan_image_3_moe": HUNYUAN_IMAGE3_PIPELINE, diff --git a/vllm_omni/deploy/pi0.yaml b/vllm_omni/deploy/pi0.yaml new file mode 100644 index 000000000..8d93855d8 --- /dev/null +++ b/vllm_omni/deploy/pi0.yaml @@ -0,0 +1,52 @@ +# π0 (Pi-Zero) VLA deploy: single diffusion stage. +# +# Topology (PI0_PIPELINE) is declared in vllm_omni/diffusion/models/pi0/pipeline_pi0.py. +# π0 takes multi-camera images + a language instruction + robot state and emits +# a continuous action chunk [chunk_size, max_action_dim] via flow matching. +# Actions are exposed through the OpenPI realtime serving API +# (/v1/realtime/robot/openpi); they ride in multimodal_output["actions"]. +# +# Default: one GPU, TP=1, eager (flow matching needs no CUDA graph), bsz 1 +# (OpenPI policy serving is one observation -> one action reply). float32 to +# match the LeRobot pi0_base reference numerics. + +pipeline: pi0 +async_chunk: false +distributed_executor_backend: mp +dtype: float32 + +stages: + - stage_id: 0 + devices: "0" + max_num_seqs: 1 + enforce_eager: true + model_class_name: Pi0Pipeline + model_config: + paligemma_variant: gemma_2b + action_expert_variant: gemma_300m + chunk_size: 50 + max_action_dim: 32 + max_state_dim: 32 + num_inference_steps: 10 + image_resolution: [224, 224] + tokenizer_max_length: 48 + max_cameras: 3 + # Camera identities (the order the model attends to). Serving maps raw obs + # keys onto these via image_key_map; leave image_key_map empty to match + # incoming keys verbatim. + image_feature_keys: + - observation.images.base_0_rgb + - observation.images.left_wrist_0_rgb + - observation.images.right_wrist_0_rgb + image_key_map: {} + # OpenPI handshake metadata advertised to the robot client. + policy_server_config: + image_resolution: [224, 224] + n_external_cameras: 1 + needs_wrist_camera: true + needs_stereo_camera: false + needs_session_id: false + action_space: joint_position + action_horizon: 50 + action_dim: 32 + max_cameras: 3 diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index a5b6d06b5..0ea85405b 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -1318,6 +1318,12 @@ class OmniDiffusionConfig: self.update_multimodal_support() else: raise + elif cfg.get("type") == "pi0": + # π0 (Pi-Zero) VLA — the LeRobot config.json uses ``type: "pi0"``. + if self.model_class_name is None: + self.model_class_name = "Pi0Pipeline" + self.set_tf_model_config(TransformerConfig()) + self.update_multimodal_support() elif architectures and len(architectures) == 1: architecture = architectures[0] from vllm_omni.diffusion.registry import DiffusionModelRegistry diff --git a/vllm_omni/diffusion/models/pi0/__init__.py b/vllm_omni/diffusion/models/pi0/__init__.py new file mode 100644 index 000000000..3862d1836 --- /dev/null +++ b/vllm_omni/diffusion/models/pi0/__init__.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""π0 (Pi-Zero) VLA model for vllm-omni. + +PaliGemma (SigLIP vision + Gemma-2B LM) + Gemma-300M action expert + a +flow-matching action head. Outputs a continuous action chunk +``[horizon, action_dim]`` rather than tokens. +""" + +from vllm_omni.diffusion.models.pi0.config import Pi0Config +from vllm_omni.diffusion.models.pi0.modeling_pi0 import ( + GemmaVariantConfig, + PaliGemmaWithActionExpert, + Pi0ForActionPrediction, + create_sinusoidal_pos_embedding, + get_gemma_config, + make_att_2d_masks, + prepare_attention_masks_4d, +) +from vllm_omni.diffusion.models.pi0.pipeline_pi0 import ( + Pi0Pipeline, + get_pi0_post_process_func, +) +from vllm_omni.diffusion.models.pi0.processor_pi0 import ( + Pi0ImageProcessor, + build_model_inputs, + pil_image_to_tensor, + resize_with_pad, + tokenize_prompt, +) + +__all__ = [ + "Pi0Config", + "Pi0ForActionPrediction", + "PaliGemmaWithActionExpert", + "GemmaVariantConfig", + "get_gemma_config", + "create_sinusoidal_pos_embedding", + "make_att_2d_masks", + "prepare_attention_masks_4d", + "Pi0ImageProcessor", + "build_model_inputs", + "pil_image_to_tensor", + "resize_with_pad", + "tokenize_prompt", + "Pi0Pipeline", + "get_pi0_post_process_func", +] diff --git a/vllm_omni/diffusion/models/pi0/config.py b/vllm_omni/diffusion/models/pi0/config.py new file mode 100644 index 000000000..ef0484f8e --- /dev/null +++ b/vllm_omni/diffusion/models/pi0/config.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Config surface for the π0 (Pi-Zero) VLA model in vllm-omni. + +Only the parameters that actually shape runtime behaviour live here; the +transformer dimensions (hidden size, head count, ...) are derived from +``paligemma_variant`` / ``action_expert_variant`` inside the model via +``get_gemma_config`` (see ``modeling_pi0``). + +The resolver reads the **raw LeRobot ``config.json``** (the field surface of +``lerobot.policies.pi0.PI0Config``): only the recognized dataclass fields are +used; any other keys in the file are ignored. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from dataclasses import fields as dataclass_fields +from pathlib import Path +from typing import Any + +# LeRobot / OpenPI observation key conventions. +ACTION = "action" +OBS_STR = "observation" +OBS_STATE = OBS_STR + ".state" +OBS_IMAGES = OBS_STR + ".images" + + +@dataclass +class Pi0Config: + """π0 VLA config (dataclass, not an HF ``PretrainedConfig``). + + Mirrors the runtime-relevant subset of LeRobot ``PI0Config`` plus a couple + of serving-side knobs (``max_cameras``, ``image_feature_keys``, + ``image_key_map``, ``norm_stats``). + """ + + # Backbone variants — mapped to Gemma dimensions by ``get_gemma_config``. + paligemma_variant: str = "gemma_2b" + action_expert_variant: str = "gemma_300m" + + # Action chunk shape. + chunk_size: int = 50 + max_action_dim: int = 32 + max_state_dim: int = 32 + + # Flow-matching denoising schedule. + num_inference_steps: int = 10 + # Sinusoidal timestep embedding periods (must match OpenPI/LeRobot). + min_period: float = 4e-3 + max_period: float = 4.0 + + # Image preprocessing. π0/SigLIP only support square inputs. + image_resolution: tuple[int, int] = (224, 224) + tokenizer_max_length: int = 48 + # Number of camera slots the model attends to (real + padded). + max_cameras: int = 3 + + # Weight dtype the checkpoint was saved in; parity passes this through to + # match LeRobot's ``PI0Config.dtype``. + dtype: str = "float32" + + # Per-dataset normalization stats, optional (schema matches LeRobot's + # ``NormalizerProcessorStep``; consumed by ``_build_norm_buffers``). ``None`` + # means identity / pass-through (the ``lerobot/pi0_base`` case). + norm_stats: dict | None = None + + # Ordered list of image feature keys (e.g. ``observation.images.base_0_rgb``) + # taken from the checkpoint's ``input_features``. This is the **camera order** + # that must be reproduced for LeRobot parity. + image_feature_keys: list[str] | None = None + + # Optional map from raw OpenPI obs keys → ``image_feature_keys`` entries, so + # serving can translate wire obs (e.g. ``observation/exterior_image_0_left``) + # into the checkpoint's camera identities. Empty → identity match on key. + image_key_map: dict[str, str] = field(default_factory=dict) + + # Stored for reference / camera-order derivation; not used at inference. + input_features: dict[str, Any] = field(default_factory=dict) + output_features: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + # Coerce list → tuple (JSON has no tuples) and validate squareness. + res = self.image_resolution + if not isinstance(res, (tuple, list)) or len(res) != 2 or res[0] != res[1]: + raise ValueError(f"π0 expects a square image_resolution (H == W); got {res!r}.") + self.image_resolution = (int(res[0]), int(res[1])) + + # Derive the camera order from input_features if not given explicitly. + if self.image_feature_keys is None and self.input_features: + self.image_feature_keys = [key for key in self.input_features if key.startswith(OBS_IMAGES + ".")] + + # ------------------------------------------------------------------ + # Constructors + # ------------------------------------------------------------------ + @classmethod + def from_pretrained(cls, checkpoint_dir: str | Path) -> Pi0Config: + """Build from a checkpoint directory's ``config.json``.""" + checkpoint_dir = Path(checkpoint_dir) + config_path = checkpoint_dir / "config.json" + if not config_path.exists(): + return cls() + with open(config_path, encoding="utf-8") as f: + raw = json.load(f) + return cls.from_model_config(raw) + + @classmethod + def from_model_config(cls, model_config: dict[str, Any] | None) -> Pi0Config: + """Build from a config dict (LeRobot ``config.json`` or deploy yaml). + + Keeps only the recognized dataclass fields (the LeRobot config also + carries many training-only keys) and coerces ``image_resolution`` to a + tuple. + """ + if not model_config: + return cls() + + raw = dict(model_config) + if "image_resolution" in raw: + raw["image_resolution"] = tuple(raw["image_resolution"]) + + allowed = {item.name for item in dataclass_fields(cls)} + filtered = {key: value for key, value in raw.items() if key in allowed} + return cls(**filtered) diff --git a/vllm_omni/diffusion/models/pi0/modeling_pi0.py b/vllm_omni/diffusion/models/pi0/modeling_pi0.py new file mode 100644 index 000000000..a45d931ef --- /dev/null +++ b/vllm_omni/diffusion/models/pi0/modeling_pi0.py @@ -0,0 +1,944 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inference-only π0 (Pi-Zero) VLA math kernel for vllm-omni. + +A self-contained inference kernel: only the math that turns a robot observation +into an action chunk, with no serving/request glue — validated bit-for-bit +against the LeRobot ``PI0Policy`` reference (``max|Δ| < 1e-4``). + +π0 = PaliGemma (SigLIP vision + Gemma 2B LM) prefix + Gemma 300M action expert +suffix + flow-matching head. Inference: + +1. Embed prefix (images + language) → bidirectional prefix tokens. +2. Forward prefix through PaliGemma → a per-layer ``list[(k, v)]`` KV cache. +3. For each Euler step ``t = 1.0, 1-dt, ..., 0``: embed the suffix + (state + noisy actions + timestep), run the action expert with cross + attention over the cached prefix K/V, predict velocity ``v_t``, and + integrate ``x_t = x_t + dt * v_t``. +4. Return ``x_0`` as the action chunk ``(batch, action_horizon, action_dim)``. + +We walk Gemma decoder layers ourselves in ``_compute_layer_*`` so we never go +through ``GemmaModel.forward`` (which has version-dependent mask/KV-cache +behaviour). Targets modern transformers (PaliGemmaForConditionalGeneration with +an inner PaliGemmaModel + DynamicCache). + +Reference implementations: + - OpenPI: openpi/src/openpi/models_pytorch/pi0_pytorch.py, gemma_pytorch.py + - LeRobot: lerobot/src/lerobot/policies/pi0/modeling_pi0.py +Weight source: https://huggingface.co/lerobot/pi0_base +""" + +from __future__ import annotations + +import logging +import math +from collections.abc import Iterable + +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers.models.auto import CONFIG_MAPPING +from transformers.models.gemma.modeling_gemma import ( + GemmaForCausalLM, + apply_rotary_pos_emb, +) +from transformers.models.paligemma.modeling_paligemma import ( + PaliGemmaForConditionalGeneration, +) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader + +logger = logging.getLogger(__name__) + +# ────────────────────────────────────────────────────────────────────── +# Constants +# ────────────────────────────────────────────────────────────────────── +DEFAULT_ACTION_DIM = 32 +DEFAULT_ACTION_HORIZON = 50 +DEFAULT_MAX_TOKEN_LEN = 48 +DEFAULT_NUM_INFERENCE_STEPS = 10 +DEFAULT_IMAGE_RESOLUTION = (224, 224) + +# Large negative value to fill masked-out positions in a float attention mask. +# Matches OpenPI's constant exactly so that numerics line up during parity. +# Ref: openpi/src/openpi/models/gemma.py +OPENPI_ATTENTION_MASK_VALUE = -2.3819763e38 + + +# ────────────────────────────────────────────────────────────────────── +# Gemma variant configs (matches openpi/models/gemma.py get_config) +# ────────────────────────────────────────────────────────────────────── + + +class GemmaVariantConfig: + def __init__(self, width, depth, mlp_dim, num_heads, num_kv_heads, head_dim): + self.width = width + self.depth = depth + self.mlp_dim = mlp_dim + self.num_heads = num_heads + self.num_kv_heads = num_kv_heads + self.head_dim = head_dim + + +def get_gemma_config(variant: str) -> GemmaVariantConfig: + if variant == "gemma_2b": + return GemmaVariantConfig(2048, 18, 16384, 8, 1, 256) + elif variant == "gemma_300m": + return GemmaVariantConfig(1024, 18, 4096, 8, 1, 256) + else: + raise ValueError(f"Unknown variant: {variant}") + + +# ────────────────────────────────────────────────────────────────────── +# Utility functions (match openpi/models_pytorch/pi0_pytorch.py) +# ────────────────────────────────────────────────────────────────────── + + +def create_sinusoidal_pos_embedding( + time: torch.Tensor, + dimension: int, + min_period: float = 4e-3, + max_period: float = 4.0, + device: torch.device = None, +) -> torch.Tensor: + """Compute a sine/cosine positional embedding for scalar timesteps. + + Ref: openpi/models_pytorch/pi0_pytorch.py create_sinusoidal_pos_embedding + """ + if dimension % 2 != 0: + raise ValueError(f"dimension ({dimension}) must be divisible by 2") + if time.ndim != 1: + raise ValueError("time tensor must be 1-D (batch_size,)") + if device is None: + device = time.device + + # Use float64 for the log-linear sweep and for the inner products, to + # match the reference implementation's numerical behaviour exactly. + dtype = torch.float64 + fraction = torch.linspace(0.0, 1.0, dimension // 2, dtype=dtype, device=device) + period = min_period * (max_period / min_period) ** fraction + scaling_factor = 1.0 / period * 2 * math.pi + sin_input = scaling_factor[None, :] * time[:, None].to(dtype) + return torch.cat([torch.sin(sin_input), torch.cos(sin_input)], dim=1) + + +def make_att_2d_masks(pad_masks: torch.Tensor, att_masks: torch.Tensor) -> torch.Tensor: + """Build a 2D attention mask from a padding mask and an autoregressive mask. + + Ref: openpi/models_pytorch/pi0_pytorch.py make_att_2d_masks + """ + if att_masks.ndim != 2: + raise ValueError(f"att_masks must be 2-D, got {att_masks.ndim}-D") + if pad_masks.ndim != 2: + raise ValueError(f"pad_masks must be 2-D, got {pad_masks.ndim}-D") + + cumsum = torch.cumsum(att_masks, dim=1) + att_2d_masks = cumsum[:, None, :] <= cumsum[:, :, None] + pad_2d_masks = pad_masks[:, None, :] * pad_masks[:, :, None] + return att_2d_masks & pad_2d_masks + + +def _build_norm_buffers(norm_stats: dict | None, key: str) -> dict[str, torch.Tensor] | None: + """Parse a ``norm_stats[key]`` entry into CPU tensors, or ``None``. + + Matches LeRobot's ``NormalizationMode``: + - ``mean_std`` : ``forward = (x - mean) / std``, ``inverse = x * std + mean`` + - ``min_max`` : forward maps into ``[-1, 1]``, inverse maps back. + """ + if not norm_stats or not isinstance(norm_stats, dict): + return None + entry = norm_stats.get(key) + if not entry: + return None + mode = str(entry.get("mode", "mean_std")).lower() + if mode == "mean_std": + mean = entry.get("mean") + std = entry.get("std") + if mean is None or std is None: + return None + return { + "mode": mode, + "mean": torch.as_tensor(mean, dtype=torch.float32), + "std": torch.as_tensor(std, dtype=torch.float32), + } + if mode == "min_max": + lo = entry.get("min") + hi = entry.get("max") + if lo is None or hi is None: + return None + return { + "mode": mode, + "min": torch.as_tensor(lo, dtype=torch.float32), + "max": torch.as_tensor(hi, dtype=torch.float32), + } + return None + + +def _apply_norm( + x: torch.Tensor, + stats: dict[str, torch.Tensor] | None, + inverse: bool, + eps: float = 1e-8, +) -> torch.Tensor: + """Apply (un)normalization using the given stats. No-op if ``stats`` is None. + + Broadcasts over leading dims; stats vectors align with the last dim. When + ``x`` has more last-dim entries than the stats (π0 pads state/action to + ``max_dim``), only the first ``len(stats)`` entries are transformed and + the padded tail is left untouched. + """ + if stats is None: + return x + mode = stats["mode"] + if mode == "mean_std": + mean = stats["mean"].to(device=x.device, dtype=x.dtype) + std = stats["std"].to(device=x.device, dtype=x.dtype) + valid = mean.shape[0] + head = x[..., :valid] + if inverse: + head = head * std + mean + else: + head = (head - mean) / (std + eps) + if valid == x.shape[-1]: + return head + return torch.cat([head, x[..., valid:]], dim=-1) + if mode == "min_max": + lo = stats["min"].to(device=x.device, dtype=x.dtype) + hi = stats["max"].to(device=x.device, dtype=x.dtype) + valid = lo.shape[0] + head = x[..., :valid] + denom = (hi - lo).clamp_min(eps) + if inverse: + head = (head + 1.0) * 0.5 * denom + lo + else: + head = 2.0 * (head - lo) / denom - 1.0 + if valid == x.shape[-1]: + return head + return torch.cat([head, x[..., valid:]], dim=-1) + return x + + +def prepare_attention_masks_4d(att_2d_masks: torch.Tensor) -> torch.Tensor: + """Convert ``(B, S, S)`` bool masks to ``(B, 1, S, S)`` float masks. + + ``True`` → 0.0 (attend), ``False`` → ``OPENPI_ATTENTION_MASK_VALUE``. + Ref: openpi PI0Pytorch._prepare_attention_masks_4d + """ + att_2d_masks_4d = att_2d_masks[:, None, :, :] + return torch.where(att_2d_masks_4d, 0.0, OPENPI_ATTENTION_MASK_VALUE) + + +# ────────────────────────────────────────────────────────────────────── +# Dual-backbone: PaliGemma + Action Expert +# Ref: openpi/models_pytorch/gemma_pytorch.py PaliGemmaWithExpertModel +# Ref: lerobot/policies/pi0/modeling_pi0.py PaliGemmaWithExpertModel +# ────────────────────────────────────────────────────────────────────── + + +def _repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """Repeat KV heads for grouped-query attention. + + ``(B, num_kv_heads, S, D) → (B, num_kv_heads * n_rep, S, D)``. + Matches ``transformers.models.gemma.modeling_gemma.repeat_kv``. + """ + if n_rep == 1: + return hidden_states + b, nh, s, d = hidden_states.shape + hidden_states = hidden_states[:, :, None, :, :].expand(b, nh, n_rep, s, d) + return hidden_states.reshape(b, nh * n_rep, s, d) + + +def _attend(query_states, key_states, value_states, attention_mask, num_kv_groups, scaling): + """Manual eager attention: ``softmax(Q Kᵀ · scale + mask) · V``. + + Important: we slice ``attention_mask[..., :key_states.shape[-2]]`` so that + the same ``(B, 1, Q, prefix+suffix)`` mask produced by ``denoise_step`` + can be re-used both in the prefix pass (K length = prefix) and in the + suffix pass (K length = prefix + suffix after we concat the cached prefix). + Stock transformers' ``eager_attention_forward`` adds the mask without + slicing and therefore breaks when the two dims disagree. + """ + k = _repeat_kv(key_states, num_kv_groups) + v = _repeat_kv(value_states, num_kv_groups) + attn_weights = torch.matmul(query_states, k.transpose(2, 3)) * scaling + if attention_mask is not None: + attn_weights = attn_weights + attention_mask[:, :, :, : k.shape[-2]] + attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + return torch.matmul(attn_weights, v) + + +def _compute_layer_prefix_only(layer_idx, hidden_states, attention_mask, position_ids, paligemma): + """Run one PaliGemma LM layer on the prefix only, returning the layer + output **and** the post-RoPE ``(k, v)`` so the caller can cache them for + the suffix pass.""" + model = paligemma.model.language_model + layer = model.layers[layer_idx] + residual = hidden_states + x = layer.input_layernorm(hidden_states) + + hidden_shape = (*x.shape[:-1], -1, layer.self_attn.head_dim) + q = layer.self_attn.q_proj(x).view(hidden_shape).transpose(1, 2) + k = layer.self_attn.k_proj(x).view(hidden_shape).transpose(1, 2) + v = layer.self_attn.v_proj(x).view(hidden_shape).transpose(1, 2) + + cos, sin = model.rotary_emb(v, position_ids) + q, k = apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1) + + att = _attend( + q, + k, + v, + attention_mask, + num_kv_groups=layer.self_attn.num_key_value_groups, + scaling=1.0 / math.sqrt(layer.self_attn.head_dim), + ) + att = att.transpose(1, 2).reshape(q.shape[0], -1, q.shape[1] * layer.self_attn.head_dim) + + out = layer.self_attn.o_proj(att) + residual + after_resid = out + out = layer.mlp(layer.post_attention_layernorm(out)) + after_resid + return out, (k, v) + + +def _compute_layer_suffix_only( + layer_idx, + hidden_states, + prefix_kv, + attention_mask, + position_ids, + paligemma, + gemma_expert, +): + """Run one action-expert layer on the suffix, attending to the cached + prefix K/V concatenated with freshly computed suffix K/V. + + ``prefix_kv`` is a ``(k_prefix, v_prefix)`` tuple, shape + ``(B, num_kv_heads, prefix_len, head_dim)``, post-RoPE, produced by the + corresponding ``_compute_layer_prefix_only`` call. Bypasses + ``GemmaModel.forward`` entirely — no HF mask rebuild, no cache format + gymnastics. + """ + layer = gemma_expert.model.layers[layer_idx] + residual = hidden_states + x = layer.input_layernorm(hidden_states) + + hidden_shape = (*x.shape[:-1], -1, layer.self_attn.head_dim) + q = layer.self_attn.q_proj(x).view(hidden_shape).transpose(1, 2) + k_suf = layer.self_attn.k_proj(x).view(hidden_shape).transpose(1, 2) + v_suf = layer.self_attn.v_proj(x).view(hidden_shape).transpose(1, 2) + + # RoPE frequencies are shared between PaliGemma and the expert. + cos, sin = gemma_expert.model.rotary_emb(v_suf, position_ids) + q, k_suf = apply_rotary_pos_emb(q, k_suf, cos, sin, unsqueeze_dim=1) + + # Concatenate cached prefix K/V (possibly different dtype) with suffix K/V. + k_prefix, v_prefix = prefix_kv + k = torch.cat([k_prefix.to(k_suf.dtype), k_suf], dim=2) + v = torch.cat([v_prefix.to(v_suf.dtype), v_suf], dim=2) + + att = _attend( + q, + k, + v, + attention_mask, + num_kv_groups=layer.self_attn.num_key_value_groups, + scaling=1.0 / math.sqrt(layer.self_attn.head_dim), + ) + att = att.transpose(1, 2).reshape(q.shape[0], -1, q.shape[1] * layer.self_attn.head_dim) + + out = layer.self_attn.o_proj(att) + residual + after_resid = out + out = layer.mlp(layer.post_attention_layernorm(out)) + after_resid + return out + + +class PaliGemmaWithActionExpert(nn.Module): + """Dual-backbone transformer: PaliGemma (Gemma 2B) + Action Expert (Gemma 300M). + + ``forward`` has two inference modes: + - **prefix_only**: ``inputs_embeds=[prefix, None]`` + ``use_cache=True`` + → compute prefix hidden states + a layer-wise K/V cache. + - **suffix_only**: ``inputs_embeds=[None, suffix]`` + the cache from the + previous prefix pass → compute expert hidden states with cross-attention + over the concatenated (prefix, suffix) K/V. + + Both modes walk Gemma decoder layers manually through ``_compute_layer_*`` + instead of ``GemmaModel.forward`` — that way we own the attention mask + handling and the KV cache format (a plain ``list[(k, v)]`` one entry + per layer). + + Ref: lerobot/policies/pi0/modeling_pi0.py PaliGemmaWithExpertModel + """ + + def __init__(self, vlm_config, action_expert_config): + super().__init__() + + # Build HF PaliGemma config from the variant dims we expose. + # NOTE: we do NOT set use_adarms / adarms_cond_dim — those are + # OpenPI-only attributes used by π0.5, not π0. + vlm_config_hf = CONFIG_MAPPING["paligemma"]() + vlm_config_hf._vocab_size = 257152 + vlm_config_hf.image_token_index = 257152 + vlm_config_hf.text_config.hidden_size = vlm_config.width + vlm_config_hf.text_config.intermediate_size = vlm_config.mlp_dim + vlm_config_hf.text_config.num_attention_heads = vlm_config.num_heads + vlm_config_hf.text_config.head_dim = vlm_config.head_dim + vlm_config_hf.text_config.num_hidden_layers = vlm_config.depth + vlm_config_hf.text_config.num_key_value_heads = vlm_config.num_kv_heads + vlm_config_hf.text_config.hidden_activation = "gelu_pytorch_tanh" + # transformers ≥ 5 uses ``dtype``; older versions still accept + # ``torch_dtype``. We set the canonical name; HF's PretrainedConfig + # normalizes one to the other internally. + vlm_config_hf.text_config.dtype = "float32" + vlm_config_hf.text_config.vocab_size = 257152 + vlm_config_hf.vision_config.intermediate_size = 4304 + vlm_config_hf.vision_config.projection_dim = 2048 + vlm_config_hf.vision_config.projector_hidden_act = "gelu_fast" + vlm_config_hf.vision_config.dtype = "float32" + + action_expert_config_hf = CONFIG_MAPPING["gemma"]( + head_dim=action_expert_config.head_dim, + hidden_size=action_expert_config.width, + intermediate_size=action_expert_config.mlp_dim, + num_attention_heads=action_expert_config.num_heads, + num_hidden_layers=action_expert_config.depth, + num_key_value_heads=action_expert_config.num_kv_heads, + vocab_size=257152, + hidden_activation="gelu_pytorch_tanh", + dtype="float32", + ) + + self.paligemma = PaliGemmaForConditionalGeneration(config=vlm_config_hf) + self.gemma_expert = GemmaForCausalLM(config=action_expert_config_hf) + # The action expert doesn't embed tokens — it only consumes the + # suffix state/action embeddings we feed in. + self.gemma_expert.model.embed_tokens = None + + def embed_image(self, pixel_values: torch.Tensor) -> torch.Tensor: + """Encode images with SigLIP vision tower + PaliGemma projector. + + We run the two steps explicitly instead of calling + ``PaliGemmaModel.get_image_features`` because that convenience + method historically has returned either the projected tensor (old) + or the raw vision-tower output (new). Being explicit makes us + independent of which transformers release we're on. + """ + # Shapes: pixel_values (B, 3, 224, 224) → SigLIP (B, 256, 1152) + # → projector (B, 256, 2048) + vision_outputs = self.paligemma.model.vision_tower(pixel_values) + image_features = vision_outputs.last_hidden_state + return self.paligemma.model.multi_modal_projector(image_features) + + def embed_language_tokens(self, tokens: torch.Tensor) -> torch.Tensor: + """Embed language tokens with PaliGemma's embedding table, returning the + ``* sqrt(hidden)``-scaled embedding (PaliGemma/Gemma convention). + + The scaling location moved across transformers releases: + * ≤ 5.3: ``embed_tokens`` is a plain ``nn.Embedding`` and the + ``* sqrt(hidden)`` normalizer lives inside ``GemmaModel.forward`` + (which we bypass) — so we apply it explicitly here. + * ≥ 5.4: ``embed_tokens`` is a ``GemmaTextScaledWordEmbedding`` that + self-applies ``embed_scale = hidden_size ** 0.5`` — applying it again + would double-scale (≈45×). We detect this and skip the manual scale. + This makes ``embed_language_tokens`` return the canonical scaled embedding + on every transformers version. + """ + embed_tokens = self.paligemma.model.language_model.embed_tokens + lang_emb = embed_tokens(tokens) + # If the embedding already self-scales (transformers ≥ 5.4), don't repeat it. + if getattr(embed_tokens, "embed_scale", None) is None: + lang_emb = lang_emb * math.sqrt(lang_emb.shape[-1]) + return lang_emb + + def forward( + self, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values=None, + inputs_embeds: list[torch.Tensor | None] | None = None, + use_cache: bool = False, + ): + """Dispatch to prefix_only / suffix_only and return + ``([prefix_out, suffix_out], past_key_values_or_None)``. + """ + num_layers = self.paligemma.config.text_config.num_hidden_layers + pali_lm = self.paligemma.model.language_model + expert_lm = self.gemma_expert.model + + if inputs_embeds[1] is None: + # Prefix-only: PaliGemma LM on (images + language) tokens; the + # per-layer post-RoPE K/V is collected into a list that the + # suffix pass will consume directly. + hidden_states = inputs_embeds[0] + kv_list: list[tuple[torch.Tensor, torch.Tensor]] = [] + for layer_idx in range(num_layers): + hidden_states, kv = _compute_layer_prefix_only( + layer_idx, + hidden_states, + attention_mask, + position_ids, + paligemma=self.paligemma, + ) + kv_list.append(kv) + hidden_states = pali_lm.norm(hidden_states) + return [hidden_states, None], (kv_list if use_cache else None) + + if inputs_embeds[0] is not None: + raise ValueError( + "PaliGemmaWithActionExpert.forward only supports prefix-only " + "or suffix-only dispatch; got both inputs_embeds populated." + ) + # Suffix-only: action expert on (state + noisy_actions + time), + # attending to the cached prefix K/V. + if not isinstance(past_key_values, list): + raise TypeError( + "suffix_only forward expects past_key_values to be the " + "list[(k, v)] produced by a previous prefix_only forward; " + f"got {type(past_key_values)}" + ) + hidden_states = inputs_embeds[1] + for layer_idx in range(num_layers): + hidden_states = _compute_layer_suffix_only( + layer_idx, + hidden_states, + past_key_values[layer_idx], + attention_mask, + position_ids, + paligemma=self.paligemma, + gemma_expert=self.gemma_expert, + ) + hidden_states = expert_lm.norm(hidden_states) + return [None, hidden_states], None + + +# ────────────────────────────────────────────────────────────────────── +# Main π0 Model +# ────────────────────────────────────────────────────────────────────── + + +class Pi0ForActionPrediction(nn.Module): + """π0 VLA model for robot action prediction via flow matching. + + Inference flow: + 1. Embed prefix (images + language) → prefix tokens. + 2. Forward prefix through PaliGemma → layer-wise KV cache. + 3. For each denoising step ``t = 1.0, 1-dt, ..., 0``: + a. Embed suffix (state + x_t + timestep) → suffix tokens. + b. Forward suffix through the action expert with the prefix cache. + c. ``x_t = x_t + dt * v_t`` (Euler integration). + 4. Return ``x_0`` as the predicted action chunk. + """ + + def __init__( + self, + config, + quant_config=None, + prefix: str = "", + ): + super().__init__() + # ``quant_config`` is accepted for interface compatibility but unused — + # π0 runs in full precision for flow-matching parity. + del quant_config + self.config = config + + self.action_dim = getattr(config, "max_action_dim", DEFAULT_ACTION_DIM) + # ``max_state_dim`` is independent from ``max_action_dim`` — e.g. Aloha + # has state_dim=14, action_dim=32, both padded to their own max. Fall + # back to action_dim for older configs that only expose a single size. + self.max_state_dim = getattr(config, "max_state_dim", self.action_dim) + self.action_horizon = getattr(config, "chunk_size", DEFAULT_ACTION_HORIZON) + self.num_inference_steps = getattr(config, "num_inference_steps", DEFAULT_NUM_INFERENCE_STEPS) + + paligemma_variant = getattr(config, "paligemma_variant", "gemma_2b") + action_expert_variant = getattr(config, "action_expert_variant", "gemma_300m") + vlm_config = get_gemma_config(paligemma_variant) + expert_config = get_gemma_config(action_expert_variant) + self.vlm_width = vlm_config.width + self.expert_width = expert_config.width + + # Dual backbone + self.paligemma_with_expert = PaliGemmaWithActionExpert(vlm_config, expert_config) + + # Action chunk projections (openpi pi0_pytorch.py). + self.action_in_proj = nn.Linear(self.action_dim, self.expert_width) + self.action_out_proj = nn.Linear(self.expert_width, self.action_dim) + + # State is projected to the expert dim once (π0 only, not π0.5). + # Ref: lerobot modeling_pi0.py ``nn.Linear(config.max_state_dim, ...)`` + self.state_proj = nn.Linear(self.max_state_dim, self.expert_width) + + # Timestep + action fusion MLP (2W → W → W). + self.action_time_mlp_in = nn.Linear(2 * self.expert_width, self.expert_width) + self.action_time_mlp_out = nn.Linear(self.expert_width, self.expert_width) + + # Optional mean/std normalization stats (LeRobot NormalizerProcessorStep + # semantics). Format on the config (all optional): + # norm_stats = { + # "state": {"mode": "mean_std", "mean": [...], "std": [...]}, + # "action": {"mode": "mean_std", "mean": [...], "std": [...]}, + # } + # ``mode`` may also be ``"min_max"``. + # + # Missing norm_stats is logged at INFO (not WARNING) because the most + # common setup is "client pre-normalizes state and post-normalizes + # actions using dataset stats" — that's a supported mode, not an error. + self._state_norm = _build_norm_buffers(getattr(config, "norm_stats", None), "state") + self._action_norm = _build_norm_buffers(getattr(config, "norm_stats", None), "action") + if self._state_norm is None: + logger.info( + "π0: no state normalization stats on config.norm_stats — state values will pass through unchanged." + ) + if self._action_norm is None: + logger.info( + "π0: no action normalization stats on config.norm_stats — " + "returned actions are in the model's normalized space." + ) + + # ── State / action normalization ───────────────────────────────── + + def _normalize_state(self, state: torch.Tensor) -> torch.Tensor: + return _apply_norm(state, self._state_norm, inverse=False) + + def _unnormalize_actions(self, actions: torch.Tensor) -> torch.Tensor: + return _apply_norm(actions, self._action_norm, inverse=True) + + # ── Prefix embedding ───────────────────────────────────────────── + + def embed_prefix( + self, + images: list[torch.Tensor], + image_masks: list[torch.Tensor], + lang_tokens: torch.Tensor, + lang_masks: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Build the prefix embeddings, per-token padding mask, and AR mask. + + Prefix tokens form a contiguous sequence + ``[img_cam_0..., img_cam_1..., ..., lang_tokens...]`` with + bidirectional attention; the returned ``att_masks`` are all zeros + because each token is free to attend to every other prefix token (the + suffix pass will put a causal boundary right before the state token). + + Ref: openpi PI0Pytorch.embed_prefix + """ + embs: list[torch.Tensor] = [] + pad_masks: list[torch.Tensor] = [] + att_masks: list[int] = [] + + for img, img_mask in zip(images, image_masks): + img_emb = self.paligemma_with_expert.embed_image(img) + bsize, num_img_embs = img_emb.shape[:2] + embs.append(img_emb) + pad_masks.append(img_mask[:, None].expand(bsize, num_img_embs)) + att_masks += [0] * num_img_embs + + # ``embed_language_tokens`` already returns the ``* sqrt(hidden)``-scaled + # embedding (version-robust: self-scaled at transformers ≥ 5.4, manually + # scaled at ≤ 5.3). The image slice went through the projector and is + # NOT scaled. See embed_language_tokens for the version details. + lang_emb = self.paligemma_with_expert.embed_language_tokens(lang_tokens) + + embs.append(lang_emb) + pad_masks.append(lang_masks) + att_masks += [0] * lang_emb.shape[1] + + embs = torch.cat(embs, dim=1) + pad_masks = torch.cat(pad_masks, dim=1) + att_masks = torch.tensor(att_masks, dtype=torch.bool, device=embs.device) + att_masks = att_masks[None, :].expand(pad_masks.shape[0], -1) + + return embs, pad_masks, att_masks + + # ── Suffix embedding ───────────────────────────────────────────── + + def embed_suffix( + self, + state: torch.Tensor, + noisy_actions: torch.Tensor, + timestep: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Build the suffix embeddings + masks: ``[state_token, action_tokens×H]``. + + AR mask layout: ``[1, 1, 0, 0, ..., 0]`` — the state token is a causal + boundary (no later token attends backwards through it onto prefix + *by mistake*), the first action token starts a new causal block, and + the rest of the action tokens attend to each other bidirectionally. + + Ref: openpi PI0Pytorch.embed_suffix + """ + model_dtype = self.state_proj.weight.dtype + state = state.to(dtype=model_dtype) + noisy_actions = noisy_actions.to(dtype=model_dtype) + device = state.device + + # State → (B, 1, W) + state_emb = self.state_proj(state)[:, None, :] + bsize = state_emb.shape[0] + + # Sinusoidal timestep → (B, W) → expand across the horizon. + time_emb = create_sinusoidal_pos_embedding( + timestep, + self.action_in_proj.out_features, + min_period=4e-3, + max_period=4.0, + device=device, + ).to(dtype=model_dtype) + + # Fuse action and time via (2W → W → W) MLP with SiLU. + action_emb = self.action_in_proj(noisy_actions) # (B, H, W) + action_time_emb = torch.cat([action_emb, time_emb[:, None, :].expand_as(action_emb)], dim=2) + action_time_emb = self.action_time_mlp_in(action_time_emb) + action_time_emb = F.silu(action_time_emb) + action_time_emb = self.action_time_mlp_out(action_time_emb) + + embs = torch.cat([state_emb, action_time_emb], dim=1) + pad_masks = torch.ones(bsize, embs.shape[1], dtype=torch.bool, device=device) + att_masks = torch.tensor( + [1] + [1] + [0] * (self.action_horizon - 1), + dtype=embs.dtype, + device=device, + )[None, :].expand(bsize, -1) + return embs, pad_masks, att_masks + + # ── Denoising step ─────────────────────────────────────────────── + + def denoise_step( + self, + state: torch.Tensor, + prefix_pad_masks: torch.Tensor, + past_key_values, + x_t: torch.Tensor, + timestep: torch.Tensor, + ) -> torch.Tensor: + """Apply one flow-matching denoising step: predict ``v_t`` from ``x_t``. + + Uses the prefix KV cache from ``sample_actions`` and only runs the + action expert. Ref: openpi PI0Pytorch.denoise_step. + """ + suffix_embs, suffix_pad_masks, suffix_att_masks = self.embed_suffix(state, x_t, timestep) + + # Build the full (B, suffix_len, prefix_len + suffix_len) boolean mask: + # * suffix queries can see every *valid* prefix key (padded camera + # slots and padded language tokens are blocked). + # * within the suffix, the state token is a causal boundary; action + # tokens attend to each other bidirectionally. + batch_size = prefix_pad_masks.shape[0] + suffix_len = suffix_pad_masks.shape[1] + prefix_len = prefix_pad_masks.shape[1] + + prefix_pad_2d_masks = prefix_pad_masks[:, None, :].expand(batch_size, suffix_len, prefix_len) + suffix_att_2d_masks = make_att_2d_masks(suffix_pad_masks, suffix_att_masks) + full_att_2d_masks = torch.cat([prefix_pad_2d_masks, suffix_att_2d_masks], dim=2) + + # Position IDs continue from where the prefix's last valid token left off. + prefix_offsets = torch.sum(prefix_pad_masks, dim=-1)[:, None] + position_ids = prefix_offsets + torch.cumsum(suffix_pad_masks, dim=1) - 1 + + full_att_2d_masks_4d = prepare_attention_masks_4d(full_att_2d_masks) + + outputs_embeds, _ = self.paligemma_with_expert.forward( + attention_mask=full_att_2d_masks_4d, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=[None, suffix_embs], + use_cache=False, + ) + + # Drop the state token, keep only the action tokens, project down. + suffix_out = outputs_embeds[1][:, -self.action_horizon :] + suffix_out = suffix_out.to(dtype=self.action_out_proj.weight.dtype) + return self.action_out_proj(suffix_out) + + # ── Full action generation ─────────────────────────────────────── + + @torch.no_grad() + def sample_actions( + self, + images: list[torch.Tensor], + image_masks: list[torch.Tensor], + lang_tokens: torch.Tensor, + lang_masks: torch.Tensor, + state: torch.Tensor, + noise: torch.Tensor | None = None, + num_steps: int | None = None, + ) -> torch.Tensor: + """Generate an action chunk via iterative flow-matching denoising. + + Convention: ``t=1`` is noise, ``t=0`` is the target — opposite of the + published π0 paper but matches both OpenPI and LeRobot. + Ref: openpi PI0Pytorch.sample_actions + """ + if num_steps is None: + num_steps = self.num_inference_steps + + bsize = state.shape[0] + device = state.device + if noise is None: + noise = torch.randn( + bsize, + self.action_horizon, + self.action_dim, + dtype=torch.float32, + device=device, + ) + + # 1. Prefix embeddings + mask building. + prefix_embs, prefix_pad_masks, prefix_att_masks = self.embed_prefix( + images, image_masks, lang_tokens, lang_masks + ) + prefix_att_2d_masks = make_att_2d_masks(prefix_pad_masks, prefix_att_masks) + prefix_position_ids = torch.cumsum(prefix_pad_masks, dim=1) - 1 + prefix_att_2d_masks_4d = prepare_attention_masks_4d(prefix_att_2d_masks) + + # 2. Forward prefix through PaliGemma LM, producing a list[(k, v)] cache. + _, past_key_values = self.paligemma_with_expert.forward( + attention_mask=prefix_att_2d_masks_4d, + position_ids=prefix_position_ids, + past_key_values=None, + inputs_embeds=[prefix_embs, None], + use_cache=True, + ) + + # 3. Euler-integrated denoising from t=1 down to t=0. + dt = -1.0 / num_steps + x_t = noise + for step in range(num_steps): + t = 1.0 + step * dt + time_tensor = torch.full((bsize,), t, dtype=torch.float32, device=device) + v_t = self.denoise_step( + state=state, + prefix_pad_masks=prefix_pad_masks, + past_key_values=past_key_values, + x_t=x_t, + timestep=time_tensor, + ) + x_t = x_t + dt * v_t + return x_t + + # ── Weight loading ─────────────────────────────────────────────── + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + """Load weights from an ``lerobot/pi0_base`` safetensors checkpoint. + + Checkpoint keys (after stripping the leading ``model.`` prefix) look + like ``paligemma_with_expert.paligemma..*``. Modern transformers + nests those sub-modules one level deeper (``paligemma.model.``), + and PaliGemma ties ``lm_head.weight`` with ``embed_tokens.weight`` at + ``post_init`` time. Two remap rules make the load lossless: + + 1. ``paligemma.{vision_tower,multi_modal_projector,language_model}.*`` + → ``paligemma.model.{...}.*`` + 2. ``paligemma.lm_head.weight`` + → ``paligemma.model.language_model.embed_tokens.weight`` + (the checkpoint does not store ``embed_tokens.weight`` on its + own — only the tied ``lm_head.weight`` copy — and + ``PaliGemmaForConditionalGeneration`` does not register + ``lm_head.weight`` as a Parameter when tied, so without this + rule the language embedding would silently remain at its + random init.) + + On any remaining mismatch the loader logs a warning listing a sample + of the unmatched checkpoint keys and model params; downstream parity + tests surface these as a hard failure. + """ + params_dict = dict(self.named_parameters()) + buffers_dict = dict(self.named_buffers()) + + _PALIGEMMA_SUBMODULES = ( + "vision_tower", + "multi_modal_projector", + "language_model", + ) + + def _remap(name: str) -> str: + # Strip the leading "model." that LeRobot's PI0Policy wrapper adds. + if name.startswith("model."): + name = name[len("model.") :] + + # Historical MLP alias used by some early π0 checkpoints. + if name.startswith("time_mlp_in."): + name = "action_time_mlp_in." + name[len("time_mlp_in.") :] + elif name.startswith("time_mlp_out."): + name = "action_time_mlp_out." + name[len("time_mlp_out.") :] + + # Nested PaliGemma layout. + for sub in _PALIGEMMA_SUBMODULES: + flat = f"paligemma_with_expert.paligemma.{sub}." + nested = f"paligemma_with_expert.paligemma.model.{sub}." + if name.startswith(flat) and not name.startswith(nested): + return nested + name[len(flat) :] + + # Tied lm_head → embed_tokens redirect (see docstring). + if name == "paligemma_with_expert.paligemma.lm_head.weight": + return "paligemma_with_expert.paligemma.model.language_model.embed_tokens.weight" + + return name + + def _fix_vision_tower(name: str) -> str: + """Reconcile the SigLIP vision-tower nesting across transformers + versions. transformers ≤5.3 wraps the encoder in an extra + ``vision_tower.vision_model.*`` module; ≥5.4 flattens it to + ``vision_tower.*``. The ``lerobot/pi0_base`` checkpoint uses the + ``vision_model`` layout, so on newer transformers we must drop that + infix (else 437 vision params silently load nothing). Adaptive: + only rewrite when the rewritten key actually exists on the model. + """ + vt = "paligemma_with_expert.paligemma.model.vision_tower." + if not name.startswith(vt): + return name + rest = name[len(vt) :] + if name in params_dict or name in buffers_dict: + return name # model already expects this exact layout + if rest.startswith("vision_model."): + # Checkpoint has the infix; model (≥5.4) doesn't → strip it. + candidate = vt + rest[len("vision_model.") :] + else: + # Checkpoint lacks the infix; model (≤5.3) has it → insert it. + candidate = vt + "vision_model." + rest + return candidate if (candidate in params_dict or candidate in buffers_dict) else name + + loaded = 0 + skipped: list[str] = [] + filled_params: set = set() + for name, loaded_weight in weights: + mapped = _fix_vision_tower(_remap(name)) + if mapped in params_dict: + param = params_dict[mapped] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded += 1 + filled_params.add(mapped) + elif mapped in buffers_dict: + buffers_dict[mapped].copy_(loaded_weight) + loaded += 1 + filled_params.add(mapped) + else: + skipped.append(mapped) + + # Reverse audit: any model param that got no checkpoint tensor at all + # would be running with random init. ``rotary_emb.inv_freq`` and + # friends are config-derived buffers, not stored in the checkpoint. + missing_params: list[str] = [] + for pname in params_dict: + if pname in filled_params: + continue + if "rotary_emb" in pname or pname.endswith(".inv_freq"): + continue + missing_params.append(pname) + + if missing_params or skipped: + parts: list[str] = [] + if skipped: + parts.append(f"{len(skipped)} checkpoint key(s) had no home in the model (first 5: {skipped[:5]})") + if missing_params: + parts.append(f"{len(missing_params)} model param(s) received NO weight (first 5: {missing_params[:5]})") + logger.warning( + "π0 load_weights: %d tensors loaded — %s.", + loaded, + "; ".join(parts), + ) + else: + logger.info( + "π0 load_weights: %d tensors loaded, 0 skipped, 0 missing.", + loaded, + ) + return filled_params diff --git a/vllm_omni/diffusion/models/pi0/pipeline_pi0.py b/vllm_omni/diffusion/models/pi0/pipeline_pi0.py new file mode 100644 index 000000000..0877df4d6 --- /dev/null +++ b/vllm_omni/diffusion/models/pi0/pipeline_pi0.py @@ -0,0 +1,277 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""π0 (Pi-Zero) VLA pipeline for vllm-omni. + +Entry point for ``DiffusionEngine.step() → pipeline.forward(req)``. Mirrors the +DreamZero contract: the pipeline owns ALL preprocessing. It reads the raw robot +observation from ``req.sampling_params.extra_args["robot_obs"]`` (delivered by +the OpenPI realtime serving layer), builds model inputs, runs flow-matching +denoising, and returns ``DiffusionOutput(output={"actions": ndarray})`` — which +``diffusion_engine`` promotes to ``multimodal_output["actions"]`` for the client. + +π0 is stateless across calls (no KV reuse), so ``session_id`` / ``reset`` from +the OpenPI protocol are accepted but ignored. +""" + +from __future__ import annotations + +import os + +import numpy as np +import torch +from torch import nn +from vllm.logger import init_logger + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) +from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.models.pi0.config import Pi0Config +from vllm_omni.diffusion.models.pi0.modeling_pi0 import Pi0ForActionPrediction +from vllm_omni.diffusion.models.pi0.processor_pi0 import build_model_inputs +from vllm_omni.diffusion.request import OmniDiffusionRequest + +logger = init_logger(__name__) + +# Default tokenizer for the PaliGemma prefix (matches LeRobot Pi0). +DEFAULT_PI0_TOKENIZER = "google/paligemma-3b-pt-224" + + +def _pi0_post_process(x): + """Module-level identity post-process (picklable across the orchestrator's + multiprocess boundary — a local closure is not).""" + return x + + +def get_pi0_post_process_func(od_config: OmniDiffusionConfig): + """π0 returns actions directly; post-processing is identity. The diffusion + engine resolves this via the registry (``_DIFFUSION_POST_PROCESS_FUNCS``) and + applies it engine-side, so the pipeline does NOT attach it to DiffusionOutput. + """ + del od_config + return _pi0_post_process + + +# Single-stage diffusion topology for online OpenPI serving. +# +# π0 is one diffusion stage (robot observation -> action chunk via flow +# matching). It is registered in ``OMNI_PIPELINES`` (rather than relying on the +# default single-stage fallback) because π0 is *online-served* via +# ``vllm serve --deploy-config pi0.yaml``; that path +# (``OmniStagePipelineFactory._create_from_registry``) requires a registered +# pipeline so the deploy yaml's ``policy_server_config`` reaches the OpenPI +# websocket layer. Offline-only models (e.g. ``internvla_a1``) need no topology. +PI0_PIPELINE = PipelineConfig( + model_type="pi0", + model_arch="Pi0Pipeline", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="diffusion", + execution_type=StageExecutionType.DIFFUSION, + input_sources=(), + final_output=True, + final_output_type="action", + model_arch="Pi0Pipeline", + ), + ), +) + + +class Pi0Pipeline(nn.Module): + """π0 VLA pipeline: raw robot obs → continuous action chunk. + + Registered as ``"Pi0Pipeline"`` in the diffusion registry. Weights are + self-loaded in ``__init__`` from the checkpoint's ``model.safetensors`` via + the kernel's ``load_weights`` (which handles the LeRobot key remaps). + """ + + def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""): + super().__init__() + self.od_config = od_config + self.prefix = prefix + # Resolve od_config.model to a LOCAL directory. A bare HF repo id (e.g. + # the documented ``lerobot/pi0_base``) must be snapshot-downloaded first; + # otherwise config/tokenizer/weights would silently fall back to random + # init (this pipeline self-loads from a local dir and has no + # ``weights_sources`` for the framework loader to download from). + self.model_dir = self._resolve_model_dir(od_config.model) + self.config = self._build_config(od_config) + + custom_args = od_config.custom_pipeline_args or {} + self.tokenizer_source = str(custom_args.get("tokenizer", self._resolve_tokenizer_source())) + + # Torch dtype/device from od_config. + self._torch_dtype = self._resolve_dtype(od_config) + self._device = self._resolve_device(od_config) + + self.tokenizer = self._load_tokenizer() + self.model = self._initialize_model() + + # ------------------------------------------------------------------ + # Construction helpers + # ------------------------------------------------------------------ + @staticmethod + def _resolve_model_dir(model: str | None) -> str | None: + """Return a local directory for ``model``; download an HF repo id if needed. + + Limits the snapshot to the files π0 actually reads (config, weights, + tokenizer) so a repo id doesn't pull unrelated artifacts. + """ + if not model: + return None + if os.path.isdir(model): + return model + from huggingface_hub import snapshot_download + + return snapshot_download( + repo_id=model, + allow_patterns=["*.json", "*.safetensors", "*.model", "tokenizer*"], + ) + + def _build_config(self, od_config: OmniDiffusionConfig) -> Pi0Config: + """Build Pi0Config from deploy-yaml model_config, falling back to the + checkpoint's config.json (raw LeRobot format).""" + if od_config.model_config: + config = Pi0Config.from_model_config(dict(od_config.model_config)) + # The deploy yaml is authoritative, but must not silently drop + # checkpoint-derived fields it omits. Backfill the camera order and + # any normalization stats the yaml didn't specify. + if self.model_dir and (not config.image_feature_keys or config.norm_stats is None): + ckpt = Pi0Config.from_pretrained(self.model_dir) + if not config.image_feature_keys: + config.image_feature_keys = ckpt.image_feature_keys + if not config.input_features: + config.input_features = ckpt.input_features + if config.norm_stats is None: + config.norm_stats = ckpt.norm_stats + return config + if self.model_dir: + return Pi0Config.from_pretrained(self.model_dir) + return Pi0Config() + + def _resolve_tokenizer_source(self) -> str: + """Prefer the checkpoint dir if it ships tokenizer files; else PaliGemma.""" + if self.model_dir and os.path.isdir(self.model_dir): + if os.path.exists(os.path.join(self.model_dir, "tokenizer_config.json")): + return self.model_dir + return DEFAULT_PI0_TOKENIZER + + @staticmethod + def _resolve_dtype(od_config: OmniDiffusionConfig) -> torch.dtype: + dt = od_config.dtype + if isinstance(dt, torch.dtype): + return dt + return getattr(torch, str(dt).split(".")[-1], torch.float32) + + @staticmethod + def _resolve_device(od_config: OmniDiffusionConfig) -> torch.device: + from vllm_omni.diffusion.distributed.utils import get_local_device + + try: + return get_local_device() + except Exception: # noqa: BLE001 + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + + def _load_tokenizer(self): + from transformers import AutoTokenizer + + return AutoTokenizer.from_pretrained(self.tokenizer_source) + + def has_real_checkpoint(self) -> bool: + return bool(self.model_dir) and os.path.exists(os.path.join(self.model_dir, "model.safetensors")) + + def _initialize_model(self) -> Pi0ForActionPrediction: + model = Pi0ForActionPrediction(self.config) + if self.has_real_checkpoint(): + self._load_checkpoint(model) + else: + logger.info("Pi0Pipeline: no model.safetensors under %s; using random init.", self.model_dir) + model.to(device=self._device, dtype=self._torch_dtype) + model.eval() + return model + + def _load_checkpoint(self, model: Pi0ForActionPrediction) -> None: + import safetensors.torch + + path = os.path.join(self.model_dir, "model.safetensors") + logger.info("Pi0Pipeline: loading π0 weights from %s", path) + state = safetensors.torch.load_file(path) + model.load_weights(list(state.items())) + + # ------------------------------------------------------------------ + # Framework weight-loading hook + # ------------------------------------------------------------------ + def load_weights(self, weights=()): # noqa: D401 + """No-op for the diffusion loader: π0 self-loads its checkpoint in + ``__init__`` (the kernel's ``load_weights`` handles the LeRobot remaps). + We expose no ``weights_sources``, so the loader passes an empty iterator + here; returning ``None`` skips its strict unloaded-weights check. + """ + for _ in weights: + pass + return None + + # ------------------------------------------------------------------ + # Inference + # ------------------------------------------------------------------ + @torch.inference_mode() + def forward(self, req: OmniDiffusionRequest, **kwargs) -> DiffusionOutput: + extra_args = getattr(req.sampling_params, "extra_args", None) or {} + robot_obs = extra_args.get("robot_obs") + + if robot_obs is None: + # Dummy warmup path (no obs): return zeros so engine warmup/capture + # doesn't crash. Mirrors DreamZero's dummy-run handling. + first_prompt = req.prompts[0] if req.prompts else "" + prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "") + num_steps = getattr(req.sampling_params, "num_inference_steps", None) + if prompt == "dummy run" or num_steps == 1: + logger.info("Pi0Pipeline: dummy warmup request without robot_obs — returning zeros.") + return DiffusionOutput( + output={ + "actions": np.zeros( + (self.config.chunk_size, self.config.max_action_dim), + dtype=np.float32, + ) + }, + ) + return DiffusionOutput( + error="Pi0Pipeline.forward requires sampling_params.extra_args['robot_obs'].", + ) + + images, image_masks, lang_tokens, lang_masks, state = build_model_inputs( + robot_obs, self.config, self.tokenizer, self._device + ) + + # State normalization (identity for pi0_base) inside the model. + state = self.model._normalize_state(state) + + noise = extra_args.get("noise") + if noise is not None and not isinstance(noise, torch.Tensor): + noise = torch.as_tensor(noise, dtype=torch.float32, device=self._device) + elif isinstance(noise, torch.Tensor): + noise = noise.to(device=self._device, dtype=torch.float32) + + num_steps = extra_args.get("num_inference_steps") + + actions = self.model.sample_actions( + images=images, + image_masks=image_masks, + lang_tokens=lang_tokens, + lang_masks=lang_masks, + state=state, + noise=noise, + num_steps=num_steps, + ) + actions = self.model._unnormalize_actions(actions) + + # (B=1, horizon, action_dim) → (horizon, action_dim) numpy for the wire. + actions_np = actions.squeeze(0).float().cpu().numpy() + + # Note: post-processing is applied engine-side via the registry + # (_DIFFUSION_POST_PROCESS_FUNCS), so we don't attach it here (a local + # closure wouldn't survive the orchestrator's multiprocess pickling). + return DiffusionOutput(output={"actions": actions_np}) diff --git a/vllm_omni/diffusion/models/pi0/processor_pi0.py b/vllm_omni/diffusion/models/pi0/processor_pi0.py new file mode 100644 index 000000000..0133e5729 --- /dev/null +++ b/vllm_omni/diffusion/models/pi0/processor_pi0.py @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Preprocessing for the π0 VLA model. + +Converts a raw robot observation (multi-camera images + language instruction + +proprioceptive state) into the tensors that ``Pi0ForActionPrediction.sample_actions`` +consumes: + + * images : list of ``(1, 3, 224, 224)`` float tensors in ``[-1, 1]``, + one per camera slot (real cameras first, then ``-1``-filled + padding) in the checkpoint's ``image_feature_keys`` order + * image_masks : list of ``(1,)`` bool tensors, ``True`` = real camera + * lang_tokens : ``(1, L)`` long, PaliGemma tokenizer, ``\\n``-terminated + * lang_masks : ``(1, L)`` bool + * state : ``(1, max_state_dim)`` float32, zero-padded + +Image/prompt preprocessing matches OpenPI and LeRobot bit-for-bit — see the +parity test (``test_pi0_parity.py``). A set of stateless helpers the diffusion +pipeline calls directly (the DreamZero contract: the pipeline owns its +preprocessing). + +Reference: + - OpenPI: openpi/src/openpi/shared/image_tools.py (resize_with_pad) + - OpenPI: openpi/src/openpi/models_pytorch/preprocessing_pytorch.py + - LeRobot: lerobot/src/lerobot/policies/pi0/modeling_pi0.py (Pi0NewLineProcessor) +""" + +from __future__ import annotations + +import logging +from typing import Any + +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image + +logger = logging.getLogger(__name__) + +# Defaults straight from the π0 reference configs. +PI0_IMAGE_SIZE = 224 # openpi/models/model.py IMAGE_RESOLUTION +PI0_NUM_IMAGE_TOKENS = 256 # SigLIP So400m/14 on 224×224 → (224/14)**2 +PI0_IMAGE_TOKEN_INDEX = 257152 # openpi/models_pytorch/gemma_pytorch.py +PI0_MAX_CAMERAS = 3 # openpi/models/model.py (3 camera slots) +PI0_MAX_TOKEN_LEN = 48 # openpi/models/pi0_config.py + + +def resize_with_pad( + images: torch.Tensor, + target_height: int, + target_width: int, + mode: str = "bilinear", +) -> torch.Tensor: + """Resize ``(B, C, H, W)`` images to the target shape, preserving aspect + ratio with -1 padding on the short side. + + Matches openpi ``image_tools.resize_with_pad_torch`` — the clamp to + [-1, 1] is what lets the padded region blend with SigLIP-normalized + pixels without adding signal at the boundary. + """ + if images.ndim != 4: + raise ValueError(f"Expected 4-D (B,C,H,W), got {images.ndim}-D") + _, _, cur_h, cur_w = images.shape + ratio = max(cur_w / target_width, cur_h / target_height) + rh, rw = int(cur_h / ratio), int(cur_w / ratio) + align_corners = False if mode == "bilinear" else None + resized = F.interpolate(images, size=(rh, rw), mode=mode, align_corners=align_corners) + resized = resized.clamp(-1.0, 1.0) + ph, rem_h = divmod(target_height - rh, 2) + pw, rem_w = divmod(target_width - rw, 2) + return F.pad(resized, (pw, pw + rem_w, ph, ph + rem_h), value=-1.0) + + +def pil_image_to_tensor(image: Image.Image) -> torch.Tensor: + """PIL → ``(1, C, H, W)`` float32 in ``[-1, 1]`` (SigLIP normalization).""" + if image.mode != "RGB": + image = image.convert("RGB") + arr = np.array(image, dtype=np.float32) / 255.0 * 2.0 - 1.0 + return torch.from_numpy(arr).permute(2, 0, 1).unsqueeze(0) + + +class Pi0ImageProcessor: + """Minimal image preprocessor: image → normalized + padded ``[-1,1]`` tensor.""" + + def __init__(self, image_size: int = PI0_IMAGE_SIZE): + self.image_size = image_size + + def preprocess_single(self, image: Any) -> torch.Tensor: + """Accept a PIL image, an HWC uint8/float ndarray, or a CHW tensor and + return a ``(1, 3, image_size, image_size)`` float tensor in ``[-1, 1]``.""" + t = self._to_tensor(image) + if t.shape[2] != self.image_size or t.shape[3] != self.image_size: + t = resize_with_pad(t, self.image_size, self.image_size) + return t + + def _to_tensor(self, image: Any) -> torch.Tensor: + if isinstance(image, Image.Image): + return pil_image_to_tensor(image) + if isinstance(image, np.ndarray): + arr = image + # HWC → normalize uint8/[0,255] to [-1,1]; assume already [-1,1] if float. + if arr.ndim == 3 and arr.shape[-1] in (1, 3): + if np.issubdtype(arr.dtype, np.integer) or arr.max() > 1.0: + arr = arr.astype(np.float32) / 255.0 * 2.0 - 1.0 + else: + arr = arr.astype(np.float32) + return torch.from_numpy(arr).permute(2, 0, 1).unsqueeze(0) + return torch.as_tensor(arr, dtype=torch.float32) + if isinstance(image, torch.Tensor): + t = image + if t.ndim == 3: # (C, H, W) → (1, C, H, W) + t = t.unsqueeze(0) + return t.to(dtype=torch.float32) + raise TypeError(f"Unsupported image type for π0 preprocessing: {type(image)}") + + def make_empty_image(self) -> torch.Tensor: + """Fill tensor for an unused camera slot — pure -1, matches OpenPI/LeRobot.""" + return torch.full((1, 3, self.image_size, self.image_size), -1.0) + + +def tokenize_prompt(tokenizer, text: str, max_token_len: int = PI0_MAX_TOKEN_LEN): + """Return ``(input_ids, attention_mask)`` lists, length ``max_token_len``. + + Matches LeRobot's ``Pi0NewLineProcessor``: append ``\\n`` if the caller + didn't, then run the PaliGemma tokenizer with right-padding. + """ + prompt = text or "" + if not prompt.endswith("\n"): + prompt = prompt + "\n" + enc = tokenizer( + prompt, + padding="max_length", + max_length=max_token_len, + truncation=True, + add_special_tokens=True, + return_tensors=None, + ) + return list(enc["input_ids"]), list(enc["attention_mask"]) + + +def _extract_images(robot_obs: dict, config) -> dict[str, Any]: + """Pull a ``{feature_key: image}`` map out of a raw robot obs. + + Accepts two shapes: + * ``robot_obs["images"]`` is a dict keyed by camera name; or + * flat ``observation/`` style keys on ``robot_obs`` itself. + Keys are translated through ``config.image_key_map`` (raw obs key → + checkpoint feature key) so serving wire names map onto ``input_features``. + """ + images = robot_obs.get("images") + if not isinstance(images, dict): + # Fall back to flat keys (anything that isn't a known scalar field). + images = {k: v for k, v in robot_obs.items() if k not in ("state", "prompt", "session_id", "reset", "images")} + key_map = getattr(config, "image_key_map", None) or {} + return {key_map.get(k, k): v for k, v in images.items()} + + +def build_model_inputs(robot_obs: dict, config, tokenizer, device: torch.device): + """Convert a raw robot observation into ``sample_actions`` inputs. + + Reproduces the LeRobot camera ordering exactly: iterate + ``config.image_feature_keys`` (the ordered camera identities from the + checkpoint's ``input_features``); for each, use the supplied image (mask + ``True``) or a ``-1``-filled empty image (mask ``False``). Then tokenize + the prompt and zero-pad/truncate the state to ``max_state_dim``. + + Returns ``(images, image_masks, lang_tokens, lang_masks, state)`` with a + leading batch dim of 1 on the tensors. + """ + image_size = int(config.image_resolution[0]) + img_proc = Pi0ImageProcessor(image_size=image_size) + + feature_keys = config.image_feature_keys or [] + if not feature_keys: + # No declared camera order — fall back to whatever the obs provides, + # capped at max_cameras (preserves insertion order). + provided = _extract_images(robot_obs, config) + feature_keys = list(provided.keys())[: config.max_cameras] + + obs_images = _extract_images(robot_obs, config) + + images: list[torch.Tensor] = [] + image_masks: list[torch.Tensor] = [] + for key in feature_keys[: config.max_cameras]: + img = obs_images.get(key) + if img is not None: + tensor = img_proc.preprocess_single(img).to(device=device) + mask = True + else: + tensor = img_proc.make_empty_image().to(device=device) + mask = False + images.append(tensor) + image_masks.append(torch.tensor([mask], dtype=torch.bool, device=device)) + + # Pad up to max_cameras with empty slots if the checkpoint declares fewer + # cameras than the model attends to. + while len(images) < config.max_cameras: + images.append(img_proc.make_empty_image().to(device=device)) + image_masks.append(torch.tensor([False], dtype=torch.bool, device=device)) + + prompt = robot_obs.get("prompt", "") or "" + ids, attn = tokenize_prompt(tokenizer, prompt, config.tokenizer_max_length) + lang_tokens = torch.tensor([ids], dtype=torch.long, device=device) + lang_masks = torch.tensor([attn], dtype=torch.bool, device=device) + + state = _build_state(robot_obs.get("state"), config.max_state_dim, device) + + return images, image_masks, lang_tokens, lang_masks, state + + +def _build_state(raw_state, max_state_dim: int, device: torch.device) -> torch.Tensor: + """Zero-pad / truncate the raw state to ``(1, max_state_dim)`` float32.""" + if raw_state is None: + return torch.zeros(1, max_state_dim, dtype=torch.float32, device=device) + state = torch.as_tensor(raw_state, dtype=torch.float32, device=device) + if state.dim() == 1: + state = state[None, :] + if state.shape[-1] < max_state_dim: + state = F.pad(state, (0, max_state_dim - state.shape[-1])) + elif state.shape[-1] > max_state_dim: + state = state[..., :max_state_dim] + return state diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index 2c4bffb4c..47063bc4a 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -181,6 +181,11 @@ _DIFFUSION_MODELS = { "pipeline_gr00t", "Gr00tN1d7Pipeline", ), + "Pi0Pipeline": ( + "pi0", + "pipeline_pi0", + "Pi0Pipeline", + ), "LongCatImageEditPipeline": ( "longcat_image", "pipeline_longcat_image_edit", @@ -358,6 +363,11 @@ _NO_CACHE_ACCELERATION = { # Pipelines that do not support cache acceleration (cache_dit / tea_cache). "NextStep11Pipeline", "AudioXPipeline", + # π0 is a flow-matching VLA with a self-contained sample_actions loop and no + # DiT-style ``.transformer`` block list, so cache_dit / tea_cache cannot apply + # to it; list it here so a stray cache_backend override disables gracefully + # instead of erroring. + "Pi0Pipeline", "LingBotWorldCausalDMDPipeline", } @@ -560,6 +570,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = { "LancePipeline": "get_lance_post_process_func", "MingImagePipeline": "get_ming_image_post_process_func", "InternVLAA1Pipeline": "get_internvla_a1_post_process_func", + "Pi0Pipeline": "get_pi0_post_process_func", "LongCatImageEditPipeline": "get_longcat_image_post_process_func", "StableDiffusion3Pipeline": "get_sd3_image_post_process_func", "FluxKontextPipeline": "get_flux_kontext_post_process_func", From 170d498771dc39a5cb90b1eacddc1337b6bad6f9 Mon Sep 17 00:00:00 2001 From: TJian Date: Mon, 17 Aug 2026 14:46:30 +0800 Subject: [PATCH 046/267] [ROCm] [CI] Fix cpu test on rocm (#6267) Signed-off-by: tjtanaa --- vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py b/vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py index a205fce6b..50d413ee7 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py @@ -203,6 +203,11 @@ def _apply_unquantized_gemm_with_weight( weight: torch.Tensor, bias: torch.Tensor | None, ) -> torch.Tensor: + # GEMM dispatch is selected from the host platform, not the tensor device. + # In CPU tests on a ROCm host that selects rocm_unquantized_gemm, which has + # no CPU kernel. Keep that combination on PyTorch's portable implementation. + if current_platform.is_rocm() and input_.device.type == "cpu": + return F.linear(input_, weight, bias) if envs.VLLM_BATCH_INVARIANT and current_platform.is_cuda_alike(): return linear_batch_invariant(input_, weight, bias) return dispatch_unquantized_gemm()(layer, input_, weight, bias) From 5d09cf27a98bb104506ee842ca81e0e76e47dc92 Mon Sep 17 00:00:00 2001 From: Zhou248 <62542847+Zhou248@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:09:51 +0800 Subject: [PATCH 047/267] [MiniCPM-o][NPU] Enable MiniCPM-o Code2Wav NPUGraph replay (#5604) Signed-off-by: Zhou248 <62542847+Zhou248@users.noreply.github.com> --- .buildkite/npu/test-npu-ready.yml | 11 + recipes/OpenBMB/MiniCPM-o-4_5.md | 39 +- .../models/minicpmo_4_5/test_pipeline.py | 25 ++ tests/platforms/npu/test_graph_tools.py | 333 ++++++++++++++++++ .../npu/test_minicpmo_code2wav_npugraph.py | 143 ++++++++ .../npu/test_step_audio2_token2wav.py | 69 ++++ vllm_omni/deploy/minicpmo_4_5.yaml | 6 + vllm_omni/deploy/minicpmo_4_5_2gpu.yaml | 6 + vllm_omni/deploy/minicpmo_4_5_3gpu.yaml | 18 +- vllm_omni/platforms/npu/__init__.py | 15 +- vllm_omni/platforms/npu/graph_tools.py | 183 ++++++++++ .../npu/models/cosyvoice2_dit_attn.py | 23 +- .../npu/models/minicpmo_4_5_code2wav.py | 262 ++++++++++++++ .../npu/models/step_audio2_token2wav.py | 22 +- vllm_omni/platforms/npu/platform.py | 4 + 15 files changed, 1129 insertions(+), 30 deletions(-) create mode 100644 tests/platforms/npu/test_graph_tools.py create mode 100644 tests/platforms/npu/test_minicpmo_code2wav_npugraph.py create mode 100644 vllm_omni/platforms/npu/graph_tools.py create mode 100644 vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py diff --git a/.buildkite/npu/test-npu-ready.yml b/.buildkite/npu/test-npu-ready.yml index f23ba7120..dbb827137 100644 --- a/.buildkite/npu/test-npu-ready.yml +++ b/.buildkite/npu/test-npu-ready.yml @@ -1,4 +1,15 @@ steps: + - label: "Platforms Test" + key: ready-platforms-test + depends_on: upload-ready-pipeline + mirror_hardwares: a3_npu_2 + commands: + - >- + pytest -s -v + tests/platforms/npu/test_minicpmo_code2wav_npugraph.py + -m 'core_model and npu and A3 and omni' + --run-level 'core_model' + - group: ":card_index_dividers: Omni Model Test" key: ready-omni-group depends_on: upload-ready-pipeline diff --git a/recipes/OpenBMB/MiniCPM-o-4_5.md b/recipes/OpenBMB/MiniCPM-o-4_5.md index 99b064310..86f0883f7 100644 --- a/recipes/OpenBMB/MiniCPM-o-4_5.md +++ b/recipes/OpenBMB/MiniCPM-o-4_5.md @@ -71,6 +71,35 @@ single-GPU entry point; `minicpmo_4_5_2gpu.yaml` is the recommended two-GPU profile. The removed fused two-stage implementation is not retained as a fallback because it would duplicate state machines and correctness paths. +### Graph execution + +Graph boundaries follow each stage's state model: + +| Stage | Graph mode on Ascend | Eager boundary | +| --- | --- | --- | +| 0 Thinker | vLLM `PIECEWISE` | multimodal preprocessing and output routing | +| 1 Talker | vLLM `PIECEWISE` | conditioning preprocessing, codec sampling, request-state updates | +| 2 Code2Wav | inner exact-shape NPUGraph for the CFM DiT estimator | encoder, timestep embedding, HiFT/RNG, request parsing, stream-state commit | + +Stage 2 keeps `enforce_eager: true` for the generation runner because its +Python request metadata and per-request cache dictionaries are not valid +outer-graph inputs. The backend captures only the deterministic CFM DiT +estimator, with an exact graph key for each tensor shape. Timestep embedding +stays eager because the upstream implementation creates a CPU frequency tensor; +HiFT stays eager because it generates random phase. The graph cache stores up +to 32 entries by default, after which unseen shapes run eagerly. + +An ACL graph capture failure is fatal to that Stage-2 process because older +torch-npu releases can leave allocator and RNG capture state invalid. Restart +the service after a capture failure. To run without capture, set +`code2wav_enable_npu_graph: false` under the Stage-2 +`platforms.npu.stages[].additional_config` block before startup. Tune the cache +limit there with `code2wav_max_npu_graphs`. Graph mode also requires +`ASCEND_LAUNCH_BLOCKING` to be unset or set to `0`. + +The inner NPUGraph is independent of the outer runner setting, so do not use a +global `--enforce-eager` override when Stage 0/1 `PIECEWISE` replay is desired. + ## GPU ### 1 x GPU (default — single command) @@ -211,9 +240,9 @@ speech output (TTS)"** checkbox on / off. model processes still share one CUDA device. - `--trust-remote-code` is required — the HF repo ships a custom `MiniCPMO` config / model class. -- Stage 0 Thinker and Stage 1 Talker enable vLLM CUDA Graphs. Stage 2 remains - eager because its request-owned Flow/HiFT caches and variable chunk/cache - shapes are not yet exposed through a static exact-shape graph wrapper. +- Stage 0 Thinker and Stage 1 Talker enable vLLM CUDA Graphs. Stage 2 keeps its + request orchestration eager while using inner CFM and HiFT CUDA Graphs. On + Ascend, the exact-shape CFM DiT estimator instead uses inner NPUGraph replay. - All default stages use `max_num_seqs: 4` to reduce cross-process GPU contention. Talker AR state and Code2Wav caches are request-owned; Code2Wav batches only @@ -224,8 +253,8 @@ speech output (TTS)"** checkbox on / off. - `StageRequestStats.batch_size` is a request-scoped placeholder, not the scheduler's execution batch. - Single-GPU co-location trades throughput for hardware density: Stage 0/1 - CUDA Graph replay and eager Stage 2 vocoder kernels compete across three - CUDA contexts. Use the 8x4090 config or a custom multi-GPU mapping for + CUDA Graph replay and Stage 2 vocoder kernels compete across three CUDA + contexts. Use the 8x4090 config or a custom multi-GPU mapping for throughput-sensitive serving. ### 8 x RTX 4090 24GB (consumer-GPU layout) diff --git a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py index 148f28b16..bbc011284 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py +++ b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py @@ -24,6 +24,7 @@ from vllm_omni.config.pipeline_registry import OMNI_PIPELINES from vllm_omni.config.stage_config import ( PipelineConfig, StageExecutionType, + _apply_platform_overrides, load_deploy_config, merge_pipeline_deploy, ) @@ -184,6 +185,30 @@ class TestDeployTopology: 0.35, ] + @pytest.mark.parametrize( + "filename", + [ + "minicpmo_4_5.yaml", + "minicpmo_4_5_2gpu.yaml", + "minicpmo_4_5_3gpu.yaml", + ], + ) + def test_npu_graph_configuration_is_stage_scoped(self, filename: str) -> None: + deploy = load_deploy_config(_DEPLOY_DIR / filename) + connector_extra = deploy.connectors["connector_of_shared_memory"]["extra"] + assert "code2wav_enable_npu_graph" not in connector_extra + assert "code2wav_max_npu_graphs" not in connector_extra + + deploy = _apply_platform_overrides(deploy, platform="npu") + stages = merge_pipeline_deploy(OMNI_PIPELINES[deploy.pipeline], deploy) + assert stages[0].yaml_engine_args["compilation_config"]["cudagraph_mode"] == "PIECEWISE" + assert stages[1].yaml_engine_args["compilation_config"]["cudagraph_mode"] == "PIECEWISE" + assert stages[2].yaml_engine_args["enforce_eager"] is True + assert stages[2].yaml_engine_args["additional_config"] == { + "code2wav_enable_npu_graph": True, + "code2wav_max_npu_graphs": 32, + } + def test_pipeline_exposes_full_and_async_payload_hooks(self) -> None: pipeline = OMNI_PIPELINES[_PIPELINE_KEY] talker = pipeline.get_stage(1) diff --git a/tests/platforms/npu/test_graph_tools.py b/tests/platforms/npu/test_graph_tools.py new file mode 100644 index 000000000..7ec36dfe9 --- /dev/null +++ b/tests/platforms/npu/test_graph_tools.py @@ -0,0 +1,333 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from contextlib import contextmanager +from types import SimpleNamespace + +import pytest +import torch + +from vllm_omni.platforms.npu.graph_tools import ( + CapturedDeviceGraph, + NPUExactGraphRunner, +) +from vllm_omni.platforms.npu.models import minicpmo_4_5_code2wav as code2wav_patch + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_captured_graph_replay_clones_persistent_outputs(): + static_input = torch.zeros(1) + static_output = torch.zeros(1) + + class _Graph: + def replay(self): + static_output.copy_(static_input * 2) + + graph = CapturedDeviceGraph( + graph=_Graph(), + static_inputs=(static_input,), + static_outputs=(static_output,), + ) + + first = graph.replay((torch.tensor([2.0]),))[0] + second = graph.replay((torch.tensor([3.0]),))[0] + + torch.testing.assert_close(first, torch.tensor([4.0])) + torch.testing.assert_close(second, torch.tensor([6.0])) + assert first.data_ptr() != second.data_ptr() + + +def test_capture_uses_vllm_global_graph_pool(monkeypatch): + runner = NPUExactGraphRunner() + global_pool = object() + seen_pools = [] + synchronizations = 0 + + class _Graph: + def replay(self): + pass + + @contextmanager + def graph_context(graph, *, pool): + del graph + seen_pools.append(pool) + yield + + def synchronize(): + nonlocal synchronizations + synchronizations += 1 + + fake_npu = SimpleNamespace( + NPUGraph=_Graph, + graph=graph_context, + synchronize=synchronize, + ) + monkeypatch.setattr(torch, "npu", fake_npu, raising=False) + + import vllm.platforms + + monkeypatch.setattr( + vllm.platforms, + "current_platform", + SimpleNamespace(get_global_graph_pool=lambda: global_pool), + ) + + captured = runner.capture( + (torch.tensor([2.0]),), + lambda value: (value + 1,), + ) + + assert seen_pools == [global_pool] + assert synchronizations == 2 + torch.testing.assert_close(captured.static_outputs[0], torch.tensor([3.0])) + + +def test_exact_signature_dispatch_captures_then_replays(monkeypatch): + runner = NPUExactGraphRunner(max_graphs=2) + replay_count = 0 + + class _FunctionalGraph: + def __init__(self, compute): + self.compute = compute + + def replay(self, inputs): + nonlocal replay_count + replay_count += 1 + return tuple(output.detach().clone() for output in self.compute(*inputs)) + + monkeypatch.setattr(runner, "_eligible", lambda inputs: True) + monkeypatch.setattr( + runner, + "capture", + lambda inputs, compute: _FunctionalGraph(compute), + ) + + def compute(value): + return (value * 2,) + + first = runner.run("unit", (torch.tensor([2.0]),), (False,), compute) + second = runner.run("unit", (torch.tensor([3.0]),), (False,), compute) + + torch.testing.assert_close(first[0], torch.tensor([4.0])) + torch.testing.assert_close(second[0], torch.tensor([6.0])) + assert runner.stats == {"captures": 1, "failed": 0, "hits": 1} + assert replay_count == 1 + + +def test_capture_failure_is_fatal_for_stage_process(monkeypatch): + runner = NPUExactGraphRunner( + component_name="test component", + disable_config_hint="disable the test graph", + ) + captures = 0 + computes = 0 + + monkeypatch.setattr(runner, "_eligible", lambda inputs: True) + + def fail_capture(inputs, compute): + nonlocal captures + captures += 1 + raise RuntimeError("unsupported op") + + monkeypatch.setattr(runner, "capture", fail_capture) + + def compute(value): + nonlocal computes + computes += 1 + return (value + 1,) + + with pytest.raises(RuntimeError, match="restart the stage process"): + runner.run("unit", (torch.tensor([1.0]),), (False,), compute) + with pytest.raises(RuntimeError, match="cannot continue"): + runner.run( + "different-shape", + (torch.tensor([1.0, 2.0]),), + (True,), + compute, + ) + + assert captures == 1 + assert computes == 1 + assert runner.stats == {"captures": 0, "failed": 1, "hits": 0} + + +def test_code2wav_runtime_disables_internal_format_and_jit(monkeypatch): + monkeypatch.delenv("ASCEND_LAUNCH_BLOCKING", raising=False) + compile_modes = [] + fake_npu = SimpleNamespace( + config=SimpleNamespace(allow_internal_format=True), + set_compile_mode=lambda **kwargs: compile_modes.append(kwargs), + ) + monkeypatch.setattr(torch, "npu", fake_npu, raising=False) + + code2wav_patch.prepare_code2wav_graph_runtime() + + assert fake_npu.config.allow_internal_format is False + assert compile_modes == [{"jit_compile": False}] + + +def test_code2wav_runtime_rejects_launch_blocking(monkeypatch): + monkeypatch.setenv("ASCEND_LAUNCH_BLOCKING", "1") + + with pytest.raises(RuntimeError, match="ASCEND_LAUNCH_BLOCKING=1"): + code2wav_patch.prepare_code2wav_graph_runtime() + + +@pytest.mark.parametrize( + ("enabled", "max_graphs", "expected_prepared"), + [ + (True, 7, 1), + (False, 7, 0), + (True, 0, 0), + ], +) +def test_code2wav_patch_reads_stage_additional_config( + monkeypatch, + enabled, + max_graphs, + expected_prepared, +): + prepared = 0 + + class _Backend: + speech_window = torch.zeros(1) + flow = SimpleNamespace(training=False) + + model = SimpleNamespace( + backend=None, + vllm_config=SimpleNamespace( + additional_config={ + "code2wav_enable_npu_graph": enabled, + "code2wav_max_npu_graphs": max_graphs, + } + ), + ) + + def build_backend(instance): + instance.backend = _Backend() + + def prepare_runtime(): + nonlocal prepared + prepared += 1 + + monkeypatch.setattr(code2wav_patch, "_original_build_backend", build_backend) + monkeypatch.setattr(code2wav_patch, "prepare_code2wav_graph_runtime", prepare_runtime) + code2wav_patch._patched_build_backend(model) + + assert prepared == expected_prepared + if expected_prepared: + graph_runner = code2wav_patch._backend_graph_runners[model.backend] + assert isinstance(graph_runner, NPUExactGraphRunner) + assert graph_runner.max_graphs == max_graphs + else: + assert model.backend not in code2wav_patch._backend_graph_runners + + +@pytest.mark.parametrize("with_cache", [False, True]) +def test_code2wav_estimator_graph_dispatch_is_platform_owned(monkeypatch, with_cache): + calls = [] + + class _Backend: + _trt_stepper = None + _cfm_graph_wrapper = None + + class _Runner: + def run(self, operation, inputs, constants, compute): + calls.append((operation, inputs, constants)) + return compute(*inputs) + + backend = _Backend() + runner = _Runner() + code2wav_patch._backend_graph_runners[backend] = runner + + def graphable_estimator_step( + instance, + estimator, + *, + x, + mu, + time_embedding, + speakers, + cond, + cnn_cache, + att_cache, + ): + del instance, estimator, time_embedding, speakers, cond + cache_marker = x.new_tensor(0 if cnn_cache is None else 1) + return x + mu, cache_marker, cache_marker.clone() + + monkeypatch.setattr(code2wav_patch, "_original_estimator_step", lambda *args, **kwargs: None) + monkeypatch.setattr(code2wav_patch, "_graphable_estimator_step", graphable_estimator_step) + value = torch.tensor([2.0]) + cache = torch.tensor([1.0]) if with_cache else None + estimator = SimpleNamespace(t_embedder=lambda time: time + 1) + outputs = code2wav_patch._patched_estimator_step( + backend, + estimator, + x=value, + mu=value, + time=value, + speakers=value, + cond=value, + cnn_cache=cache, + att_cache=cache, + ) + + assert calls[0][0] == "cfm_estimator" + assert calls[0][2] == (with_cache,) + assert len(calls[0][1]) == (7 if with_cache else 5) + torch.testing.assert_close(calls[0][1][2], torch.tensor([[3.0]])) + torch.testing.assert_close(outputs[0], torch.tensor([4.0])) + torch.testing.assert_close(outputs[1], torch.tensor(1.0 if with_cache else 0.0)) + + +def test_code2wav_platform_wraps_flow_execution_context(monkeypatch): + entered = [] + + class _Backend: + pass + + class _Context: + def __init__(self, device, require_math): + self.device = device + self.require_math = require_math + + def __enter__(self): + entered.append((self.device, self.require_math)) + + def __exit__(self, exc_type, exc, traceback): + return False + + backend = _Backend() + code2wav_patch._backend_graph_runners[backend] = NPUExactGraphRunner() + monkeypatch.setattr( + code2wav_patch, + "_flow_execution_context", + lambda device, *, require_math: _Context(device, require_math), + ) + monkeypatch.setattr( + code2wav_patch, + "_original_setup_batch", + lambda instance, features, batch_size: ("setup", batch_size), + ) + monkeypatch.setattr( + code2wav_patch, + "_original_decode_batch", + lambda instance, tokens, features, states, *, last_chunk, flush_encoder: ( + "decode", + last_chunk, + flush_encoder, + ), + ) + features = SimpleNamespace(speech_tokens=torch.zeros(1)) + + assert code2wav_patch._patched_setup_batch(backend, features, 3) == ("setup", 3) + assert code2wav_patch._patched_decode_batch( + backend, + torch.zeros(1), + features, + [], + last_chunk=True, + ) == ("decode", True, False) + assert entered == [(torch.device("cpu"), True), (torch.device("cpu"), True)] diff --git a/tests/platforms/npu/test_minicpmo_code2wav_npugraph.py b/tests/platforms/npu/test_minicpmo_code2wav_npugraph.py new file mode 100644 index 000000000..1c35c37aa --- /dev/null +++ b/tests/platforms/npu/test_minicpmo_code2wav_npugraph.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Ascend hardware validation for MiniCPM-o Code2Wav NPUGraph replay.""" + +from math import prod +from types import SimpleNamespace + +import pytest +import torch +import torch.nn as nn + +from tests.helpers.mark import hardware_marks +from vllm_omni.model_executor.models.minicpmo_4_5.batched_token2wav import ( + BatchedToken2Wav, +) +from vllm_omni.platforms.npu.graph_tools import NPUExactGraphRunner +from vllm_omni.platforms.npu.models.minicpmo_4_5_code2wav import ( + _graphable_estimator_step, + prepare_code2wav_graph_runtime, +) +from vllm_omni.platforms.npu.models.step_audio2_token2wav import ( + npu_token2wav_sdpa_context, +) + +pytestmark = [ + pytest.mark.core_model, + pytest.mark.omni, + *hardware_marks(res={"npu": "A3"}, num_cards=1), +] + + +class _Flow(nn.Module): + def __init__(self, estimator: nn.Module): + super().__init__() + self.decoder = SimpleNamespace(estimator=estimator) + + +class _Token2Wav: + def __init__(self, estimator: nn.Module): + self.flow = _Flow(estimator).eval() + self.hift = nn.Identity().eval() + self.float16 = False + self.n_timesteps = 2 + self.mel_cache_len = 1 + self.source_cache_len = 2 + self.speech_window = torch.ones(4, device="npu") + + +def test_minicpmo_cfm_estimator_npugraph_matches_eager(): + decoder_dit = pytest.importorskip("cosyvoice2.flow.decoder_dit") + prepare_code2wav_graph_runtime() + + estimator = ( + decoder_dit.DiT( + in_channels=6, + out_channels=2, + depth=2, + num_heads=2, + head_dim=8, + hidden_size=16, + mlp_ratio=2.0, + ) + .npu() + .eval() + ) + graph_runner = NPUExactGraphRunner(max_graphs=4) + adapter = BatchedToken2Wav(_Token2Wav(estimator)) + + def inputs(seed: int): + def make(shape: tuple[int, ...], offset: float): + values = torch.arange( + prod(shape), + device="npu", + dtype=torch.float32, + ) + return torch.sin(values * 0.17 + seed + offset).reshape(shape) + + return ( + make((2, 2, 6), 0.0), + make((2, 2, 6), 0.5), + make((2, 1, 16), 1.0), + make((2, 1), 1.5), + make((2, 1, 6), 2.0), + ) + + def compute(*values, caches=None): + cnn_cache, att_cache = (None, None) if caches is None else caches + return _graphable_estimator_step( + adapter, + estimator, + x=values[0], + mu=values[1], + time_embedding=values[2], + speakers=values[3], + cond=values[4], + cnn_cache=cnn_cache, + att_cache=att_cache, + ) + + with torch.inference_mode(), npu_token2wav_sdpa_context(require_math=True): + warm_inputs = inputs(11) + warm_outputs = graph_runner.run( + "cfm_estimator", + warm_inputs, + (False,), + lambda *values: compute(*values), + ) + replay_inputs = inputs(13) + expected = compute(*replay_inputs) + actual = graph_runner.run( + "cfm_estimator", + replay_inputs, + (False,), + lambda *values: compute(*values), + ) + for eager, replayed in zip(expected, actual, strict=True): + torch.testing.assert_close(replayed, eager, rtol=1e-3, atol=1e-3) + + cached_inputs = (*inputs(17), warm_outputs[1], warm_outputs[2]) + graph_runner.run( + "cfm_estimator", + cached_inputs, + (True,), + lambda *values: compute(*values[:5], caches=(values[5], values[6])), + ) + replay_cached_inputs = ( + *inputs(19), + warm_outputs[1] + 0.1, + warm_outputs[2] + 0.1, + ) + expected_cached = compute( + *replay_cached_inputs[:5], + caches=(replay_cached_inputs[5], replay_cached_inputs[6]), + ) + actual_cached = graph_runner.run( + "cfm_estimator", + replay_cached_inputs, + (True,), + lambda *values: compute(*values[:5], caches=(values[5], values[6])), + ) + for eager, replayed in zip(expected_cached, actual_cached, strict=True): + torch.testing.assert_close(replayed, eager, rtol=1e-3, atol=1e-3) + assert graph_runner.stats == {"captures": 2, "failed": 0, "hits": 2} diff --git a/tests/platforms/npu/test_step_audio2_token2wav.py b/tests/platforms/npu/test_step_audio2_token2wav.py index ae86f92d9..23baa2c3e 100644 --- a/tests/platforms/npu/test_step_audio2_token2wav.py +++ b/tests/platforms/npu/test_step_audio2_token2wav.py @@ -7,6 +7,7 @@ from __future__ import annotations import importlib.util import sys import types +from contextlib import contextmanager from pathlib import Path from types import SimpleNamespace @@ -194,3 +195,71 @@ def test_hift_patch_reports_incompatible_layout(monkeypatch: pytest.MonkeyPatch) with pytest.raises(TypeError, match=r"m_source\.l_sin_gen\._f02sine"): module.patch_step_audio2_hift_for_npu(SimpleNamespace()) + + +def test_sdpa_context_preserves_exceptions_from_token2wav_body( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_patch_module(monkeypatch) + fake_attention_patch = types.ModuleType("vllm_omni.platforms.npu.models.cosyvoice2_dit_attn") + fake_attention_patch.apply_cosyvoice2_dit_attn_npu_patch = lambda: None + + @contextmanager + def fake_math_context(*, require_available=False): + del require_available + yield + + fake_attention_patch.npu_math_sdpa_context = fake_math_context + monkeypatch.setitem( + sys.modules, + "vllm_omni.platforms.npu.models.cosyvoice2_dit_attn", + fake_attention_patch, + ) + + with pytest.raises(RuntimeError, match="original graph failure"): + with module.npu_token2wav_sdpa_context(): + raise RuntimeError("original graph failure") + + +def test_sdpa_context_requires_math_for_npu_graph(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_patch_module(monkeypatch) + required = [] + fake_attention_patch = types.ModuleType("vllm_omni.platforms.npu.models.cosyvoice2_dit_attn") + fake_attention_patch.apply_cosyvoice2_dit_attn_npu_patch = lambda: None + + @contextmanager + def fake_math_context(*, require_available=False): + required.append(require_available) + yield + + fake_attention_patch.npu_math_sdpa_context = fake_math_context + monkeypatch.setitem( + sys.modules, + "vllm_omni.platforms.npu.models.cosyvoice2_dit_attn", + fake_attention_patch, + ) + + with module.npu_token2wav_sdpa_context(require_math=True): + pass + + assert required == [True] + + +def test_sdpa_context_does_not_hide_patch_setup_failures(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_patch_module(monkeypatch) + fake_attention_patch = types.ModuleType("vllm_omni.platforms.npu.models.cosyvoice2_dit_attn") + + def fail_patch(): + raise RuntimeError("attention patch failed") + + fake_attention_patch.apply_cosyvoice2_dit_attn_npu_patch = fail_patch + fake_attention_patch.npu_math_sdpa_context = contextmanager(lambda: iter((None,))) + monkeypatch.setitem( + sys.modules, + "vllm_omni.platforms.npu.models.cosyvoice2_dit_attn", + fake_attention_patch, + ) + + with pytest.raises(RuntimeError, match="attention patch failed"): + with module.npu_token2wav_sdpa_context(): + pass diff --git a/vllm_omni/deploy/minicpmo_4_5.yaml b/vllm_omni/deploy/minicpmo_4_5.yaml index bc34bfe9f..04ddad608 100644 --- a/vllm_omni/deploy/minicpmo_4_5.yaml +++ b/vllm_omni/deploy/minicpmo_4_5.yaml @@ -111,3 +111,9 @@ platforms: max_num_batched_tokens: 8192 compilation_config: cudagraph_mode: PIECEWISE + - stage_id: 2 + # Stage 2 stays outer-eager and captures only exact-shape CFM DiT + # estimator kernels through the NPU platform patch. + additional_config: + code2wav_enable_npu_graph: true + code2wav_max_npu_graphs: 32 diff --git a/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml b/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml index 2a4abed42..5b43206b8 100644 --- a/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml +++ b/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml @@ -93,3 +93,9 @@ platforms: max_num_batched_tokens: 8192 compilation_config: cudagraph_mode: PIECEWISE + - stage_id: 2 + # Stage 2 stays outer-eager and captures only exact-shape CFM DiT + # estimator kernels through the NPU platform patch. + additional_config: + code2wav_enable_npu_graph: true + code2wav_max_npu_graphs: 32 diff --git a/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml b/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml index b34673910..edbde0c5d 100644 --- a/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml +++ b/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml @@ -86,4 +86,20 @@ stages: max_tokens: 65536 detokenize: true -platforms: {} +platforms: + npu: + stages: + - stage_id: 0 + max_num_batched_tokens: 8192 + compilation_config: + cudagraph_mode: PIECEWISE + - stage_id: 1 + max_num_batched_tokens: 8192 + compilation_config: + cudagraph_mode: PIECEWISE + - stage_id: 2 + # Stage 2 stays outer-eager and captures only exact-shape CFM DiT + # estimator kernels through the NPU platform patch. + additional_config: + code2wav_enable_npu_graph: true + code2wav_max_npu_graphs: 32 diff --git a/vllm_omni/platforms/npu/__init__.py b/vllm_omni/platforms/npu/__init__.py index fb95f7b29..74dfc419c 100644 --- a/vllm_omni/platforms/npu/__init__.py +++ b/vllm_omni/platforms/npu/__init__.py @@ -1,6 +1,19 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from vllm_omni.platforms.npu.platform import NPUOmniPlatform +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from vllm_omni.platforms.npu.platform import NPUOmniPlatform __all__ = ["NPUOmniPlatform"] + + +def __getattr__(name: str): + if name != "NPUOmniPlatform": + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + from vllm_omni.platforms.npu.platform import NPUOmniPlatform + + globals()[name] = NPUOmniPlatform + return NPUOmniPlatform diff --git a/vllm_omni/platforms/npu/graph_tools.py b/vllm_omni/platforms/npu/graph_tools.py new file mode 100644 index 000000000..d943d1584 --- /dev/null +++ b/vllm_omni/platforms/npu/graph_tools.py @@ -0,0 +1,183 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Reusable exact-signature NPUGraph capture and replay helpers.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Protocol + +import torch +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +class _ReplayGraph(Protocol): + def replay(self) -> None: ... + + +@dataclass +class CapturedDeviceGraph: + graph: _ReplayGraph + static_inputs: tuple[torch.Tensor, ...] + static_outputs: tuple[torch.Tensor, ...] + + def replay(self, inputs: tuple[torch.Tensor, ...]) -> tuple[torch.Tensor, ...]: + with torch.inference_mode(): + for static, current in zip(self.static_inputs, inputs, strict=True): + static.copy_(current) + self.graph.replay() + # Graph outputs are persistent and overwritten by the next replay. + # Clone before they become request-owned streaming cache entries. + return tuple(output.detach().clone() for output in self.static_outputs) + + +def _tensor_signature(value: torch.Tensor) -> tuple[tuple[int, ...], str, str]: + return tuple(value.shape), str(value.dtype), str(value.device) + + +class NPUExactGraphRunner: + """Capture and replay tensor-only functions for exact NPU signatures.""" + + def __init__( + self, + *, + max_graphs: int = 32, + component_name: str = "device graph", + disable_config_hint: str = "disable graph capture", + ) -> None: + self.max_graphs = max(0, int(max_graphs)) + self.component_name = component_name + self.disable_config_hint = disable_config_hint + self._enabled = self.max_graphs > 0 + self._graphs: dict[tuple[object, ...], CapturedDeviceGraph] = {} + self._failed_keys: set[tuple[object, ...]] = set() + self._hits = 0 + self._graph_pool: object | None = None + + @staticmethod + def is_supported() -> bool: + npu = getattr(torch, "npu", None) + return npu is not None and all( + hasattr(npu, name) + for name in ( + "NPUGraph", + "graph", + "is_current_stream_capturing", + "synchronize", + ) + ) + + @staticmethod + def _stream_is_capturing() -> bool: + npu = getattr(torch, "npu", None) + is_capturing = getattr(npu, "is_current_stream_capturing", None) + if not callable(is_capturing): + return False + try: + return bool(is_capturing()) + except (RuntimeError, TypeError): + return False + + def _eligible(self, inputs: tuple[torch.Tensor, ...]) -> bool: + return ( + self._enabled + and bool(inputs) + and all(value.device.type == "npu" for value in inputs) + and self.is_supported() + and not self._stream_is_capturing() + ) + + @property + def stats(self) -> dict[str, int]: + return { + "captures": len(self._graphs), + "failed": len(self._failed_keys), + "hits": self._hits, + } + + def capture( + self, + inputs: tuple[torch.Tensor, ...], + compute: Callable[..., tuple[torch.Tensor, ...]], + ) -> CapturedDeviceGraph: + npu = torch.npu + static_inputs = tuple(value.detach().clone() for value in inputs) + npu.synchronize() + graph = npu.NPUGraph() + if self._graph_pool is None: + from vllm.platforms import current_platform + + self._graph_pool = current_platform.get_global_graph_pool() + with torch.inference_mode(), npu.graph(graph, pool=self._graph_pool): + static_outputs = compute(*static_inputs) + npu.synchronize() + return CapturedDeviceGraph( + graph=graph, + static_inputs=static_inputs, + static_outputs=static_outputs, + ) + + def run( + self, + operation: str, + inputs: tuple[torch.Tensor, ...], + constants: tuple[object, ...], + compute: Callable[..., tuple[torch.Tensor, ...]], + ) -> tuple[torch.Tensor, ...]: + if self._failed_keys: + raise RuntimeError( + f"{self.component_name} cannot continue after a failed NPUGraph capture; " + f"restart the stage process and {self.disable_config_hint} before retrying." + ) + if not self._eligible(inputs): + return compute(*inputs) + + key = ( + operation, + constants, + tuple(_tensor_signature(value) for value in inputs), + ) + graph = self._graphs.get(key) + if graph is not None: + self._hits += 1 + if self._hits == 1: + logger.info("%s started NPUGraph replay", self.component_name) + return graph.replay(inputs) + + # Prime lazy kernels and allocator state before capture. The next call + # with the same exact tensor signature replays this graph. + eager_outputs = compute(*inputs) + if len(self._graphs) >= self.max_graphs: + logger.warning_once( + "%s reached the %d-entry NPUGraph limit; new tensor shapes will use eager execution.", + self.component_name, + self.max_graphs, + ) + return eager_outputs + try: + self._graphs[key] = self.capture(inputs, compute) + except Exception as exc: + self._failed_keys.add(key) + self._enabled = False + logger.exception( + "%s failed to capture NPUGraph for %s; the torch-npu " + "allocator/RNG capture state may be invalid and this stage " + "process must be restarted.", + self.component_name, + operation, + ) + raise RuntimeError( + f"{self.component_name} NPUGraph capture failed for {operation}; " + f"restart the stage process. To run eagerly, {self.disable_config_hint}." + ) from exc + logger.info( + "%s captured NPUGraph %d/%d for %s", + self.component_name, + len(self._graphs), + self.max_graphs, + operation, + ) + return eager_outputs diff --git a/vllm_omni/platforms/npu/models/cosyvoice2_dit_attn.py b/vllm_omni/platforms/npu/models/cosyvoice2_dit_attn.py index 7e1d5d540..7cf280a00 100644 --- a/vllm_omni/platforms/npu/models/cosyvoice2_dit_attn.py +++ b/vllm_omni/platforms/npu/models/cosyvoice2_dit_attn.py @@ -18,7 +18,7 @@ This module: from __future__ import annotations from collections.abc import Iterator -from contextlib import contextmanager, nullcontext +from contextlib import ExitStack, contextmanager import torch import torch.nn.functional as F @@ -107,17 +107,20 @@ def _patched_attention_forward_chunk( @contextmanager -def npu_math_sdpa_context() -> Iterator[None]: +def npu_math_sdpa_context(*, require_available: bool = False) -> Iterator[None]: """Force SDPA MATH backend so Ascend does not call fused FA.""" - try: - from torch.nn.attention import SDPBackend, sdpa_kernel + with ExitStack() as stack: + try: + from torch.nn.attention import SDPBackend, sdpa_kernel - with sdpa_kernel(SDPBackend.MATH): - yield - except Exception: - # Older torch / missing backend enum — just run as-is. - with nullcontext(): - yield + stack.enter_context(sdpa_kernel(SDPBackend.MATH)) + except Exception as exc: + if require_available: + raise RuntimeError( + "MiniCPM-o Code2Wav NPU graph capture requires torch.nn.attention.sdpa_kernel(SDPBackend.MATH)" + ) from exc + logger.warning_once("MATH SDPA selection is unavailable; using the default Ascend SDPA backend.") + yield def _disable_upsample_encoder_compile() -> None: diff --git a/vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py b/vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py new file mode 100644 index 000000000..c6effe6f8 --- /dev/null +++ b/vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py @@ -0,0 +1,262 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inject MiniCPM-o Code2Wav NPUGraph acceleration on Ascend.""" + +from __future__ import annotations + +import os +from collections.abc import Mapping +from contextlib import nullcontext +from weakref import WeakKeyDictionary + +import torch +from vllm.logger import init_logger + +from vllm_omni.platforms.npu.graph_tools import NPUExactGraphRunner + +logger = init_logger(__name__) + +_PATCHED = False +_original_build_backend = None +_original_estimator_step = None +_original_setup_batch = None +_original_decode_batch = None +_backend_graph_runners: WeakKeyDictionary[object, NPUExactGraphRunner] = WeakKeyDictionary() +_ENABLE_KEY = "code2wav_enable_npu_graph" +_MAX_GRAPHS_KEY = "code2wav_max_npu_graphs" + + +def _config_bool(value: object, default: bool) -> bool: + if value is None: + return default + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "on"} + return bool(value) + + +def _graph_config(model: object) -> dict[str, object]: + config = getattr(getattr(model, "vllm_config", None), "additional_config", None) + return dict(config) if isinstance(config, Mapping) else {} + + +def prepare_code2wav_graph_runtime() -> None: + """Select graph-capturable ACLNN kernels before Token2Wav is loaded.""" + if os.environ.get("ASCEND_LAUNCH_BLOCKING") == "1": + raise RuntimeError( + "MiniCPM-o Code2Wav NPUGraph capture is incompatible with " + "ASCEND_LAUNCH_BLOCKING=1; unset it or set it to 0 before startup." + ) + npu = torch.npu + npu.config.allow_internal_format = False + npu.set_compile_mode(jit_compile=False) + logger.info("Configured MiniCPM-o Code2Wav NPUGraph runtime (allow_internal_format=False, jit_compile=False)") + + +def _flow_execution_context(device: torch.device, *, require_math: bool): + if device.type != "npu": + return nullcontext() + from vllm_omni.platforms.npu.models.step_audio2_token2wav import ( + npu_token2wav_sdpa_context, + ) + + return npu_token2wav_sdpa_context(require_math=require_math) + + +def _graphable_estimator_step( + backend, + estimator, + *, + x, + mu, + time_embedding, + speakers, + cond, + cnn_cache, + att_cache, +): + """Run the CFM estimator body after host-backed timestep embedding.""" + width = int(x.shape[-1]) + speaker_features = speakers.unsqueeze(-1).expand(-1, -1, width) + estimator_input = torch.cat((x, mu, speaker_features, cond), dim=1) + cnn_out, att_out = backend._estimator_buffers(estimator, estimator_input, att_cache) + old_cnn = cnn_cache if cnn_cache is not None else [None] * len(estimator.blocks) + old_att = att_cache if att_cache is not None else [None] * len(estimator.blocks) + result = estimator.blocks_forward_chunk( + estimator_input, + time_embedding, + None, + old_cnn, + old_att, + cnn_out, + att_out, + ) + return result, cnn_out, att_out + + +def _patched_estimator_step( + self, + estimator, + *, + x, + mu, + time, + speakers, + cond, + cnn_cache, + att_cache, +): + assert _original_estimator_step is not None + graph_runner = _backend_graph_runners.get(self) + if graph_runner is None or self._trt_stepper is not None or self._cfm_graph_wrapper is not None: + return _original_estimator_step( + self, + estimator, + x=x, + mu=mu, + time=time, + speakers=speakers, + cond=cond, + cnn_cache=cnn_cache, + att_cache=att_cache, + ) + if (cnn_cache is None) != (att_cache is None): + raise ValueError("estimator CNN and attention caches must both be present or absent") + + # The upstream embedder creates a frequency tensor on the host. Keep it + # outside capture while retaining the tensor-only estimator body in graph. + time_embedding = estimator.t_embedder(time).unsqueeze(1) + if cnn_cache is None: + return graph_runner.run( + "cfm_estimator", + (x, mu, time_embedding, speakers, cond), + (False,), + lambda step_x, step_mu, step_time, step_speakers, step_cond: _graphable_estimator_step( + self, + estimator, + x=step_x, + mu=step_mu, + time_embedding=step_time, + speakers=step_speakers, + cond=step_cond, + cnn_cache=None, + att_cache=None, + ), + ) + + return graph_runner.run( + "cfm_estimator", + (x, mu, time_embedding, speakers, cond, cnn_cache, att_cache), + (True,), + lambda step_x, step_mu, step_time, step_speakers, step_cond, step_cnn, step_att: _graphable_estimator_step( + self, + estimator, + x=step_x, + mu=step_mu, + time_embedding=step_time, + speakers=step_speakers, + cond=step_cond, + cnn_cache=step_cnn, + att_cache=step_att, + ), + ) + + +def _patched_setup_batch(self, features, batch_size): + assert _original_setup_batch is not None + with _flow_execution_context( + features.speech_tokens.device, + require_math=self in _backend_graph_runners, + ): + return _original_setup_batch(self, features, batch_size) + + +def _patched_decode_batch( + self, + tokens, + features, + states, + *, + last_chunk, + flush_encoder=False, +): + assert _original_decode_batch is not None + with _flow_execution_context( + tokens.device, + require_math=self in _backend_graph_runners, + ): + return _original_decode_batch( + self, + tokens, + features, + states, + last_chunk=last_chunk, + flush_encoder=flush_encoder, + ) + + +def _patched_build_backend(self) -> None: + if self.backend is not None: + return + + config = _graph_config(self) + max_graphs = max(0, int(config.get(_MAX_GRAPHS_KEY, 32))) + graph_enabled = max_graphs > 0 and _config_bool(config.get(_ENABLE_KEY), False) + if graph_enabled: + # NPUOmniPlatform enables internal format for quantized LLM kernels. + # Code2Wav uses regular convolution kernels that must remain in the + # graph-capturable ACLNN path. + prepare_code2wav_graph_runtime() + + assert _original_build_backend is not None + _original_build_backend(self) + + graph_runner = None + if graph_enabled: + graph_runner = NPUExactGraphRunner( + max_graphs=max_graphs, + component_name="MiniCPM-o Code2Wav", + disable_config_hint=( + "set platforms.npu.stages[stage_id=2].additional_config.code2wav_enable_npu_graph=false" + ), + ) + if self.backend.speech_window.device.type == "npu" and not graph_runner.is_supported(): + raise RuntimeError( + "MiniCPM-o Code2Wav NPUGraph capture requires torch.npu " + "NPUGraph, graph, is_current_stream_capturing, and synchronize APIs." + ) + if self.backend.flow.training: + raise ValueError("MiniCPM-o Code2Wav NPUGraph capture requires flow.eval()") + _backend_graph_runners[self.backend] = graph_runner + + if graph_enabled: + logger.info( + "MiniCPM-o Code2Wav NPUGraph replay enabled (max_graphs=%d)", + max_graphs, + ) + + +def apply_minicpmo_4_5_code2wav_patch() -> None: + """Patch the generic Code2Wav backend builder with Ascend acceleration.""" + global _PATCHED, _original_build_backend + global _original_decode_batch, _original_estimator_step, _original_setup_batch + if _PATCHED: + return + + from vllm_omni.model_executor.models.minicpmo_4_5.batched_token2wav import ( + BatchedToken2Wav, + ) + from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_code2wav import ( + MiniCPMO45Code2Wav, + ) + + _original_build_backend = MiniCPMO45Code2Wav._build_backend + _original_estimator_step = BatchedToken2Wav._estimator_step + _original_setup_batch = BatchedToken2Wav.setup_batch + _original_decode_batch = BatchedToken2Wav.decode_batch + + MiniCPMO45Code2Wav._build_backend = _patched_build_backend # type: ignore[method-assign] + BatchedToken2Wav._estimator_step = _patched_estimator_step # type: ignore[method-assign] + BatchedToken2Wav.setup_batch = _patched_setup_batch # type: ignore[method-assign] + BatchedToken2Wav.decode_batch = _patched_decode_batch # type: ignore[method-assign] + _PATCHED = True + logger.debug("Applied NPU patch for MiniCPM-o 4.5 Code2Wav") diff --git a/vllm_omni/platforms/npu/models/step_audio2_token2wav.py b/vllm_omni/platforms/npu/models/step_audio2_token2wav.py index 1641fee27..1999a08a0 100644 --- a/vllm_omni/platforms/npu/models/step_audio2_token2wav.py +++ b/vllm_omni/platforms/npu/models/step_audio2_token2wav.py @@ -13,7 +13,7 @@ Ascend-specific workarounds that must not live in the shared GPU model file: from __future__ import annotations from collections.abc import Iterator -from contextlib import contextmanager, nullcontext +from contextlib import contextmanager from types import MethodType import numpy as np @@ -113,20 +113,16 @@ def patch_step_audio2_hift_for_npu(hift: torch.nn.Module) -> None: @contextmanager -def npu_token2wav_sdpa_context() -> Iterator[None]: +def npu_token2wav_sdpa_context(*, require_math: bool = False) -> Iterator[None]: """Expand CosyVoice masks + force MATH SDPA to avoid FA 161001.""" - try: - from vllm_omni.platforms.npu.models.cosyvoice2_dit_attn import ( - apply_cosyvoice2_dit_attn_npu_patch, - npu_math_sdpa_context, - ) + from vllm_omni.platforms.npu.models.cosyvoice2_dit_attn import ( + apply_cosyvoice2_dit_attn_npu_patch, + npu_math_sdpa_context, + ) - apply_cosyvoice2_dit_attn_npu_patch() - with npu_math_sdpa_context(): - yield - except Exception: - with nullcontext(): - yield + apply_cosyvoice2_dit_attn_npu_patch() + with npu_math_sdpa_context(require_available=require_math): + yield def _patched_ensure_models_loaded(self) -> None: diff --git a/vllm_omni/platforms/npu/platform.py b/vllm_omni/platforms/npu/platform.py index b17f447c4..38c35f1f7 100644 --- a/vllm_omni/platforms/npu/platform.py +++ b/vllm_omni/platforms/npu/platform.py @@ -36,6 +36,9 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform): from vllm_ascend.utils import adapt_patch from vllm_omni.platforms.npu._310p import apply_patches as apply_310p_patches + from vllm_omni.platforms.npu.models.minicpmo_4_5_code2wav import ( + apply_minicpmo_4_5_code2wav_patch, + ) from vllm_omni.platforms.npu.models.minimax_h3 import apply_minimax_h3_qwen3vl_patch from vllm_omni.platforms.npu.models.qwen3_tts_code2wav import ( apply_qwen3_tts_code2wav_patch, @@ -46,6 +49,7 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform): adapt_patch(is_global_patch=True) apply_minimax_h3_qwen3vl_patch() + apply_minicpmo_4_5_code2wav_patch() apply_qwen3_tts_code2wav_patch() apply_qwen3_tts_tokenizer_v2_patch() apply_310p_patches() From 265d80b4dd87f86f7cf5cbde07d248855383ce90 Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Mon, 17 Aug 2026 17:38:44 +0800 Subject: [PATCH 048/267] [Docs] Classify PD disaggregation as an experimental feature (#6115) Signed-off-by: Hongsheng Liu --- docs/.nav.yml | 1 + docs/configuration/README.md | 4 - docs/configuration/pd_disaggregation.md | 171 ------------------------ docs/features/README.md | 12 +- docs/features/pd_disaggregation.md | 140 +++++++++++++++++++ mkdocs.yml | 1 + 6 files changed, 151 insertions(+), 178 deletions(-) delete mode 100644 docs/configuration/pd_disaggregation.md create mode 100644 docs/features/pd_disaggregation.md diff --git a/docs/.nav.yml b/docs/.nav.yml index d367ded05..a2c93d534 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -43,6 +43,7 @@ nav: - Overview: features/README.md - Runtime and Stage Execution: - Execution Modes and Streaming: user_guide/diffusion/execution_modes.md + - Prefill-Decode Disaggregation (experimental): features/pd_disaggregation.md - Sleep Mode: features/sleep_mode.md - Quantization: - Overview: user_guide/quantization/overview.md diff --git a/docs/configuration/README.md b/docs/configuration/README.md index 09772ccd1..b59c29a7f 100644 --- a/docs/configuration/README.md +++ b/docs/configuration/README.md @@ -14,10 +14,6 @@ For introduction, please check [Introduction for stage config](./stage_configs.m - **[GPU Memory Calculation and Configuration](./gpu_memory_utilization.md)** - Guide on how to calculate memory requirements and set up `gpu_memory_utilization` for optimal performance -## Multi-Stage Recipes - -- **[Prefill-Decode Disaggregation](./pd_disaggregation.md)** - How to derive a PD-aware Qwen3-Omni stage config from the default config without introducing another bundled YAML - ## Optimization Features - **[Diffusion Features Overview](../user_guide/diffusion_features.md)** - Complete overview of all diffusion model features and supported models diff --git a/docs/configuration/pd_disaggregation.md b/docs/configuration/pd_disaggregation.md deleted file mode 100644 index 93c623ebd..000000000 --- a/docs/configuration/pd_disaggregation.md +++ /dev/null @@ -1,171 +0,0 @@ -# Prefill-Decode (PD) Disaggregation - -PD disaggregation splits the Qwen3-Omni thinker into separate prefill and decode -stages so prompt processing and token generation can run on different workers. - -This is documented as a stage-config recipe instead of a bundled YAML because the -deployment-specific values usually change per environment: - -- GPU placement -- `tensor_parallel_size` -- connector backend and connector ports -- connector IPs or bootstrap addresses - -Start from the [default Qwen3-Omni stage config](gh-file:vllm_omni/deploy/qwen3_omni_moe.yaml) -and copy it to your own file, for example `qwen3_omni_pd.yaml`. Then apply the -changes below. - -## Requirements - -- 3+ GPUs for a basic layout: prefill, decode, and talker+code2wav -- A KV connector supported by vLLM, such as `MooncakeConnector` -- Matching `tensor_parallel_size` on the prefill and decode thinker stages - -## 1. Split the thinker into prefill and decode stages - -Replace the original thinker stage with two stages: - -```yaml -stage_args: - - stage_id: 0 - stage_type: llm - is_prefill_only: true - runtime: - devices: "0" - engine_args: - max_num_seqs: 16 - model_stage: thinker - model_arch: Qwen3OmniMoeForConditionalGeneration - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.9 - enforce_eager: true - trust_remote_code: true - engine_output_type: latent - distributed_executor_backend: "mp" - enable_prefix_caching: false - max_num_batched_tokens: 32768 - hf_config_name: thinker_config - tensor_parallel_size: 1 - kv_transfer_config: - kv_connector: "MooncakeConnector" - kv_role: "kv_producer" - kv_rank: 0 - kv_parallel_size: 2 - kv_connector_extra_config: - mooncake_bootstrap_port: 25201 - final_output: false - is_comprehension: true - default_sampling_params: - temperature: 0.4 - top_p: 0.9 - top_k: 1 - max_tokens: 2048 - seed: 42 - detokenize: True - repetition_penalty: 1.05 - - - stage_id: 1 - stage_type: llm - is_decode_only: true - runtime: - devices: "1" - engine_args: - max_num_seqs: 64 - model_stage: thinker - model_arch: Qwen3OmniMoeForConditionalGeneration - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.9 - enforce_eager: true - trust_remote_code: true - engine_output_type: latent - distributed_executor_backend: "mp" - enable_prefix_caching: false - max_num_batched_tokens: 32768 - hf_config_name: thinker_config - tensor_parallel_size: 1 - kv_transfer_config: - kv_connector: "MooncakeConnector" - kv_role: "kv_consumer" - kv_rank: 1 - kv_parallel_size: 2 - kv_connector_extra_config: - mooncake_bootstrap_port: 25202 - engine_input_source: [0] - final_output: true - final_output_type: text - is_comprehension: true - default_sampling_params: - temperature: 0.4 - top_p: 0.9 - top_k: 1 - max_tokens: 2048 - seed: 42 - detokenize: True - repetition_penalty: 1.05 -``` - -Notes: - -- `is_prefill_only: true` marks the thinker stage that only saves KV. -- `is_decode_only: true` marks the thinker stage that resumes from remote KV. -- `kv_transfer_config` is required on both stages. -- The orchestrator forces the prefill stage to run with `max_tokens=1`, so the - prefill side only processes the prompt and exports KV. - -## 2. Shift the downstream stages by one index - -After inserting the extra thinker stage, renumber the remaining stages: - -```yaml - - stage_id: 2 - runtime: - devices: "2" - engine_input_source: [1] - sync_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_token_only - - - stage_id: 3 - runtime: - devices: "2" - engine_args: - max_num_seqs: 1 - engine_input_source: [2] - sync_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_token_only -``` - -Compared with the default Qwen3-Omni config: - -- the talker becomes stage `2` instead of stage `1` -- the code2wav stage becomes stage `3` instead of stage `2` -- the talker now reads from decode stage `1` - -## 3. Add runtime edges for the four-stage pipeline - -```yaml -runtime: - enabled: true - edges: - - from: 0 - to: 1 - - from: 1 - to: 2 - - from: 2 - to: 3 -``` - -## 4. Launch with your custom config - -```bash -vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \ - --deploy-config /path/to/qwen3_omni_pd.yaml -``` - -## Operational Notes - -- `MooncakeConnector` does not support heterogeneous TP sizes across the PD - pair. Keep prefill and decode at the same `tensor_parallel_size`. -- If the thinker requires TP=2, both thinker stages must use TP=2 and be given - separate GPU sets, for example `"0,1"` for prefill and `"2,3"` for decode. -- Choose connector ports and addresses that match your deployment. The values - shown above are examples only. diff --git a/docs/features/README.md b/docs/features/README.md index 8090b783f..7c3e89676 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -20,13 +20,19 @@ an implementation contract has a user-facing workflow. | Choose serial, batched, step-wise, or streaming diffusion execution | [Execution Modes and Streaming](../user_guide/diffusion/execution_modes.md) | [Diffusion Continuous Batching](../design/feature/diffusion_continuous_batching.md), [Async Diffusion Output](../design/feature/async_diffusion_output.md) | | Reclaim stage memory without restarting the server | [Sleep Mode](sleep_mode.md) | Runtime lifecycle behavior is documented in the user guide | +The [Prefill-Decode Disaggregation design reference](pd_disaggregation.md) +describes the experimental Qwen3-Omni runtime topology and its current +integration boundaries. It is not yet a supported launch recipe. + Some runtime designs are deliberately not promoted as standalone User Guide features yet: -- [Disaggregated Inference](../design/feature/disaggregated_inference.md) and +- Generic, OmniConnector-based + [Disaggregated Inference](../design/feature/disaggregated_inference.md) is not + yet a standalone User Guide feature. Its [OmniConnector implementations](../design/index.md#communication) describe - topology and transport contracts. Practical deployment configuration remains - under [Pipeline and deploy configurations](../configuration/stage_configs.md). + transport contracts, while practical connector configuration remains under + [Pipeline and deploy configurations](../configuration/stage_configs.md). - [Async Chunk](../design/feature/async_chunk.md) and [Async Omni Output Materialization](../design/feature/omni_async_output_materialization.md) are model- and pipeline-dependent. Use the selected model's deploy diff --git a/docs/features/pd_disaggregation.md b/docs/features/pd_disaggregation.md new file mode 100644 index 000000000..ae8497e81 --- /dev/null +++ b/docs/features/pd_disaggregation.md @@ -0,0 +1,140 @@ +# Prefill-Decode (PD) Disaggregation (experimental) + +!!! warning "Experimental design reference" + PD disaggregation is only partially integrated on the current `main` branch. + The runtime contains PD detection and routing scaffolding, but there is no + supported deploy overlay or end-to-end validated launch recipe yet. Treat this + page as a description of the current design, not as production guidance. + +Prefill-decode disaggregation splits the Qwen3-Omni Thinker into two logical +stages: a prefill worker that processes the prompt and produces KV cache, and a +decode worker that imports that cache and generates tokens. The Talker and +Code2Wav stages remain downstream; PD does not disaggregate the entire +Qwen3-Omni pipeline. + +## Current design + +### Runtime topology + +```mermaid +flowchart LR + client["Client request"] --> orchestrator["Omni Orchestrator"] + orchestrator -->|"original prompt
max_tokens = 1"| prefill["Thinker prefill
KV producer"] + prefill -->|"KV blocks"| connector[("vLLM KV connector")] + prefill -->|"KV routing metadata"| orchestrator + orchestrator -->|"original prompt
remote KV parameters"| decode["Thinker decode
KV consumer"] + connector -->|"remote KV load"| decode + decode -->|"generated text"| client +``` + +The orchestrator owns the request lifecycle, while the vLLM KV connector owns +the bulk KV transfer. The original prompt is submitted to both Thinker stages: +the decode worker still needs the request tokens and multimodal features to +construct its request, while the imported KV cache prevents it from repeating +the completed prefill computation. + +For the full Qwen3-Omni pipeline, splitting the Thinker adds one logical stage: + +| Stage | Role | Output or handoff | +| --- | --- | --- | +| 0 | Thinker prefill | Saves prompt KV and prompt-side multimodal output | +| 1 | Thinker decode | Loads remote KV, generates text, and produces decode-side conditioning | +| 2 | Talker | Converts combined Thinker conditioning into codec codes | +| 3 | Code2Wav | Converts codec codes into audio | + +Text remains a final output of the decode stage, and audio remains a final +output of Code2Wav. + +### Request lifecycle + +1. At startup, PD detection looks for one stage marked `is_prefill_only` and + one downstream stage marked `is_decode_only`. The decode stage must list the + prefill stage as an input source. +2. The orchestrator clones the Thinker sampling parameters for prefill, forces + `max_tokens=1`, clears stop conditions, and sets producer-side + `kv_transfer_params`. This makes prefill finish by length after exporting KV. +3. When prefill finishes, the orchestrator captures its KV routing metadata and + Qwen multimodal output under the original request ID. +4. The orchestrator submits the original prompt to decode with consumer-side + `kv_transfer_params`. Those parameters identify the transfer, prefill engine, + bootstrap endpoint, and, when required by the connector, remote request. +5. The decode engine imports the KV cache and continues generation. The normal + Omni stage path then carries Thinker conditioning through Talker and + Code2Wav. + +### Control and data paths + +PD uses two different connector layers. They solve different handoffs and are +not interchangeable. + +| Path | Payload | Owner | +| --- | --- | --- | +| PD control path | Request ID, original prompt, `transfer_id`, bootstrap address, engine ID, and connector-specific request metadata | Omni Orchestrator | +| PD data path | Thinker attention KV blocks | vLLM KV connector | +| Downstream model path | Thinker hidden states and embeddings, Talker codec codes, and audio tensors | vLLM-Omni stage connectors and processors | + +### Detection and validation + +The current runtime accepts one logical prefill/decode pair per pipeline. Each +stage may use tensor parallelism, but both sides of the pair must satisfy these +invariants: + +- both stages define `kv_transfer_config` +- prefill uses `kv_producer` or `kv_both` +- decode uses `kv_consumer` or `kv_both` +- both stages use the same KV connector +- `kv_buffer_device` and `kv_buffer_size` agree when set on both stages +- both stages use the same `tensor_parallel_size` + +These checks happen before request processing so an inconsistent PD topology +fails during engine initialization. + +## What is implemented today + +| Area | Current state | +| --- | --- | +| Pair detection and topology validation | Implemented in the shared PD helper | +| Prefill sampling preparation | Implemented for sync and async entrypoints | +| Prefill-to-decode routing | Implemented in the current Orchestrator | +| Deploy-based configuration | Not yet migrated; the supported `stages:` deploy schema cannot currently expand Qwen3-Omni into the four-stage PD topology | +| Qwen3 Thinker-to-Talker handoff | Bridge state and embedding-merge helpers exist, but the merge is not wired into the live stage processor | +| Connector compatibility | Incomplete; the Orchestrator currently requires `remote_request_id`, while the pinned Mooncake connector completes prefill without returning that metadata | +| Validation coverage | PD entrypoint tests and the PD deploy overlay in online end-to-end tests are temporarily disabled during migration | +| Performance | No non-regression or throughput/latency result is established for the current path | + +Because these pieces are still converging, the previous legacy `stage_args` YAML +and `vllm serve` command have been removed. The serve CLI now accepts the +deploy-based `stages:` schema, so retaining the legacy recipe would imply a +launch path that current `main` rejects. + +## Configuration contract under development + +The intended deploy integration starts from the +[default Qwen3-Omni deploy config](gh-file:vllm_omni/deploy/qwen3_omni_moe.yaml) +and expands its frozen three-stage pipeline into the four stages shown above. +A complete integration must: + +- split the Thinker definition without duplicating the Talker or Code2Wav model + contracts +- remap downstream stage IDs, input sources, and stage connectors +- inject producer and consumer `kv_transfer_config` values into the two Thinker + stages +- allocate separate device sets for prefill and decode +- preserve both final outputs: decode text and Code2Wav audio +- provide connector-specific routing metadata without hard-coding one connector + contract + +The required number of GPUs depends on tensor-parallel sizes and whether Talker +and Code2Wav share a device. A basic TP=1 layout uses one device for prefill, one +for decode, and one for the two downstream audio stages. + +## Implementation map + +- PD detection, validation, and sampling-parameter preparation: + [`PDDisaggregationMixin`](gh-file:vllm_omni/entrypoints/pd_utils.py) +- startup detection and connector metadata: + [`AsyncOmniEngine`](gh-file:vllm_omni/engine/async_omni_engine.py) +- request-scoped prefill-to-decode routing: + [`Orchestrator`](gh-file:vllm_omni/engine/orchestrator.py) +- Qwen3 downstream conditioning helpers: + [Qwen3-Omni stage input processor](gh-file:vllm_omni/model_executor/stage_input_processors/qwen3_omni.py) diff --git a/mkdocs.yml b/mkdocs.yml index 1a96ca67d..8a0e322e3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -79,6 +79,7 @@ plugins: user_guide/diffusion/step_execution.md: user_guide/diffusion/execution_modes.md user_guide/diffusion/cpu_offload_diffusion.md: user_guide/diffusion/cpu_offload.md user_guide/quantization/index.md: user_guide/quantization/overview.md + configuration/pd_disaggregation.md: features/pd_disaggregation.md design/feature/diffusion_request_level_batching.md: design/feature/diffusion_continuous_batching.md design/feature/diffusion_step_execution.md: design/feature/diffusion_continuous_batching.md design/feature/cpu_offloading.md: design/feature/offloader/README.md From 2fdbf2234aeb76715618a5b236f0016f115e3e64 Mon Sep 17 00:00:00 2001 From: psv666 <150513104+psv666@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:32:55 +0800 Subject: [PATCH 049/267] [CI/Build][MiniCPM-o] Pin Seed-TTS scorer to npu:1 in the NPU accuracy job (#6275) Signed-off-by: psv666 <2693925048@qq.com> Co-authored-by: amy-why-3459 --- .buildkite/npu/test-npu-nightly.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index 2cdf0cd54..b89f61a39 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -69,6 +69,12 @@ steps: VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" HF_TOKEN: "${HF_TOKEN}" commands: + # Pin the Seed-TTS scorer to the second card: _get_eval_device() otherwise + # auto-picks npu:0, which the omni server under test already fills with its + # KV cache, so Whisper OOMs on `.to(device)` and every clip reports + # "ASR / WER failed". + - export SEED_TTS_WER_EVAL=1 + - export SEED_TTS_EVAL_DEVICE=npu:1 - | set +e pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and npu and A3" --run-level full_model From d7216ef5144866c2f039947e1cc396b1984fcf09 Mon Sep 17 00:00:00 2001 From: JackyLiu Date: Tue, 18 Aug 2026 06:53:06 +0800 Subject: [PATCH 050/267] Add speech token usage headers (#4499) Signed-off-by: JackyLiu Co-authored-by: Hongsheng Liu --- docs/serving/speech_api.md | 15 ++++++- .../openai_api/test_serving_speech.py | 42 ++++++++++++++++++- .../entrypoints/openai/serving_speech.py | 34 +++++++++++++-- 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index 0deea7f8c..11c943ca4 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -141,7 +141,20 @@ The response shape depends on the streaming parameters: **Non-streaming (default).** With `stream=false` and no `stream_format`, returns the complete clip as binary audio data with an appropriate `Content-Type` header (e.g. -`audio/wav`). The raw-bytes body has no JSON carrier, so no `usage` is reported. +`audio/wav`). Because the raw-bytes body has no JSON carrier, successful +non-streaming responses from non-diffusion speech servers report usage through +response headers: + +| Header | Description | +| --- | --- | +| `x-vllm-omni-input-tokens` | Total input tokens (`text_tokens` + `audio_tokens`). | +| `x-vllm-omni-output-tokens` | Generated codec/audio tokens. | +| `x-vllm-omni-total-tokens` | `input_tokens` + `output_tokens`. | +| `x-vllm-omni-input-text-tokens` | Tokens from the synthesized text (`input` plus `instructions`). | +| `x-vllm-omni-input-audio-tokens` | Reference-audio codec frames, non-zero only for in-context voice cloning. | + +Diffusion-mode speech servers route through a separate response path and do not +emit these headers. **Raw audio stream** (`stream_format="audio"`). Streams raw audio bytes (PCM or WAV) as they are decoded. diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 934c0ac64..604eb54f3 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -33,6 +33,8 @@ from vllm_omni.entrypoints.openai.protocol.audio import ( OpenAICreateAudioGenerateRequest, OpenAICreateSpeechRequest, SpeechBatchItem, + SpeechInputTokenDetails, + SpeechTokenUsage, StreamingSpeechSessionConfig, ) from vllm_omni.entrypoints.openai.serving_speech import ( @@ -155,6 +157,7 @@ class TestAudioMixin: # Helper to create mock model output for endpoint tests def create_mock_audio_output_for_test( request_id: str = "speech-mock-123", + metrics: dict | None = None, ) -> OmniRequestOutput: class MockCompletionOutput: def __init__(self, index: int = 0): @@ -181,11 +184,16 @@ def create_mock_audio_output_for_test( audio_tensor = torch.sin(torch.linspace(0, 440 * 2 * torch.pi, num_samples)) mock_request_output = MockRequestOutput(request_id=request_id, audio_tensor=audio_tensor) - return OmniRequestOutput.from_stage_output( + if metrics is None: + metrics = {"stage_metrics": {"0": {"num_tokens_in": 0, "num_tokens_out": 5}}} + + output = OmniRequestOutput.from_stage_output( mock_request_output, stage_id=0, final_output_type="audio", ) + output.metrics = metrics + return output def _write_custom_voice_manifest(root: Path, *, model_type: str, voices: dict) -> None: @@ -233,6 +241,7 @@ def test_app(mocker: MockerFixture): models=mock_models, request_logger=mock_request_logger, ) + speech_server._tts_tokenizer = lambda text, padding=False: {"input_ids": text.split()} # Skip TTS validation in tests (mock doesn't set up supported_speakers) speech_server._validate_tts_request = mocker.MagicMock(return_value=None) @@ -370,6 +379,37 @@ class TestSpeechAPI: assert response.headers["content-type"] == "audio/wav" assert len(response.content) > 0 + def test_create_speech_includes_token_usage_and_detail_headers(self, client): + payload = { + "input": "Hello world", + "model": "tts-model", + "voice": "alloy", + "response_format": "wav", + } + response = client.post("/v1/audio/speech", json=payload) + assert response.status_code == 200 + assert response.headers["x-vllm-omni-input-tokens"] == "2" + assert response.headers["x-vllm-omni-output-tokens"] == "5" + assert response.headers["x-vllm-omni-total-tokens"] == "7" + assert response.headers["x-vllm-omni-input-text-tokens"] == "2" + assert response.headers["x-vllm-omni-input-audio-tokens"] == "0" + + def test_build_speech_usage_headers_uses_usage_and_detail_field_names(self): + usage = SpeechTokenUsage( + input_tokens=7, + output_tokens=11, + total_tokens=18, + input_token_details=SpeechInputTokenDetails(text_tokens=3, audio_tokens=4), + ) + headers = OmniOpenAIServingSpeech._build_speech_usage_headers(usage) + assert headers == { + "X-VLLM-OMNI-INPUT-TOKENS": "7", + "X-VLLM-OMNI-OUTPUT-TOKENS": "11", + "X-VLLM-OMNI-TOTAL-TOKENS": "18", + "X-VLLM-OMNI-INPUT-TEXT-TOKENS": "3", + "X-VLLM-OMNI-INPUT-AUDIO-TOKENS": "4", + } + def test_create_speech_mp3_format(self, client): payload = { "input": "Hello world", diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 1b44d397d..a3069c38c 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -80,6 +80,12 @@ from vllm_omni.utils.speaker_cache import ( logger = init_logger(__name__) +_SPEECH_USAGE_INPUT_TOKENS_HEADER = "X-VLLM-OMNI-INPUT-TOKENS" +_SPEECH_USAGE_OUTPUT_TOKENS_HEADER = "X-VLLM-OMNI-OUTPUT-TOKENS" +_SPEECH_USAGE_TOTAL_TOKENS_HEADER = "X-VLLM-OMNI-TOTAL-TOKENS" +_SPEECH_USAGE_INPUT_TEXT_TOKENS_HEADER = "X-VLLM-OMNI-INPUT-TEXT-TOKENS" +_SPEECH_USAGE_INPUT_AUDIO_TOKENS_HEADER = "X-VLLM-OMNI-INPUT-AUDIO-TOKENS" + # TTS Configuration # # The stage-key -> model-type mapping is NOT declared here: it is derived from @@ -950,6 +956,24 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): details = self._compute_speech_input_details(request, tts_params) return build_speech_usage(details, output_tokens) + @staticmethod + def _build_speech_usage_headers(usage: SpeechTokenUsage | None) -> dict[str, str]: + """Map speech usage into non-streaming response headers. + + Returns an empty dict when usage is unavailable, allowing callers to + merge these headers with other optional response headers. + """ + if usage is None: + return {} + + return { + _SPEECH_USAGE_INPUT_TOKENS_HEADER: str(usage.input_tokens), + _SPEECH_USAGE_OUTPUT_TOKENS_HEADER: str(usage.output_tokens), + _SPEECH_USAGE_TOTAL_TOKENS_HEADER: str(usage.total_tokens), + _SPEECH_USAGE_INPUT_TEXT_TOKENS_HEADER: str(usage.input_token_details.text_tokens), + _SPEECH_USAGE_INPUT_AUDIO_TOKENS_HEADER: str(usage.input_token_details.audio_tokens), + } + def _estimate_fish_ref_code_len(self, ref_audio: object) -> int | None: """Estimate Fish Speech semantic token length from raw reference audio.""" from vllm_omni.model_executor.models.fish_speech.dac_utils import ( @@ -3170,7 +3194,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # ``usage_out`` is an opt-in output channel: when a list is passed, the # computed SpeechTokenUsage is appended to it. The return stays a # 2-tuple so existing callers (and their test mocks) are unaffected; - # only the batch path, which surfaces per-item usage, opts in. + # batch and non-streaming response-header paths opt in when surfacing + # usage outside the raw audio body. request_id, generator, bytes_tts_params = await self._prepare_speech_generation( request, request_id=request_id, has_inline_ref_audio=has_inline_ref_audio ) @@ -3576,7 +3601,10 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): ) collect: dict = {} - audio_bytes, media_type = await self._generate_audio_bytes(request, request_id=request_id, collect=collect) + usage_box: list[SpeechTokenUsage] = [] + audio_bytes, media_type = await self._generate_audio_bytes( + request, request_id=request_id, usage_out=usage_box, collect=collect + ) total_ms = (time.perf_counter() - request_start_s) * 1000.0 logger.info( "[SpeechE2E] request_id=%s stream=false status=ok total_ms=%.2f response_bytes=%d", @@ -3584,7 +3612,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): total_ms, len(audio_bytes) if isinstance(audio_bytes, (bytes, bytearray)) else len(str(audio_bytes)), ) - headers = {} + headers = self._build_speech_usage_headers(usage_box[0] if usage_box else None) if collect.get("word_timestamps") is not None: # Default ensure_ascii keeps the header latin-1 encodable (non-ASCII words \uXXXX-escaped). ts_json = json.dumps(collect["word_timestamps"]) From 3c6648d8a54902297ca7d46815e9883c1de6b204 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Tue, 18 Aug 2026 10:11:10 +0800 Subject: [PATCH 051/267] [Bugfix][NPU] Break the pytest DiffusionOutput circular import (#6293) Signed-off-by: amy-why-3459 --- tests/config/test_config_import_cycle.py | 95 ++++++++++++++++++++++++ vllm_omni/config/__init__.py | 27 ++++++- vllm_omni/platforms/npu/platform.py | 13 ++-- 3 files changed, 128 insertions(+), 7 deletions(-) create mode 100644 tests/config/test_config_import_cycle.py diff --git a/tests/config/test_config_import_cycle.py b/tests/config/test_config_import_cycle.py new file mode 100644 index 000000000..d73f6265e --- /dev/null +++ b/tests/config/test_config_import_cycle.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Guard the NPU pytest circular import through DiffusionOutput.""" + +from __future__ import annotations + +import ast +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _top_level_import_modules(path: Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8")) + modules: set[str] = set() + for node in tree.body: + if isinstance(node, ast.ImportFrom) and node.module: + modules.add(node.module) + elif isinstance(node, ast.Import): + modules.update(alias.name for alias in node.names) + return modules + + +def _assert_isolated_import_succeeds(script: str) -> None: + env = os.environ.copy() + env.pop("CUDA_VISIBLE_DEVICES", None) + env.pop("HIP_VISIBLE_DEVICES", None) + result = subprocess.run( + [sys.executable, "-c", script], + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + timeout=120, + check=False, + ) + assert result.returncode == 0, result.stderr + + +def test_config_package_does_not_eagerly_import_pipeline_registry() -> None: + imported = _top_level_import_modules(REPO_ROOT / "vllm_omni" / "config" / "__init__.py") + assert "vllm_omni.config.config_factory" not in imported + assert "vllm_omni.config.pipeline_registry" not in imported + + +def test_config_package_lazy_exports_still_resolve() -> None: + from vllm_omni.config import StageConfigFactory, register_pipeline + from vllm_omni.config.config_factory import StageConfigFactory as DirectStageConfigFactory + from vllm_omni.config.pipeline_registry import register_pipeline as direct_register_pipeline + + assert StageConfigFactory is DirectStageConfigFactory + assert register_pipeline is direct_register_pipeline + + +def test_npu_platform_defers_minimax_h3_encoder_patch() -> None: + src = (REPO_ROOT / "vllm_omni" / "platforms" / "npu" / "platform.py").read_text(encoding="utf-8") + init_start = src.index("def __init__(self) -> None:") + runtime_start = src.index("def init_diffusion_model_runner_runtime") + init_src = src[init_start:runtime_start] + runtime_src = src[runtime_start:] + assert "apply_minimax_h3_qwen3vl_patch" not in init_src + assert "apply_minimax_h3_qwen3vl_patch" in runtime_src + assert "apply_minimax_h3_qwen3vl_swiglu_patch" in runtime_src + + +def test_lora_config_import_does_not_load_pipeline_registry() -> None: + """Import LoRAConfig and diffusion.data in a fresh interpreter. + + The NPU failure was ``data.py`` importing LoRAConfig while still loading, + then ``vllm_omni.config`` re-entering ``pipeline_registry`` → PI0 → + DiffusionOutput. This child process must not preload those exports. + """ + _assert_isolated_import_succeeds( + """ +import sys + +from vllm_omni.config.lora import LoRAConfig + +if "vllm_omni.config.pipeline_registry" in sys.modules: + raise SystemExit("LoRAConfig import loaded pipeline_registry") +if "vllm_omni.diffusion.models.pi0.pipeline_pi0" in sys.modules: + raise SystemExit("LoRAConfig import loaded PI0_PIPELINE") + +from vllm_omni.diffusion.data import DiffusionOutput + +if DiffusionOutput is None or LoRAConfig is None: + raise SystemExit("expected LoRAConfig and DiffusionOutput") +""" + ) diff --git a/vllm_omni/config/__init__.py b/vllm_omni/config/__init__.py index 3069025be..f53a1ef74 100644 --- a/vllm_omni/config/__init__.py +++ b/vllm_omni/config/__init__.py @@ -2,7 +2,6 @@ Configuration module for vLLM-Omni. """ -from vllm_omni.config.config_factory import StageConfigFactory from vllm_omni.config.lora import LoRAConfig from vllm_omni.config.model import OmniModelConfig from vllm_omni.config.omni_config import ( @@ -22,7 +21,6 @@ from vllm_omni.config.omni_config import ( VllmOmniGenerationStageConfig, VllmOmniOrchestratorConfig, ) -from vllm_omni.config.pipeline_registry import register_pipeline from vllm_omni.config.stage_config import ( PIPELINE_WIDE_ENGINE_FIELDS, DeployConfig, @@ -42,6 +40,15 @@ from vllm_omni.config.yaml_util import ( to_dict, ) +# StageConfigFactory / register_pipeline pull pipeline_registry, which eagerly +# imports PI0_PIPELINE → diffusion.data. Keep those lazy so +# `from vllm_omni.config.lora import LoRAConfig` (used while data.py is still +# loading) cannot close a circular import through DiffusionOutput. +_LAZY_ATTRS = { + "StageConfigFactory": ("vllm_omni.config.config_factory", "StageConfigFactory"), + "register_pipeline": ("vllm_omni.config.pipeline_registry", "register_pipeline"), +} + __all__ = [ # Legacy model-level configs. "LoRAConfig", @@ -82,3 +89,19 @@ __all__ = [ "merge_configs", "to_dict", ] + + +def __getattr__(name: str): + spec = _LAZY_ATTRS.get(name) + if spec is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module_name, attr_name = spec + import importlib + + value = getattr(importlib.import_module(module_name), attr_name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted(set(__all__) | set(globals())) diff --git a/vllm_omni/platforms/npu/platform.py b/vllm_omni/platforms/npu/platform.py index 38c35f1f7..9a6a278e4 100644 --- a/vllm_omni/platforms/npu/platform.py +++ b/vllm_omni/platforms/npu/platform.py @@ -39,7 +39,6 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform): from vllm_omni.platforms.npu.models.minicpmo_4_5_code2wav import ( apply_minicpmo_4_5_code2wav_patch, ) - from vllm_omni.platforms.npu.models.minimax_h3 import apply_minimax_h3_qwen3vl_patch from vllm_omni.platforms.npu.models.qwen3_tts_code2wav import ( apply_qwen3_tts_code2wav_patch, ) @@ -48,7 +47,6 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform): ) adapt_patch(is_global_patch=True) - apply_minimax_h3_qwen3vl_patch() apply_minicpmo_4_5_code2wav_patch() apply_qwen3_tts_code2wav_patch() apply_qwen3_tts_tokenizer_v2_patch() @@ -87,12 +85,17 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform): from vllm_ascend.ascend_forward_context import set_mc2_mask, set_mc2_tokens_capacity from vllm_omni.platforms.npu.models.minimax_h3 import ( + apply_minimax_h3_qwen3vl_patch, apply_minimax_h3_qwen3vl_swiglu_patch, ) - # The patch imports the MiniMax encoder, which depends on - # current_omni_platform. Run it only after platform construction has - # completed, but before the diffusion pipeline is loaded. + # Both patches import the MiniMax encoder package, whose __init__ loads + # pipeline_minimax_h3 → diffusion.data. Doing that during platform + # construction races vllm_omni/__init__.py (patch before config) and + # closes a cycle through pipeline_registry → PI0_PIPELINE → + # DiffusionOutput. Apply them only after the platform exists, before + # the diffusion pipeline is loaded. + apply_minimax_h3_qwen3vl_patch() apply_minimax_h3_qwen3vl_swiglu_patch() set_mc2_tokens_capacity(vllm_config, od_config.max_num_seqs, 1) set_mc2_mask(vllm_config, device) From 199ef7779c1b7650a0e9df0951e0a2b9da4e0190 Mon Sep 17 00:00:00 2001 From: psv666 <150513104+psv666@users.noreply.github.com> Date: Tue, 18 Aug 2026 10:53:55 +0800 Subject: [PATCH 052/267] [Bugfix][MiniCPM-o] Load Daily-Omni QA metadata from the hub cache (#6276) Signed-off-by: psv666 <2693925048@qq.com> Co-authored-by: amy-why-3459 --- .../test_daily_omni_local_source.py | 60 +++++++++ .../data_modules/daily_omni_dataset.py | 117 +++++++++++++++--- 2 files changed, 163 insertions(+), 14 deletions(-) diff --git a/tests/benchmarks/test_daily_omni_local_source.py b/tests/benchmarks/test_daily_omni_local_source.py index 35045a04b..13ae0637a 100644 --- a/tests/benchmarks/test_daily_omni_local_source.py +++ b/tests/benchmarks/test_daily_omni_local_source.py @@ -9,13 +9,17 @@ from pathlib import Path import pytest from vllm_omni.benchmarks.data_modules.daily_omni_dataset import ( + DailyOmniDataset, daily_omni_local_qa_json, daily_omni_local_videos_dir, + ensure_daily_omni_hub_root, resolve_daily_omni_local_root, ) pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark] +_HUB_ID = "liarliar/Daily-Omni" + def _write_qa_json(root: Path) -> Path: qa = root / "qa.json" @@ -23,6 +27,28 @@ def _write_qa_json(root: Path) -> Path: return qa +def _stage_offline_hub_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, with_qa: bool = True) -> Path: + """Build a hub cache holding ``liarliar/Daily-Omni`` and cut off all network access. + + Reproduces the air-gapped NPU runner: the dataset is pre-staged with ``hf download``, so it + only exists in the *hub* cache -- never in the ``datasets`` cache that ``load_dataset`` reads. + """ + constants = pytest.importorskip("huggingface_hub").constants + + cache = tmp_path / "hub" + repo_dir = cache / "datasets--liarliar--Daily-Omni" + snapshot = repo_dir / "snapshots" / "deadbeef" + snapshot.mkdir(parents=True) + if with_qa: + _write_qa_json(snapshot) + (repo_dir / "refs").mkdir() + (repo_dir / "refs" / "main").write_text("deadbeef", encoding="utf-8") + + monkeypatch.setattr(constants, "HF_HUB_CACHE", str(cache)) + monkeypatch.setattr(constants, "HF_HUB_OFFLINE", True) + return snapshot + + def test_resolve_local_root_returns_none_for_hub_id() -> None: assert resolve_daily_omni_local_root("liarliar/Daily-Omni") is None assert resolve_daily_omni_local_root("") is None @@ -90,3 +116,37 @@ def test_missing_qa_json_reports_none(tmp_path: Path) -> None: root = resolve_daily_omni_local_root(str(tmp_path)) assert daily_omni_local_qa_json(root) is None assert daily_omni_local_videos_dir(root) is None + + +def test_hub_id_resolves_to_offline_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + snapshot = _stage_offline_hub_cache(tmp_path, monkeypatch) + + root = ensure_daily_omni_hub_root(_HUB_ID) + + assert root == snapshot.resolve() + assert daily_omni_local_qa_json(root) == (snapshot / "qa.json").resolve() + + +def test_dataset_loads_hub_id_without_network(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The regression this file exists for: a Hub id must not require ``load_dataset``.""" + snapshot = _stage_offline_hub_cache(tmp_path, monkeypatch) + # Any use of `datasets` here means we went back to the path that cannot read the hub cache. + monkeypatch.setattr("vllm_omni.benchmarks.data_modules.daily_omni_dataset.load_dataset", None) + + dataset = DailyOmniDataset(dataset_path=_HUB_ID, random_seed=0) + + assert dataset.qa_json_path == (snapshot / "qa.json").resolve() + assert [row["video_id"] for row in dataset.data] == ["vid0"] + + +def test_uncached_hub_id_reports_both_attempts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _stage_offline_hub_cache(tmp_path, monkeypatch, with_qa=False) + monkeypatch.setattr("vllm_omni.benchmarks.data_modules.daily_omni_dataset.load_dataset", None) + + with pytest.raises(RuntimeError) as excinfo: + DailyOmniDataset(dataset_path="someone-else/Daily-Omni", random_seed=0) + + message = str(excinfo.value) + assert "someone-else/Daily-Omni" in message + assert "snapshot_download failed" in message + assert "--daily-omni-qa-json" in message diff --git a/vllm_omni/benchmarks/data_modules/daily_omni_dataset.py b/vllm_omni/benchmarks/data_modules/daily_omni_dataset.py index 512e86538..287a9cbca 100644 --- a/vllm_omni/benchmarks/data_modules/daily_omni_dataset.py +++ b/vllm_omni/benchmarks/data_modules/daily_omni_dataset.py @@ -7,7 +7,8 @@ Dataset source: https://huggingface.co/datasets/liarliar/Daily-Omni Supports loading QA metadata from: - Local JSON file (``qa_json_path``): recommended for offline/air-gapped environments -- HuggingFace datasets (``dataset_path``): legacy online mode +- HuggingFace Hub id (``dataset_path``): resolved with ``snapshot_download`` to the repo's + ``qa.json``, which reads the hub cache and therefore also works under ``HF_HUB_OFFLINE`` Video/audio files normally come from extracted ``Videos.tar``. When ``--daily-omni-video-dir`` is not set, the first request that needs on-disk media downloads that archive from the Hugging Face @@ -24,9 +25,9 @@ Why ``BenchmarkDataset`` instead of ``HuggingFaceDataset``? This class therefore inherits only ``BenchmarkDataset`` (minimal: ``dataset_path``, ``random_seed``, ``self.data``) and implements **two explicit loaders**: ``_load_from_local_json`` (default path for air-gapped runs) and ``_load_from_huggingface`` - (optional legacy path for users who prefer ``datasets`` + Hub cache). The latter is **not** + (Hub ids, resolved to the repo's ``qa.json`` through the hub cache). The latter is **not** inheritance; it is the same Hub rows as before, factored into a helper so one class can - serve both deployment modes without mandatory ``datasets`` when using ``qa_json_path``. + serve both deployment modes without mandatory ``datasets``. Usage: from vllm_omni.benchmarks.data_modules.daily_omni_dataset import DailyOmniDataset @@ -38,7 +39,7 @@ Usage: random_seed=42, ) - # HuggingFace mode (legacy, requires network) + # Hub id mode (works offline once the repo is in the HF hub cache) dataset = DailyOmniDataset( dataset_path="liarliar/Daily-Omni", dataset_split="train", @@ -210,6 +211,68 @@ def _resolve_hf_cache_snapshot(path: Path) -> Path: return path +#: QA metadata pulled by :func:`ensure_daily_omni_hub_root`. ``Videos.tar`` is deliberately left +#: out: it is ~60GB and only :func:`ensure_daily_omni_hub_videos_dir` needs it, lazily, once a +#: request actually asks for media. +_DAILY_OMNI_QA_ALLOW_PATTERNS = ["qa.json", "QA.json"] + + +def ensure_daily_omni_hub_root(repo_id: str) -> Path: + """Resolve a Daily-Omni Hub id to its snapshot directory, downloading QA metadata if needed. + + Mirrors :func:`...seed_tts_dataset.resolve_seed_tts_root`: ``snapshot_download`` resolves the + repo through the **hub cache**, so an air-gapped run (``HF_HUB_OFFLINE=1``) keeps working as + long as the dataset was pre-staged there. ``datasets.load_dataset`` cannot do this — it only + consults the separate *datasets* cache, which a ``hf download`` never populates. + + Raises: + ImportError: if ``huggingface_hub`` is not installed. + ValueError: if ``repo_id`` is empty. + """ + rid = (repo_id or "").strip() + if not rid: + raise ValueError("repo_id is required to download Daily-Omni from Hugging Face") + + try: + from huggingface_hub import snapshot_download + except ImportError as e: + raise ImportError( + "Install huggingface_hub to load Daily-Omni from the Hub, or pass a local " + "--daily-omni-qa-json / --daily-omni-video-dir mirror." + ) from e + + cache = snapshot_download( + repo_id=rid, + repo_type="dataset", + allow_patterns=_DAILY_OMNI_QA_ALLOW_PATTERNS, + ) + root = _resolve_hf_cache_snapshot(Path(cache).resolve()) + logger.info("Daily-Omni Hub snapshot ready at %s (repo=%s)", root, rid) + return root + + +def _daily_omni_qa_load_error( + repo_id: str | None, + snapshot_error: Exception | None, + datasets_error: Exception, +) -> RuntimeError: + """Compose an actionable error after both QA-metadata sources failed. + + Offline CI hits this when the dataset is missing from the hub cache *or* staged under a + different repo id, so the message names both attempts and the flags that bypass them. + """ + parts = [f"Could not load Daily-Omni QA metadata for {repo_id!r}."] + if snapshot_error is not None: + parts.append(f"huggingface_hub snapshot_download failed: {type(snapshot_error).__name__}: {snapshot_error}") + parts.append(f"datasets.load_dataset failed: {type(datasets_error).__name__}: {datasets_error}") + parts.append( + "For offline runs, pre-stage the dataset in the HF hub cache " + "(`hf download --repo-type dataset `) and make sure the repo id matches, " + "or pass --daily-omni-qa-json / --daily-omni-video-dir pointing at a local mirror." + ) + return RuntimeError(" ".join(parts)) + + def resolve_daily_omni_local_root(dataset_path: str | None) -> Path | None: """Return the local Daily-Omni mirror root for ``dataset_path``, or ``None`` if not local. @@ -433,7 +496,8 @@ class DailyOmniDataset(BenchmarkDataset): QA metadata can be loaded from: - Local JSON file (``qa_json_path``): recommended for offline/air-gapped environments - - HuggingFace datasets (``dataset_path``): legacy online mode + - HuggingFace Hub id (``dataset_path``): the repo's ``qa.json`` via ``snapshot_download`` + (hub-cache backed, offline-safe); ``datasets.load_dataset`` only as a parquet fallback Video/audio files normally come from extracted ``Videos.tar``. When ``video_dir`` is not set, the first sample that needs on-disk media downloads that archive from the Hugging Face dataset @@ -444,7 +508,8 @@ class DailyOmniDataset(BenchmarkDataset): qa_json_path: Path to local qa.json file (offline mode, preferred). When provided, ``dataset_path`` and ``dataset_split`` are ignored. dataset_path: HuggingFace dataset path (e.g., "liarliar/Daily-Omni"). Used only if - ``qa_json_path`` is not provided (legacy online mode). + ``qa_json_path`` is not provided; resolved through the hub cache, so a pre-staged + repo also loads with ``HF_HUB_OFFLINE=1``. dataset_split: Dataset split to use (default: "train"). Used only in online mode. random_seed: Random seed for shuffling video_dir: Directory containing extracted video files (default: None; may be filled lazily @@ -602,21 +667,42 @@ class DailyOmniDataset(BenchmarkDataset): self.data = _ListDatasetIterator(data) def _load_from_huggingface(self) -> None: - """Load QA rows via ``datasets.load_dataset`` (legacy / convenience path). + """Load QA rows for a Hub id, preferring the repo's ``qa.json`` over ``datasets``. - Kept for backward compatibility: callers can still pass ``dataset_path=liarliar/Daily-Omni`` - and get the same parquet-backed rows as the Hub dataset card, with streaming (or - non-streaming if ``no_stream=True``) and shuffle. + The Hub repo ships ``qa.json`` (1,197 rows) next to ``Videos.tar``, so the primary path is + :func:`ensure_daily_omni_hub_root` → :meth:`_load_from_local_json`: ``snapshot_download`` + reads the hub cache and therefore works under ``HF_HUB_OFFLINE``, the same contract as + Seed-TTS. ``datasets.load_dataset`` remains only as a fallback for mirrors + that publish parquet rows instead of ``qa.json``; it consults the *datasets* cache rather + than the hub cache and streams shards over HTTP, so it cannot serve air-gapped runs. This is intentionally **not** implemented by subclassing ``HuggingFaceDataset``: that base always runs Hub ``load_dataset`` from its constructor and expects a Hub id as the primary API; Daily-Omni instead chooses the source in ``load_data()`` (JSON vs Hub) while sharing one ``sample()`` / request-building implementation for both. """ + snapshot_error: Exception | None = None + try: + root = ensure_daily_omni_hub_root(self.dataset_path) + except Exception as e: # noqa: BLE001 - any failure just falls through to `datasets` + snapshot_error = e + else: + qa_json = daily_omni_local_qa_json(root) + if qa_json is not None: + self.qa_json_path = qa_json + logger.info("Loading Daily-Omni QA metadata from Hub snapshot: %s", qa_json) + self._load_from_local_json() + return + logger.info( + "Hub snapshot %s ships no qa.json; falling back to `datasets.load_dataset`", + root, + ) + if load_dataset is None: - raise ImportError( - "datasets library is required for HuggingFace mode. " - "Install with: pip install datasets, or use local JSON mode instead." + raise _daily_omni_qa_load_error( + self.dataset_path, + snapshot_error, + ImportError("datasets library is not installed (pip install datasets)"), ) load_kw: dict[str, Any] = { @@ -626,7 +712,10 @@ class DailyOmniDataset(BenchmarkDataset): } if self.dataset_subset is not None: load_kw["name"] = self.dataset_subset - ds = load_dataset(self.dataset_path, **load_kw) + try: + ds = load_dataset(self.dataset_path, **load_kw) + except Exception as e: + raise _daily_omni_qa_load_error(self.dataset_path, snapshot_error, e) from e if not getattr(self, "disable_shuffle", False): ds = ds.shuffle(seed=self.random_seed) self.data = ds From b283215c28d56d24b3f72a120b8f34222699aeda Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Mon, 17 Aug 2026 21:33:13 -0700 Subject: [PATCH 053/267] [Model] Add MiniMax Music 3 text-to-music (#6186) Signed-off-by: Yueqian Lin --- docs/models/supported_models.md | 1 + recipes/MiniMaxAI/MiniMax-Music3.md | 173 +++ .../test_invalid_audio_speech.py | 65 ++ .../e2e/online_serving/test_minimax_music3.py | 112 ++ tests/engine/test_async_omni_engine_input.py | 103 ++ .../openai_api/test_tts_detection.py | 1 + .../test_minimax_music3_batch_payload.py | 132 +++ .../models/test_minimax_music3_frame_state.py | 214 ++++ .../models/test_minimax_music3_prompt.py | 203 ++++ .../models/test_minimax_music3_row_binding.py | 217 ++++ tests/model_executor/test_cfg_pairing.py | 389 +++++++ vllm_omni/config/model.py | 5 + vllm_omni/config/pipeline_registry.py | 2 + vllm_omni/data_entry_keys.py | 6 + vllm_omni/deploy/minimax_music3.yaml | 130 +++ vllm_omni/deploy/minimax_music3_2gpu.yaml | 97 ++ vllm_omni/engine/arg_utils.py | 2 + vllm_omni/engine/async_omni_engine.py | 86 +- vllm_omni/engine/stage_engine_core_proc.py | 9 +- vllm_omni/engine/stage_init_utils.py | 57 +- .../openai/tts_adapters/__init__.py | 1 + .../openai/tts_adapters/minimax_music3.py | 163 +++ vllm_omni/model_executor/models/audex/cfg.py | 391 +------ .../models/common/cfg_pairing.py | 460 ++++++++ .../models/minimax_music3/__init__.py | 34 + .../models/minimax_music3/acoustic.py | 459 ++++++++ .../models/minimax_music3/chunking.py | 96 ++ .../models/minimax_music3/constants.py | 111 ++ .../models/minimax_music3/dav.py | 294 ++++++ .../models/minimax_music3/depth_graph.py | 137 +++ .../models/minimax_music3/dit.py | 484 +++++++++ .../models/minimax_music3/frame_state.py | 158 +++ .../models/minimax_music3/pipeline.py | 103 ++ .../models/minimax_music3/prompt.py | 172 +++ .../models/minimax_music3/rvq_decoder.py | 179 ++++ .../models/minimax_music3/sampling.py | 223 ++++ .../models/minimax_music3/staging.py | 66 ++ .../models/minimax_music3/talker.py | 990 ++++++++++++++++++ .../models/minimax_music3/weights.py | 133 +++ vllm_omni/model_executor/models/registry.py | 11 + .../stage_input_processors/minimax_music3.py | 544 ++++++++++ .../transformers_utils/configs/__init__.py | 3 + .../configs/minimax_music3.py | 65 ++ 43 files changed, 6876 insertions(+), 405 deletions(-) create mode 100644 recipes/MiniMaxAI/MiniMax-Music3.md create mode 100644 tests/e2e/online_serving/test_minimax_music3.py create mode 100644 tests/model_executor/models/test_minimax_music3_batch_payload.py create mode 100644 tests/model_executor/models/test_minimax_music3_frame_state.py create mode 100644 tests/model_executor/models/test_minimax_music3_prompt.py create mode 100644 tests/model_executor/models/test_minimax_music3_row_binding.py create mode 100644 tests/model_executor/test_cfg_pairing.py create mode 100644 vllm_omni/deploy/minimax_music3.yaml create mode 100644 vllm_omni/deploy/minimax_music3_2gpu.yaml create mode 100644 vllm_omni/entrypoints/openai/tts_adapters/minimax_music3.py create mode 100644 vllm_omni/model_executor/models/common/cfg_pairing.py create mode 100644 vllm_omni/model_executor/models/minimax_music3/__init__.py create mode 100644 vllm_omni/model_executor/models/minimax_music3/acoustic.py create mode 100644 vllm_omni/model_executor/models/minimax_music3/chunking.py create mode 100644 vllm_omni/model_executor/models/minimax_music3/constants.py create mode 100644 vllm_omni/model_executor/models/minimax_music3/dav.py create mode 100644 vllm_omni/model_executor/models/minimax_music3/depth_graph.py create mode 100644 vllm_omni/model_executor/models/minimax_music3/dit.py create mode 100644 vllm_omni/model_executor/models/minimax_music3/frame_state.py create mode 100644 vllm_omni/model_executor/models/minimax_music3/pipeline.py create mode 100644 vllm_omni/model_executor/models/minimax_music3/prompt.py create mode 100644 vllm_omni/model_executor/models/minimax_music3/rvq_decoder.py create mode 100644 vllm_omni/model_executor/models/minimax_music3/sampling.py create mode 100644 vllm_omni/model_executor/models/minimax_music3/staging.py create mode 100644 vllm_omni/model_executor/models/minimax_music3/talker.py create mode 100644 vllm_omni/model_executor/models/minimax_music3/weights.py create mode 100644 vllm_omni/model_executor/stage_input_processors/minimax_music3.py create mode 100644 vllm_omni/transformers_utils/configs/minimax_music3.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index d9fdb09a4..31f3ac35d 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -78,6 +78,7 @@ th { | `SoulXSingerPipeline` | SoulX-Singer (SVS) | `Soul-AILab/SoulX-Singer` | ✅︎ | | | | — | | `SoulXSingerSVCPipeline` | SoulX-Singer-SVC | `Soul-AILab/SoulX-Singer` (`model-svc.pt`) | ✅︎ | | | | — | | `AudioXPipeline` | AudioX | `zhangj1an/AudioX` | ✅︎ | ✅︎ | | | — | +| `MiniMaxMusic3ForConditionalGeneration` | MiniMax Music 3 (text-to-music) | `MiniMaxAI/MiniMax-Music3` | ✅︎ | | | | — | | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | | | | [Published](https://recipes.vllm.ai/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) | | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | diff --git a/recipes/MiniMaxAI/MiniMax-Music3.md b/recipes/MiniMaxAI/MiniMax-Music3.md new file mode 100644 index 000000000..3d078f645 --- /dev/null +++ b/recipes/MiniMaxAI/MiniMax-Music3.md @@ -0,0 +1,173 @@ +# MiniMax Music 3 + +> Text-to-music: lyrics plus a style caption in, a 32 kHz stereo song out + +## Summary + +- Vendor: MiniMax +- Model: `MiniMaxAI/MiniMax-Music3` +- Task: Text-to-music generation (vocals and instrumental together) +- Mode: Online serving through the OpenAI-compatible `/v1/audio/speech` API +- Maintainer: @linyueqian + +## When to use this recipe + +Use it to serve `MiniMaxAI/MiniMax-Music3` for full-song generation up to six +minutes. This is not a TTS model with a musical voice: there is no reference +speaker, no `voice`, and no `temperature`. What you control is the lyrics, the +caption, the seed, and the length. + +## References + +- Model card: [MiniMaxAI/MiniMax-Music3](https://huggingface.co/MiniMaxAI/MiniMax-Music3) +- Demo samples: [MiniMax Music 3 demo](https://minimax-ai.github.io/music3-demo/) +- Deploy config: `vllm_omni/deploy/minimax_music3.yaml` (single GPU), + `vllm_omni/deploy/minimax_music3_2gpu.yaml` (split across two) + +## Architecture + +| Component | Spec | +|---|---| +| Backbone | Qwen3 decoder, 36 layers, hidden 4096, GQA 32/8, vocab 200k | +| Depth decoder | 4 layers, hidden 4096, 16 heads; 7 residual codebooks of 1024 | +| Frame | 8 codebooks; `c0` in the backbone vocabulary, `c1..c7` in the depth decoder | +| AR guidance | Classifier-free, scale 1.5, `c0` masked to the conditioned branch's top 50 | +| Acoustic stage | Flow-matching transformer (36 layers, dim 2048) plus a DAC-style decoder, 512x upsampling | +| Solver | Euler, 30 steps, acoustic guidance scale 1.7 | +| Window | 200 frames per acoustic chunk, 100-frame hop | +| Frame rate | 25 frames per second | +| Context length | 10,240 tokens | +| Output | 32 kHz stereo WAV | + +Generation is two-stage. The backbone predicts one audio frame per decode step, +each frame eight RVQ codebooks deep. Every 200 frames are handed to the +flow-matching transformer, which solves a latent that the decoder turns into +waveform. + +**Guidance doubles the rows.** Every request decodes twice: one row sees the +real prompt, the other sees the same prompt with the caption-and-lyrics span +replaced by `<|audio_cfg|>`. Both rows hold their own KV cache for the whole +song, so a request costs twice the KV its length suggests. Size the pool for +rows, not requests. + +## Hardware Support + +## GPU + +### 1x H200 141GB + +#### Environment + +- OS: Linux +- Python: 3.12 +- CUDA: 12.9 +- vLLM version: 0.27.0 +- vLLM-Omni version or commit: branch `feat/minimax-music3` + +#### Command + +```bash +vllm serve MiniMaxAI/MiniMax-Music3 \ + --host 0.0.0.0 --port 8000 \ + --trust-remote-code --omni +``` + +The default deploy config `vllm_omni/deploy/minimax_music3.yaml` is discovered +automatically from the checkpoint's `model_type=minimax_music3`. It colocates +both stages on device 0. + +#### Verification + +```bash +curl -X POST http://localhost:8000/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "model": "MiniMaxAI/MiniMax-Music3", + "input": "[Verse]\nWalking down the empty street at midnight\nStreetlights flicker like a broken dream\n[Chorus]\nAnd I keep on walking\nTill the morning finds me", + "instructions": "A melancholic lo-fi hip-hop track at 85 BPM in F minor: mellow Rhodes piano riff, soft vinyl crackle, dusty boom-bap drums with a laid-back swing, warm upright bass. Intimate bedroom production, gentle tape saturation, no bright cymbals.", + "seed": 1, + "max_new_tokens": 750 + }' \ + --output song.wav +``` + +Expected: a 30 second 32 kHz stereo WAV. `max_new_tokens` counts audio frames +at 25 per second, so 750 frames is at most 30 seconds. It is a cap, not a +target: the model ends the song itself when it emits the audio-end token, and a +response shorter than the cap is the model finishing rather than a truncation. + +#### Notes + +- **Memory:** stage 0 takes `gpu_memory_utilization: 0.6` for the bf16 backbone + and its KV pool; stage 1 takes 0.2 for the transformer and decoder weights, + which sit outside any KV budget. +- **Key flags:** none required. Guidance, the solver step count and the window + geometry are fixed by the checkpoint and are not request parameters. +- **Known limitations:** + - The external API is non-streaming; `stream: true` is rejected. + - `max_model_len` is 10,240, so a maximum-length prompt (5,000 tokens) and a + maximum-length song (9,000 frames) cannot both fit. Long captions shorten + the maximum song. This is a property of the checkpoint. + - The acoustic stage runs in float32. bfloat16 is accepted but measurably + degrades the solver. + +### 2x H200 141GB + +Use `minimax_music3_2gpu.yaml` to put the acoustic stage on the second device: + +```bash +vllm serve MiniMaxAI/MiniMax-Music3 \ + --host 0.0.0.0 --port 8000 \ + --trust-remote-code --omni \ + --deploy-config minimax_music3_2gpu.yaml +``` + +Only the placement differs; both layouts run the acoustic stage in float32. + +## Writing the request + +Two fields carry the request, and both are required. + +- `input` is the **lyrics**. Structure tags (`[Verse]`, `[Chorus]`, `[Bridge]`, + `[Outro]`) steer the arrangement and are part of the prompt contract. + **Put a tag on its own line.** Normalization keeps only the tags on a line + that starts with one and drops the rest of that line, so lyrics written next + to a tag are silently lost: + + ```text + "[Verse]\nWalking down the street" -> [start] [verse] Walking down the street + "[Verse] Walking down the street" -> [start] [verse] + ``` + +- `instructions` is the **caption** describing genre, instrumentation, tempo, + mood and production. It is the strongest control available: vague captions + give generic arrangements. Long structured captions work well, since the + model was trained on descriptions covering global attributes, emotional + progression, vocal detail and arrangement. + +A request is deterministic in its seed: the same lyrics, caption, seed and +length return the same audio. Changing only the seed gives another take of the +same song. + +## Request parameters + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `input` | string | required | Lyrics, non-empty. Tags on their own lines | +| `instructions` | string | required | Caption, non-empty | +| `seed` | int | `0` | Non-negative. Fixes the output for a given request | +| `max_new_tokens` | int | `9000` | Cap on audio frames, 25 per second. Maximum 9,000 | +| `response_format` | string | `"wav"` | Output container | + +## Parameters the model rejects + +The request is refused rather than silently ignored, so a mistake is visible +immediately. + +| Parameter | Reason | +|---|---| +| `temperature`, `top_p`, `top_k`, `repetition_penalty` | Sampling is fixed: guidance at 1.5, then a seeded top-k 50 draw | +| `voice` | There is no speaker to select; the vocal comes from the caption | +| `ref_audio`, `ref_text`, `language`, `task_type` | No reference-audio conditioning and no language tag | +| `speed` | Only `1.0`. Tempo belongs in the caption, e.g. "at 92 BPM" | +| `stream: true` | The external API is non-streaming | diff --git a/tests/dfx/reliability/invalid_param_test/test_invalid_audio_speech.py b/tests/dfx/reliability/invalid_param_test/test_invalid_audio_speech.py index 61cd30ec8..e30591c19 100644 --- a/tests/dfx/reliability/invalid_param_test/test_invalid_audio_speech.py +++ b/tests/dfx/reliability/invalid_param_test/test_invalid_audio_speech.py @@ -976,3 +976,68 @@ def test_voices_delete_invalid_requests( "err_message": err_message, } ) + + +# ─── POST /v1/audio/speech · MiniMax Music 3 ─── + +# Text-to-music on the speech endpoint. Most of the speech contract does not +# apply: there is no speaker, no reference audio and no temperature, and the +# lyrics alone are not a request. Each case below pins a rejection so a caller +# who assumes the usual TTS shape finds out immediately. +_MINIMAX_MUSIC3_SPEECH = [ + pytest.param( + OmniServerParams( + model="MiniMaxAI/MiniMax-Music3", + stage_config_path=get_deploy_config_path("minimax_music3.yaml"), + server_args=_SPEECH_SERVER_ARGS, + ), + id="minimax_music3", + marks=hardware_marks(res={"cuda": "H100"}), + ), +] + +_MINIMAX_MUSIC3_LYRICS = "[Verse]\nWalking down the empty street at midnight" +_MINIMAX_MUSIC3_CAPTION = "A melancholic lo-fi hip-hop track at 85 BPM in F minor." +# 25 frames per second, capped at six minutes of song. +_MINIMAX_MUSIC3_MAX_FRAMES = 9000 + + +@pytest.mark.parametrize( + "overrides, err_message", + [ + pytest.param({"instructions": None}, ("instructions",), id="caption_missing"), + pytest.param({"instructions": " "}, ("instructions",), id="caption_blank"), + pytest.param({"input": " "}, ("input",), id="lyrics_blank"), + pytest.param({"voice": "alloy"}, ("voice",), id="voice_rejected"), + pytest.param({"speed": 1.5}, ("speed",), id="speed_rejected"), + pytest.param( + {"max_new_tokens": _MINIMAX_MUSIC3_MAX_FRAMES + 1}, + ("max_new_tokens", str(_MINIMAX_MUSIC3_MAX_FRAMES)), + id="max_new_tokens_over_cap", + ), + pytest.param({"max_new_tokens": 0}, ("max_new_tokens",), id="max_new_tokens_zero"), + ], +) +@pytest.mark.parametrize("omni_server", _MINIMAX_MUSIC3_SPEECH, indirect=True) +def test_minimax_music3_speech_invalid_field_values( + omni_server: OmniServer, + openai_client: OpenAIClientHandler, + overrides: dict[str, object], + err_message: str | tuple[str, ...], +) -> None: + body: dict[str, Any] = { + "model": omni_server.model, + "input": _MINIMAX_MUSIC3_LYRICS, + "instructions": _MINIMAX_MUSIC3_CAPTION, + "seed": 1, + "max_new_tokens": 250, + "response_format": "wav", + } + for key, value in overrides.items(): + if value is None: + body.pop(key, None) + else: + body[key] = value + openai_client.send_audio_speech_http_request( + {"json": body, "timeout": 120, "err_code": 400, "err_message": err_message} + ) diff --git a/tests/e2e/online_serving/test_minimax_music3.py b/tests/e2e/online_serving/test_minimax_music3.py new file mode 100644 index 000000000..3d8718e70 --- /dev/null +++ b/tests/e2e/online_serving/test_minimax_music3.py @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""E2E online tests for MiniMax Music 3 text-to-music. + +This is a music model on the speech endpoint, so the request shape differs +from every other TTS model here: ``input`` carries the lyrics, ``instructions`` +carries the caption that decides genre and arrangement, and both are required. + +Weekly rather than per-PR: the checkpoint is 27 GB across five components and a +single request occupies two decode rows, because classifier-free guidance is +mandatory for this model. Rejection cases live in +``tests/dfx/reliability/invalid_param_test/test_invalid_audio_speech.py``. +""" + +import os + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServerParams +from tests.helpers.stage_config import get_deploy_config_path + +MODEL = "MiniMaxAI/MiniMax-Music3" +DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 600.0 + +# 25 frames per second, so 250 frames is ten seconds and 750 is thirty. The +# byte floors sit below the true payload (32 kHz stereo, 16-bit) to leave room +# for the model ending a song before the cap. +_FRAMES_10S = 250 +_FRAMES_30S = 750 +_MIN_BYTES_10S = 500_000 +_MIN_BYTES_30S = 1_500_000 + +LYRICS = ( + "[Verse]\nWalking down the empty street at midnight\n" + "Streetlights flicker like a broken dream\n" + "[Chorus]\nAnd I keep on walking\nTill the morning finds me" +) + +CAPTION = ( + "A melancholic lo-fi hip-hop track at 85 BPM in F minor: mellow Rhodes piano riff, " + "soft vinyl crackle, dusty boom-bap drums with a laid-back swing, warm upright bass. " + "Intimate bedroom production, gentle tape saturation, no bright cymbals." +) + + +tts_server_params = [ + pytest.param( + OmniServerParams( + model=MODEL, + stage_config_path=get_deploy_config_path("minimax_music3.yaml"), + server_args=["--trust-remote-code", "--disable-log-stats"], + ), + id="minimax_music3", + ) +] + + +@pytest.mark.slow +@pytest.mark.tts +@hardware_test(res={"cuda": "H100"}, num_cards={"cuda": 1}) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_text_to_music_001(omni_server, openai_client) -> None: + """Baseline: lyrics plus caption in, one ten-second song out. + + Deploy Setting: minimax_music3.yaml + Input Modal: text (lyrics) + text (caption) + Output Modal: audio, 32 kHz stereo + Input Setting: stream=False, seed pinned, 250 frames + """ + request_config = { + "model": omni_server.model, + "input": LYRICS, + "instructions": CAPTION, + "seed": 1, + "max_new_tokens": _FRAMES_10S, + "stream": False, + "response_format": "wav", + "timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S, + "min_audio_bytes": _MIN_BYTES_10S, + } + openai_client.send_audio_speech_request(request_config) + + +@pytest.mark.slow +@pytest.mark.tts +@hardware_test(res={"cuda": "H100"}, num_cards={"cuda": 1}) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_text_to_music_multi_window_002(omni_server, openai_client) -> None: + """A song long enough to span several acoustic windows. + + The acoustic stage decodes overlapping 200-frame windows on a 100-frame + hop and crops the overlap back off, so a request that fits in one window + never exercises the stitching. 750 frames covers seven of them. + + Deploy Setting: minimax_music3.yaml + Input Setting: stream=False, seed pinned, 750 frames + """ + request_config = { + "model": omni_server.model, + "input": LYRICS, + "instructions": CAPTION, + "seed": 1, + "max_new_tokens": _FRAMES_30S, + "stream": False, + "response_format": "wav", + "timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S, + "min_audio_bytes": _MIN_BYTES_30S, + } + openai_client.send_audio_speech_request(request_config) diff --git a/tests/engine/test_async_omni_engine_input.py b/tests/engine/test_async_omni_engine_input.py index 740064173..ee6d322ca 100644 --- a/tests/engine/test_async_omni_engine_input.py +++ b/tests/engine/test_async_omni_engine_input.py @@ -371,3 +371,106 @@ def test_build_add_request_message_releases_preselected_replica_on_preprocess_er ) assert stage_pool.get_bound_replica_id("req-error") is None + + +def test_cfg_companion_build_failure_admits_nothing(mocker: MockerFixture): + """A guided request is all-or-nothing. + + Admitting the parent before its companion is built leaves an orphan: a + model whose guidance is mandatory never completes the pair, so the request + holds scheduler and KV capacity for the scheduler's whole hold budget and + then produces no audio. The build has to raise before anything is enqueued. + """ + engine = object.__new__(AsyncOmniEngine) + params = SamplingParams(max_tokens=8) + engine.prompt_expand_func = mocker.Mock(side_effect=ValueError("cannot build the null twin")) + engine.supported_tasks = ("generate",) + engine.input_processor = mocker.Mock() + engine.request_queue = mocker.Mock() + + with pytest.raises(ValueError, match="null twin"): + engine._build_cfg_companions( + parent_id="req", + original_prompt={"prompt": "positive"}, + stage0_params=params, + sampling_params_list=[params], + ) + + engine.request_queue.sync_q.put.assert_not_called() + + +def test_cfg_companion_processing_failure_admits_nothing(mocker: MockerFixture): + """The same holds when expansion succeeds but input processing does not.""" + engine = object.__new__(AsyncOmniEngine) + params = SamplingParams(max_tokens=8) + engine.prompt_expand_func = lambda *_args: [ + ExpandedPrompt( + prompt={"prompt": "negative"}, + role="cfg_text", + request_id_suffix="__cfg_text", + ) + ] + engine.supported_tasks = ("generate",) + engine.input_processor = mocker.Mock() + engine.input_processor.process_inputs.side_effect = RuntimeError("tokenizer rejected the prompt") + engine.request_queue = mocker.Mock() + + with pytest.raises(RuntimeError, match="tokenizer rejected"): + engine._build_cfg_companions( + parent_id="req", + original_prompt={"prompt": "positive"}, + stage0_params=params, + sampling_params_list=[params], + ) + + engine.request_queue.sync_q.put.assert_not_called() + + +def test_cfg_companion_build_returns_messages_without_enqueueing(mocker: MockerFixture): + """Construction and admission are separate steps.""" + engine = object.__new__(AsyncOmniEngine) + params = SamplingParams(max_tokens=8) + engine.prompt_expand_func = lambda *_args: [ + ExpandedPrompt( + prompt={"prompt": "negative"}, + role="cfg_text", + request_id_suffix="__cfg_text", + ) + ] + engine.supported_tasks = ("generate",) + engine.input_processor = mocker.Mock() + engine.input_processor.process_inputs.return_value = _make_engine_core_request("req__cfg_text") + engine.request_queue = mocker.Mock() + + companions = engine._build_cfg_companions( + parent_id="req", + original_prompt={"prompt": "positive"}, + stage0_params=params, + sampling_params_list=[params], + ) + + assert len(companions) == 1 + assert companions[0].companion_id == "req__cfg_text" + assert companions[0].parent_id == "req" + engine.request_queue.sync_q.put.assert_not_called() + + +def test_cfg_expansion_returning_nothing_is_not_an_error(mocker: MockerFixture): + """An unguided request expands to no companions and admits normally.""" + engine = object.__new__(AsyncOmniEngine) + params = SamplingParams(max_tokens=8) + engine.prompt_expand_func = lambda *_args: [] + engine.supported_tasks = ("generate",) + engine.input_processor = mocker.Mock() + engine.request_queue = mocker.Mock() + + assert ( + engine._build_cfg_companions( + parent_id="req", + original_prompt={"prompt": "positive"}, + stage0_params=params, + sampling_params_list=[params], + ) + == [] + ) + engine.request_queue.sync_q.put.assert_not_called() diff --git a/tests/entrypoints/openai_api/test_tts_detection.py b/tests/entrypoints/openai_api/test_tts_detection.py index 2ac27bb18..6b8b97849 100644 --- a/tests/entrypoints/openai_api/test_tts_detection.py +++ b/tests/entrypoints/openai_api/test_tts_detection.py @@ -153,6 +153,7 @@ _PIPELINE_STAGES = [ "indextts2_talker", "latent_generator", "llm", + "minimax_music3_ar", "ming_tts", "moss_tts", "moss_tts_codec", diff --git a/tests/model_executor/models/test_minimax_music3_batch_payload.py b/tests/model_executor/models/test_minimax_music3_batch_payload.py new file mode 100644 index 000000000..531500cd4 --- /dev/null +++ b/tests/model_executor/models/test_minimax_music3_batch_payload.py @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression tests for MiniMax Music 3 batched payload partitioning. + +The stage-0 payload is split per request by indexing list values: a list is +indexed by batch row, anything else is handed to every request unchanged. So +per-request metadata has to be carried as lists. Merging it into one dict made +every request in a batch receive the last request's seed, window offset and +terminal flag, which crosses seeds between songs, misplaces acoustic windows, +and lets one request's terminal flag end another request's stream early. + +These drive ``make_omni_output`` directly with a hand-built batch, so no GPU +and no engine are needed. +""" + +from __future__ import annotations + +import pytest +import torch + +from vllm_omni.model_executor.models.minimax_music3.talker import ( + MiniMaxMusic3TalkerForConditionalGeneration as Talker, +) +from vllm_omni.utils.mm_outputs import to_payload_element + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_WIDTH = 4 + + +def _info(*, seed: int, start: int, frames: int, last: bool) -> dict: + """One batch row's intermediate buffer, as ``postprocess`` leaves it.""" + return { + "latent": torch.full((frames, _WIDTH), float(seed)), + "codes": {"audio": torch.full((frames, 8), seed)}, + "meta": { + "audio_seed": seed, + "num_processed_tokens": start, + "codec_chunk_frames": frames, + "last_chunk": last, + }, + } + + +def _payload_for(output, row: int) -> dict: + """Partition the batch payload down to one request, as the runner does.""" + flat = {} + for key, value in output.multimodal_outputs.items(): + if isinstance(value, dict): + for qual, val in value.items(): + flat[f"{key}.{qual}"] = val + else: + flat[key] = value + return {key: to_payload_element(value, row, 1, 1) for key, value in flat.items()} + + +class TestBatchedMetadataPartitioning: + """Two requests in one batch must keep their own metadata.""" + + @staticmethod + def _two_request_batch(): + infos = [ + _info(seed=111, start=100, frames=3, last=False), + _info(seed=222, start=200, frames=5, last=True), + ] + talker = Talker.__new__(Talker) + return talker.make_omni_output(torch.zeros(2, _WIDTH), model_intermediate_buffer=infos) + + def test_seeds_are_not_crossed(self) -> None: + """A song decoded with another request's seed is a different song.""" + output = self._two_request_batch() + assert _payload_for(output, 0)["meta.audio_seed"] == 111 + assert _payload_for(output, 1)["meta.audio_seed"] == 222 + + def test_window_offsets_stay_with_their_request(self) -> None: + """The start frame decides where a window lands in the song.""" + output = self._two_request_batch() + assert _payload_for(output, 0)["meta.num_processed_tokens"] == 100 + assert _payload_for(output, 1)["meta.num_processed_tokens"] == 200 + + def test_terminal_flag_does_not_leak_across_requests(self) -> None: + """One request finishing must not end another request's stream.""" + output = self._two_request_batch() + assert _payload_for(output, 0)["meta.last_chunk"] is False + assert _payload_for(output, 1)["meta.last_chunk"] is True + + def test_span_length_matches_its_own_latent(self) -> None: + output = self._two_request_batch() + for row, frames in ((0, 3), (1, 5)): + entry = _payload_for(output, row) + assert entry["meta.codec_chunk_frames"] == frames + assert entry["latent"].shape[0] == frames + + def test_latent_and_codes_follow_the_same_row(self) -> None: + """The tensors were already per-request; pin that they stay aligned.""" + output = self._two_request_batch() + for row, seed in ((0, 111), (1, 222)): + entry = _payload_for(output, row) + assert entry["latent"][0, 0].item() == pytest.approx(float(seed)) + assert int(entry["codes.audio"][0, 0]) == seed + + +class TestPartialBatches: + """Only some rows carry a span on any given step.""" + + def test_row_without_a_span_does_not_take_another_row_metadata(self) -> None: + infos = [{}, _info(seed=777, start=300, frames=2, last=True)] + talker = Talker.__new__(Talker) + output = talker.make_omni_output(torch.zeros(2, _WIDTH), model_intermediate_buffer=infos) + + idle = _payload_for(output, 0) + assert idle["latent"].numel() == 0 + assert idle["meta.audio_seed"] is None + assert idle["meta.last_chunk"] is None + + active = _payload_for(output, 1) + assert active["meta.audio_seed"] == 777 + assert active["meta.last_chunk"] is True + + def test_batch_with_no_spans_emits_no_payload(self) -> None: + talker = Talker.__new__(Talker) + output = talker.make_omni_output(torch.zeros(2, _WIDTH), model_intermediate_buffer=[{}, {}]) + assert output.multimodal_outputs is None + + def test_consumed_entries_are_cleared_from_the_buffer(self) -> None: + """The runner never clears this buffer, so a span left behind would be + re-emitted on every later step.""" + infos = [_info(seed=1, start=0, frames=2, last=False)] + talker = Talker.__new__(Talker) + talker.make_omni_output(torch.zeros(1, _WIDTH), model_intermediate_buffer=infos) + assert "latent" not in infos[0] + assert "meta" not in infos[0] diff --git a/tests/model_executor/models/test_minimax_music3_frame_state.py b/tests/model_executor/models/test_minimax_music3_frame_state.py new file mode 100644 index 000000000..73dc9215b --- /dev/null +++ b/tests/model_executor/models/test_minimax_music3_frame_state.py @@ -0,0 +1,214 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for MiniMax Music 3 per-request AR frame bookkeeping. + +The talker emits overlapping windows while dropping frames it can no longer +need, and flushes whatever is left as one span when the song ends. Getting the +eviction boundary wrong by one frame silently corrupts the conditioning handed +to the acoustic stage from the second window onward, which is audible but hard +to attribute, so the lifecycle is simulated here frame by frame. +""" + +from __future__ import annotations + +import pytest +import torch + +from vllm_omni.model_executor.models.minimax_music3.chunking import chunk_windows +from vllm_omni.model_executor.models.minimax_music3.constants import ( + AR_CHUNK_FRAMES, + AR_CHUNK_HOP_FRAMES, + NUM_CODEBOOKS, +) +from vllm_omni.model_executor.models.minimax_music3.frame_state import ( + FrameBuffer, + RequestARState, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +# One scalar per frame instead of 32768 dims: the bookkeeping is what is under +# test, and a frame's value doubles as its identity. +_WIDTH = 4 + + +def _frame(index: int) -> torch.Tensor: + return torch.full((1, _WIDTH), float(index)) + + +def _identities(chunk: torch.Tensor) -> list[int]: + return [int(row[0].item()) for row in chunk] + + +class TestFrameBuffer: + def test_slice_returns_the_requested_frames(self) -> None: + buffer = FrameBuffer() + for index in range(10): + buffer.append(_frame(index)) + assert _identities(buffer.slice(3, 7)) == [3, 4, 5, 6] + + def test_slice_is_relative_to_the_live_base(self) -> None: + buffer = FrameBuffer() + for index in range(10): + buffer.append(_frame(index)) + buffer.discard_before(4) + assert buffer.base_frame == 4 + assert _identities(buffer.slice(4, 8)) == [4, 5, 6, 7] + + def test_slicing_a_discarded_range_raises(self) -> None: + buffer = FrameBuffer() + for index in range(10): + buffer.append(_frame(index)) + buffer.discard_before(5) + with pytest.raises(RuntimeError, match="unavailable"): + buffer.slice(3, 6) + + def test_slicing_past_the_end_raises(self) -> None: + buffer = FrameBuffer() + for index in range(4): + buffer.append(_frame(index)) + with pytest.raises(RuntimeError, match="unavailable"): + buffer.slice(2, 9) + + def test_discard_is_clamped_at_both_ends(self) -> None: + buffer = FrameBuffer() + for index in range(5): + buffer.append(_frame(index)) + buffer.discard_before(-3) + assert buffer.base_frame == 0 + buffer.discard_before(99) + assert buffer.base_frame == 5 + assert buffer.end_frame == 5 + + def test_peak_tracks_the_high_water_mark(self) -> None: + buffer = FrameBuffer() + for index in range(10): + buffer.append(_frame(index)) + buffer.discard_before(8) + assert buffer.peak_frames == 10 + + +class TestWindowLifecycle: + """Drive the state exactly as the talker does, one frame per step.""" + + @staticmethod + def _run(total_frames: int) -> tuple[list[list[int]], int]: + """Generate ``total_frames`` frames, returning the emitted spans. + + Returns: + The frame identities of every emitted span, and the peak buffer + occupancy reached along the way. + """ + state = RequestARState(seed=1, max_frames=total_frames) + spans: list[list[int]] = [] + for index in range(total_frames): + state.frames.append(_frame(index)) + state.codes.append(torch.full((1, NUM_CODEBOOKS), index)) + state.generated_frames += 1 + window = state.ready_window() + if window is not None: + spans.append(_identities(state.take_window(window, is_final=False))) + + # Terminal flush: everything not yet emitted, as one contiguous span. + start = state.emitted_windows * AR_CHUNK_HOP_FRAMES + end = state.frames.end_frame + if end > start: + spans.append(_identities(state.frames.slice(start, end))) + return spans, state.frames.peak_frames + + def test_short_song_is_one_span(self) -> None: + spans, _ = self._run(50) + assert spans == [list(range(50))] + + def test_every_frame_is_delivered_exactly_once_in_order(self) -> None: + """Spans overlap by design, but their union must be the whole song. + + Reconstruct by taking each span's non-overlapping remainder, which is + what the acoustic stage does after cropping. + """ + for total in (250, 437, 600, 901): + spans, _ = self._run(total) + rebuilt: list[int] = [] + for span in spans: + rebuilt.extend(frame for frame in span if frame >= len(rebuilt)) + assert rebuilt == list(range(total)), f"gap or duplicate at {total} frames" + + def test_windows_advance_by_the_hop(self) -> None: + spans, _ = self._run(600) + for index, span in enumerate(spans[:-1]): + assert span[0] == index * AR_CHUNK_HOP_FRAMES + assert len(span) == AR_CHUNK_FRAMES + + def test_mid_stream_windows_are_never_the_final_frame(self) -> None: + """A window is held back until a further frame proves it is not last.""" + total = 600 + spans, _ = self._run(total) + for span in spans[:-1]: + assert span[-1] < total - 1 + + def test_terminal_span_covers_the_tail(self) -> None: + total = 437 + spans, _ = self._run(total) + assert spans[-1][-1] == total - 1 + + def test_terminal_span_is_bounded(self) -> None: + """The tail is at most one full window plus one hop.""" + for total in (250, 301, 437, 600, 901, 1000): + spans, _ = self._run(total) + assert len(spans[-1]) <= AR_CHUNK_FRAMES + AR_CHUNK_HOP_FRAMES + + def test_buffer_stays_bounded_over_a_long_song(self) -> None: + """Peak occupancy must not grow with song length; that is the point + of evicting retired frames.""" + _, peak_short = self._run(600) + _, peak_long = self._run(3000) + assert peak_long == peak_short + assert peak_long <= AR_CHUNK_FRAMES + AR_CHUNK_HOP_FRAMES + + def test_emitted_window_count_matches_the_window_plan(self) -> None: + """What the stream emitted must reconcile with chunk_windows().""" + for total in (250, 437, 600): + spans, _ = self._run(total) + planned = chunk_windows(total) + # Every window is accounted for: the pre-terminal spans are one + # window each, and the terminal span carries whatever remains. + assert len(spans) <= len(planned) + assert spans[0][0] == planned[0].start + + +class TestSamplingPositions: + def test_each_frame_reserves_one_slot_per_codebook(self) -> None: + """Codebook streams must never collide, or codes repeat across frames.""" + state = RequestARState(seed=1, max_frames=10) + positions = [state.advance_position() for _ in range(5)] + assert positions == [0, 8, 16, 24, 32] + + def test_draw_slots_are_globally_unique(self) -> None: + state = RequestARState(seed=1, max_frames=64) + slots: set[int] = set() + for _ in range(64): + base = state.advance_position() + for codebook in range(NUM_CODEBOOKS): + slot = base + codebook + assert slot not in slots + slots.add(slot) + assert len(slots) == 64 * NUM_CODEBOOKS + + +class TestCodeRetention: + def test_codes_span_matches_the_frame_span(self) -> None: + state = RequestARState(seed=1, max_frames=10) + for index in range(10): + state.codes.append(torch.full((1, NUM_CODEBOOKS), index)) + codes = state.take_codes(2, 6) + assert codes is not None + assert codes.shape == (4, NUM_CODEBOOKS) + assert [int(row[0]) for row in codes] == [2, 3, 4, 5] + + def test_out_of_range_span_is_dropped_not_raised(self) -> None: + """Codes are diagnostic; the audio path must not fail without them.""" + state = RequestARState(seed=1, max_frames=10) + for index in range(4): + state.codes.append(torch.full((1, NUM_CODEBOOKS), index)) + assert state.take_codes(2, 99) is None + assert state.take_codes(0, 0) is None diff --git a/tests/model_executor/models/test_minimax_music3_prompt.py b/tests/model_executor/models/test_minimax_music3_prompt.py new file mode 100644 index 000000000..8cd9e41b0 --- /dev/null +++ b/tests/model_executor/models/test_minimax_music3_prompt.py @@ -0,0 +1,203 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for MiniMax Music 3 prompt assembly and window arithmetic. + +The prompt transformations are not cosmetic: they decide token positions, and +token positions decide the audio. These tests pin the documented behaviours +that a well-meaning cleanup would otherwise "fix". +""" + +from __future__ import annotations + +import pytest + +from vllm_omni.model_executor.models.minimax_music3.chunking import ( + chunk_windows, + crop_sample_bounds, + overlap_mel_length, +) +from vllm_omni.model_executor.models.minimax_music3.constants import ( + AR_CHUNK_FRAMES, + AR_CHUNK_HOP_FRAMES, + BLEND_MEL_FRAMES, + DAV_HOP_SAMPLES, + HOP_MEL_FRAMES, +) +from vllm_omni.model_executor.models.minimax_music3.prompt import ( + SPECIAL_TOKEN_IDS, + build_cfg_null_token_ids, + build_prompt, + clean_caption, + normalize_lyrics, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class TestNormalizeLyrics: + def test_prepends_start_tag(self) -> None: + assert normalize_lyrics("[Verse]\nhello").startswith("[start]\n") + + def test_lowercases_structure_tags(self) -> None: + assert "[verse]" in normalize_lyrics("[Verse]\nhello") + assert "[Verse]" not in normalize_lyrics("[Verse]\nhello") + + def test_text_after_a_leading_tag_is_dropped(self) -> None: + """Documented contract: a tag must sit on its own line. + + Lyrics written next to a tag are silently lost, which is why the + serving docs tell callers to put tags on their own line. + """ + out = normalize_lyrics("[Verse] Walking down the street") + assert "Walking" not in out + assert "[verse]" in out + + def test_text_on_its_own_line_survives(self) -> None: + out = normalize_lyrics("[Verse]\nWalking down the street") + assert "Walking down the street" in out + + def test_bracket_space_becomes_newline(self) -> None: + assert normalize_lyrics("a [chorus]") == "[start]\na\n[chorus]" + + +class TestCleanCaption: + def test_special_token_form_is_rewritten_not_passed_through(self) -> None: + """A caption must not be able to inject a real special token.""" + assert clean_caption("<|bpm 92|>") == "bpm is 92" + assert "<|" not in clean_caption("<|bpm 92|>") + + def test_lone_tag_keeps_its_inner_text(self) -> None: + assert clean_caption("<|instrumental|>") == "instrumental" + + def test_markdown_is_stripped(self) -> None: + assert clean_caption("## Heading") == "Heading" + assert clean_caption("**bold**") == "bold" + assert clean_caption("- item") == "item" + + def test_blank_lines_collapse_at_two_not_three(self) -> None: + """The source collapses runs of two or more, not three or more. + + This shifts blank-line token positions and therefore the audio, so it + is pinned deliberately. + """ + assert clean_caption("a\n\nb") == "a\nb" + assert clean_caption("a\n\n\n\nb") == "a\nb" + + +class TestBuildPrompt: + def test_framing_tokens_are_present_and_ordered(self) -> None: + prompt = build_prompt("a bright pop song", "[Verse]\nhello") + assert prompt.startswith("<|im_start|><|caption_start|>") + assert prompt.endswith("<|im_end|><|audio_start|>") + assert prompt.index("<|caption_end|>") < prompt.index("<|lyrics_start|>") + + +class TestBuildCfgNullTokenIds: + @staticmethod + def _prompt_ids(body_len: int) -> list[int]: + return ( + [SPECIAL_TOKEN_IDS["<|im_start|>"]] + + list(range(1000, 1000 + body_len)) + + [SPECIAL_TOKEN_IDS["<|im_end|>"], SPECIAL_TOKEN_IDS["<|audio_start|>"]] + ) + + def test_length_is_preserved(self) -> None: + """Equal length is what lets the two rows decode in lockstep.""" + ids = self._prompt_ids(12) + assert len(build_cfg_null_token_ids(ids)) == len(ids) + + def test_body_is_replaced_and_framing_survives(self) -> None: + ids = self._prompt_ids(12) + null = build_cfg_null_token_ids(ids) + assert null[0] == SPECIAL_TOKEN_IDS["<|im_start|>"] + assert null[-2] == SPECIAL_TOKEN_IDS["<|im_end|>"] + assert null[-1] == SPECIAL_TOKEN_IDS["<|audio_start|>"] + assert set(null[1:-2]) == {SPECIAL_TOKEN_IDS["<|audio_cfg|>"]} + + def test_input_is_not_mutated(self) -> None: + ids = self._prompt_ids(6) + before = list(ids) + build_cfg_null_token_ids(ids) + assert ids == before + + def test_minimal_prompt_with_empty_body(self) -> None: + ids = [ + SPECIAL_TOKEN_IDS["<|im_start|>"], + SPECIAL_TOKEN_IDS["<|audio_cfg|>"], + SPECIAL_TOKEN_IDS["<|im_end|>"], + SPECIAL_TOKEN_IDS["<|audio_start|>"], + ] + assert build_cfg_null_token_ids(ids) == ids + + def test_too_short_is_rejected(self) -> None: + with pytest.raises(ValueError, match="too short"): + build_cfg_null_token_ids([1, 2, 3]) + + def test_wrong_framing_is_rejected(self) -> None: + with pytest.raises(ValueError, match="framing"): + build_cfg_null_token_ids([9, 9, 9, 9, 9]) + + +class TestChunkWindows: + def test_no_frames_yields_nothing(self) -> None: + assert chunk_windows(0) == [] + + def test_short_song_is_one_final_window(self) -> None: + windows = chunk_windows(50) + assert len(windows) == 1 + assert (windows[0].start, windows[0].end) == (0, 50) + assert windows[0].is_first and windows[0].is_last + + def test_exactly_one_window(self) -> None: + windows = chunk_windows(AR_CHUNK_FRAMES) + assert len(windows) == 1 + assert windows[0].is_last + + def test_windows_advance_by_the_hop(self) -> None: + windows = chunk_windows(450) + starts = [w.start for w in windows] + assert starts == list(range(0, starts[-1] + 1, AR_CHUNK_HOP_FRAMES)) + assert windows[-1].is_last + assert not any(w.is_last for w in windows[:-1]) + + def test_windows_cover_every_frame(self) -> None: + frames = 437 + windows = chunk_windows(frames) + covered: set[int] = set() + for window in windows: + covered.update(range(window.start, window.end)) + assert covered == set(range(frames)) + + def test_final_window_is_clamped_to_the_song(self) -> None: + windows = chunk_windows(437) + assert windows[-1].end == 437 + + def test_only_the_first_window_is_first(self) -> None: + windows = chunk_windows(600) + assert windows[0].is_first + assert not any(w.is_first for w in windows[1:]) + + +class TestCropBounds: + def test_single_window_song_is_not_cropped(self) -> None: + window = chunk_windows(50)[0] + assert crop_sample_bounds(window) == (0, 0) + + def test_first_window_keeps_its_head(self) -> None: + left, right = crop_sample_bounds(chunk_windows(600)[0]) + assert left == 0 + assert right == (HOP_MEL_FRAMES - BLEND_MEL_FRAMES) * DAV_HOP_SAMPLES + + def test_last_window_keeps_its_tail(self) -> None: + left, right = crop_sample_bounds(chunk_windows(600)[-1]) + assert left == BLEND_MEL_FRAMES * DAV_HOP_SAMPLES + assert right == 0 + + def test_middle_window_is_cropped_on_both_sides(self) -> None: + windows = chunk_windows(900) + middle = windows[len(windows) // 2] + left, right = crop_sample_bounds(middle) + assert left > 0 and right > 0 + + def test_overlap_is_half_a_window(self) -> None: + assert overlap_mel_length() == HOP_MEL_FRAMES // 2 diff --git a/tests/model_executor/models/test_minimax_music3_row_binding.py b/tests/model_executor/models/test_minimax_music3_row_binding.py new file mode 100644 index 000000000..94ff30199 --- /dev/null +++ b/tests/model_executor/models/test_minimax_music3_row_binding.py @@ -0,0 +1,217 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression tests for MiniMax Music 3 row-to-request binding. + +vLLM compacts and reorders the persistent batch whenever a request finishes or +is preempted, so a batch row number is only meaningful within the step that +produced it. Every failure pinned here was a real defect: state keyed by row +survived into a step where the row belonged to someone else, or state keyed by +one spelling of the request id was released under another and leaked. + +These exercise the pure helpers and the binding arithmetic directly, so they +need no GPU and no engine. +""" + +from __future__ import annotations + +import pytest + +from vllm_omni.model_executor.models.minimax_music3.constants import ( + DEFAULT_MAX_AUDIO_FRAMES, +) +from vllm_omni.model_executor.models.minimax_music3.talker import ( + _CFG_UNCOND_SUFFIX, + _first_int, + _pair_id_from_request, + _request_key, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class TestRequestKey: + def test_engine_list_form_is_unwrapped(self) -> None: + """The engine injects global_request_id as a single-element list. + + Keying state on the list's ``str()`` would produce "['req-1']", which + never matches the bare id that ``on_requests_finished`` delivers, so + every finished song would leak its frame buffer. + """ + assert _request_key(["req-1"]) == "req-1" + + def test_bare_string_is_unchanged(self) -> None: + assert _request_key("req-1") == "req-1" + + def test_list_and_bare_forms_agree(self) -> None: + """Allocation and release must land on the same key.""" + assert _request_key(["req-1"]) == _request_key("req-1") + + def test_empty_and_missing_are_stable(self) -> None: + assert _request_key(None) == "" + assert _request_key([]) == "" + + def test_tuple_form(self) -> None: + assert _request_key(("req-9",)) == "req-9" + + +class TestPairIdDerivation: + def test_conditioned_request_is_its_own_pair(self) -> None: + assert _pair_id_from_request("req-1") == "req-1" + + def test_companion_maps_to_its_partner(self) -> None: + assert _pair_id_from_request(f"req-1{_CFG_UNCOND_SUFFIX}") == "req-1" + + def test_pair_members_agree_on_the_pair_id(self) -> None: + cond = "req-42" + uncond = f"req-42{_CFG_UNCOND_SUFFIX}" + assert _pair_id_from_request(cond) == _pair_id_from_request(uncond) + + def test_a_request_ending_in_a_similar_string_is_not_confused(self) -> None: + assert _pair_id_from_request("req-cfg_uncond") == "req-cfg_uncond" + + +class TestFirstInt: + def test_serving_layer_list_wrapping(self) -> None: + """tts_params values arrive wrapped in a single-element list.""" + assert _first_int([750], DEFAULT_MAX_AUDIO_FRAMES) == 750 + + def test_bare_value(self) -> None: + assert _first_int(750, DEFAULT_MAX_AUDIO_FRAMES) == 750 + + def test_missing_falls_back(self) -> None: + assert _first_int(None, DEFAULT_MAX_AUDIO_FRAMES) == DEFAULT_MAX_AUDIO_FRAMES + assert _first_int([], DEFAULT_MAX_AUDIO_FRAMES) == DEFAULT_MAX_AUDIO_FRAMES + + def test_nonsense_falls_back_rather_than_raising(self) -> None: + assert _first_int("abc", DEFAULT_MAX_AUDIO_FRAMES) == DEFAULT_MAX_AUDIO_FRAMES + assert _first_int(0, DEFAULT_MAX_AUDIO_FRAMES) == DEFAULT_MAX_AUDIO_FRAMES + assert _first_int(-5, DEFAULT_MAX_AUDIO_FRAMES) == DEFAULT_MAX_AUDIO_FRAMES + + +def _feedback_plan( + request_ids: list[str], + computed: list[int], + scheduled: list[int], + prompt_tokens: dict[str, int], + has_codes: set[str], +) -> list[int]: + """Mirror of the talker's feedback row selection. + + Returns the flat token offsets whose embedding gets replaced by an audio + frame. Kept as a standalone function so the arithmetic can be checked + without constructing a model. + """ + offsets: list[int] = [] + running = 0 + for count in scheduled: + offsets.append(running) + running += count + + rows: list[int] = [] + for row, request_id in enumerate(request_ids): + if request_id not in has_codes: + continue + if scheduled[row] != 1 or computed[row] < prompt_tokens.get(request_id, 0): + continue + rows.append(offsets[row]) + return rows + + +class TestFeedbackRowSelection: + """The guard that decides which token embeddings become audio frames.""" + + def test_decode_only_step_replaces_every_row(self) -> None: + ids = ["a", "a__cfg_uncond", "b", "b__cfg_uncond"] + rows = _feedback_plan( + ids, + computed=[120, 120, 90, 90], + scheduled=[1, 1, 1, 1], + prompt_tokens={i: 50 for i in ids}, + has_codes=set(ids), + ) + assert rows == [0, 1, 2, 3] + + def test_prefill_row_is_never_overwritten(self) -> None: + """A prompt token must keep its real embedding. + + The earlier guard compared the flat token count against a row budget, + so a short prompt slipped through and had its first embeddings + replaced with a previous request's audio frame. + """ + rows = _feedback_plan( + ["a", "b"], + computed=[0, 300], + scheduled=[8, 1], + prompt_tokens={"a": 8, "b": 40}, + has_codes={"b"}, + ) + # Row 0 occupies flat tokens 0..7 and must be untouched; row 1's single + # decode token sits at flat offset 8. + assert rows == [8] + + def test_short_prefill_below_the_row_budget_is_still_safe(self) -> None: + """The exact shape that defeated the old token-count guard.""" + rows = _feedback_plan( + ["a"], + computed=[0], + scheduled=[10], + prompt_tokens={"a": 10}, + has_codes={"a"}, + ) + assert rows == [] + + def test_chunked_prefill_midway_is_not_fed_back(self) -> None: + rows = _feedback_plan( + ["a"], + computed=[2048], + scheduled=[2048], + prompt_tokens={"a": 5000}, + has_codes={"a"}, + ) + assert rows == [] + + def test_final_prefill_chunk_is_not_fed_back(self) -> None: + """The step that completes a prompt still carries prompt tokens.""" + rows = _feedback_plan( + ["a"], + computed=[4096], + scheduled=[904], + prompt_tokens={"a": 5000}, + has_codes={"a"}, + ) + assert rows == [] + + def test_mixed_batch_offsets_follow_the_flat_layout(self) -> None: + """Tokens are a flat concatenation, so offsets are a running sum.""" + ids = ["p", "d1", "d2"] + rows = _feedback_plan( + ids, + computed=[0, 500, 700], + scheduled=[64, 1, 1], + prompt_tokens={"p": 64, "d1": 40, "d2": 40}, + has_codes={"d1", "d2"}, + ) + assert rows == [64, 65] + + def test_request_without_codes_yet_is_skipped(self) -> None: + """The first decode step after prefill has no previous frame.""" + rows = _feedback_plan( + ["a"], + computed=[50], + scheduled=[1], + prompt_tokens={"a": 50}, + has_codes=set(), + ) + assert rows == [] + + def test_reordered_batch_binds_by_request_not_position(self) -> None: + """The same two requests in swapped rows get swapped offsets. + + This is the compaction case: holding codes in a row-indexed buffer + would have handed each request the other's frame. + """ + prompt_tokens = {"a": 50, "b": 50} + first = _feedback_plan(["a", "b"], [100, 200], [1, 1], prompt_tokens, {"a"}) + swapped = _feedback_plan(["b", "a"], [200, 100], [1, 1], prompt_tokens, {"a"}) + assert first == [0] + assert swapped == [1] diff --git a/tests/model_executor/test_cfg_pairing.py b/tests/model_executor/test_cfg_pairing.py new file mode 100644 index 000000000..c00313a2a --- /dev/null +++ b/tests/model_executor/test_cfg_pairing.py @@ -0,0 +1,389 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for the model-neutral CFG request-pairing scheduler patches. + +The patches keep a cond/uncond request pair step-locked inside one engine: +admitted together, kept at equal ``num_computed_tokens``, finished together. +They are installed onto a lightweight fake scheduler here; vLLM's real +``Scheduler`` needs a full engine (and a GPU) to construct. +""" + +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass, field +from types import SimpleNamespace +from typing import Any + +import pytest + +from vllm_omni.model_executor.models.common import cfg_pairing +from vllm_omni.model_executor.models.common.cfg_pairing import ( + _equalize_cfg_pair_progress, + _patch_scheduler, + cfg_uncond_suffixes, + register_cfg_uncond_suffix, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +# ───────── fakes ───────── + + +class _FakeRequest: + """Minimal stand-in for ``vllm.v1.request.Request`` (identity equality).""" + + def __init__(self, request_id: str, extra_args: dict[str, Any] | None = None, step_tokens: int = 1) -> None: + self.request_id = request_id + self.sampling_params = SimpleNamespace(extra_args=extra_args) + self.num_computed_tokens = 0 + self.step_tokens = step_tokens + self.skip_reading_prefix_cache = False + + +class _FakeWaitingQueue(deque): + """FCFS waiting queue: vLLM's is also a ``deque`` subclass.""" + + def remove_requests(self, requests: list[_FakeRequest]) -> None: + keep = [req for req in self if not any(req is dropped for dropped in requests)] + self.clear() + self.extend(keep) + + def prepend_request(self, request: _FakeRequest) -> None: + self.appendleft(request) + + +@dataclass +class _FakeSchedulerOutput: + num_scheduled_tokens: dict[str, int] = field(default_factory=dict) + total_num_scheduled_tokens: int = 0 + + +def _make_scheduler_cls() -> type: + """Fresh class per test so the in-place patches never stack.""" + + class FakeScheduler: + def __init__(self) -> None: + self.waiting = _FakeWaitingQueue() + self.running: list[_FakeRequest] = [] + self.requests: dict[str, _FakeRequest] = {} + self.finished: list[set[str]] = [] + self.observed_waiting: list[list[str]] = [] + self.max_num_scheduled_tokens = 512 + self.scheduler_config = SimpleNamespace(long_prefill_token_threshold=4096) + + def add_request(self, request: _FakeRequest) -> None: + self.requests[request.request_id] = request + self.waiting.append(request) + + def finish_requests(self, request_ids: Any, finished_status: Any) -> None: + if request_ids is None: + ids = set(self.requests) + elif isinstance(request_ids, str): + ids = {request_ids} + else: + ids = set(request_ids) + self.finished.append(ids) + self.running = [req for req in self.running if req.request_id not in ids] + for req_id in ids: + self.requests.pop(req_id, None) + + def schedule(self) -> _FakeSchedulerOutput: + self.observed_waiting.append([req.request_id for req in self.waiting]) + while self.waiting: + self.running.append(self.waiting.popleft()) + output = _FakeSchedulerOutput() + for req in self.running: + req.num_computed_tokens += req.step_tokens + output.num_scheduled_tokens[req.request_id] = req.step_tokens + output.total_num_scheduled_tokens += req.step_tokens + return output + + return FakeScheduler + + +@pytest.fixture +def scheduler() -> Any: + cls = _make_scheduler_cls() + _patch_scheduler(cls) + return cls() + + +@pytest.fixture(autouse=True) +def _restore_suffix_registry(): + original = set(cfg_pairing._UNCOND_REQUEST_ID_SUFFIXES) + yield + cfg_pairing._UNCOND_REQUEST_ID_SUFFIXES.clear() + cfg_pairing._UNCOND_REQUEST_ID_SUFFIXES.update(original) + + +def _add_pair( + scheduler: Any, + pair_id: str | None = None, + base_id: str = "req-1", + suffix: str = "__cfg_uncond", +) -> tuple[_FakeRequest, _FakeRequest]: + cond_extra: dict[str, Any] = {"cfg_role": "cond"} + uncond_extra: dict[str, Any] = {"cfg_role": "uncond"} + if pair_id is not None: + cond_extra["cfg_pair_id"] = pair_id + uncond_extra["cfg_pair_id"] = pair_id + cond = _FakeRequest(base_id, cond_extra) + uncond = _FakeRequest(f"{base_id}{suffix}", uncond_extra) + scheduler.add_request(cond) + scheduler.add_request(uncond) + return cond, uncond + + +# ───────── pair-id derivation ───────── + + +def test_pair_id_is_derived_from_request_id_when_absent(scheduler: Any) -> None: + cond, uncond = _add_pair(scheduler) + + assert scheduler._cfg_pairs == {"req-1": {"cond": cond.request_id, "uncond": uncond.request_id}} + assert scheduler._cfg_req_to_pair == {cond.request_id: "req-1", uncond.request_id: "req-1"} + + +def test_registered_companion_suffix_is_stripped(scheduler: Any) -> None: + register_cfg_uncond_suffix("::negative") + assert "::negative" in cfg_uncond_suffixes() + + cond, uncond = _add_pair(scheduler, base_id="music-7", suffix="::negative") + + assert scheduler._cfg_pairs == {"music-7": {"cond": cond.request_id, "uncond": uncond.request_id}} + + +def test_register_rejects_empty_suffix() -> None: + with pytest.raises(ValueError): + register_cfg_uncond_suffix("") + + +def test_uncond_without_known_suffix_keeps_its_own_request_id(scheduler: Any) -> None: + # Degenerate input: the companion never joins the cond pair, so the pair + # stays incomplete and the hold budget eventually releases it. + scheduler.add_request(_FakeRequest("solo-uncond", {"cfg_role": "uncond"})) + + assert scheduler._cfg_pairs == {"solo-uncond": {"uncond": "solo-uncond"}} + + +def test_explicit_pair_id_takes_precedence(scheduler: Any) -> None: + cond, uncond = _add_pair(scheduler, pair_id="pair-A") + + assert scheduler._cfg_pairs == {"pair-A": {"cond": cond.request_id, "uncond": uncond.request_id}} + assert cond.request_id not in scheduler._cfg_pairs + assert scheduler._cfg_req_to_pair[uncond.request_id] == "pair-A" + + +def test_cfg_requests_skip_prefix_cache_reads(scheduler: Any) -> None: + cond, uncond = _add_pair(scheduler) + + assert cond.skip_reading_prefix_cache is True + assert uncond.skip_reading_prefix_cache is True + + +# ───────── admission holding ───────── + + +def test_lone_member_is_held_then_released(scheduler: Any) -> None: + scheduler.add_request(_FakeRequest("req-1", {"cfg_role": "cond"})) + + scheduler.schedule() + + assert scheduler.observed_waiting == [[]] # the base scheduler saw nothing + assert [req.request_id for req in scheduler.waiting] == ["req-1"] + assert scheduler.running == [] + + +def test_complete_pair_is_admitted_together(scheduler: Any) -> None: + scheduler.add_request(_FakeRequest("req-1", {"cfg_role": "cond"})) + scheduler.schedule() + scheduler.add_request(_FakeRequest("req-1__cfg_uncond", {"cfg_role": "uncond"})) + + scheduler.schedule() + + assert scheduler.observed_waiting[-1] == ["req-1", "req-1__cfg_uncond"] + assert [req.request_id for req in scheduler.running] == ["req-1", "req-1__cfg_uncond"] + assert list(scheduler.waiting) == [] + + +def test_hold_budget_expiry_drops_the_stale_pair(scheduler: Any, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(cfg_pairing, "_MAX_PAIR_HOLD_STEPS", 2) + scheduler.add_request(_FakeRequest("req-1", {"cfg_role": "cond"})) + + scheduler.schedule() + scheduler.schedule() + assert scheduler.running == [] + assert scheduler._cfg_pairs # still hoping for the partner + + scheduler.schedule() + + assert scheduler._cfg_pairs == {} + assert scheduler._cfg_req_to_pair == {} + assert [req.request_id for req in scheduler.running] == ["req-1"] + + +def test_waiting_queue_reorder_puts_partners_adjacent(scheduler: Any) -> None: + scheduler.add_request(_FakeRequest("req-1", {"cfg_role": "cond"})) + scheduler.add_request(_FakeRequest("other", None)) + scheduler.add_request(_FakeRequest("req-1__cfg_uncond", {"cfg_role": "uncond"})) + + scheduler.schedule() + + assert scheduler.observed_waiting[-1] == ["req-1", "req-1__cfg_uncond", "other"] + + +# ───────── progress equalization ───────── + + +def _equalize_fixture( + cond_computed: int, + uncond_computed: int, + cond_sched: int, + uncond_sched: int, +) -> tuple[Any, _FakeSchedulerOutput, _FakeRequest, _FakeRequest]: + cond = _FakeRequest("req-1", {"cfg_role": "cond"}) + uncond = _FakeRequest("req-1__cfg_uncond", {"cfg_role": "uncond"}) + cond.num_computed_tokens = cond_computed + uncond.num_computed_tokens = uncond_computed + fake = SimpleNamespace( + _cfg_pairs={"req-1": {"cond": cond.request_id, "uncond": uncond.request_id}}, + requests={cond.request_id: cond, uncond.request_id: uncond}, + ) + output = _FakeSchedulerOutput( + num_scheduled_tokens={cond.request_id: cond_sched, uncond.request_id: uncond_sched}, + total_num_scheduled_tokens=cond_sched + uncond_sched, + ) + return fake, output, cond, uncond + + +def test_equalize_pulls_the_ahead_member_back() -> None: + fake, output, cond, uncond = _equalize_fixture(10, 6, 5, 4) + + _equalize_cfg_pair_progress(fake, output) + + assert cond.num_computed_tokens == 6 + assert uncond.num_computed_tokens == 6 + assert output.num_scheduled_tokens == {cond.request_id: 1, uncond.request_id: 4} + assert output.total_num_scheduled_tokens == 5 + + +def test_equalize_is_a_noop_when_the_pair_is_already_aligned() -> None: + fake, output, cond, uncond = _equalize_fixture(8, 8, 4, 4) + + _equalize_cfg_pair_progress(fake, output) + + assert (cond.num_computed_tokens, uncond.num_computed_tokens) == (8, 8) + assert output.total_num_scheduled_tokens == 8 + + +def test_equalize_skips_an_infeasible_rollback() -> None: + # Clawing back 4 tokens would leave the cond row with 0 scheduled tokens, + # which the scheduler output cannot express; leave the pair to _drop_split_pairs. + fake, output, cond, uncond = _equalize_fixture(10, 6, 4, 4) + + _equalize_cfg_pair_progress(fake, output) + + assert cond.num_computed_tokens == 10 + assert output.total_num_scheduled_tokens == 8 + + +# ───────── finish ───────── + + +def test_finish_requests_finishes_the_partner(scheduler: Any) -> None: + cond, uncond = _add_pair(scheduler) + scheduler.schedule() + + scheduler.finish_requests(cond.request_id, "FINISHED_STOPPED") + + assert scheduler.finished == [{cond.request_id, uncond.request_id}] + assert scheduler._cfg_pairs == {} + assert scheduler._cfg_req_to_pair == {} + assert scheduler.running == [] + + +def test_finish_all_requests_clears_the_registry(scheduler: Any) -> None: + _add_pair(scheduler) + scheduler.schedule() + + scheduler.finish_requests(None, "FINISHED_ABORTED") + + assert scheduler._cfg_pairs == {} + assert scheduler._cfg_req_to_pair == {} + + +# ───────── non-CFG traffic ───────── + + +def test_plain_requests_are_untouched(scheduler: Any) -> None: + scheduler.add_request(_FakeRequest("plain-1", None)) + scheduler.add_request(_FakeRequest("plain-2", {"seed": 3})) + + output = scheduler.schedule() + + assert scheduler._cfg_pairs == {} + assert scheduler.observed_waiting == [["plain-1", "plain-2"]] + assert output.num_scheduled_tokens == {"plain-1": 1, "plain-2": 1} + assert output.total_num_scheduled_tokens == 2 + # No pair ever existed, so the hold bookkeeping is never allocated. + assert not hasattr(scheduler, "_cfg_hold_counts") + + +def test_default_suffix_matches_the_audex_stage_processor() -> None: + from vllm_omni.model_executor.stage_input_processors.audex import CFG_UNCOND_SUFFIX + + assert CFG_UNCOND_SUFFIX in cfg_uncond_suffixes() + + +def test_audex_cfg_module_still_exports_the_patch_entry_points() -> None: + from vllm_omni.model_executor.models.audex.cfg import apply_cfg_patches, cfg_patches_applied + + assert apply_cfg_patches is cfg_pairing.apply_cfg_patches + assert cfg_patches_applied is cfg_pairing.cfg_patches_applied + + +# ───────── install gate ───────── + + +def test_gate_triggers_on_a_cfg_logits_processor() -> None: + from vllm_omni.engine.stage_init_utils import _stage_declares_cfg_pairs + + class AudexCFGLogitsProcessor: + pass + + assert _stage_declares_cfg_pairs(SimpleNamespace(logits_processors=[AudexCFGLogitsProcessor])) + # vLLM keeps unresolved entries as dotted strings. + assert _stage_declares_cfg_pairs( + SimpleNamespace(logits_processors=["vllm_omni.model_executor.models.audex.cfg:AudexCFGLogitsProcessor"]) + ) + + +def test_gate_triggers_on_a_declared_cfg_role_default() -> None: + from vllm_omni.engine.stage_init_utils import _stage_declares_cfg_pairs + + assert _stage_declares_cfg_pairs( + SimpleNamespace(logits_processors=None, sampling_extra_args_keys=("cfg_role", "cfg_scale")) + ) + + +def test_gate_stays_off_for_non_cfg_stages() -> None: + from vllm_omni.engine.stage_init_utils import _stage_declares_cfg_pairs + + assert not _stage_declares_cfg_pairs(SimpleNamespace(logits_processors=[], sampling_extra_args_keys=())) + assert not _stage_declares_cfg_pairs( + SimpleNamespace(logits_processors=["some.module:MinPLogitsProcessor"], sampling_extra_args_keys=("tta_rvq",)) + ) + assert not _stage_declares_cfg_pairs(SimpleNamespace()) + + +def test_sampling_extra_args_keys_are_derived_from_stage_defaults() -> None: + from vllm_omni.engine.stage_init_utils import _sampling_extra_args_keys + + assert _sampling_extra_args_keys({"extra_args": {"cfg_scale": 1.5, "cfg_role": "cond"}}) == ( + "cfg_role", + "cfg_scale", + ) + assert _sampling_extra_args_keys({"temperature": 0.1}) == () + assert _sampling_extra_args_keys(None) == () diff --git a/vllm_omni/config/model.py b/vllm_omni/config/model.py index 9164e1352..ef08dde8d 100644 --- a/vllm_omni/config/model.py +++ b/vllm_omni/config/model.py @@ -146,6 +146,11 @@ class OmniModelConfig(ModelConfig): task_type: str | None = None enable_sleep_mode: bool = False has_sampling_extra_args: bool = False + # Key names (not values) of the stage's default sampling ``extra_args``. + # Engine-core code runs before any request arrives, so this is the only + # place it can learn which request-shaping conventions a stage uses (e.g. + # ``cfg_role`` for classifier-free-guidance request pairs). + sampling_extra_args_keys: tuple[str, ...] = () @property def registry(self): diff --git a/vllm_omni/config/pipeline_registry.py b/vllm_omni/config/pipeline_registry.py index e8e3448b2..f498b7bde 100644 --- a/vllm_omni/config/pipeline_registry.py +++ b/vllm_omni/config/pipeline_registry.py @@ -85,6 +85,7 @@ from vllm_omni.model_executor.models.ming_tts.pipeline import ( MING_TTS_PIPELINE, ) from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO_4_5_PIPELINE +from vllm_omni.model_executor.models.minimax_music3.pipeline import MINIMAX_MUSIC3_PIPELINE from vllm_omni.model_executor.models.moss_tts.pipeline import ( MOSS_TTS_LOCAL_PIPELINE, MOSS_TTS_PIPELINE, @@ -177,6 +178,7 @@ OMNI_PIPELINES: dict[str, PipelineConfig | PipelineResolverFunc] = { "moss_tts_realtime": MOSS_TTS_REALTIME_PIPELINE, "moss_tts_local": MOSS_TTS_LOCAL_PIPELINE, "minicpmo_4_5": MINICPMO_4_5_PIPELINE, + "minimax_music3": MINIMAX_MUSIC3_PIPELINE, "higgs_audio_v2": HIGGS_AUDIO_V2_PIPELINE, "higgs_multimodal_qwen3": HIGGS_AUDIO_V3_PIPELINE, "dynin_omni": DYNIN_OMNI_PIPELINE, diff --git a/vllm_omni/data_entry_keys.py b/vllm_omni/data_entry_keys.py index db2cf15ea..98575b689 100644 --- a/vllm_omni/data_entry_keys.py +++ b/vllm_omni/data_entry_keys.py @@ -91,6 +91,11 @@ class OmniPayloadMeta(TypedDict, total=False): ref_context_included: bool talker_prefill_offset: int omni_final_stage_id: int + # Per-request audio seed. Stages that draw their own noise (flow-matching + # / diffusion decoders) need the producing stage's seed to stay + # reproducible, and the producing stage's SamplingParams do not travel + # with the payload. + audio_seed: int class OmniPayload(TypedDict, total=False): @@ -189,6 +194,7 @@ class MetaStruct(_StructBase): codec_left_context_frames: int | None = None code_flat_numel: int | None = None omni_final_stage_id: int | None = None + audio_seed: int | None = None class OmniPayloadStruct(_StructBase): diff --git a/vllm_omni/deploy/minimax_music3.yaml b/vllm_omni/deploy/minimax_music3.yaml new file mode 100644 index 000000000..6e8a13d9d --- /dev/null +++ b/vllm_omni/deploy/minimax_music3.yaml @@ -0,0 +1,130 @@ +# MiniMax-Music3 deploy: Stage 0 (AR talker) -> Stage 1 (acoustic decoder), +# both on one 140 GB card, streaming 200-frame windows over shared memory. +# +# Auto-discovered default for model_type=minimax_music3. +# +# Classifier-free guidance is mandatory for this checkpoint: every request +# decodes as a cond/uncond pair in stage 0, so stage-0 KV, max_num_seqs and +# prefill budget are all sized for TWO rows per user request. The stage-0 +# extra_args block below is what turns on has_sampling_extra_args (so per-row +# sampling extra args reach the model's forward) and is also how the CFG +# scheduler patches learn this stage is paired. +pipeline: minimax_music3 +async_chunk: true + +# The repo-root config.json reports model_type=minimax_music3, which upstream +# transformers does not ship a config class for. +trust_remote_code: true + +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + # Stream stereo PCM back to the client as stage 1 finishes each window. + codec_streaming: true + # Window geometry is fixed by the checkpoint (200-frame windows on a + # 100-frame hop, models/minimax_music3/constants.py), so there are no + # codec_chunk_frames knobs here: changing the window changes the audio. + shm_threshold_bytes: 65536 + connector_get_sleep_s: 0.01 + # One window is 200 AR frames = 8 s of music, so stage 1 waits far longer + # for a chunk than a per-step codec stage would. First chunk additionally + # covers stage-0 prefill of both CFG rows. + connector_get_max_wait_first_chunk: 30000 + connector_get_max_wait: 10000 + +stages: + - stage_id: 0 + # 16 rows = 8 concurrent guided requests, since CFG doubles every request. + max_num_seqs: 16 + # ~16 GB of bf16 Qwen3-8B weights plus KV for 16 full-length rows + # (36 layers x 8 KV heads x 128 head_dim ~= 144 KB/token, so ~1.4 GB per + # 10240-token row) fits comfortably inside 0.6 of a 140 GB card. + gpu_memory_utilization: 0.6 + dtype: bfloat16 + # The checkpoint's max_position_embeddings is 10240 and the whole song is + # decoded in-context. Note this cannot hold a max-length prompt AND a + # max-length song at the same time: the prompt cap is 5000 tokens and the + # frame budget is 9000, which overruns 10240. That is a property of the + # checkpoint, not of this file; long captions shorten the maximum song. + max_model_len: 10240 + # One scheduler step must be able to prefill both rows of a pair + # (2 x 5000 prompt tokens) so the cond/uncond pair stays step-locked. + max_num_batched_tokens: 10240 + # Two rows at 8B is a launch-bound decode step, so the backbone is + # captured. FULL_DECODE_ONLY only ever dispatches a graph for a uniform + # decode batch; prefill and mixed steps stay eager, which is what the + # talker's frame-feedback fallback is for. The talker's own sampler and + # depth decoder run after the graph, not inside it. + # + # What makes the forward capturable: the frame feedback writes a + # fixed-size slab in prepare_runner_inputs (outside the captured region) + # and the forward selects over it with torch.where, instead of an + # index_copy against an index tensor whose address and length changed + # every step. The row-to-request binding the sampler reads is likewise + # rebuilt in prepare_runner_inputs, because a replayed step never runs + # the forward's Python body. + enforce_eager: false + compilation_config: + # No compilation. The talker walks the backbone's decoder layers itself, + # so the compiled Qwen3Model.forward is never entered and inductor has + # nothing to do; what the mode still decides is which kernels vLLM + # defaults to. Left at its non-eager default it picks the elementwise + # PyTorch norms over the fused vllm_c ones, which is both slower and + # numerically different: measured here, that alone moved the sampled + # codes off the eager trajectory by frame 4. Graph capture is + # independent of this and still happens. + mode: 0 + # 16 rows is the stage ceiling, i.e. 8 guided requests. + cudagraph_capture_sizes: [1, 2, 4, 8, 16] + cudagraph_mode: FULL_DECODE_ONLY + cudagraph_num_of_warmups: 1 + # Belt and braces on the same point: the non-eager default is "none", + # which hands the norm and activation ops to elementwise PyTorch. + custom_ops: ["all"] + # A CFG pair must be sampled inside one engine step for the logits blend + # and the cond->uncond token copy to line up. + async_scheduling: false + # Off during bring-up: the uncond row is a long run of identical + # <|audio_cfg|> tokens and would take a much larger prefix-cache hit than + # its cond partner, which is exactly the num_computed_tokens skew the CFG + # pair patches then have to undo. + enable_prefix_caching: false + devices: "0" + output_connectors: + to_stage_1: connector_of_shared_memory + default_sampling_params: + temperature: 1.0 + # The frame budget is enforced by the model. This ceiling only has to + # leave room for the extra step that delivers the final window. + max_tokens: 9002 + extra_args: + cfg_role: cond + cfg_scale: 1.5 + + - stage_id: 1 + # One row per user request: the uncond companion is stage-0-final and + # never reaches the acoustic stage. + max_num_seqs: 8 + # No KV cache on this stage; the budget only covers the DiT and DAV + # weights plus per-window latents and activations. + gpu_memory_utilization: 0.2 + # The flow-matching solver degrades measurably in bfloat16 + # (SUPPORTED_ACOUSTIC_DTYPES in models/minimax_music3/constants.py). + dtype: float32 + enforce_eager: true + # Async scheduling builds per-request payloads against the NEXT step's + # input batch, which breaks when chunk arrival staggers requests. + async_scheduling: false + enable_prefix_caching: false + # One placeholder prompt token per arriving window, so a six-minute song + # (90 windows) needs a tiny context. + max_model_len: 4096 + max_num_batched_tokens: 4096 + devices: "0" + input_connectors: + from_stage_0: connector_of_shared_memory + default_sampling_params: + temperature: 0.0 + max_tokens: 4096 + seed: 0 diff --git a/vllm_omni/deploy/minimax_music3_2gpu.yaml b/vllm_omni/deploy/minimax_music3_2gpu.yaml new file mode 100644 index 000000000..8c463e0e7 --- /dev/null +++ b/vllm_omni/deploy/minimax_music3_2gpu.yaml @@ -0,0 +1,97 @@ +# MiniMax-Music3 two-GPU layout: AR talker on device 0, acoustic decoder on +# device 1. +# +# Splitting the stages removes the memory-budget compromise of the single-card +# file: the talker's CFG-doubled KV no longer competes with the DiT and DAV +# weights, and the flow-matching solver no longer competes with AR decode for +# SMs. Everything else (window geometry, CFG contract, sampling defaults) +# matches minimax_music3.yaml, so the two files stay behaviorally identical. +pipeline: minimax_music3 +async_chunk: true + +# The repo-root config.json reports model_type=minimax_music3, which upstream +# transformers does not ship a config class for. +trust_remote_code: true + +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + # Stream stereo PCM back to the client as stage 1 finishes each window. + codec_streaming: true + # Window geometry is fixed by the checkpoint (200-frame windows on a + # 100-frame hop, models/minimax_music3/constants.py), so there are no + # codec_chunk_frames knobs here: changing the window changes the audio. + shm_threshold_bytes: 65536 + connector_get_sleep_s: 0.01 + # One window is 200 AR frames = 8 s of music, so stage 1 waits far longer + # for a chunk than a per-step codec stage would. First chunk additionally + # covers stage-0 prefill of both CFG rows. + connector_get_max_wait_first_chunk: 30000 + connector_get_max_wait: 10000 + +stages: + - stage_id: 0 + # 32 rows = 16 concurrent guided requests, since CFG doubles every request. + # Twice the single-card figure: this stage now owns its whole device. + max_num_seqs: 32 + gpu_memory_utilization: 0.85 + dtype: bfloat16 + # The checkpoint's max_position_embeddings is 10240 and the whole song is + # decoded in-context. Note this cannot hold a max-length prompt AND a + # max-length song at the same time: the prompt cap is 5000 tokens and the + # frame budget is 9000, which overruns 10240. That is a property of the + # checkpoint, not of this file; long captions shorten the maximum song. + max_model_len: 10240 + # One scheduler step must be able to prefill both rows of a pair + # (2 x 5000 prompt tokens) so the cond/uncond pair stays step-locked. + max_num_batched_tokens: 10240 + # The talker owns its sampler and drives a depth decoder per AR step; + # leave graph capture off until that path is verified graph-safe. + enforce_eager: true + # A CFG pair must be sampled inside one engine step for the logits blend + # and the cond->uncond token copy to line up. + async_scheduling: false + # Off during bring-up: the uncond row is a long run of identical + # <|audio_cfg|> tokens and would take a much larger prefix-cache hit than + # its cond partner, which is exactly the num_computed_tokens skew the CFG + # pair patches then have to undo. + enable_prefix_caching: false + devices: "0" + output_connectors: + to_stage_1: connector_of_shared_memory + default_sampling_params: + temperature: 1.0 + # The frame budget is enforced by the model. This ceiling only has to + # leave room for the extra step that delivers the final window. + max_tokens: 9002 + extra_args: + cfg_role: cond + cfg_scale: 1.5 + + - stage_id: 1 + # One row per user request: the uncond companion is stage-0-final and + # never reaches the acoustic stage. + max_num_seqs: 16 + # No KV cache on this stage, but it now has a device to itself, so the + # solver can hold more windows in flight than the shared-card profile. + gpu_memory_utilization: 0.6 + # The flow-matching solver degrades measurably in bfloat16 + # (SUPPORTED_ACOUSTIC_DTYPES in models/minimax_music3/constants.py). + dtype: float32 + enforce_eager: true + # Async scheduling builds per-request payloads against the NEXT step's + # input batch, which breaks when chunk arrival staggers requests. + async_scheduling: false + enable_prefix_caching: false + # One placeholder prompt token per arriving window, so a six-minute song + # (90 windows) needs a tiny context. + max_model_len: 4096 + max_num_batched_tokens: 4096 + devices: "1" + input_connectors: + from_stage_0: connector_of_shared_memory + default_sampling_params: + temperature: 0.0 + max_tokens: 4096 + seed: 0 diff --git a/vllm_omni/engine/arg_utils.py b/vllm_omni/engine/arg_utils.py index 29c84a1c9..fcd11b173 100644 --- a/vllm_omni/engine/arg_utils.py +++ b/vllm_omni/engine/arg_utils.py @@ -232,6 +232,7 @@ class OmniEngineArgs(EngineArgs): log_stats: bool = False custom_pipeline_args: dict[str, Any] | None = None has_sampling_extra_args: bool = False + sampling_extra_args_keys: tuple[str, ...] = () def __post_init__(self) -> None: if self.worker_cls is None: @@ -409,6 +410,7 @@ class OmniEngineArgs(EngineArgs): omni_kv_config=self.omni_kv_config, task_type=self.task_type, has_sampling_extra_args=self.has_sampling_extra_args, + sampling_extra_args_keys=tuple(self.sampling_extra_args_keys or ()), ) return omni_config diff --git a/vllm_omni/engine/async_omni_engine.py b/vllm_omni/engine/async_omni_engine.py index 38eea768e..e35315f7e 100644 --- a/vllm_omni/engine/async_omni_engine.py +++ b/vllm_omni/engine/async_omni_engine.py @@ -743,23 +743,31 @@ class AsyncOmniEngine: enqueue_ts=time.perf_counter(), ) - def _enqueue_cfg_companions( + def _build_cfg_companions( self, parent_id: str, original_prompt: Any, stage0_params: Any, sampling_params_list: list[Any], - ) -> None: - """Expand prompt into CFG companions, process through InputProcessor, and enqueue.""" - try: - expanded = self.prompt_expand_func(original_prompt, stage0_params) - except Exception: - logger.exception("[AsyncOmniEngine] prompt_expand_func failed for req %s", parent_id) - return + ) -> list[AddCompanionRequestMessage]: + """Expand a prompt into its CFG companions, without enqueueing any. + Construction is separated from admission so a guided request is + all-or-nothing. A model whose guidance is mandatory cannot decode a + request whose companion never arrived: the pair never completes, the + request occupies scheduler and KV capacity for the scheduler's whole + hold budget, and then produces no audio. Raising here instead means the + caller learns immediately and nothing was admitted. + + Raises: + Exception: Whatever prompt expansion or input processing raised. + The caller is expected to let it reach the client. + """ + expanded = self.prompt_expand_func(original_prompt, stage0_params) if not expanded: - return + return [] + companions: list[AddCompanionRequestMessage] = [] for ep in expanded: cid = f"{parent_id}{ep.request_id_suffix}" companion_prompt = ep.prompt @@ -775,6 +783,14 @@ class AsyncOmniEngine: params=companion_params, supported_tasks=self.supported_tasks, ) + # Same restore the parent request gets: the upstream input + # processor drops omni-only prompt fields, so without this the + # companion reaches the worker with no additional_information at + # all. That is where ``global_request_id`` lives, and it is what + # was just injected above, so skipping it silently undoes the + # injection: the model sees the companion row with no id and + # cannot match it to its conditioned partner. + request = upgrade_to_omni_request(request, companion_prompt) request.external_req_id = cid # Companions are stage-0-final for ordinary downstream payloads, # but diffusion still needs their CFG KV caches. @@ -783,7 +799,7 @@ class AsyncOmniEngine: # Registration of this companion on stage-0's output processor is # deferred to Orchestrator._handle_add_companion, which routes # admission through StagePool.submit_initial(..., affinity_request_id=...). - self.request_queue.sync_q.put( + companions.append( AddCompanionRequestMessage( companion_id=cid, parent_id=parent_id, @@ -793,11 +809,35 @@ class AsyncOmniEngine: sampling_params_list=companion_spl, ) ) + return companions + + def _enqueue_cfg_companions( + self, + parent_id: str, + original_prompt: Any, + stage0_params: Any, + sampling_params_list: list[Any], + ) -> None: + """Build and enqueue CFG companions, tolerating a build failure. + + Kept for callers that admit the parent first and cannot roll it back. + Prefer building with :meth:`_build_cfg_companions` before the parent is + admitted, so the pair is atomic. + """ + try: + companions = self._build_cfg_companions(parent_id, original_prompt, stage0_params, sampling_params_list) + except Exception: + logger.exception("[AsyncOmniEngine] CFG companion build failed for req %s", parent_id) + return + for companion in companions: + self.request_queue.sync_q.put(companion) + if not companions: + return logger.info( "[AsyncOmniEngine] CFG expansion for req %s: %d companions", parent_id, - len(expanded), + len(companions), ) @staticmethod @@ -1279,16 +1319,28 @@ class AsyncOmniEngine: reasoning_ended=reasoning_ended, resumable=resumable, ) - self.request_queue.sync_q.put(msg) - - # CFG companion expansion: create and enqueue companion requests - # so the AR stage also generates their KV caches. + # CFG companions are built before the parent is admitted, so the group + # is all-or-nothing: a build failure raises here, nothing is enqueued, + # and the caller sees the error. Admitting the parent first would leave + # an orphan holding scheduler and KV capacity that can never complete, + # because a model whose guidance is mandatory cannot decode a request + # whose companion never arrived. + companions: list[AddCompanionRequestMessage] = [] if self.prompt_expand_func is not None and final_stage_id > 0: - original_prompt = msg.original_prompt effective_spl = msg.sampling_params_list stage0_params = effective_spl[0] if effective_spl else None if stage0_params is not None: - self._enqueue_cfg_companions(request_id, original_prompt, stage0_params, effective_spl) + companions = self._build_cfg_companions(request_id, msg.original_prompt, stage0_params, effective_spl) + + self.request_queue.sync_q.put(msg) + for companion in companions: + self.request_queue.sync_q.put(companion) + if companions: + logger.info( + "[AsyncOmniEngine] CFG expansion for req %s: %d companions", + request_id, + len(companions), + ) async def add_request_async( self, diff --git a/vllm_omni/engine/stage_engine_core_proc.py b/vllm_omni/engine/stage_engine_core_proc.py index 3ac719a6b..4bcf47d3b 100644 --- a/vllm_omni/engine/stage_engine_core_proc.py +++ b/vllm_omni/engine/stage_engine_core_proc.py @@ -31,7 +31,7 @@ from vllm.v1.engine.utils import ( from vllm_omni.distributed.omni_coordinator import create_stage_coord_client from vllm_omni.engine import OmniEngineCoreRequest from vllm_omni.engine.stage_init_utils import ( - maybe_apply_audex_cfg_patches, + maybe_apply_cfg_scheduler_patches, set_death_signal, ) @@ -134,9 +134,10 @@ class StageEngineCoreProc(EngineCoreProc): _vllm_engine_core_module.EngineCoreRequest, ) - # Audex CFG scheduler patches must land before EngineCore builds - # its Scheduler; gated on the stage's logits_processors config. - maybe_apply_audex_cfg_patches(kwargs.get("vllm_config")) + # CFG pairing scheduler patches must land before EngineCore builds + # its Scheduler; gated on the stage's logits_processors and its + # default sampling extra_args. + maybe_apply_cfg_scheduler_patches(kwargs.get("vllm_config")) engine_core = StageEngineCoreProc( *args, diff --git a/vllm_omni/engine/stage_init_utils.py b/vllm_omni/engine/stage_init_utils.py index 054a2fb22..9da6f2cfa 100644 --- a/vllm_omni/engine/stage_init_utils.py +++ b/vllm_omni/engine/stage_init_utils.py @@ -871,6 +871,20 @@ def _project_omni_stage_engine_args( return engine_args +def _sampling_extra_args_keys(default_sampling_params: Any) -> tuple[str, ...]: + """Key names of a stage's default sampling ``extra_args``, sorted. + + Only the keys travel into the engine config: engine-core code needs to know + which request-shaping conventions a stage uses (e.g. CFG request pairing) + before any request exists, while the values stay a serving-layer concern. + """ + extra_args = _to_dict(default_sampling_params or {}).get("extra_args") or {} + try: + return tuple(sorted(str(key) for key in extra_args)) + except TypeError: + return () + + def _finalize_engine_args_dict( engine_args_dict: dict[str, Any], *, @@ -880,6 +894,7 @@ def _finalize_engine_args_dict( stage_connector_spec: dict[str, Any] | None, cli_tokenizer: str | None, has_sampling_extra_args: bool, + sampling_extra_args_keys: tuple[str, ...] = (), ) -> dict[str, Any]: """Apply representation-independent engine adapter behavior.""" pipeline_model_root = model @@ -960,6 +975,7 @@ def _finalize_engine_args_dict( engine_args_dict.setdefault("enable_prefix_caching", False) engine_args_dict["has_sampling_extra_args"] = has_sampling_extra_args + engine_args_dict["sampling_extra_args_keys"] = sampling_extra_args_keys # TODO: Remove this after the performance regression is fixed # Set VLLM_USE_FLASHINFER_MOE_FP16=0 for Qwen3-Omni to avoid performance regression @@ -990,6 +1006,7 @@ def build_legacy_engine_args_dict( stage_connector_spec=stage_connector_spec, cli_tokenizer=cli_tokenizer, has_sampling_extra_args=bool(default_sp.get("extra_args")), + sampling_extra_args_keys=_sampling_extra_args_keys(default_sp), ) @@ -1036,6 +1053,7 @@ def build_engine_args_dict_from_omni_stage_config( stage_connector_spec=stage_connector_spec, cli_tokenizer=cli_tokenizer, has_sampling_extra_args=stage_config.model_config.has_sampling_extra_args, + sampling_extra_args_keys=_sampling_extra_args_keys(stage_config.model_config.default_sampling_params), ) @@ -1445,24 +1463,45 @@ def initialize_diffusion_stage( return create_diffusion_client(model, od_config, metadata, stage_init_timeout, batch_size, use_inline) -def maybe_apply_audex_cfg_patches(vllm_config: Any) -> None: - """Install the Audex CFG scheduler patches for CFG-configured engines. +def _stage_declares_cfg_pairs(model_config: Any) -> bool: + """Whether this stage submits classifier-free-guidance request pairs. + + Two independent declarations, because a model may own either side of the + mechanism without the other: + + * a CFG logits processor is configured (blending implies pairing), or + * the stage's default sampling ``extra_args`` carry ``cfg_role``, which is + how a model declares paired requests without owning a logits processor. + Only the *key set* survives into the engine config (see + ``sampling_extra_args_keys``); the values stay in the serving layer. + """ + processors = getattr(model_config, "logits_processors", None) or [] + if any("CFGLogitsProcessor" in getattr(proc, "__name__", str(proc)) for proc in processors): + return True + return "cfg_role" in (getattr(model_config, "sampling_extra_args_keys", None) or ()) + + +def maybe_apply_cfg_scheduler_patches(vllm_config: Any) -> None: + """Install the CFG pairing scheduler patches for CFG-configured engines. Must run in the engine-core process BEFORE ``Scheduler`` is constructed: the patch wraps ``Scheduler.__init__`` to add the pair registry, so a - scheduler built earlier would never become pair-aware. Gated on the - engine's ``logits_processors`` so non-CFG stages stay untouched. + scheduler built earlier would never become pair-aware. Gated so non-CFG + stages stay untouched. """ model_config = getattr(vllm_config, "model_config", None) - processors = getattr(model_config, "logits_processors", None) or [] - if not any("AudexCFGLogitsProcessor" in getattr(proc, "__name__", str(proc)) for proc in processors): + if model_config is None or not _stage_declares_cfg_pairs(model_config): return - from vllm_omni.model_executor.models.audex.cfg import apply_cfg_patches + from vllm_omni.model_executor.models.common.cfg_pairing import apply_cfg_patches apply_cfg_patches() from vllm.v1.core.sched.scheduler import Scheduler - if not getattr(Scheduler.schedule, "_audex_cfg_patched", False): - raise RuntimeError("Audex CFG scheduler patches failed to install before Scheduler construction") + if not getattr(Scheduler.schedule, "_cfg_pairing_patched", False): + raise RuntimeError("CFG pairing scheduler patches failed to install before Scheduler construction") + + +# Name kept for callers written against the Audex-only gate. +maybe_apply_audex_cfg_patches = maybe_apply_cfg_scheduler_patches diff --git a/vllm_omni/entrypoints/openai/tts_adapters/__init__.py b/vllm_omni/entrypoints/openai/tts_adapters/__init__.py index 74597a2d1..e725e65cd 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/__init__.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/__init__.py @@ -111,6 +111,7 @@ from vllm_omni.entrypoints.openai.tts_adapters import ( # noqa: E402,F401 indextts2, ming_flash_omni_tts, ming_tts, + minimax_music3, moss_tts, omnivoice, qwen3_tts, diff --git a/vllm_omni/entrypoints/openai/tts_adapters/minimax_music3.py b/vllm_omni/entrypoints/openai/tts_adapters/minimax_music3.py new file mode 100644 index 000000000..50b9c2d71 --- /dev/null +++ b/vllm_omni/entrypoints/openai/tts_adapters/minimax_music3.py @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MiniMax Music 3 serving adapter for ``/v1/audio/speech``. + +This is a text-to-music model on a speech endpoint, so most of the speech +contract does not apply. ``input`` carries the lyrics and ``instructions`` the +caption that describes genre, instrumentation, tempo and production. There is +no speaker to select, no reference audio, and no temperature: sampling is +fixed by the checkpoint. Unsupported parameters are rejected rather than +silently ignored, so a mistaken request fails immediately instead of quietly +returning something the caller did not ask for. +""" + +from typing import TYPE_CHECKING, Any + +from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.model_executor.models.minimax_music3.constants import ( + AUDIO_FRAME_RATE, + MAX_AUDIO_FRAMES, + MAX_PROMPT_TOKENS, +) +from vllm_omni.model_executor.models.minimax_music3.prompt import ( + build_prompt, + validate_tokenizer_ids, +) + +if TYPE_CHECKING: + from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest + +# Captions are long by design: the checkpoint was trained on structured +# descriptions covering global attributes, emotional progression, vocal detail +# and arrangement. The reference captions run to roughly 5,000 characters, so +# the generic 500-character instructions cap does not apply here. The real +# limit is the tokenized prompt cap, enforced downstream. +_MAX_CAPTION_CHARS = 20_000 + +# Sampling is fixed by the checkpoint: guidance at 1.5, then a seeded top-k 50 +# draw. Accepting these would imply a control the model does not have. +_UNSUPPORTED_SAMPLING_PARAMS = ("temperature", "top_p", "top_k", "repetition_penalty") + +# CFG pair plumbing is derived from the request id by the scheduler. A caller +# supplying these would desync the guided pair. +_INTERNAL_CFG_KEYS = ("cfg_role", "cfg_pair_id", "cfg_null_prompt") + + +@register_tts_adapter +class MiniMaxMusic3Adapter(ARTTSAdapter): + """Lyrics plus caption in, one 32 kHz stereo song out.""" + + name = "minimax_music3" + stage_keys = frozenset({"minimax_music3_ar"}) + # 9,000 frames at 25 fps is six minutes, the checkpoint's own ceiling. + max_new_tokens_min = 1 + max_new_tokens_max = MAX_AUDIO_FRAMES + + def __init__(self, ctx: Any) -> None: + super().__init__(ctx) + self._cached_tokenizer: Any = None + + def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: + if not request.input or not request.input.strip(): + return "MiniMax Music 3 requires non-empty 'input' carrying the lyrics" + + instructions = request.instructions + if not isinstance(instructions, str) or not instructions.strip(): + return ( + "MiniMax Music 3 requires 'instructions' describing the music " + "(genre, instrumentation, tempo, mood); it is what decides the arrangement" + ) + if len(instructions) > _MAX_CAPTION_CHARS: + return f"'instructions' exceeds the maximum length of {_MAX_CAPTION_CHARS} characters" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"'max_new_tokens' must be at least {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return ( + f"'max_new_tokens' counts audio frames at {AUDIO_FRAME_RATE} per second " + f"and cannot exceed {self.max_new_tokens_max}" + ) + + voice = (request.voice or "").strip().lower() + if voice not in ("", "default"): + return ( + "MiniMax Music 3 has no speaker to select; the vocal comes from " + f"'instructions'. Omit 'voice' or pass 'default'; got {request.voice!r}" + ) + if request.ref_audio is not None or request.ref_text is not None: + return "MiniMax Music 3 does not support reference-audio conditioning" + if request.language is not None: + return "MiniMax Music 3 has no language tag; the language follows the lyrics" + if request.task_type is not None: + return "MiniMax Music 3 does not support 'task_type'" + if request.speed is not None and float(request.speed) != 1.0: + return "MiniMax Music 3 only supports speed=1.0; put the tempo in 'instructions', e.g. 'at 92 BPM'" + + extra: dict[str, Any] = request.extra_params or {} + for key in _INTERNAL_CFG_KEYS: + if key in extra: + return f"extra_params.{key} is managed internally by the server and cannot be set" + rejected = sorted(key for key in _UNSUPPORTED_SAMPLING_PARAMS if extra.get(key) is not None) + if rejected: + return "MiniMax Music 3 has fixed sampling and does not accept: " + ", ".join(rejected) + return None + + async def build( + self, + request: "OpenAICreateSpeechRequest", + sampling_params_list: list, + has_inline_ref_audio: bool, + ) -> PreparedRequest: + del sampling_params_list, has_inline_ref_audio + prompt = build_prompt(request.instructions, request.input) + # Submit token ids, not text. Guidance needs a companion whose prompt is + # the same length with the caption-and-lyrics span replaced, and that + # length match can only be guaranteed on ids. Handing over text would + # leave the expansion with nothing to build the twin from, and a request + # that decodes unguided produces a different song. + token_ids = self._tokenizer().encode(prompt) + if len(token_ids) > MAX_PROMPT_TOKENS: + raise ValueError( + f"MiniMax Music 3 prompt is {len(token_ids)} tokens; " + f"the maximum is {MAX_PROMPT_TOKENS}. Shorten the caption or the lyrics." + ) + # The frame budget is enforced by the model, not by the engine's + # max_tokens. The model needs one decode step beyond the last frame to + # deliver the song's final window, so capping the engine at exactly + # this many tokens would cut the tail off. + max_frames = int(request.max_new_tokens or MAX_AUDIO_FRAMES) + # Carried on the prompt rather than through tts_params: the adapter + # path hands PreparedRequest.prompt to the engine untouched, so only + # what is attached here reaches the model's per-request info. + return PreparedRequest( + prompt={ + "prompt_token_ids": token_ids, + "additional_information": {"max_audio_frames": [max_frames]}, + }, + tts_params={"max_audio_frames": [max_frames]}, + model_type=self.name, + ) + + def _tokenizer(self) -> Any: + """The music tokenizer, loaded once per process. + + Its special-token ids are pinned: a checkpoint whose tokenizer + disagrees is not this model, and quietly producing noise is worse than + failing at the first request. + """ + if self._cached_tokenizer is None: + from transformers import AutoTokenizer + + stage = getattr(self.ctx.server, "_tts_stage", None) + model_path = getattr(getattr(stage, "engine_args", None), "tokenizer", None) + if not model_path: + model_path = self.ctx.server.engine_client.model_config.tokenizer + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + validate_tokenizer_ids(tokenizer) + self._cached_tokenizer = tokenizer + return self._cached_tokenizer + + +__all__ = ["MiniMaxMusic3Adapter"] diff --git a/vllm_omni/model_executor/models/audex/cfg.py b/vllm_omni/model_executor/models/audex/cfg.py index 488097216..3c1837140 100644 --- a/vllm_omni/model_executor/models/audex/cfg.py +++ b/vllm_omni/model_executor/models/audex/cfg.py @@ -3,8 +3,7 @@ # # Adapted from nvidia/Nemotron-Labs-Audex-2B (Apache-2.0): # inference_scripts_vllm/audiogen_scripts/cfg_logits_processor.py -# inference_scripts_vllm/audiogen_scripts/vllm_cfg_patch.py -# Divergences from the official files are commented at the divergence site. +# Divergences from the official file are commented at the divergence site. """Classifier-free guidance for the Audex thinker stage. CFG pairs a conditional request with an unconditional (null-prompt) @@ -20,16 +19,13 @@ Requests opt in via ``SamplingParams.extra_args``: uncond: {"cfg_scale": 1.5, "cfg_role": "uncond", "cfg_pair_id": } Blending is only correct when both pair members decode the same position in -the same engine step, so ``apply_cfg_patches()`` makes the vLLM scheduler -pair-aware: a lone member waits for its partner, partners stay adjacent, -their ``num_computed_tokens`` are equalized after every schedule, and they -finish together. The patch is a no-op for engines that never see a CFG -request (the pair registry stays empty). +the same engine step. Keeping the pair step-locked is model-neutral and lives +in :mod:`vllm_omni.model_executor.models.common.cfg_pairing`; the scheduler +patches are re-exported here so existing imports keep working. """ from __future__ import annotations -from collections import deque from typing import Any import torch @@ -42,18 +38,20 @@ from vllm.v1.sample.logits_processor import ( MoveDirectionality, ) +from vllm_omni.model_executor.models.common.cfg_pairing import ( + _COND, + _UNCOND, + apply_cfg_patches, + cfg_patches_applied, +) + logger = init_logger(__name__) -_COND = "cond" -_UNCOND = "uncond" - - -def _cfg_extra(request: Any, key: str) -> Any: - sampling_params = getattr(request, "sampling_params", None) - extra_args = getattr(sampling_params, "extra_args", None) - if extra_args: - return extra_args.get(key) - return None +__all__ = [ + "AudexCFGLogitsProcessor", + "apply_cfg_patches", + "cfg_patches_applied", +] class AudexCFGLogitsProcessor(LogitsProcessor): @@ -208,360 +206,3 @@ class AudexCFGLogitsProcessor(LogitsProcessor): logits[uncond_idx] = blended return logits - - -# --------------------------------------------------------------------------- -# Scheduler pair synchronization -# --------------------------------------------------------------------------- - - -def _wait_queues(scheduler: Any) -> list[Any]: - # Divergence from the official (vLLM 0.20-era) file: vLLM 0.24 parks - # structurally not-ready requests in a second ``skipped_waiting`` queue; - # pair-hold must cover both queues or a lone member parked there would - # be admitted without its partner. - queues = [scheduler.waiting] - skipped = getattr(scheduler, "skipped_waiting", None) - if skipped is not None: - queues.append(skipped) - return queues - - -def _pair_complete(scheduler: Any, request_id: str, blocked_ids: set[str] | None = None) -> bool: - pair_id = scheduler._cfg_req_to_pair.get(request_id) - if pair_id is None: - return True - roles = scheduler._cfg_pairs.get(pair_id, {}) - if len(roles) != 2 or not all(rid in scheduler.requests for rid in roles.values()): - return False - if blocked_ids: - # A partner parked in skipped_waiting (e.g. waiting for remote KVs) - # may fail promotion this step while this member gets scheduled from - # waiting — hold this member until the partner is schedulable. - for rid in roles.values(): - if rid != request_id and rid in blocked_ids: - return False - return True - - -# Schedule steps a lone pair member may wait for its partner. Partners arrive -# within a handful of steps in practice; exhausting this budget means the -# companion was never created (e.g. prompt expansion failed), and the request -# is released to decode unguided instead of hanging forever. -_MAX_PAIR_HOLD_STEPS = 512 - - -def _drop_stale_pair(scheduler: Any, request_id: str) -> None: - pair_id = scheduler._cfg_req_to_pair.get(request_id) - if pair_id is None: - return - for rid in scheduler._cfg_pairs.pop(pair_id, {}).values(): - scheduler._cfg_req_to_pair.pop(rid, None) - logger.warning( - "CFG pair %s: partner of %s never arrived after %d schedule steps; releasing the request unguided", - pair_id, - request_id, - _MAX_PAIR_HOLD_STEPS, - ) - - -def _hold_incomplete_pairs(scheduler: Any) -> list[tuple[Any, Any]]: - """Pull CFG requests whose partner is not yet admitted out of the wait queues. - - The engine schedules as soon as the first member of a pair arrives over - IPC; without the hold, the lone member prefills one step early and the - pair is permanently offset by one token. Held requests are re-prepended - after the schedule step. - """ - hold_counts: dict[str, int] = getattr(scheduler, "_cfg_hold_counts", None) or {} - scheduler._cfg_hold_counts = hold_counts - - skipped = getattr(scheduler, "skipped_waiting", None) - blocked_ids = {req.request_id for req in list(skipped)} if skipped is not None else set() - - held: list[tuple[Any, Any]] = [] - for queue in _wait_queues(scheduler): - # A member is only "blocked" for its partner's sake when it sits in - # the OTHER queue; members of the queue being scanned move together. - partner_blockers = blocked_ids if queue is not skipped else set() - held_here = [] - for request in list(queue): - if _pair_complete(scheduler, request.request_id, partner_blockers): - hold_counts.pop(request.request_id, None) - continue - count = hold_counts.get(request.request_id, 0) + 1 - if count > _MAX_PAIR_HOLD_STEPS: - hold_counts.pop(request.request_id, None) - _drop_stale_pair(scheduler, request.request_id) - continue - hold_counts[request.request_id] = count - held_here.append(request) - if held_here: - queue.remove_requests(held_here) - held.extend((queue, req) for req in held_here) - return held - - -def _release_held(held: list[tuple[Any, Any]]) -> None: - for queue, request in reversed(held): - queue.prepend_request(request) - - -def _reorder_waiting_for_cfg(scheduler: Any) -> None: - """Move CFG pair partners adjacent in the FCFS waiting queue.""" - waiting = scheduler.waiting - # Only the FCFS queue (a deque) has caller-controlled ordering; priority - # queues order themselves and pair sync then relies on hold + equalize. - if not isinstance(waiting, deque) or len(waiting) < 2: - return - - requests = list(waiting) - waiting.clear() - - seen: set[str] = set() - result: list[Any] = [] - for request in requests: - rid = request.request_id - if rid in seen: - continue - seen.add(rid) - result.append(request) - - pair_id = scheduler._cfg_req_to_pair.get(rid) - if pair_id is None: - continue - for partner_id in scheduler._cfg_pairs.get(pair_id, {}).values(): - if partner_id != rid and partner_id not in seen: - for candidate in requests: - if candidate.request_id == partner_id: - seen.add(partner_id) - result.append(candidate) - break - - waiting.extend(result) - - -def _equalize_cfg_pair_progress(scheduler: Any, scheduler_output: Any) -> None: - """Bring both members of every CFG pair to the same ``num_computed_tokens``. - - Prefix-cache hits or unequal chunked-prefill budget can leave one member - ahead after a schedule step; reduce the faster member's allocation so - both land on the same position. Over-allocated KV blocks remain and are - consumed next step — nothing leaks. - """ - for roles in scheduler._cfg_pairs.values(): - cond_id = roles.get(_COND) - uncond_id = roles.get(_UNCOND) - if not cond_id or not uncond_id: - continue - - cond_sched = scheduler_output.num_scheduled_tokens.get(cond_id, 0) - uncond_sched = scheduler_output.num_scheduled_tokens.get(uncond_id, 0) - if cond_sched == 0 or uncond_sched == 0: - continue - - cond_req = scheduler.requests.get(cond_id) - uncond_req = scheduler.requests.get(uncond_id) - if not cond_req or not uncond_req: - continue - - if cond_req.num_computed_tokens == uncond_req.num_computed_tokens: - continue - - target = min(cond_req.num_computed_tokens, uncond_req.num_computed_tokens) - - feasible = all( - req.num_computed_tokens - target < sched - for req, sched in ((cond_req, cond_sched), (uncond_req, uncond_sched)) - ) - if not feasible: - continue - - for req_id, req, orig_sched in ( - (cond_id, cond_req, cond_sched), - (uncond_id, uncond_req, uncond_sched), - ): - diff = req.num_computed_tokens - target - if diff > 0: - req.num_computed_tokens = target - scheduler_output.num_scheduled_tokens[req_id] = orig_sched - diff - scheduler_output.total_num_scheduled_tokens -= diff - logger.debug( - "CFG equalize %s: %d -> %d scheduled, computed -> %d", - req_id, - orig_sched, - orig_sched - diff, - target, - ) - - -def _drop_split_pairs(scheduler: Any) -> None: - """Deregister a desynced CFG pair (one member preempted / left behind). - - Manual re-preemption surgery is unsafe against the upstream scheduler. - Instead, a pair that split anyway (KV-pressure preemption of exactly one - member, or unrecoverable progress skew) is dropped from the registry: - the cond request continues UNGUIDED (its logits are never blended again) - rather than being re-paired against a desynced partner, and the - companion's discarded output no longer matters. - """ - if not scheduler._cfg_pairs: - return - - running_ids = {req.request_id for req in scheduler.running} - to_drop: list[tuple[str, str, str, str]] = [] - for pair_id, roles in scheduler._cfg_pairs.items(): - cond_id = roles.get(_COND) - uncond_id = roles.get(_UNCOND) - if cond_id is None or uncond_id is None: - continue - if cond_id not in scheduler.requests or uncond_id not in scheduler.requests: - continue - - cond_running = cond_id in running_ids - uncond_running = uncond_id in running_ids - if cond_running != uncond_running: - # Split only counts once decoding started: a lone member that has - # computed tokens while its partner was pushed back is desynced. - runner = scheduler.requests.get(cond_id if cond_running else uncond_id) - if runner is not None and runner.num_computed_tokens > 0: - to_drop.append((pair_id, cond_id, uncond_id, "one member preempted")) - continue - - if cond_running and uncond_running: - cond_req = scheduler.requests.get(cond_id) - uncond_req = scheduler.requests.get(uncond_id) - if cond_req and uncond_req and cond_req.num_computed_tokens != uncond_req.num_computed_tokens: - logger.warning( - "CFG pair progress mismatch (equalize will retry): %s@%d vs %s@%d", - cond_id, - cond_req.num_computed_tokens, - uncond_id, - uncond_req.num_computed_tokens, - ) - - for pair_id, cond_id, uncond_id, reason in to_drop: - scheduler._cfg_pairs.pop(pair_id, None) - scheduler._cfg_req_to_pair.pop(cond_id, None) - scheduler._cfg_req_to_pair.pop(uncond_id, None) - logger.warning( - "CFG pair %s split (%s); releasing cond=%s unguided, uncond=%s output is discarded", - pair_id, - reason, - cond_id, - uncond_id, - ) - - -_patches_applied = False - - -def cfg_patches_applied() -> bool: - """True once :func:`apply_cfg_patches` has patched the scheduler in this process.""" - return _patches_applied - - -def _patch_scheduler() -> None: - from vllm.v1.core.sched.scheduler import Scheduler - - orig_init = Scheduler.__init__ - orig_add = Scheduler.add_request - orig_finish = Scheduler.finish_requests - orig_schedule = Scheduler.schedule - - def _init(self, *args: Any, **kwargs: Any) -> None: - orig_init(self, *args, **kwargs) - self._cfg_pairs: dict[str, dict[str, str]] = {} - self._cfg_req_to_pair: dict[str, str] = {} - - def _add(self, request: Any) -> None: - orig_add(self, request) - pair_id = _cfg_extra(request, "cfg_pair_id") - role = _cfg_extra(request, "cfg_role") - if pair_id and role: - self._cfg_pairs.setdefault(pair_id, {})[role] = request.request_id - self._cfg_req_to_pair[request.request_id] = pair_id - # An asymmetric prefix-cache hit (the cond prompt shares cached - # branches; the null prompt does not) can advance one - # member past what post-schedule equalization can claw back. - # Skip prefix-cache reads for CFG requests entirely. - if hasattr(request, "skip_reading_prefix_cache"): - request.skip_reading_prefix_cache = True - - def _finish(self, request_ids: Any, finished_status: Any) -> Any: - if request_ids is None: - result = orig_finish(self, request_ids, finished_status) - self._cfg_pairs.clear() - self._cfg_req_to_pair.clear() - return result - - if isinstance(request_ids, str): - request_ids = {request_ids} - else: - request_ids = set(request_ids) - - # Finish both members together: a surviving partner would decode - # unpaired garbage (its logits are never blended again). - partner_ids: set[str] = set() - for req_id in request_ids: - pair_id = self._cfg_req_to_pair.get(req_id) - if pair_id is None: - continue - for rid in self._cfg_pairs.get(pair_id, {}).values(): - if rid != req_id and rid in self.requests: - partner_ids.add(rid) - - all_ids = request_ids | partner_ids - result = orig_finish(self, all_ids, finished_status) - - for req_id in all_ids: - pair_id = self._cfg_req_to_pair.pop(req_id, None) - if pair_id: - self._cfg_pairs.pop(pair_id, None) - return result - - def _schedule(self, *args: Any, **kwargs: Any) -> Any: - if not self._cfg_pairs: - return orig_schedule(self, *args, **kwargs) - - _reorder_waiting_for_cfg(self) - held = _hold_incomplete_pairs(self) - - # Chunked prefill can split one pair member's prompt across steps - # while the other fits in a single step; halving the long-prefill - # threshold makes both chunk, and equalize aligns the remainder. - scheduler_config = getattr(self, "scheduler_config", None) - orig_threshold = getattr(scheduler_config, "long_prefill_token_threshold", None) - if scheduler_config is not None and orig_threshold is not None: - scheduler_config.long_prefill_token_threshold = self.max_num_scheduled_tokens // 2 - try: - scheduler_output = orig_schedule(self, *args, **kwargs) - finally: - if scheduler_config is not None and orig_threshold is not None: - scheduler_config.long_prefill_token_threshold = orig_threshold - _release_held(held) - - _equalize_cfg_pair_progress(self, scheduler_output) - _drop_split_pairs(self) - return scheduler_output - - _schedule._audex_cfg_patched = True # type: ignore[attr-defined] - - Scheduler.__init__ = _init - Scheduler.add_request = _add - Scheduler.finish_requests = _finish - Scheduler.schedule = _schedule - - -def apply_cfg_patches() -> None: - """Make the vLLM v1 scheduler CFG-pair-aware. Idempotent, per process. - - Must run in the engine-core process before ``Scheduler`` is constructed - (the ``__init__`` wrapper installs the pair registry). - """ - global _patches_applied - if _patches_applied: - return - _patches_applied = True - logger.info("Applying Audex CFG scheduler patches to vLLM v1") - _patch_scheduler() diff --git a/vllm_omni/model_executor/models/common/cfg_pairing.py b/vllm_omni/model_executor/models/common/cfg_pairing.py new file mode 100644 index 000000000..ae4b3ac38 --- /dev/null +++ b/vllm_omni/model_executor/models/common/cfg_pairing.py @@ -0,0 +1,460 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# Adapted from nvidia/Nemotron-Labs-Audex-2B (Apache-2.0): +# inference_scripts_vllm/audiogen_scripts/vllm_cfg_patch.py +# Divergences from the official file are commented at the divergence site. +"""Model-neutral classifier-free-guidance request pairing for the vLLM v1 scheduler. + +A guided request decodes together with an unconditional companion so their +logits can be blended every step. Blending is only correct when both members +decode the same position in the same engine step, so :func:`apply_cfg_patches` +makes the scheduler pair-aware: a lone member waits for its partner, partners +stay adjacent in the waiting queue, their ``num_computed_tokens`` are equalized +after every schedule, and they finish together. + +Requests opt in through ``SamplingParams.extra_args``:: + + cond: {"cfg_role": "cond", "cfg_pair_id": } + uncond: {"cfg_role": "uncond", "cfg_pair_id": } + +``cfg_pair_id`` is optional. A model whose serving layer cannot mint a pair id +(the companion request id is only known inside the engine) may send ``cfg_role`` +alone; the pair id is then derived from the request id, stripping the +companion's registered suffix (see :func:`register_cfg_uncond_suffix`). + +Logit blending itself is the caller's concern: this module only keeps the two +rows step-locked. The patch is a no-op for engines that never see a CFG request +(the pair registry stays empty). +""" + +from __future__ import annotations + +from collections import deque +from typing import Any + +from vllm.logger import init_logger + +logger = init_logger(__name__) + +__all__ = [ + "apply_cfg_patches", + "cfg_patches_applied", + "cfg_uncond_suffixes", + "register_cfg_uncond_suffix", +] + +_COND = "cond" +_UNCOND = "uncond" + +# Request-id suffix appended to the companion request by the Audex stage input +# processor (``CFG_UNCOND_SUFFIX`` in +# vllm_omni/model_executor/stage_input_processors/audex.py). Models that mint +# the companion request id differently register their own suffix. +_UNCOND_REQUEST_ID_SUFFIXES: set[str] = {"__cfg_uncond"} + + +def register_cfg_uncond_suffix(suffix: str) -> None: + """Register a companion request-id suffix used to derive a missing pair id.""" + if not suffix or not isinstance(suffix, str): + raise ValueError(f"CFG uncond request-id suffix must be a non-empty string, got {suffix!r}") + _UNCOND_REQUEST_ID_SUFFIXES.add(suffix) + + +def cfg_uncond_suffixes() -> frozenset[str]: + """Companion request-id suffixes registered in this process.""" + return frozenset(_UNCOND_REQUEST_ID_SUFFIXES) + + +def _derive_pair_id(request_id: str, role: str) -> str: + """Pair id for a request that carries ``cfg_role`` but no ``cfg_pair_id``. + + The cond request id IS the pair id; its companion carries the same id plus + a registered suffix, so stripping the suffix recovers it. Longest suffix + first, so a registered suffix extending another still resolves. + """ + if role == _UNCOND: + for suffix in sorted(_UNCOND_REQUEST_ID_SUFFIXES, key=len, reverse=True): + if request_id.endswith(suffix): + return request_id[: -len(suffix)] + return request_id + + +def _cfg_extra(request: Any, key: str) -> Any: + sampling_params = getattr(request, "sampling_params", None) + extra_args = getattr(sampling_params, "extra_args", None) + if extra_args: + return extra_args.get(key) + return None + + +def _wait_queues(scheduler: Any) -> list[Any]: + # Divergence from the official (vLLM 0.20-era) file: vLLM 0.24 parks + # structurally not-ready requests in a second ``skipped_waiting`` queue; + # pair-hold must cover both queues or a lone member parked there would + # be admitted without its partner. + queues = [scheduler.waiting] + skipped = getattr(scheduler, "skipped_waiting", None) + if skipped is not None: + queues.append(skipped) + return queues + + +def _pair_complete(scheduler: Any, request_id: str, blocked_ids: set[str] | None = None) -> bool: + pair_id = scheduler._cfg_req_to_pair.get(request_id) + if pair_id is None: + return True + roles = scheduler._cfg_pairs.get(pair_id, {}) + if len(roles) != 2 or not all(rid in scheduler.requests for rid in roles.values()): + return False + if blocked_ids: + # A partner parked in skipped_waiting (e.g. waiting for remote KVs) + # may fail promotion this step while this member gets scheduled from + # waiting; hold this member until the partner is schedulable. + for rid in roles.values(): + if rid != request_id and rid in blocked_ids: + return False + return True + + +# Schedule steps a lone pair member may wait for its partner. Partners arrive +# within a handful of steps in practice; exhausting this budget means the +# companion was never created (e.g. prompt expansion failed), and the request +# is released to decode unguided instead of hanging forever. +_MAX_PAIR_HOLD_STEPS = 512 + + +def _drop_stale_pair(scheduler: Any, request_id: str) -> None: + pair_id = scheduler._cfg_req_to_pair.get(request_id) + if pair_id is None: + return + for rid in scheduler._cfg_pairs.pop(pair_id, {}).values(): + scheduler._cfg_req_to_pair.pop(rid, None) + logger.warning( + "CFG pair %s: partner of %s never arrived after %d schedule steps; releasing the request unguided", + pair_id, + request_id, + _MAX_PAIR_HOLD_STEPS, + ) + + +def _hold_incomplete_pairs(scheduler: Any) -> list[tuple[Any, Any]]: + """Pull CFG requests whose partner is not yet admitted out of the wait queues. + + The engine schedules as soon as the first member of a pair arrives over + IPC; without the hold, the lone member prefills one step early and the + pair is permanently offset by one token. Held requests are re-prepended + after the schedule step. + """ + hold_counts: dict[str, int] = getattr(scheduler, "_cfg_hold_counts", None) or {} + scheduler._cfg_hold_counts = hold_counts + + skipped = getattr(scheduler, "skipped_waiting", None) + blocked_ids = {req.request_id for req in list(skipped)} if skipped is not None else set() + + held: list[tuple[Any, Any]] = [] + for queue in _wait_queues(scheduler): + # A member is only "blocked" for its partner's sake when it sits in + # the OTHER queue; members of the queue being scanned move together. + partner_blockers = blocked_ids if queue is not skipped else set() + held_here = [] + for request in list(queue): + if _pair_complete(scheduler, request.request_id, partner_blockers): + hold_counts.pop(request.request_id, None) + continue + count = hold_counts.get(request.request_id, 0) + 1 + if count > _MAX_PAIR_HOLD_STEPS: + hold_counts.pop(request.request_id, None) + _drop_stale_pair(scheduler, request.request_id) + continue + hold_counts[request.request_id] = count + held_here.append(request) + if held_here: + queue.remove_requests(held_here) + held.extend((queue, req) for req in held_here) + return held + + +def _release_held(held: list[tuple[Any, Any]]) -> None: + for queue, request in reversed(held): + queue.prepend_request(request) + + +def _reorder_waiting_for_cfg(scheduler: Any) -> None: + """Move CFG pair partners adjacent in the FCFS waiting queue.""" + waiting = scheduler.waiting + # Only the FCFS queue (a deque) has caller-controlled ordering; priority + # queues order themselves and pair sync then relies on hold + equalize. + if not isinstance(waiting, deque) or len(waiting) < 2: + return + + requests = list(waiting) + waiting.clear() + + seen: set[str] = set() + result: list[Any] = [] + for request in requests: + rid = request.request_id + if rid in seen: + continue + seen.add(rid) + result.append(request) + + pair_id = scheduler._cfg_req_to_pair.get(rid) + if pair_id is None: + continue + for partner_id in scheduler._cfg_pairs.get(pair_id, {}).values(): + if partner_id != rid and partner_id not in seen: + for candidate in requests: + if candidate.request_id == partner_id: + seen.add(partner_id) + result.append(candidate) + break + + waiting.extend(result) + + +def _equalize_cfg_pair_progress(scheduler: Any, scheduler_output: Any) -> None: + """Bring both members of every CFG pair to the same ``num_computed_tokens``. + + Prefix-cache hits or unequal chunked-prefill budget can leave one member + ahead after a schedule step; reduce the faster member's allocation so + both land on the same position. Over-allocated KV blocks remain and are + consumed next step, so nothing leaks. + """ + for roles in scheduler._cfg_pairs.values(): + cond_id = roles.get(_COND) + uncond_id = roles.get(_UNCOND) + if not cond_id or not uncond_id: + continue + + cond_sched = scheduler_output.num_scheduled_tokens.get(cond_id, 0) + uncond_sched = scheduler_output.num_scheduled_tokens.get(uncond_id, 0) + if cond_sched == 0 or uncond_sched == 0: + continue + + cond_req = scheduler.requests.get(cond_id) + uncond_req = scheduler.requests.get(uncond_id) + if not cond_req or not uncond_req: + continue + + if cond_req.num_computed_tokens == uncond_req.num_computed_tokens: + continue + + target = min(cond_req.num_computed_tokens, uncond_req.num_computed_tokens) + + feasible = all( + req.num_computed_tokens - target < sched + for req, sched in ((cond_req, cond_sched), (uncond_req, uncond_sched)) + ) + if not feasible: + continue + + for req_id, req, orig_sched in ( + (cond_id, cond_req, cond_sched), + (uncond_id, uncond_req, uncond_sched), + ): + diff = req.num_computed_tokens - target + if diff > 0: + req.num_computed_tokens = target + scheduler_output.num_scheduled_tokens[req_id] = orig_sched - diff + scheduler_output.total_num_scheduled_tokens -= diff + logger.debug( + "CFG equalize %s: %d -> %d scheduled, computed -> %d", + req_id, + orig_sched, + orig_sched - diff, + target, + ) + + +def _drop_split_pairs(scheduler: Any) -> None: + """Deregister a desynced CFG pair (one member preempted / left behind). + + Manual re-preemption surgery is unsafe against the upstream scheduler. + Instead, a pair that split anyway (KV-pressure preemption of exactly one + member, or unrecoverable progress skew) is dropped from the registry: + the cond request continues UNGUIDED (its logits are never blended again) + rather than being re-paired against a desynced partner, and the + companion's discarded output no longer matters. + """ + if not scheduler._cfg_pairs: + return + + running_ids = {req.request_id for req in scheduler.running} + to_drop: list[tuple[str, str, str, str]] = [] + for pair_id, roles in scheduler._cfg_pairs.items(): + cond_id = roles.get(_COND) + uncond_id = roles.get(_UNCOND) + if cond_id is None or uncond_id is None: + continue + if cond_id not in scheduler.requests or uncond_id not in scheduler.requests: + continue + + cond_running = cond_id in running_ids + uncond_running = uncond_id in running_ids + if cond_running != uncond_running: + # Split only counts once decoding started: a lone member that has + # computed tokens while its partner was pushed back is desynced. + runner = scheduler.requests.get(cond_id if cond_running else uncond_id) + if runner is not None and runner.num_computed_tokens > 0: + to_drop.append((pair_id, cond_id, uncond_id, "one member preempted")) + continue + + if cond_running and uncond_running: + cond_req = scheduler.requests.get(cond_id) + uncond_req = scheduler.requests.get(uncond_id) + if cond_req and uncond_req and cond_req.num_computed_tokens != uncond_req.num_computed_tokens: + logger.warning( + "CFG pair progress mismatch (equalize will retry): %s@%d vs %s@%d", + cond_id, + cond_req.num_computed_tokens, + uncond_id, + uncond_req.num_computed_tokens, + ) + + for pair_id, cond_id, uncond_id, reason in to_drop: + scheduler._cfg_pairs.pop(pair_id, None) + scheduler._cfg_req_to_pair.pop(cond_id, None) + scheduler._cfg_req_to_pair.pop(uncond_id, None) + logger.warning( + "CFG pair %s split (%s); releasing cond=%s unguided, uncond=%s output is discarded", + pair_id, + reason, + cond_id, + uncond_id, + ) + + +_patches_applied = False + + +def cfg_patches_applied() -> bool: + """True once :func:`apply_cfg_patches` has patched the scheduler in this process.""" + return _patches_applied + + +def _patch_scheduler(scheduler_cls: type | None = None) -> None: + """Wrap the scheduler methods that keep a CFG pair step-locked. + + ``scheduler_cls`` defaults to vLLM's v1 ``Scheduler``; accepting a class + keeps the wrappers exercisable without building a real engine. + """ + if scheduler_cls is None: + from vllm.v1.core.sched.scheduler import Scheduler + + scheduler_cls = Scheduler + + orig_init = scheduler_cls.__init__ + orig_add = scheduler_cls.add_request + orig_finish = scheduler_cls.finish_requests + orig_schedule = scheduler_cls.schedule + + def _init(self, *args: Any, **kwargs: Any) -> None: + orig_init(self, *args, **kwargs) + self._cfg_pairs: dict[str, dict[str, str]] = {} + self._cfg_req_to_pair: dict[str, str] = {} + + def _add(self, request: Any) -> None: + orig_add(self, request) + pair_id = _cfg_extra(request, "cfg_pair_id") + role = _cfg_extra(request, "cfg_role") + if role and not pair_id: + # Models that cannot inject a pair id from the serving layer send + # the role alone and let the request id carry the pairing. It has + # to be the user-facing id: the engine renames every request to an + # internal UUID on admission, and the companion suffix survives + # only on ``external_req_id``. Deriving from the UUID would give + # the two members unrelated pair ids, so they would never be held + # for each other and could be scheduled apart. + external_id = getattr(request, "external_req_id", None) or request.request_id + pair_id = _derive_pair_id(str(external_id), role) + if pair_id and role: + self._cfg_pairs.setdefault(pair_id, {})[role] = request.request_id + self._cfg_req_to_pair[request.request_id] = pair_id + # An asymmetric prefix-cache hit (the cond prompt shares cached + # branches; the null prompt does not) can advance one member past + # what post-schedule equalization can claw back. Skip prefix-cache + # reads for CFG requests entirely. + if hasattr(request, "skip_reading_prefix_cache"): + request.skip_reading_prefix_cache = True + + def _finish(self, request_ids: Any, finished_status: Any) -> Any: + if request_ids is None: + result = orig_finish(self, request_ids, finished_status) + self._cfg_pairs.clear() + self._cfg_req_to_pair.clear() + return result + + if isinstance(request_ids, str): + request_ids = {request_ids} + else: + request_ids = set(request_ids) + + # Finish both members together: a surviving partner would decode + # unpaired garbage (its logits are never blended again). + partner_ids: set[str] = set() + for req_id in request_ids: + pair_id = self._cfg_req_to_pair.get(req_id) + if pair_id is None: + continue + for rid in self._cfg_pairs.get(pair_id, {}).values(): + if rid != req_id and rid in self.requests: + partner_ids.add(rid) + + all_ids = request_ids | partner_ids + result = orig_finish(self, all_ids, finished_status) + + for req_id in all_ids: + pair_id = self._cfg_req_to_pair.pop(req_id, None) + if pair_id: + self._cfg_pairs.pop(pair_id, None) + return result + + def _schedule(self, *args: Any, **kwargs: Any) -> Any: + if not self._cfg_pairs: + return orig_schedule(self, *args, **kwargs) + + _reorder_waiting_for_cfg(self) + held = _hold_incomplete_pairs(self) + + # Chunked prefill can split one pair member's prompt across steps + # while the other fits in a single step; halving the long-prefill + # threshold makes both chunk, and equalize aligns the remainder. + scheduler_config = getattr(self, "scheduler_config", None) + orig_threshold = getattr(scheduler_config, "long_prefill_token_threshold", None) + if scheduler_config is not None and orig_threshold is not None: + scheduler_config.long_prefill_token_threshold = self.max_num_scheduled_tokens // 2 + try: + scheduler_output = orig_schedule(self, *args, **kwargs) + finally: + if scheduler_config is not None and orig_threshold is not None: + scheduler_config.long_prefill_token_threshold = orig_threshold + _release_held(held) + + _equalize_cfg_pair_progress(self, scheduler_output) + _drop_split_pairs(self) + return scheduler_output + + _schedule._cfg_pairing_patched = True # type: ignore[attr-defined] + # Legacy marker name kept for callers written against the Audex-only patch. + _schedule._audex_cfg_patched = True # type: ignore[attr-defined] + + scheduler_cls.__init__ = _init + scheduler_cls.add_request = _add + scheduler_cls.finish_requests = _finish + scheduler_cls.schedule = _schedule + + +def apply_cfg_patches() -> None: + """Make the vLLM v1 scheduler CFG-pair-aware. Idempotent, per process. + + Must run in the engine-core process before ``Scheduler`` is constructed + (the ``__init__`` wrapper installs the pair registry). + """ + global _patches_applied + if _patches_applied: + return + _patches_applied = True + logger.info("Applying CFG pairing scheduler patches to vLLM v1") + _patch_scheduler() diff --git a/vllm_omni/model_executor/models/minimax_music3/__init__.py b/vllm_omni/model_executor/models/minimax_music3/__init__.py new file mode 100644 index 000000000..39e617d04 --- /dev/null +++ b/vllm_omni/model_executor/models/minimax_music3/__init__.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MiniMax Music 3 text-to-music. + +Stage 0 is a Qwen3 backbone with an eight-codebook RVQ frame; stage 1 turns +the resulting conditioning frames into 32 kHz stereo audio through a +flow-matching transformer and a DAC decoder. +""" + +from .constants import ( + AR_CHUNK_FRAMES, + AR_CHUNK_HOP_FRAMES, + AR_HIDDEN_SIZE, + MAX_AUDIO_FRAMES, + OUTPUT_SAMPLE_RATE, +) +from .prompt import ( + AUDIO_CODE_OFFSET, + SPECIAL_TOKEN_IDS, + build_cfg_null_token_ids, + build_prompt, +) + +__all__ = [ + "AR_CHUNK_FRAMES", + "AR_CHUNK_HOP_FRAMES", + "AR_HIDDEN_SIZE", + "AUDIO_CODE_OFFSET", + "MAX_AUDIO_FRAMES", + "OUTPUT_SAMPLE_RATE", + "SPECIAL_TOKEN_IDS", + "build_cfg_null_token_ids", + "build_prompt", +] diff --git a/vllm_omni/model_executor/models/minimax_music3/acoustic.py b/vllm_omni/model_executor/models/minimax_music3/acoustic.py new file mode 100644 index 000000000..879185a57 --- /dev/null +++ b/vllm_omni/model_executor/models/minimax_music3/acoustic.py @@ -0,0 +1,459 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Stage-1 acoustic decoder for MiniMax Music 3. + +Consumes the AR stage's conditioning spans and returns 32 kHz stereo audio. +Each span is cut into 200-frame windows that overlap by half; every window is +solved by the flow-matching DiT with the previous window's tail latent pinned +as a prompt, decoded by the vocoder, and cropped so that concatenating the +survivors reconstructs the song exactly once. + +The window arithmetic lives in :mod:`.chunking` and is driven entirely by the +span's global frame range, so the streaming path (one window per engine step) +and the non-streaming path (the whole song in one payload) produce identical +audio for the same seed. + +The stage runs in float32. TF32 keeps that affordable, and bfloat16 measurably +degrades a 30-step solve. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Iterable +from dataclasses import dataclass, field +from typing import Any + +import torch +from torch import Tensor, nn +from vllm.config import VllmConfig +from vllm.logger import init_logger + +from vllm_omni.model_executor.models.output_templates import OmniOutput + +from .chunking import ChunkWindow, chunk_windows, crop_sample_bounds, overlap_mel_length +from .constants import ( + AR_CHUNK_HOP_FRAMES, + AR_HIDDEN_SIZE, + DEFAULT_DIT_CFG_SCALE, + DEFAULT_DIT_STEPS, + OUTPUT_CHANNELS, + OUTPUT_SAMPLE_RATE, +) +from .dav import MiniMaxMusic3Vocoder, StreamingResampler, remove_weight_norm +from .dit import MiniMaxMusic3FlowMatchingDiT, remap_transformer_state +from .weights import ( + CONDITION_ENCODER_DIR, + TRANSFORMER_DIR, + VOCODER_DIR, + load_component_state, + resolve_repo_root, +) + +logger = init_logger(__name__) + +__all__ = ["MiniMaxMusic3AcousticForConditionalGeneration"] + +# Namespaced so a request seed reused by another component cannot collide with +# the noise draw, and so the draw is reproducible across processes. Python's +# hash() is randomized per interpreter and must not be used here. +_SEED_PERSON = b"minimax-ttm" +_SEED_BYTES = 8 + + +def _derive_seed(seed: int, *parts: object) -> int: + """Derive a stable 63-bit sub-seed from a request seed and some labels.""" + digest = hashlib.blake2b(digest_size=_SEED_BYTES, person=_SEED_PERSON) + digest.update(int(seed).to_bytes(8, "little", signed=False)) + for part in parts: + raw = str(part).encode("utf-8") + digest.update(len(raw).to_bytes(4, "little")) + digest.update(raw) + return int.from_bytes(digest.digest(), "little") & ((1 << 63) - 1) + + +def _meta_int(value: Any, default: int = 0) -> int: + """Read an int that transport may have wrapped in a tensor or a list.""" + if isinstance(value, torch.Tensor): + return int(value.reshape(-1)[0].item()) if value.numel() else default + if isinstance(value, list | tuple): + return _meta_int(value[0], default) if value else default + try: + return int(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return default + + +def _meta_bool(value: Any) -> bool: + """Read a bool that transport may have wrapped in a tensor or a list.""" + if isinstance(value, torch.Tensor): + return bool(value.reshape(-1)[0].item()) if value.numel() else False + if isinstance(value, list | tuple): + return _meta_bool(value[0]) if value else False + return bool(value) + + +def _meta_str(value: Any) -> str | None: + """Read a string that transport may have wrapped in a list.""" + if isinstance(value, list | tuple): + return _meta_str(value[0]) if value else None + return value if isinstance(value, str) else None + + +@dataclass +class _RequestState: + """Cross-window streaming state, keyed by request id and never by row.""" + + last_latent: Tensor | None = None + last_condition: Tensor | None = None + next_window_index: int = 0 + decoded_windows: int = 0 + resampler: StreamingResampler = field(default_factory=StreamingResampler) + + def release(self) -> None: + self.last_latent = None + self.last_condition = None + self.resampler.reset() + + +class MiniMaxMusic3AcousticForConditionalGeneration(nn.Module): + """Flow-matching DiT plus DAC vocoder, driven by the generation runner.""" + + input_modalities = "audio" + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + del prefix + self.vllm_config = vllm_config + self.model_path = vllm_config.model_config.model + + self.have_multimodal_outputs = True + self.has_preprocess = False + self.has_postprocess = False + self.enable_update_additional_information = True + self.requires_raw_input_tokens = True + + self._configure_backends() + + # Skeleton only. vLLM's memory profiler walks the module tree at + # startup, before any weight loader has run. + self.dit = MiniMaxMusic3FlowMatchingDiT() + self.vocoder = MiniMaxMusic3Vocoder() + + extra = self._connector_extra_config() + self.dit_steps = max(1, _meta_int(extra.get("dit_steps"), DEFAULT_DIT_STEPS)) + cfg_scale = extra.get("dit_cfg_scale") + self.dit_cfg_scale = ( + float(cfg_scale) + if isinstance(cfg_scale, int | float) and not isinstance(cfg_scale, bool) + else DEFAULT_DIT_CFG_SCALE + ) + + self._states: dict[str, _RequestState] = {} + self._loaded = False + + @staticmethod + def _configure_backends() -> None: + """Pin the numeric backends this checkpoint was validated against. + + These are process-global switches, which is safe here only because a + stage owns its own engine-core process. Never call this from a module + shared with the AR stage: cuDNN's fused attention is not + bit-reproducible for grouped-query shapes, so flipping it underneath + the backbone makes the same seed produce a different song depending on + which stage loaded first. + + cuDNN itself is off because its 1-D convolution paths are slower than + the native ones for the vocoder's dilated stacks, and its SDPA backend + is not needed because the DiT runs the flash path. TF32 keeps + float32's range with a 10-bit mantissa, which is what makes a float32 + solve affordable. + """ + torch.backends.cuda.matmul.allow_tf32 = True + torch.set_float32_matmul_precision("high") + if torch.cuda.is_available(): + torch.backends.cudnn.enabled = False + torch.backends.cuda.enable_cudnn_sdp(False) + + def _connector_extra_config(self) -> dict[str, Any]: + """Return the stage connector's ``extra`` block, or an empty mapping.""" + connector = getattr(self.vllm_config.model_config, "stage_connector_config", None) + if isinstance(connector, dict): + extra = connector.get("extra", connector) + return extra if isinstance(extra, dict) else {} + extra = getattr(connector, "extra", None) + return extra if isinstance(extra, dict) else {} + + # ------------------------------------------------------------- vLLM hooks + + def embed_input_ids(self, input_ids: Tensor, **_: Any) -> Tensor: + """Stable dummy embedding: this stage never reads token embeddings.""" + if input_ids.numel() == 0: + return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32) + return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32) + + def compute_logits(self, hidden_states: Tensor | OmniOutput, sampling_metadata: Any = None) -> None: + del hidden_states, sampling_metadata + return None + + def load_weights(self, weights: Iterable[tuple[str, Tensor]]) -> set[str]: + """Load the three acoustic components from their own folders. + + The iterator vLLM hands over covers the stage's own checkpoint folder, + which holds nothing this stage needs. It is drained anyway: the loader + streams from a file handle the caller closes only once the iterator is + exhausted, so abandoning it wedges startup. + """ + try: + for _ in weights: + pass + except RuntimeError as exc: + # The iterator is lazy, so draining it is what actually asks the + # default loader for root-level weight files. This repository keeps + # every component in its own subfolder and has none at the root, so + # that lookup raises. Nothing here needs those weights; any other + # RuntimeError is a real problem and is re-raised. + if "Cannot find any model weights" not in str(exc): + raise + logger.debug("MiniMax Music 3 acoustic stage has no root-level weights, as expected") + return self._load_components() + + def _ensure_loaded(self) -> None: + """Load on first use when the weight loader skipped ``load_weights``. + + ``load_format=dummy`` (used by warmup and memory profiling) never calls + ``load_weights``, which would leave the modules holding their random + skeleton and make the dummy run produce noise or crash. + """ + if not self._loaded: + self._load_components() + + def _load_components(self) -> set[str]: + """Read ``condition_encoder/``, ``transformer/`` and ``vocoder/``. + + Every component loads with ``strict=True``. A silently partial load + would produce audio that sounds plausible but is wrong, which is far + worse than a startup failure. + """ + root = resolve_repo_root(self.model_path) + encoder = self.dit.condition_encoder + encoder.load_state_dict(load_component_state(root, CONDITION_ENCODER_DIR), strict=True) + transformer_state = remap_transformer_state(load_component_state(root, TRANSFORMER_DIR)) + self.dit.transformer.load_state_dict(transformer_state, strict=True) + self.vocoder.load_state_dict(load_component_state(root, VOCODER_DIR), strict=True) + + device = self.vllm_config.device_config.device + self.dit.to(device=device, dtype=torch.float32).eval() + self.vocoder.to(device=device, dtype=torch.float32).eval() + folded = remove_weight_norm(self.vocoder) + self._loaded = True + + loaded = {f"dit.{name}" for name, _ in self.dit.named_parameters()} + loaded |= {f"vocoder.{name}" for name, _ in self.vocoder.named_parameters()} + logger.info( + "MiniMax Music 3 acoustic loaded from %s device=%s dit_steps=%d " + "dit_cfg_scale=%.3f attention=%s folded_weight_norms=%d parameters=%d", + root, + device, + self.dit_steps, + self.dit_cfg_scale, + self.dit.transformer.attention_backend_name, + folded, + sum(p.numel() for p in self.parameters()), + ) + return loaded + + def on_requests_finished(self, finished_req_ids: Iterable[str]) -> None: + """Drop the streaming state of finished or aborted requests.""" + for req_id in finished_req_ids: + state = self._states.pop(str(req_id), None) + if state is not None: + logger.info( + "MiniMax Music 3 acoustic done request=%s windows=%d", + req_id, + state.decoded_windows, + ) + state.release() + + def make_omni_output(self, model_outputs: Tensor | OmniOutput | tuple, **kwargs: Any) -> OmniOutput: + del kwargs + if isinstance(model_outputs, OmniOutput): + return model_outputs + # The runner may unpack the NamedTuple in transit; rebuild it. + if isinstance(model_outputs, tuple) and len(model_outputs) == len(OmniOutput._fields): + return OmniOutput(*model_outputs) + raise TypeError(f"MiniMaxMusic3AcousticForConditionalGeneration expected OmniOutput, got {type(model_outputs)}") + + # --------------------------------------------------------------- decoding + + @staticmethod + def _silence() -> Tensor: + """Placeholder for a step that produced no audio. + + Deliberately rank 2. A rank-1 empty tensor mixed with rank-2 chunks + makes the multimodal collector's ``torch.cat(..., dim=-1)`` fall back + to a flattening path that would planar-ize the stereo. + """ + return torch.zeros((OUTPUT_CHANNELS, 0), dtype=torch.float32) + + def _decode_window(self, state: _RequestState, hidden: Tensor, window: ChunkWindow, seed: int) -> Tensor: + """Solve and vocode one window, cropped to its unique contribution. + + Everything stays on the device: the crop is a slice and the result is + handed straight to the resampler, so a step costs one device-to-host + copy however many windows it decoded. + """ + device = self.vllm_config.device_config.device + hidden = hidden.unsqueeze(0).to(device=device, dtype=torch.float32) + generator = torch.Generator(device=device).manual_seed(_derive_seed(seed, "dit", window.index)) + align = self.dit.aligned_condition(hidden) + latent = self.dit( + align, + generator=generator, + initial_latent=state.last_latent, + initial_condition=state.last_condition, + num_steps=self.dit_steps, + cfg_scale=self.dit_cfg_scale, + ) + waveform = self.vocoder(latent) + + # Carry the middle of the overlap forward. The solver re-imposes it on + # the next window, so the two windows agree where they meet. + overlap = overlap_mel_length() + start = max(0, latent.shape[-1] - 2 * overlap) + end = max(start, latent.shape[-1] - overlap) + state.last_latent = latent[..., start:end].clone() + state.last_condition = align[..., start:end].clone() + + wave = waveform[0].clamp(-1.0, 1.0).float() + left, right = crop_sample_bounds(window) + stop = wave.shape[-1] - right if right else wave.shape[-1] + return wave[:, left:stop] + + def _decode_span(self, req_id: str, span: Tensor | None, meta: dict[str, Any]) -> Tensor: + """Decode every window a span covers and return this step's audio. + + A span is normally one window, but the terminal emission carries the + held-back complete window plus the truncated final one, and the + non-streaming path carries the whole song. Window boundaries and the + crop are re-derived from the span's global frame range so all three + cases go through one code path. + + Raises: + ValueError: If the span is malformed or lands on a window other + than the one this request is waiting for. + """ + span_start = _meta_int(meta.get("num_processed_tokens"), 0) + is_terminal = _meta_bool(meta.get("last_chunk")) or _meta_bool(meta.get("stream_finished")) + seed = _meta_int(meta.get("audio_seed"), 0) + state = self._states.setdefault(req_id, _RequestState()) + + # A metadata-only terminal marker carries no conditioning; it decodes + # zero windows and exists only to flush the resampler. + windows = chunk_windows(int(span.shape[0])) if span is not None else [] + pieces: list[Tensor] = [] + with torch.inference_mode(): + for local in windows: + start = span_start + local.start + index = start // AR_CHUNK_HOP_FRAMES + if index != state.next_window_index: + # Each window is prompted by its predecessor's tail, so a + # gap or a replay corrupts the join. Fail the request here + # rather than return audio with a seam in it. + raise ValueError( + f"MiniMax Music 3 acoustic request {req_id} expected window " + f"{state.next_window_index} but received window {index} " + f"(span starts at AR frame {span_start}); the overlap prompt " + f"for this window is stale and the audio would not join" + ) + window = ChunkWindow( + index=index, + start=start, + end=span_start + local.end, + is_first=start == 0, + is_last=local.is_last and is_terminal, + ) + pieces.append(self._decode_window(state, span[local.start : local.end], window, seed)) + state.next_window_index = index + 1 + state.decoded_windows += 1 + + emitted = state.resampler.push(pieces, flush=is_terminal) + return emitted.cpu() + + def _span_from_info(self, info: dict[str, Any], req_id: str) -> Tensor | None: + """Return the conditioning span as ``[T, 32768]``, or ``None``. + + ``None`` means this step carried no conditioning, which is normal: a + request can be scheduled before its span arrives, and the terminal + marker is metadata only. A span that is present but the wrong shape is + not normal and stops the request. + + Raises: + ValueError: If a span is present but is not ``[T, 32768]``. + """ + latent = info.get("latent") + if not isinstance(latent, Tensor) or latent.numel() == 0: + return None + if latent.ndim == 3 and latent.shape[0] == 1: + latent = latent[0] + if latent.ndim != 2 or latent.shape[1] != AR_HIDDEN_SIZE: + raise ValueError( + f"MiniMax Music 3 acoustic request {req_id} carries a conditioning " + f"span of shape {tuple(latent.shape)}; expected [T,{AR_HIDDEN_SIZE}]" + ) + return latent + + def forward( + self, + input_ids: Tensor | None = None, + positions: Tensor | None = None, + intermediate_tensors: Any = None, + inputs_embeds: Tensor | None = None, + runtime_additional_information: list[dict[str, Any]] | None = None, + **kwargs: Any, + ) -> OmniOutput: + """Decode this step's spans into one stereo waveform per request. + + Raises: + ValueError: If a payload is malformed. Failing the step is + deliberate: a request whose conditioning cannot be decoded has + no audio to return, and reporting an error is better than + leaving the caller waiting on a stream that will never end. + """ + del input_ids, positions, intermediate_tensors, inputs_embeds + self._ensure_loaded() + + infos = runtime_additional_information + if infos is None: + infos = kwargs.get("model_intermediate_buffer") + infos = infos or [] + + # One entry per scheduled request, including steps where a request's + # span has not arrived yet. + seq_token_counts = kwargs.get("seq_token_counts") + num_reqs = len(seq_token_counts) if seq_token_counts is not None else len(infos) + + audios: list[Tensor] = [] + sr_tensor = torch.tensor(OUTPUT_SAMPLE_RATE, dtype=torch.int32) + for index in range(num_reqs): + info = infos[index] if index < len(infos) else {} + info = info if isinstance(info, dict) else {} + meta = info.get("meta") + meta = meta if isinstance(meta, dict) else {} + req_id = _meta_str(meta.get("req_id")) or _meta_str(info.get("req_id")) + has_span = isinstance(info.get("latent"), Tensor) and info["latent"].numel() > 0 + is_terminal = _meta_bool(meta.get("last_chunk")) or _meta_bool(meta.get("stream_finished")) + if not has_span and not is_terminal: + audios.append(self._silence()) + continue + if req_id is None: + raise ValueError( + "MiniMax Music 3 acoustic payload carries no meta.req_id, so its " + "window cannot be joined to the rest of its request" + ) + audios.append(self._decode_span(req_id, self._span_from_info(info, req_id), meta)) + + return OmniOutput( + text_hidden_states=None, + multimodal_outputs={"model_outputs": audios, "sr": [sr_tensor] * num_reqs}, + ) diff --git a/vllm_omni/model_executor/models/minimax_music3/chunking.py b/vllm_omni/model_executor/models/minimax_music3/chunking.py new file mode 100644 index 000000000..ba917f39d --- /dev/null +++ b/vllm_omni/model_executor/models/minimax_music3/chunking.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Chunk boundary and overlap arithmetic for MiniMax Music 3. + +The AR stage emits fixed 200-frame windows that overlap by half. The acoustic +stage decodes each window with the previous window's latent as a prompt, then +crops the overlap back off so the concatenated waveform is seamless. All of +that is pure integer arithmetic and lives here so it can be unit tested +without a GPU. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from .constants import ( + AR_CHUNK_FRAMES, + AR_CHUNK_HOP_FRAMES, + BLEND_MEL_FRAMES, + DAV_HOP_SAMPLES, + HOP_MEL_FRAMES, +) + + +@dataclass(frozen=True) +class ChunkWindow: + """One acoustic window, in AR frame coordinates.""" + + index: int + start: int + end: int + is_first: bool + is_last: bool + + @property + def length(self) -> int: + return self.end - self.start + + +def chunk_windows(frames: int) -> list[ChunkWindow]: + """Return the fixed-size windows covering ``frames``, overlapping by half. + + A song shorter than one window produces a single short window; otherwise + windows advance by ``AR_CHUNK_HOP_FRAMES`` and the final one is truncated + at the end of the song. + """ + frames = int(frames) + if frames <= 0: + return [] + if frames <= AR_CHUNK_FRAMES: + return [ChunkWindow(0, 0, frames, True, True)] + + windows: list[ChunkWindow] = [] + index = 0 + start = 0 + while start < frames: + end = min(start + AR_CHUNK_FRAMES, frames) + windows.append( + ChunkWindow( + index=index, + start=start, + end=end, + is_first=index == 0, + is_last=end >= frames, + ) + ) + if end >= frames: + break + index += 1 + start += AR_CHUNK_HOP_FRAMES + return windows + + +def overlap_mel_length() -> int: + """Latent frames carried forward as the next window's flow-matching prompt.""" + return HOP_MEL_FRAMES // 2 + + +def crop_sample_bounds(window: ChunkWindow) -> tuple[int, int]: + """Return ``(left, right)`` waveform samples to trim from a decoded window. + + The first window keeps its head and the last keeps its tail; every other + window is cropped on both sides so that concatenating the survivors + reconstructs the song exactly once. + """ + left = 0 if window.is_first else BLEND_MEL_FRAMES * DAV_HOP_SAMPLES + right = 0 if window.is_last else (HOP_MEL_FRAMES - BLEND_MEL_FRAMES) * DAV_HOP_SAMPLES + return left, right + + +__all__ = [ + "ChunkWindow", + "chunk_windows", + "crop_sample_bounds", + "overlap_mel_length", +] diff --git a/vllm_omni/model_executor/models/minimax_music3/constants.py b/vllm_omni/model_executor/models/minimax_music3/constants.py new file mode 100644 index 000000000..4e93139c2 --- /dev/null +++ b/vllm_omni/model_executor/models/minimax_music3/constants.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inference-contract constants for MiniMax Music 3. + +These values are fixed by the checkpoint, not tuning knobs. Changing any of +them changes the audio, so they are declared once here and imported +everywhere rather than re-derived per call site. +""" + +# --- Generation limits ----------------------------------------------------- + +# 25 audio frames per second, capped at six minutes of song. +AUDIO_FRAME_RATE = 25 +MAX_AUDIO_FRAMES = 9_000 +DEFAULT_MAX_AUDIO_FRAMES = 9_000 + +# The backbone was trained with a 10,240-token context; the prompt is capped +# well below it so there is room for the audio frames that follow. +MAX_PROMPT_TOKENS = 5_000 + +# --- Frame layout ---------------------------------------------------------- + +# Eight RVQ codebooks per frame: c0 lives in the backbone vocabulary, c1..c7 +# in the depth decoder's own 1024-entry books. +NUM_CODEBOOKS = 8 +C0_VOCAB_SIZE = 16_384 +AUDIO_VOCAB_SIZE = 1_024 + +# The acoustic stage is conditioned on the backbone hidden state concatenated +# with the seven depth-decoder hidden states. +BACKBONE_HIDDEN_SIZE = 4_096 +AR_HIDDEN_SIZE = BACKBONE_HIDDEN_SIZE * NUM_CODEBOOKS # 32768 + +# --- Sampling -------------------------------------------------------------- + +# Classifier-free guidance is not optional and has no request field: this is +# how the reference implementation samples this checkpoint. +AR_CFG_SCALE = 1.5 +AR_CFG_TOP_K = 50 +AR_TOP_K = 50 + +# --- Acoustic stage -------------------------------------------------------- + +DEFAULT_DIT_STEPS = 30 +DEFAULT_DIT_CFG_SCALE = 1.7 +DIT_LATENT_CHANNELS = 128 + +# Windowing: the AR stage hands the acoustic stage 200-frame windows that +# overlap by half, and the acoustic stage crops the overlap back off. +AR_CHUNK_FRAMES = 200 +AR_CHUNK_HOP_FRAMES = 100 + +# DAV runs at 44.1 kHz; the response is resampled to 32 kHz stereo. +DAV_SAMPLE_RATE = 44_100 +OUTPUT_SAMPLE_RATE = 32_000 +OUTPUT_CHANNELS = 2 + +# Condition resampling: AR frames land on a 24 kHz / 960-hop grid and are +# stretched onto the DAV latent's 44.1 kHz / 512-hop grid. +CONDITION_INPUT_SAMPLE_RATE = 24_000 +CONDITION_INPUT_HOP = 960 +CONDITION_OUTPUT_SAMPLE_RATE = 44_100 +CONDITION_OUTPUT_HOP = 512 + +# Mel frames per AR frame: 44100/24000 * 960/512 = 3.4453125. So a 100-frame +# hop is 344 mel frames and a 200-frame window is 689. +# +# HOP_MEL_FRAMES is the hop, not the window: the crop and carry-forward +# arithmetic is expressed in units of the hop, and a window is two of them +# plus one frame of rounding. Reading this as the window length is the easy +# mistake here, and it is off by a factor of two. +HOP_MEL_FRAMES = 344 +BLEND_MEL_FRAMES = HOP_MEL_FRAMES // 4 +DAV_HOP_SAMPLES = 512 + +# Only these two dtypes are supported for the acoustic stage. float32 is the +# default: TF32 keeps it affordable, and bfloat16 measurably degrades the +# solver. +SUPPORTED_ACOUSTIC_DTYPES = frozenset({"float32", "bfloat16"}) + + +__all__ = [ + "AR_CFG_SCALE", + "AR_CFG_TOP_K", + "AR_CHUNK_FRAMES", + "AR_CHUNK_HOP_FRAMES", + "AR_HIDDEN_SIZE", + "AR_TOP_K", + "AUDIO_FRAME_RATE", + "AUDIO_VOCAB_SIZE", + "BACKBONE_HIDDEN_SIZE", + "BLEND_MEL_FRAMES", + "C0_VOCAB_SIZE", + "CONDITION_INPUT_HOP", + "CONDITION_INPUT_SAMPLE_RATE", + "CONDITION_OUTPUT_HOP", + "CONDITION_OUTPUT_SAMPLE_RATE", + "DAV_HOP_SAMPLES", + "DAV_SAMPLE_RATE", + "DEFAULT_DIT_CFG_SCALE", + "DEFAULT_DIT_STEPS", + "DEFAULT_MAX_AUDIO_FRAMES", + "DIT_LATENT_CHANNELS", + "HOP_MEL_FRAMES", + "MAX_AUDIO_FRAMES", + "MAX_PROMPT_TOKENS", + "NUM_CODEBOOKS", + "OUTPUT_CHANNELS", + "OUTPUT_SAMPLE_RATE", + "SUPPORTED_ACOUSTIC_DTYPES", +] diff --git a/vllm_omni/model_executor/models/minimax_music3/dav.py b/vllm_omni/model_executor/models/minimax_music3/dav.py new file mode 100644 index 000000000..bc0a478fd --- /dev/null +++ b/vllm_omni/model_executor/models/minimax_music3/dav.py @@ -0,0 +1,294 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DAC-style vocoder for MiniMax Music 3. + +Only the decoder half of the autoencoder ships in the checkpoint, which is all +inference needs: the flow-matching stage produces the latent directly, so the +encoder and the quantizer never run. + +Stereo is folded into the batch. A ``[B, 128, T]`` latent is read as two +64-channel mono latents per item, decoded as ``2B`` independent mono streams, +and unfolded again at the end. The four transposed-convolution blocks upsample +by 8, 8, 4 and 2, so one latent frame becomes 512 samples at 44.1 kHz. + +Module names match the ``vocoder/`` component folder key for key, so its state +dict loads with ``strict=True`` and no remapping. The convolutions keep the +checkpoint's weight-norm parameterization (``weight_g``/``weight_v``) through +loading; :func:`remove_weight_norm` folds it away afterwards, since the +factorization is fixed once the weights stop training. + +:class:`StreamingResampler` carries the 44.1 kHz output down to the 32 kHz +response rate across chunk boundaries, which is not something a plain +per-chunk resample can do. +""" + +from __future__ import annotations + +import math +from collections.abc import Sequence + +import torch +from torch import Tensor, nn +from vllm.logger import init_logger + +from .constants import DAV_SAMPLE_RATE, DIT_LATENT_CHANNELS, OUTPUT_CHANNELS, OUTPUT_SAMPLE_RATE + +logger = init_logger(__name__) + +__all__ = ["MiniMaxMusic3Vocoder", "StreamingResampler", "remove_weight_norm", "snake"] + +# Fixed by vocoder/config.json. +_DECODER_INPUT_DIM = 1_024 +_DECODER_HIDDEN_DIM = 1_536 +_UPSAMPLING_RATIOS = (8, 8, 4, 2) +_RESIDUAL_DILATIONS = (1, 3, 9) +_KERNEL_SIZE = 7 +# Each vocoder stream sees half the latent channels; the other half is the +# opposite stereo channel. +_STREAM_CHANNELS = DIT_LATENT_CHANNELS // OUTPUT_CHANNELS + + +def snake(x: Tensor, alpha: Tensor) -> Tensor: + """Periodic ``x + sin^2(alpha x) / alpha`` activation. + + The reshape to three dimensions is what the reference does and is a no-op + for the rank-3 activations used here; it is kept so the numerics are + identical for any rank. + """ + shape = x.shape + flat = x.reshape(shape[0], shape[1], -1) + flat = flat + (alpha + 1e-9).reciprocal() * torch.sin(alpha * flat).pow(2) + return flat.reshape(shape) + + +class Snake1d(nn.Module): + """Snake activation with one learned frequency per channel.""" + + def __init__(self, channels: int) -> None: + super().__init__() + self.alpha = nn.Parameter(torch.ones(1, channels, 1)) + + def forward(self, x: Tensor) -> Tensor: + return snake(x, self.alpha) + + +def _wn_conv(*args: object, **kwargs: object) -> nn.Module: + """A weight-normalized ``Conv1d``, named ``weight_g``/``weight_v``.""" + return nn.utils.weight_norm(nn.Conv1d(*args, **kwargs)) # type: ignore[arg-type] + + +def _wn_conv_transpose(*args: object, **kwargs: object) -> nn.Module: + """A weight-normalized ``ConvTranspose1d``, named ``weight_g``/``weight_v``.""" + return nn.utils.weight_norm(nn.ConvTranspose1d(*args, **kwargs)) # type: ignore[arg-type] + + +def remove_weight_norm(module: nn.Module) -> int: + """Fold weight normalization into every convolution weight in place. + + Returns: + How many submodules were folded. + """ + removed = 0 + for child in module.modules(): + try: + nn.utils.remove_weight_norm(child) + except (ValueError, RuntimeError): + continue + removed += 1 + return removed + + +class ResidualUnit(nn.Module): + """Dilated snake-gated residual block; preserves length and width.""" + + def __init__(self, dim: int, dilation: int) -> None: + super().__init__() + pad = (_KERNEL_SIZE - 1) * dilation // 2 + self.snake1 = Snake1d(dim) + self.conv1 = _wn_conv(dim, dim, kernel_size=_KERNEL_SIZE, dilation=dilation, padding=pad) + self.snake2 = Snake1d(dim) + self.conv2 = _wn_conv(dim, dim, kernel_size=1) + + def forward(self, x: Tensor) -> Tensor: + y = self.conv2(self.snake2(self.conv1(self.snake1(x)))) + if y.shape[-1] != x.shape[-1]: + # Defensive centre crop. The padding above makes this a no-op for + # every dilation the checkpoint ships. + pad = (x.shape[-1] - y.shape[-1]) // 2 + x = x[..., pad : x.shape[-1] - pad] + return x + y + + +class DecoderBlock(nn.Module): + """One upsampling stage: activate, stretch by ``stride``, then refine.""" + + def __init__(self, input_dim: int, output_dim: int, stride: int) -> None: + super().__init__() + self.snake1 = Snake1d(input_dim) + self.conv_t1 = _wn_conv_transpose( + input_dim, + output_dim, + kernel_size=2 * stride, + stride=stride, + padding=math.ceil(stride / 2), + ) + self.res_unit1 = ResidualUnit(output_dim, _RESIDUAL_DILATIONS[0]) + self.res_unit2 = ResidualUnit(output_dim, _RESIDUAL_DILATIONS[1]) + self.res_unit3 = ResidualUnit(output_dim, _RESIDUAL_DILATIONS[2]) + + def forward(self, x: Tensor) -> Tensor: + x = self.conv_t1(self.snake1(x)) + return self.res_unit3(self.res_unit2(self.res_unit1(x))) + + +class MiniMaxMusic3Vocoder(nn.Module): + """Latent to waveform: ``[B, 128, T] -> [B, 2, T * 512]`` at 44.1 kHz.""" + + def __init__(self) -> None: + super().__init__() + self.dec_in_proj = nn.Conv1d(_STREAM_CHANNELS, _DECODER_INPUT_DIM, kernel_size=1) + self.conv_in = _wn_conv( + _DECODER_INPUT_DIM, + _DECODER_HIDDEN_DIM, + kernel_size=_KERNEL_SIZE, + padding=_KERNEL_SIZE // 2, + ) + blocks: list[nn.Module] = [] + output_dim = _DECODER_HIDDEN_DIM + for index, stride in enumerate(_UPSAMPLING_RATIOS): + input_dim = _DECODER_HIDDEN_DIM // (2**index) + output_dim = _DECODER_HIDDEN_DIM // (2 ** (index + 1)) + blocks.append(DecoderBlock(input_dim, output_dim, stride)) + self.blocks = nn.ModuleList(blocks) + self.snake_out = Snake1d(output_dim) + self.conv_out = _wn_conv(output_dim, 1, kernel_size=_KERNEL_SIZE, padding=_KERNEL_SIZE // 2) + + @property + def upsampling_factor(self) -> int: + """Waveform samples produced per latent frame.""" + return math.prod(_UPSAMPLING_RATIOS) + + @torch.inference_mode() + def forward(self, latent: Tensor) -> Tensor: + """Decode a latent to interleaved stereo. + + Args: + latent: ``[B, 128, T]`` vocoder latent. + + Returns: + ``[B, 2, T * 512]`` waveform at 44.1 kHz, unclamped. + + Raises: + ValueError: If the latent is not ``[B, 128, T]``. + """ + if latent.ndim != 3 or latent.shape[1] != DIT_LATENT_CHANNELS: + raise ValueError( + f"MiniMax Music 3 vocoder latent must be [B,{DIT_LATENT_CHANNELS},T], got {tuple(latent.shape)}" + ) + bsz, _, frames = latent.shape + folded = latent.reshape(bsz * OUTPUT_CHANNELS, _STREAM_CHANNELS, frames) + h = self.conv_in(self.dec_in_proj(folded)) + for block in self.blocks: + h = block(h) + wave = torch.tanh(self.conv_out(self.snake_out(h))) + return wave.reshape(bsz, OUTPUT_CHANNELS, -1) + + +class StreamingResampler: + """Resample a chunked stream to the response rate without a seam. + + A polyphase resampler is time-invariant but not offset-invariant: it only + reproduces the same output grid when the input offset is a whole multiple + of ``orig // gcd(orig, new)`` samples, and it zero-pads both ends of every + call. Resampling each decoded chunk on its own therefore shifts the grid + by a fraction of a sample at every join and tapers the signal across it. + Measured against a single whole-stream resample on 20 s of broadband + noise, that costs a peak error of 2.2 (full scale) and drifts the length. + + This buffers instead. Only whole blocks are emitted, so every boundary + lands on an exact output sample; the previously emitted tail is prepended + as real left context and the not-yet-emitted head appended as real right + context, and both are dropped from the result. On the same measurement + the emitted stream is bit-identical to the whole-stream resample. + + One instance per request. It holds at most a couple of blocks of audio. + """ + + def __init__( + self, + *, + orig_freq: int = DAV_SAMPLE_RATE, + new_freq: int = OUTPUT_SAMPLE_RATE, + channels: int = OUTPUT_CHANNELS, + context_blocks: int = 1, + margin_blocks: int = 1, + ) -> None: + self.orig_freq = int(orig_freq) + self.new_freq = int(new_freq) + self.channels = int(channels) + # The output grid repeats every this many input samples. + self._block = self.orig_freq // math.gcd(self.orig_freq, self.new_freq) + self._context_len = self._block * context_blocks + self._margin = self._block * margin_blocks + self._pending: Tensor | None = None + self._context: Tensor | None = None + + def _empty(self, like: Tensor | None) -> Tensor: + if like is None: + return torch.zeros((self.channels, 0), dtype=torch.float32) + return like.new_zeros((self.channels, 0)) + + def reset(self) -> None: + """Drop all buffered audio.""" + self._pending = None + self._context = None + + @torch.inference_mode() + def push(self, chunks: Sequence[Tensor], *, flush: bool = False) -> Tensor: + """Append source-rate audio and return whatever is now emittable. + + Args: + chunks: New ``[channels, T]`` segments at ``orig_freq``, in order. + flush: Emit the remainder too, because the stream has ended. + + Returns: + ``[channels, T]`` at ``new_freq``; empty when nothing is ready. + """ + import torchaudio.functional as audio_functional + + parts = [chunk for chunk in chunks if chunk.numel()] + if self._pending is not None: + parts.insert(0, self._pending) + self._pending = None + if not parts: + return self._empty(self._context) + + buffered = torch.cat(parts, dim=-1) if len(parts) > 1 else parts[0] + total = int(buffered.shape[-1]) + if flush: + emit_len, lookahead = total, 0 + else: + emit_len = max(0, ((total - self._margin) // self._block) * self._block) + lookahead = min(self._margin, total - emit_len) + if emit_len == 0: + self._pending = buffered + return self._empty(buffered) + + padded = buffered[..., : emit_len + lookahead] + start = 0 + if self._context is not None: + padded = torch.cat((self._context, padded), dim=-1) + # Exact because the context is a whole number of blocks. + start = int(self._context.shape[-1]) * self.new_freq // self.orig_freq + resampled = audio_functional.resample(padded.float(), self.orig_freq, self.new_freq) + # Round up, matching how a whole-stream resample sizes its output. This + # only bites on the flush, where the remainder is not a whole block; + # every other emission divides exactly. + stop = start + -(-emit_len * self.new_freq // self.orig_freq) + + if flush: + self.reset() + else: + self._context = buffered[..., max(0, emit_len - self._context_len) : emit_len].clone() + self._pending = buffered[..., emit_len:].clone() if emit_len < total else None + return resampled[..., start:stop] diff --git a/vllm_omni/model_executor/models/minimax_music3/depth_graph.py b/vllm_omni/model_executor/models/minimax_music3/depth_graph.py new file mode 100644 index 000000000..9c9a4f8e5 --- /dev/null +++ b/vllm_omni/model_executor/models/minimax_music3/depth_graph.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CUDA graph capture for the RVQ depth loop. + +The depth loop is seven sequential forwards of a four-layer block over a +sequence that grows from two to eight positions, plus a guided draw after each +one. At the batch sizes this stage runs, none of those kernels is anywhere near +large enough to hide its own launch: measured in isolation on one L20X, the +loop costs 8.4 ms per frame when driven from Python and 4.0 ms when replayed +from a graph, and that figure barely moves between one and eight guided pairs. +It is launch-bound, so it is worth recording. + +Capture is lazy and keyed on the exact pair count, never padded. That keeps the +recorded kernels the same ones the eager loop selected for that shape, which is +what makes a replay bit-identical to the loop it replaces rather than merely +close to it. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass + +import torch +from vllm.logger import init_logger +from vllm.platforms import current_platform + +logger = init_logger(__name__) + +__all__ = ["DepthGraphCache"] + +# Inputs, in the order the eager loop takes them. +_DepthInputs = tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor] +_DepthOutputs = tuple[torch.Tensor, torch.Tensor] + + +@dataclass +class _CapturedDepth: + """One recorded depth loop and the buffers it reads and writes.""" + + graph: torch.cuda.CUDAGraph + inputs: _DepthInputs + codes: torch.Tensor + depth_hidden: torch.Tensor + + +class DepthGraphCache: + """Replay the depth loop from a graph once its shape has settled. + + Args: + run_eager: The loop itself, taking ``(cond_hidden, uncond_hidden, c0, + seeds, positions)`` and returning ``(codes, depth_hidden)``. + warmup_steps: How many times a pair count runs eagerly before it is + recorded. + """ + + def __init__( + self, + run_eager: Callable[..., _DepthOutputs], + *, + warmup_steps: int = 2, + ) -> None: + self._run_eager = run_eager + self._warmup_steps = warmup_steps + self._captured: dict[int, _CapturedDepth] = {} + self._eager_steps: dict[int, int] = {} + self._disabled: set[int] = set() + + def run(self, *inputs: torch.Tensor) -> _DepthOutputs: + """Run one depth loop, recording or replaying it where possible.""" + pairs = int(inputs[2].shape[0]) + entry = self._captured.get(pairs) + if entry is None: + if not self._should_capture(pairs, inputs[0]): + return self._run_eager(*inputs) + entry = self._capture(pairs, inputs) + if entry is None: + return self._run_eager(*inputs) + + for static, live in zip(entry.inputs, inputs, strict=True): + static.copy_(live) + entry.graph.replay() + # The codes are fed back as the next frame's input and appended to the + # request's trajectory, so they outlive the step; the graph overwrites + # its output buffer on the next replay. The depth hidden states are + # consumed inside this step by the concatenation that builds the + # conditioning frame, which allocates, so that one can be handed back + # as is. + return entry.codes.clone(), entry.depth_hidden + + def _should_capture(self, pairs: int, sample: torch.Tensor) -> bool: + if pairs in self._disabled or not sample.is_cuda: + return False + if torch.cuda.is_current_stream_capturing(): + # Already inside somebody else's capture; nested graphs are not a + # thing and the outer recording covers this work anyway. + return False + count = self._eager_steps.get(pairs, 0) + 1 + self._eager_steps[pairs] = count + # Let a new pair count run eagerly a few times first. Capture records + # whatever the kernels do at that moment, and the first call into a + # cuBLAS or SDPA path for an unseen shape can allocate a workspace or + # pick an algorithm, neither of which belongs inside the recording. + return count > self._warmup_steps + + def _capture(self, pairs: int, inputs: _DepthInputs) -> _CapturedDepth | None: + try: + static = tuple(torch.empty_like(tensor) for tensor in inputs) + for target, live in zip(static, inputs, strict=True): + target.copy_(live) + graph = torch.cuda.CUDAGraph() + with ( + torch.inference_mode(), + torch.cuda.graph( + graph, + pool=current_platform.get_global_graph_pool(), + # The engine's connector threads keep their own streams + # busy while this runs. They never touch the depth loop's + # buffers, so restrict the legality check to this thread + # rather than failing the capture on their activity. + capture_error_mode="thread_local", + ), + ): + codes, depth_hidden = self._run_eager(*static) + entry = _CapturedDepth(graph=graph, inputs=static, codes=codes, depth_hidden=depth_hidden) + self._captured[pairs] = entry + logger.info("MiniMax Music 3 captured the RVQ depth CUDA graph for %d guided pair(s)", pairs) + return entry + except Exception as exc: # noqa: BLE001 - any capture failure must fall back, not crash + self._disabled.add(pairs) + logger.warning( + "MiniMax Music 3 RVQ depth CUDA graph disabled for %d guided pair(s), " + "falling back to the eager loop: %s", + pairs, + exc, + ) + return None diff --git a/vllm_omni/model_executor/models/minimax_music3/dit.py b/vllm_omni/model_executor/models/minimax_music3/dit.py new file mode 100644 index 000000000..cf180ac52 --- /dev/null +++ b/vllm_omni/model_executor/models/minimax_music3/dit.py @@ -0,0 +1,484 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Condition encoder and flow-matching transformer for MiniMax Music 3. + +Stage 1 turns the AR stage's conditioning frames into a DAC latent in two +steps. The condition encoder collapses the eight 4096-wide layer slices of a +frame into one 2048-wide vector and stretches the result from the AR frame +grid onto the vocoder's latent grid. The transformer is then solved as a +flow-matching velocity field: a fixed-step Euler integrator walks Gaussian +noise to the latent the vocoder expects, under classifier-free guidance. + +Module names mirror the checkpoint's ``condition_encoder/`` and +``transformer/`` component folders key for key, with one deliberate +exception: the per-block ``to_q``/``to_k``/``to_v`` projections are fused into +a single ``to_qkv`` so each block runs one GEMM instead of three. +:func:`remap_transformer_state` performs that fusion, and is the only place +the checkpoint's layout is reinterpreted. + +Attention runs through the project's own diffusion attention layer, so this +model picks up the same backend selection, sequence-parallel plumbing and +KV-quant policy as every other transformer in the tree. That layer resolves +its backend from the diffusion config, which an ``LLM_GENERATION`` stage such +as this one does not set; it tolerates that and falls through to the platform +default. If it cannot be built at all the block runs plain SDPA instead, a +choice made once when the blocks are built and never inside ``forward``. +""" + +from __future__ import annotations + +import math + +import torch +from torch import Tensor, nn +from torch.nn import functional as F +from vllm.logger import init_logger + +from .constants import ( + AR_HIDDEN_SIZE, + BACKBONE_HIDDEN_SIZE, + CONDITION_INPUT_HOP, + CONDITION_INPUT_SAMPLE_RATE, + CONDITION_OUTPUT_HOP, + CONDITION_OUTPUT_SAMPLE_RATE, + DEFAULT_DIT_CFG_SCALE, + DEFAULT_DIT_STEPS, + DIT_LATENT_CHANNELS, + NUM_CODEBOOKS, +) + +logger = init_logger(__name__) + +__all__ = [ + "MiniMaxMusic3ConditionEncoder", + "MiniMaxMusic3FlowMatchingDiT", + "MiniMaxMusic3Transformer1DModel", + "remap_transformer_state", +] + +# Fixed by transformer/config.json. Declared rather than read from the file so +# the skeleton can be built before any checkpoint is on disk, which is what +# vLLM's memory profiler needs at startup. +_DIM = 2_048 +_NUM_LAYERS = 36 +_HEAD_DIM = 64 +_FF_INNER_DIM = 8_192 +_FOURIER_DIM = 256 +_ROTARY_DIM = 32 +_ROTARY_BASE = 10_000.0 +_CONDITION_DIM = 2_048 +# The solver feeds the transformer the current latent, a zero block of the +# same width, and the condition: 128 + 128 + 2048 = 2304. +_TRANSFORMER_IN_DIM = DIT_LATENT_CHANNELS * 2 + _CONDITION_DIM + +# The prompt interpolation floor from the reference sampler. Sigma never +# reaches exactly zero, so the noise term never fully vanishes. +_SIGMA_FLOOR = 1e-6 + + +class FourierFeatures(nn.Module): + """Random-Fourier timestep features: ``[B, 1] -> [B, out_features]``.""" + + def __init__(self, in_features: int, out_features: int) -> None: + super().__init__() + self.weight = nn.Parameter(torch.empty(out_features // 2, in_features)) + + def forward(self, value: Tensor) -> Tensor: + f = 2.0 * math.pi * value @ self.weight.T + return torch.cat((f.cos(), f.sin()), dim=-1) + + +class TimestepEmbedding(nn.Module): + """The checkpoint's ``time_embed``: Linear, SiLU, Linear.""" + + def __init__(self, in_dim: int, dim: int) -> None: + super().__init__() + self.linear_1 = nn.Linear(in_dim, dim) + self.act = nn.SiLU() + self.linear_2 = nn.Linear(dim, dim) + + def forward(self, x: Tensor) -> Tensor: + return self.linear_2(self.act(self.linear_1(x))) + + +def _rotate_half(x: Tensor) -> Tensor: + x1, x2 = x.chunk(2, dim=-1) + return torch.cat((-x2, x1), dim=-1) + + +def _apply_rope(x: Tensor, rope_cos: Tensor, rope_sin: Tensor) -> Tensor: + """Rotate the leading ``rope_cos.shape[-1]`` channels of ``[B, S, H, D]``. + + Only the first 32 of each head's 64 channels are rotated; the rest pass + through untouched, which is what this checkpoint was trained with. The + tables are cached at the exact sequence length, so there is nothing to + slice off the front. + """ + rot_dim = rope_cos.shape[-1] + rotated = x[..., :rot_dim] + rotated = rotated * rope_cos + _rotate_half(rotated) * rope_sin + return torch.cat((rotated, x[..., rot_dim:]), dim=-1) + + +# Set to the failure reason the first time the native attention layer cannot be +# built, so the 36 blocks agree on one path and the reason is logged once. +_NATIVE_ATTENTION_UNAVAILABLE: str | None = None + + +def _build_native_attention(*, num_heads: int, head_size: int, softmax_scale: float, prefix: str) -> nn.Module | None: + """Build the project's diffusion attention layer, or ``None`` to use SDPA. + + Anything that goes wrong here is a legitimate reason to run plain SDPA, + and is reported once rather than once per block. + """ + global _NATIVE_ATTENTION_UNAVAILABLE + if _NATIVE_ATTENTION_UNAVAILABLE is not None: + return None + try: + from vllm_omni.diffusion.attention.layer import Attention as DiffusionAttention + + return DiffusionAttention( + num_heads=num_heads, + head_size=head_size, + causal=False, + softmax_scale=softmax_scale, + prefix=prefix, + role="self", + role_category="self", + ) + except Exception as exc: # noqa: BLE001 - any failure means: use SDPA + _NATIVE_ATTENTION_UNAVAILABLE = repr(exc) + logger.warning( + "MiniMax Music 3 DiT could not build the diffusion attention layer " + "(%s); running torch SDPA instead. The stage decodes in float32, " + "for which the diffusion layer dispatches to SDPA anyway, so this " + "changes performance bookkeeping and not the audio.", + exc, + ) + return None + + +class Attention(nn.Module): + """Bidirectional self-attention with partial rotary position embeddings. + + Q, K and V are kept in ``[B, S, H, D]``, which is the layout the diffusion + attention backends consume and return. + """ + + def __init__(self, dim: int, *, head_dim: int, prefix: str = "") -> None: + super().__init__() + self.num_heads = dim // head_dim + self.head_dim = head_dim + self.softmax_scale = float(head_dim**-0.5) + # Fused from the checkpoint's separate to_q/to_k/to_v; see + # remap_transformer_state for the concatenation order. + self.to_qkv = nn.Linear(dim, dim * 3, bias=False) + self.to_out = nn.Linear(dim, dim, bias=False) + self.backend = _build_native_attention( + num_heads=self.num_heads, + head_size=head_dim, + softmax_scale=self.softmax_scale, + prefix=prefix, + ) + self.backend_name = self.backend.attn_backend.get_name() if self.backend is not None else "TORCH_SDPA" + + def _attend(self, q: Tensor, k: Tensor, v: Tensor) -> Tensor: + if self.backend is not None: + return self.backend(q, k, v) + q, k, v = (t.transpose(1, 2) for t in (q, k, v)) + out = F.scaled_dot_product_attention(q, k, v, is_causal=False, scale=self.softmax_scale) + return out.transpose(1, 2) + + def forward(self, x: Tensor, rope_cos: Tensor, rope_sin: Tensor) -> Tensor: + bsz, seq, dim = x.shape + q, k, v = self.to_qkv(x).chunk(3, dim=-1) + q = q.reshape(bsz, seq, self.num_heads, self.head_dim) + k = k.reshape(bsz, seq, self.num_heads, self.head_dim) + v = v.reshape(bsz, seq, self.num_heads, self.head_dim) + q = _apply_rope(q, rope_cos, rope_sin) + k = _apply_rope(k, rope_cos, rope_sin) + out = self._attend(q, k, v) + return self.to_out(out.contiguous().reshape(bsz, seq, dim)) + + +class TransformerBlock(nn.Module): + """Pre-norm attention followed by a pre-norm gated feed-forward.""" + + def __init__(self, dim: int, *, head_dim: int, inner_dim: int, prefix: str = "") -> None: + super().__init__() + self.norm1 = nn.LayerNorm(dim) + self.attn = Attention(dim, head_dim=head_dim, prefix=prefix) + self.norm2 = nn.LayerNorm(dim) + # One fused projection produces the GLU's value and gate halves. + self.ff_in = nn.Linear(dim, inner_dim * 2) + self.ff_out = nn.Linear(inner_dim, dim) + + def forward(self, x: Tensor, rope_cos: Tensor, rope_sin: Tensor) -> Tensor: + x = x + self.attn(self.norm1(x), rope_cos, rope_sin) + value, gate = self.ff_in(self.norm2(x)).chunk(2, dim=-1) + return x + self.ff_out(value * F.silu(gate)) + + +class MiniMaxMusic3Transformer1DModel(nn.Module): + """The 36-block velocity field: ``([B,128,T], [B], [B,2048,T]) -> [B,128,T]``.""" + + def __init__(self) -> None: + super().__init__() + self.preprocess_conv = nn.Conv1d(_TRANSFORMER_IN_DIM, _TRANSFORMER_IN_DIM, 1, bias=False) + self.postprocess_conv = nn.Conv1d(DIT_LATENT_CHANNELS, DIT_LATENT_CHANNELS, 1, bias=False) + self.proj_in = nn.Linear(_TRANSFORMER_IN_DIM, _DIM, bias=False) + self.proj_out = nn.Linear(_DIM, DIT_LATENT_CHANNELS, bias=False) + self.time_proj = FourierFeatures(1, _FOURIER_DIM) + self.time_embed = TimestepEmbedding(_FOURIER_DIM, _DIM) + self.transformer_blocks = nn.ModuleList( + TransformerBlock( + _DIM, + head_dim=_HEAD_DIM, + inner_dim=_FF_INNER_DIM, + prefix=f"transformer_blocks.{index}.attn", + ) + for index in range(_NUM_LAYERS) + ) + inv_freq = 1.0 / (_ROTARY_BASE ** (torch.arange(0, _ROTARY_DIM, 2).float() / _ROTARY_DIM)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self._rope_cache: dict[tuple[int, torch.dtype, torch.device], tuple[Tensor, Tensor]] = {} + + @property + def attention_backend_name(self) -> str: + """The attention path the blocks actually run, for startup logging.""" + return self.transformer_blocks[0].attn.backend_name + + def _rope(self, seq_len: int, *, dtype: torch.dtype, device: torch.device) -> tuple[Tensor, Tensor]: + """Return cached ``(cos, sin)`` shaped ``[seq_len, 1, 32]``. + + The head axis is left as a singleton so the tables broadcast over + ``[B, S, H, D]`` without transposing Q and K. + """ + key = (seq_len, dtype, device) + cached = self._rope_cache.get(key) + if cached is None: + t = torch.arange(seq_len, device=self.inv_freq.device, dtype=torch.float32) + freqs = torch.einsum("i,j->ij", t, self.inv_freq) + freqs = torch.cat((freqs, freqs), dim=-1).to(dtype=dtype, device=device).unsqueeze(1) + cached = (freqs.cos(), freqs.sin()) + self._rope_cache[key] = cached + return cached + + def forward(self, x: Tensor, t: Tensor, condition: Tensor) -> Tensor: + """Predict the flow velocity at latent ``x`` and time ``t``. + + Args: + x: Current latent, ``[B, 128, T_mel]``. + t: Flow time in ``[0, 1)``, ``[B]``. + condition: Aligned condition, ``[B, 2048, T_mel]``. + + Returns: + The velocity, ``[B, 128, T_mel]``. + """ + full = torch.cat((x, torch.zeros_like(x), condition), dim=1) + full = self.preprocess_conv(full) + full + timestep_embed = self.time_embed(self.time_proj(t[:, None])) + h = self.proj_in(full.transpose(1, 2)) + # The timestep rides as an extra leading position and is dropped again + # after the stack, so it shifts every rotary phase by one step. + h = torch.cat((timestep_embed.unsqueeze(1), h), dim=1) + rope_cos, rope_sin = self._rope(h.shape[1], dtype=h.dtype, device=h.device) + for block in self.transformer_blocks: + h = block(h, rope_cos, rope_sin) + out = self.proj_out(h[:, 1:]).transpose(1, 2) + return self.postprocess_conv(out) + out + + +def remap_transformer_state(state: dict[str, Tensor]) -> dict[str, Tensor]: + """Rewrite ``transformer/`` checkpoint keys onto this module tree. + + Two rewrites, both confined to the attention block: + + * ``attn.to_q/to_k/to_v.weight`` are concatenated along the output axis + into ``attn.to_qkv.weight``. ``nn.Linear`` output feature ``i`` reads + weight row ``i``, and :meth:`Attention.forward` splits the projection + with ``chunk(3, dim=-1)`` into ``q, k, v``, so query rows come first and + value rows last. + * ``attn.to_out.0.weight`` loses its index: the checkpoint models + ``to_out`` as a two-element list whose second element is a parameterless + dropout, and inference only needs the projection. + + Every other key is copied through untouched, including the fused ``ff_in`` + whose halves are consumed as ``value, gate`` in that order. + + Raises: + ValueError: If a block is missing one of its three projections. + """ + remapped: dict[str, Tensor] = {} + for key, value in state.items(): + if key.endswith(".attn.to_out.0.weight"): + remapped[key.replace(".to_out.0.weight", ".to_out.weight")] = value + continue + if ".attn.to_q.weight" in key: + prefix = key[: -len("to_q.weight")] + try: + fused = [state[f"{prefix}to_{name}.weight"] for name in "qkv"] + except KeyError as exc: + raise ValueError( + f"MiniMax Music 3 transformer block {prefix!r} is missing attention projection {exc}" + ) from exc + remapped[f"{prefix}to_qkv.weight"] = torch.cat(fused, dim=0) + continue + if ".attn.to_k.weight" in key or ".attn.to_v.weight" in key: + continue + remapped[key] = value + return remapped + + +class MiniMaxMusic3ConditionEncoder(nn.Module): + """Collapse an AR frame's eight layer slices and resample onto mel time.""" + + def __init__(self) -> None: + super().__init__() + self.layer_weight_logits = nn.Parameter(torch.zeros(NUM_CODEBOOKS)) + self.layer_scale = nn.Parameter(torch.ones(1)) + self.proj = nn.Conv1d(BACKBONE_HIDDEN_SIZE, _CONDITION_DIM, kernel_size=3, padding=1) + + def aligned_mel_length(self, frames: int) -> int: + """Return how many vocoder latent frames ``frames`` AR frames cover. + + AR frames sit on a 24 kHz / 960-hop grid (25 fps) and the vocoder + latent on a 44.1 kHz / 512-hop grid (about 86 fps), so one AR frame is + roughly 3.445 latent frames. The result is truncated, never rounded. + """ + return max( + 1, + int( + frames + * CONDITION_OUTPUT_SAMPLE_RATE + / CONDITION_INPUT_SAMPLE_RATE + * CONDITION_INPUT_HOP + / CONDITION_OUTPUT_HOP + ), + ) + + def condition(self, hidden: Tensor) -> Tensor: + """Project ``[B, T, 32768]`` AR frames to ``[B, 2048, T]``. + + Raises: + ValueError: If the conditioning is not ``[B, T, 32768]``. + """ + if hidden.ndim != 3 or hidden.shape[-1] != AR_HIDDEN_SIZE: + raise ValueError(f"MiniMax Music 3 condition must be [B,T,{AR_HIDDEN_SIZE}], got {tuple(hidden.shape)}") + hidden_cf = hidden.transpose(1, 2) + bsz, _, frames = hidden_cf.shape + hidden_cf = hidden_cf.reshape(bsz, NUM_CODEBOOKS, BACKBONE_HIDDEN_SIZE, frames) + weights = torch.softmax(self.layer_weight_logits, dim=0).to(hidden_cf.dtype) + hidden_cf = torch.einsum("blht,l->bht", hidden_cf, weights) + hidden_cf = self.layer_scale.to(hidden_cf.dtype) * hidden_cf + return self.proj(hidden_cf) + + def aligned_condition(self, hidden: Tensor) -> Tensor: + """Project AR hidden states and resample them onto the latent grid.""" + hidden = hidden.to(dtype=next(self.parameters()).dtype) + align = self.condition(hidden) + mel_len = self.aligned_mel_length(hidden.shape[1]) + return F.interpolate(align, size=mel_len, mode="nearest") + + +class MiniMaxMusic3FlowMatchingDiT(nn.Module): + """Condition encoder plus velocity field, with the Euler solver on top.""" + + def __init__(self) -> None: + super().__init__() + self.condition_encoder = MiniMaxMusic3ConditionEncoder() + self.transformer = MiniMaxMusic3Transformer1DModel() + + def aligned_mel_length(self, frames: int) -> int: + return self.condition_encoder.aligned_mel_length(frames) + + def condition(self, hidden: Tensor) -> Tensor: + return self.condition_encoder.condition(hidden) + + def aligned_condition(self, hidden: Tensor) -> Tensor: + return self.condition_encoder.aligned_condition(hidden) + + @torch.inference_mode() + def forward( + self, + align: Tensor, + *, + generator: torch.Generator, + initial_latent: Tensor | None = None, + initial_condition: Tensor | None = None, + num_steps: int = DEFAULT_DIT_STEPS, + cfg_scale: float = DEFAULT_DIT_CFG_SCALE, + ) -> Tensor: + """Solve an aligned condition into a vocoder latent ``[1, 128, T_mel]``. + + The previous window's latent is re-imposed at every step rather than + only at the start: the leading ``left`` frames are pinned to the exact + noise-to-latent interpolation the solver would have produced had it + generated them, so the two windows join without a seam. + + Args: + align: Aligned condition, ``[1, 2048, T_mel]``. Overwritten in + place over the prompt span when ``initial_condition`` is + given, which the caller relies on when it saves the condition + for the next window. + generator: Seeded generator for the initial noise draw. + initial_latent: Previous window's tail latent, or ``None``. + initial_condition: Previous window's tail condition, or ``None``. + num_steps: Euler steps. + cfg_scale: Classifier-free guidance weight. + + Returns: + The solved latent, ``[1, 128, T_mel]``. + + Raises: + ValueError: If ``num_steps`` is not positive. + """ + if num_steps < 1: + raise ValueError("MiniMax Music 3 DiT num_steps must be positive") + mel_len = align.shape[-1] + x = torch.randn( + (align.shape[0], DIT_LATENT_CHANNELS, mel_len), + device=align.device, + dtype=align.dtype, + generator=generator, + ) + + left = 0 + latent_prompt: Tensor | None = None + noise_prompt: Tensor | None = None + if initial_latent is not None: + left = min(initial_latent.shape[-1], mel_len) + if left <= 0: + left = 0 + else: + latent_prompt = initial_latent[..., :left].to(x) + noise_prompt = x[..., :left].clone() + if initial_condition is not None: + align[..., :left] = initial_condition[..., :left].to(align) + + dt = 1.0 / num_steps + # Row 0 carries the condition, row 1 stays zero: one batched forward + # evaluates both guidance branches. + cond_cfg = torch.zeros((2, *align.shape[1:]), device=align.device, dtype=align.dtype) + cond_cfg[0].copy_(align[0]) + # Every timestep in one host-to-device copy. Built from the same + # Python divisions the per-step construction would have used, so the + # values are identical, but the solver loop then touches the host only + # to launch kernels. + schedule = torch.tensor( + [step / num_steps for step in range(num_steps)], + device=align.device, + dtype=align.dtype, + ) + + for step in range(num_steps): + t = schedule[step].expand(x.shape[0]) + if left and latent_prompt is not None and noise_prompt is not None: + x[..., :left] = (1.0 - (1.0 - _SIGMA_FLOOR) * t[0]) * noise_prompt + t[0] * latent_prompt + d = self.transformer(x.expand(2, -1, -1), t.expand(2), cond_cfg) + d = cfg_scale * d[:1] + (1.0 - cfg_scale) * d[1:2] + x = x + dt * d + + if left and latent_prompt is not None: + x[..., :left] = latent_prompt + return x diff --git a/vllm_omni/model_executor/models/minimax_music3/frame_state.py b/vllm_omni/model_executor/models/minimax_music3/frame_state.py new file mode 100644 index 000000000..5191cebac --- /dev/null +++ b/vllm_omni/model_executor/models/minimax_music3/frame_state.py @@ -0,0 +1,158 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Per-request AR state for MiniMax Music 3. + +The talker accumulates one 32768-dimensional conditioning frame per decode +step and hands the acoustic stage overlapping 200-frame windows. Windows are +emitted one frame late on purpose: a window can only be declared non-final +once a further frame proves the song did not end at its boundary, and the +tail is flushed when the request finishes. + +State is keyed by request id rather than batch row, because a row's identity +is not stable across engine steps. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import torch + +from .chunking import ChunkWindow +from .constants import AR_CHUNK_FRAMES, AR_CHUNK_HOP_FRAMES, NUM_CODEBOOKS + + +class FrameBuffer: + """Rolling buffer of conditioning frames addressed by absolute frame index. + + Frames before an emitted window's hop point are dropped, so peak memory is + bounded by the window size rather than by the length of the song. + """ + + def __init__(self) -> None: + self.base_frame = 0 + self._frames: list[torch.Tensor] = [] + self.peak_frames = 0 + + @property + def end_frame(self) -> int: + return self.base_frame + len(self._frames) + + def append(self, frame: torch.Tensor) -> None: + """Append one ``[1, hidden]`` frame.""" + self._frames.append(frame) + self.peak_frames = max(self.peak_frames, len(self._frames)) + + def slice(self, start: int, end: int) -> torch.Tensor: + """Return frames ``[start, end)`` stacked as ``[end - start, hidden]``. + + Raises: + RuntimeError: If the range has already been discarded or has not + been produced yet. + """ + rel_start = start - self.base_frame + rel_end = end - self.base_frame + if rel_start < 0 or rel_end > len(self._frames): + raise RuntimeError( + f"MiniMax Music 3 frame range [{start}, {end}) is unavailable; " + f"the buffer holds [{self.base_frame}, {self.end_frame})" + ) + return torch.cat(self._frames[rel_start:rel_end], dim=0) + + def discard_before(self, frame: int) -> None: + drop = min(max(frame - self.base_frame, 0), len(self._frames)) + if drop: + del self._frames[:drop] + self.base_frame += drop + + def clear(self) -> None: + self._frames.clear() + self.base_frame = 0 + + +@dataclass +class RequestARState: + """Everything the talker tracks for one conditioned request.""" + + seed: int + max_frames: int + # Prompt length in tokens, used to tell a decode row apart from a row that + # is still working through a chunked prefill. + prompt_tokens: int = 0 + sampling_position: int = 0 + generated_frames: int = 0 + emitted_windows: int = 0 + finished: bool = False + # The engine is told to stop one step after the song ends. Payloads are + # built from the buffer that ``postprocess`` filled on the previous step, + # so without that extra step the terminal span would be written and then + # discarded with the request. + pending_stop: bool = False + finish_reason: str = "length" + frames: FrameBuffer = field(default_factory=FrameBuffer) + codes: list[torch.Tensor] = field(default_factory=list) + codes_base_frame: int = 0 + # The previous frame's eight codes, fed back as the next step's input + # embedding. Held on the request rather than on a batch row, because the + # row a request occupies is not stable between steps. + last_codes: torch.Tensor | None = None + + def advance_position(self) -> int: + """Return this frame's base draw position and reserve the frame's slot. + + Each frame consumes ``NUM_CODEBOOKS`` positions so that the ``c0`` draw + and the seven depth draws never share an RNG stream. + """ + position = self.sampling_position + self.sampling_position += NUM_CODEBOOKS + return position + + def ready_window(self) -> ChunkWindow | None: + """Return the next window that a lookahead frame has proven non-final.""" + start = self.emitted_windows * AR_CHUNK_HOP_FRAMES + if self.frames.end_frame <= start + AR_CHUNK_FRAMES: + return None + return ChunkWindow( + index=self.emitted_windows, + start=start, + end=start + AR_CHUNK_FRAMES, + is_first=self.emitted_windows == 0, + is_last=False, + ) + + def take_window(self, window: ChunkWindow, *, is_final: bool) -> torch.Tensor: + """Materialize ``window`` and release the frames it retires.""" + chunk = self.frames.slice(window.start, window.end) + self.emitted_windows = max(self.emitted_windows, window.index + 1) + if not is_final: + self.frames.discard_before(window.start + AR_CHUNK_HOP_FRAMES) + return chunk + + def take_codes(self, start: int, end: int) -> torch.Tensor | None: + """Return the sampled codes for frames ``[start, end)``, if retained. + + Codes are carried alongside the conditioning frames so a downstream + stage or a debugging harness can recover the exact RVQ trajectory. + They are dropped rather than raising when a span predates what is + still buffered, since nothing in the audio path depends on them. + """ + if not self.codes or end <= start: + return None + offset = self.codes_base_frame + lo, hi = start - offset, end - offset + if lo < 0 or hi > len(self.codes): + return None + return torch.cat(self.codes[lo:hi], dim=0) + + def discard_codes_before(self, frame: int) -> None: + drop = min(max(frame - self.codes_base_frame, 0), len(self.codes)) + if drop: + del self.codes[:drop] + self.codes_base_frame += drop + + def release(self) -> None: + self.frames.clear() + self.codes.clear() + + +__all__ = ["FrameBuffer", "RequestARState"] diff --git a/vllm_omni/model_executor/models/minimax_music3/pipeline.py b/vllm_omni/model_executor/models/minimax_music3/pipeline.py new file mode 100644 index 000000000..1a8279f05 --- /dev/null +++ b/vllm_omni/model_executor/models/minimax_music3/pipeline.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MiniMax Music 3 pipeline topology: AR talker -> acoustic decoder. + +Stage 0 ``minimax_music3_ar``: a Qwen3 backbone (the repo's ``language_model/`` +subfolder is a plain ``Qwen3ForCausalLM``) plus a depth decoder, wrapped by a +class that owns its sampler. It emits 200-frame windows of frame hidden states +and the matching 8-codebook RVQ codes. + +Stage 1 ``minimax_music3_acoustic``: flow-matching DiT plus DAV vocoder. Pure +compute, no KV cache, weights loaded from the repo's ``condition_encoder/``, +``transformer/`` and ``vocoder/`` subfolders. Output is 32 kHz stereo. + +Classifier-free guidance is mandatory for this checkpoint: every request +decodes as a cond/uncond pair, and the uncond row is produced by +``prompt_expand_func``. The deploy YAMLs therefore carry +``default_sampling_params.extra_args`` with ``cfg_role`` and ``cfg_scale`` so +that ``has_sampling_extra_args`` is on for the stage and per-row extra args +reach the model's forward. +""" + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +_PROC = "vllm_omni.model_executor.stage_input_processors.minimax_music3" + +# ``<|audio_end|>`` in the checkpoint tokenizer (pinned by +# ``models/minimax_music3/prompt.SPECIAL_TOKEN_IDS``). The talker's own sampler +# forces this at the frame budget; the stop id is the engine-side safety net. +MINIMAX_MUSIC3_AUDIO_END_TOKEN_ID = 151670 + +# MiniMax captions carry the full style description and run to roughly 5000 +# characters, an order of magnitude past the 500-character serving default. +# ``tts_args`` is a pipeline-level stage extra, not a deploy-YAML field: it +# reaches the serving layer through ``StageConfig.yaml_extras``, which is where +# ``ServingSpeech._compute_max_instructions_length`` reads it from. +_MAX_INSTRUCTIONS_LENGTH = 20_000 + +MINIMAX_MUSIC3_PIPELINE = PipelineConfig( + model_type="minimax_music3", + model_arch="MiniMaxMusic3TalkerForConditionalGeneration", + hf_architectures=("MiniMaxMusic3ForConditionalGeneration",), + default_deploy_config_name="minimax_music3.yaml", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="minimax_music3_ar", + execution_type=StageExecutionType.LLM_AR, + input_sources=(), + owns_tokenizer=True, + engine_output_type="latent", + # The repo root config.json only names the composite architecture; + # the AR weights and the tokenizer live in their own subfolders. + model_subdir="language_model", + tokenizer_subdir="tokenizer", + # detokenize: the sampled stream is audio codes, never text. + # stop_token_ids: <|audio_end|> closes the song. + sampling_constraints={ + "detokenize": False, + "stop_token_ids": [MINIMAX_MUSIC3_AUDIO_END_TOKEN_ID], + }, + # CFG is mandatory here, so this expands on essentially every + # request rather than only on an opt-in flag. + prompt_expand_func=f"{_PROC}.expand_cfg_prompts", + async_chunk_process_next_stage_input_func=f"{_PROC}.ar2acoustic_async_chunk", + # retains_state_across_chunks stays False. It is read only by + # OmniGenerationScheduler, so it is a no-op on an LLM_AR stage; + # more to the point, stage 0 produces chunks rather than parking on + # them, so it never holds a runner slot while waiting for one. + extras={"tts_args": {"max_instructions_length": _MAX_INSTRUCTIONS_LENGTH}}, + ), + StagePipelineConfig( + stage_id=1, + model_stage="minimax_music3_acoustic", + execution_type=StageExecutionType.LLM_GENERATION, + input_sources=(0,), + final_output=True, + final_output_type="audio", + # "audio" (not "latent"): the accumulator concatenates audio over + # time, whereas the latent path would stack the two stereo channels. + engine_output_type="audio", + model_arch="MiniMaxMusic3AcousticForConditionalGeneration", + # This stage reads the repo ROOT so it can reach condition_encoder/, + # transformer/ and vocoder/, and the root ships no tokenizer files. + tokenizer_subdir="tokenizer", + # The acoustic stage carries the previous window's latent forward as + # the next window's flow-matching prompt, so a request parked between + # chunks still owns its model state and must keep counting against + # max_num_seqs. + retains_state_across_chunks=True, + sync_process_input_func=f"{_PROC}.ar2acoustic", + sampling_constraints={"detokenize": True}, + ), + ), +) + +__all__ = [ + "MINIMAX_MUSIC3_AUDIO_END_TOKEN_ID", + "MINIMAX_MUSIC3_PIPELINE", +] diff --git a/vllm_omni/model_executor/models/minimax_music3/prompt.py b/vllm_omni/model_executor/models/minimax_music3/prompt.py new file mode 100644 index 000000000..b9093eae9 --- /dev/null +++ b/vllm_omni/model_executor/models/minimax_music3/prompt.py @@ -0,0 +1,172 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# Prompt normalization follows the MiniMaxAI/MiniMax-Music3 input contract. +# The transformations are reproduced byte-for-byte because any divergence +# shifts token positions and therefore changes the generated audio. +"""MiniMax Music 3 prompt construction. + +The prompt is a single flat string, not a chat template:: + + <|im_start|><|caption_start|>{caption}<|caption_end|> + <|lyrics_start|>{lyrics}<|lyrics_end|><|im_end|><|audio_start|> + +The classifier-free-guidance twin reuses the same token ids with the whole +caption-and-lyrics span (delimiters included) overwritten by +``<|audio_cfg|>``, so only the chat markers and ``<|audio_start|>`` survive. +Both rows therefore have identical length and identical audio-start position. +""" + +from __future__ import annotations + +import re +from typing import Any + +# Pinned by ``validate_tokenizer_ids``: a checkpoint whose tokenizer disagrees +# is not this model, and silently producing noise is worse than failing. +SPECIAL_TOKEN_IDS: dict[str, int] = { + "<|im_start|>": 151644, + "<|im_end|>": 151645, + "<|audio_cfg|>": 151654, + "<|audio_start|>": 151669, + "<|audio_end|>": 151670, + "<|caption_start|>": 151671, + "<|caption_end|>": 151672, + "<|lyrics_start|>": 151673, + "<|lyrics_end|>": 151674, +} + +# c0 codes occupy the backbone vocabulary immediately above the text tokens. +AUDIO_CODE_OFFSET = 151675 + +_SPECIAL_TAG_RE = re.compile(r"<\|([^|]*)\|>") +_LEADING_TAGS_RE = re.compile(r"^[ \t]*((?:\[[^\]]+\][ \t]*)+)") + + +def _strip_markdown_residuals(text: str) -> str: + """Strip the Markdown syntax the model input contract does not accept.""" + if not text: + return text + lines_out: list[str] = [] + for raw_line in text.splitlines(): + line = raw_line + line = re.sub(r"^\s{0,3}#{1,6}\s+", "", line) + line = re.sub(r"^\s*[*+-]\s+", "", line) + line = re.sub(r"^\s*\*\s+", "", line) + while "**" in line: + updated = re.sub(r"\*\*([^*]+)\*\*", r"\1", line) + if updated == line: + break + line = updated + line = re.sub(r"(? str: + return _strip_markdown_residuals(text).replace("• ", "").replace(" ", "") + + +def clean_caption(caption: str) -> str: + """Apply the source order: special tokens, then Markdown, then newlines. + + ``<|tag value|>`` forms are rewritten to ``tag is value`` so a caption + cannot inject a real special token into the prompt. + """ + + def _special(match: re.Match[str]) -> str: + inner = match.group(1).strip() + parts = inner.split(None, 1) + return f"{parts[0]} is {parts[1]}" if len(parts) == 2 else inner + + text = _SPECIAL_TAG_RE.sub(_special, caption) + text = _remove_markdown_format(text) + # The source collapses runs of two or more newlines, not three or more; + # this shifts blank-line token positions and therefore the audio. + return re.sub(r"\n{2,}", "\n", text) + + +def _strip_text_after_leading_tags(text: str) -> str: + """Keep only the consecutive structural tags that start a line. + + Lyrics written on the same line as a ``[Verse]`` tag are dropped, which is + the documented contract: a tag must sit on its own line. + """ + if not text: + return text + output: list[str] = [] + for line in text.split("\n"): + match = _LEADING_TAGS_RE.match(line) + output.append(match.group(1).strip() if match else line) + return "\n".join(output) + + +def normalize_lyrics(lyrics: str) -> str: + """Normalize lyrics to the form the checkpoint was trained on.""" + text = _strip_text_after_leading_tags(lyrics) + text = text.replace("] ", "]\n") + text = text.replace(" [", "\n[") + text = text.replace(" ^ ", "\n") + text = re.sub(r"\[([^\]]+)\]", lambda match: f"[{match.group(1).lower()}]", text) + return f"[start]\n{text}" + + +def build_prompt(caption: str, lyrics: str) -> str: + """Build the conditioned prompt string.""" + return ( + "<|im_start|><|caption_start|>" + f"{clean_caption(caption)}" + "<|caption_end|><|lyrics_start|>" + f"{normalize_lyrics(lyrics)}" + "<|lyrics_end|><|im_end|><|audio_start|>" + ) + + +def build_cfg_null_token_ids(prompt_token_ids: list[int]) -> list[int]: + """Return the unconditioned twin's token ids for a conditioned prompt. + + Everything between the leading ``<|im_start|>`` and the trailing + ``<|im_end|><|audio_start|>`` becomes ``<|audio_cfg|>``. Length and the + audio-start position are preserved, which is what lets the two rows decode + in lockstep. + + Raises: + ValueError: If the prompt is too short to carry a caption and lyrics, + or does not have the expected framing tokens. + """ + if len(prompt_token_ids) < 4: + raise ValueError(f"MiniMax Music 3 prompt is too short to build a CFG twin: {len(prompt_token_ids)} tokens") + expected = ( + SPECIAL_TOKEN_IDS["<|im_start|>"], + SPECIAL_TOKEN_IDS["<|im_end|>"], + SPECIAL_TOKEN_IDS["<|audio_start|>"], + ) + actual = (prompt_token_ids[0], prompt_token_ids[-2], prompt_token_ids[-1]) + if actual != expected: + raise ValueError( + "MiniMax Music 3 prompt framing is wrong: expected " + f"(im_start, im_end, audio_start)={expected}, got {actual}" + ) + null_ids = list(prompt_token_ids) + null_ids[1:-2] = [SPECIAL_TOKEN_IDS["<|audio_cfg|>"]] * (len(null_ids) - 3) + return null_ids + + +def validate_tokenizer_ids(tokenizer: Any) -> None: + """Fail fast when a tokenizer is not the supported music tokenizer.""" + for token, expected in SPECIAL_TOKEN_IDS.items(): + token_id = tokenizer.convert_tokens_to_ids(token) + if token_id != expected: + raise ValueError(f"MiniMax Music 3 tokenizer mismatch for {token}: expected {expected}, got {token_id}") + + +__all__ = [ + "AUDIO_CODE_OFFSET", + "SPECIAL_TOKEN_IDS", + "build_cfg_null_token_ids", + "build_prompt", + "clean_caption", + "normalize_lyrics", + "validate_tokenizer_ids", +] diff --git a/vllm_omni/model_executor/models/minimax_music3/rvq_decoder.py b/vllm_omni/model_executor/models/minimax_music3/rvq_decoder.py new file mode 100644 index 000000000..64acc9b62 --- /dev/null +++ b/vllm_omni/model_executor/models/minimax_music3/rvq_decoder.py @@ -0,0 +1,179 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""RVQ depth decoder for MiniMax Music 3 (codebooks c1..c7). + +Each audio frame is eight codebooks deep. The backbone's own ``lm_head`` +emits ``c0``; this four-layer decoder walks ``c1..c7``, conditioning each code +on the previous ones. Its per-codebook hidden states are also the acoustic +stage's conditioning signal, so this module returns them alongside the codes. + +The sequence fed to the decoder is:: + + [proj(backbone_hidden), proj(embed(c0)), proj(embed(c1)), ...] + +and after step ``i`` the head ``audio_heads[i - 1]`` predicts ``c_i``. The +whole thing runs over at most eight positions, so it is written as a dense +causal block rather than anything with a KV cache. +""" + +from __future__ import annotations + +import torch +from torch import Tensor, nn +from torch.nn import functional as F +from torch.nn.attention import SDPBackend, sdpa_kernel + +from .constants import AUDIO_VOCAB_SIZE, BACKBONE_HIDDEN_SIZE, NUM_CODEBOOKS + +# cuDNN's fused attention is not bit-reproducible on this hardware: two +# identical replays of the same input can differ by one bfloat16 ULP, which is +# enough to flip a sampled code and send the rest of the song somewhere else. +# The choice is pinned per call rather than through a process-global switch so +# it cannot be undone by whatever else shares the process. +_SDPA_BACKENDS = [SDPBackend.FLASH_ATTENTION, SDPBackend.MATH] + + +class DepthDecoderBlock(nn.Module): + """Pre-norm transformer block with fused QKV and a SwiGLU MLP.""" + + def __init__(self, hidden_size: int, num_heads: int, intermediate_size: int) -> None: + super().__init__() + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + self.input_layernorm = nn.RMSNorm(hidden_size, eps=1e-6) + # Stored fused so the three projections are one matmul; the checkpoint + # ships them split and ``load_checkpoint_state`` concatenates. + self.qkv_proj = nn.Linear(hidden_size, hidden_size * 3, bias=False) + self.o_proj = nn.Linear(hidden_size, hidden_size, bias=False) + self.post_attention_layernorm = nn.RMSNorm(hidden_size, eps=1e-6) + self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False) + self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False) + self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False) + + def forward(self, x: Tensor) -> Tensor: + h = self.input_layernorm(x) + bsz, seq, hidden = h.shape + q, k, v = self.qkv_proj(h).chunk(3, dim=-1) + q = q.reshape(bsz, seq, self.num_heads, self.head_dim).transpose(1, 2) + k = k.reshape(bsz, seq, self.num_heads, self.head_dim).transpose(1, 2) + v = v.reshape(bsz, seq, self.num_heads, self.head_dim).transpose(1, 2) + with sdpa_kernel(_SDPA_BACKENDS): + h = F.scaled_dot_product_attention(q, k, v, is_causal=True) + h = h.transpose(1, 2).contiguous().reshape(bsz, seq, hidden) + x = x + self.o_proj(h) + h = self.post_attention_layernorm(x) + h = self.down_proj(F.silu(self.gate_proj(h)) * self.up_proj(h)) + return x + h + + +class RVQDepthDecoder(nn.Module): + """Depth-autoregressive decoder over the seven residual codebooks.""" + + def __init__( + self, + *, + hidden_size: int = BACKBONE_HIDDEN_SIZE, + num_layers: int = 4, + num_heads: int = 16, + intermediate_size: int = 6144, + audio_vocab_size: int = AUDIO_VOCAB_SIZE, + num_codebooks: int = NUM_CODEBOOKS, + max_seq_len: int = 16, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.audio_vocab_size = audio_vocab_size + self.num_codebooks = num_codebooks + self.projection = nn.Linear(hidden_size, hidden_size, bias=False) + self.pos_embedding = nn.Embedding(max_seq_len, hidden_size) + self.layers = nn.ModuleList( + DepthDecoderBlock(hidden_size, num_heads, intermediate_size) for _ in range(num_layers) + ) + self.norm = nn.RMSNorm(hidden_size, eps=1e-6) + self.audio_heads = nn.ModuleList( + nn.Linear(hidden_size, audio_vocab_size, bias=False) for _ in range(num_codebooks - 1) + ) + + def forward(self, seq: Tensor) -> Tensor: + positions = torch.arange(seq.shape[1], device=seq.device) + x = seq + self.pos_embedding(positions).unsqueeze(0) + for layer in self.layers: + x = layer(x) + return self.norm(x) + + def load_checkpoint_state(self, state: dict[str, Tensor]) -> None: + """Load the component checkpoint, failing loudly on any mismatch. + + The checkpoint keeps ``q``/``k``/``v`` split and names attention + projections ``attn.to_*``; both are reshaped here. A silent partial + load would produce plausible-sounding but wrong audio, so every key is + accounted for and anything left over is an error. + + Args: + state: Tensors from the ``rvq_depth_decoder`` component, with the + ``audio_embeddings.weight`` entry already removed by the + caller. + + Raises: + ValueError: If any expected key is missing, any shape disagrees, + or the checkpoint carries keys this module does not consume. + """ + loaded: set[str] = set() + missing: list[str] = [] + mismatched: list[str] = [] + + def take(key: str) -> Tensor | None: + value = state.get(key) + if value is None: + missing.append(key) + return value + + def assign(key: str, target: Tensor, source: Tensor, keys: tuple[str, ...]) -> None: + if tuple(source.shape) != tuple(target.shape): + mismatched.append(f"{key}: checkpoint={tuple(source.shape)} model={tuple(target.shape)}") + return + target.data.copy_(source.to(dtype=target.dtype)) + loaded.update(keys) + + def copy_one(key: str, target: Tensor) -> None: + source = take(key) + if source is not None: + assign(key, target, source, (key,)) + + copy_one("projection.weight", self.projection.weight) + copy_one("pos_embedding.weight", self.pos_embedding.weight) + copy_one("norm.weight", self.norm.weight) + for index, head in enumerate(self.audio_heads): + copy_one(f"audio_heads.{index}.weight", head.weight) + + for index, layer in enumerate(self.layers): + prefix = f"layers.{index}." + copy_one(prefix + "input_layernorm.weight", layer.input_layernorm.weight) + copy_one( + prefix + "post_attention_layernorm.weight", + layer.post_attention_layernorm.weight, + ) + qkv_keys = tuple(prefix + f"attn.to_{name}.weight" for name in ("q", "k", "v")) + parts = [take(key) for key in qkv_keys] + if all(part is not None for part in parts): + assign( + prefix + "attn.qkv", + layer.qkv_proj.weight, + torch.cat(parts, dim=0), + qkv_keys, + ) + copy_one(prefix + "attn.to_out.weight", layer.o_proj.weight) + copy_one(prefix + "gate_proj.weight", layer.gate_proj.weight) + copy_one(prefix + "up_proj.weight", layer.up_proj.weight) + copy_one(prefix + "down_proj.weight", layer.down_proj.weight) + + unexpected = sorted(set(state) - loaded) + if missing or mismatched or unexpected: + raise ValueError( + "MiniMax Music 3 RVQ depth decoder failed to load: " + f"missing={missing[:8]}, mismatched={mismatched[:8]}, " + f"unexpected={unexpected[:8]}" + ) + + +__all__ = ["DepthDecoderBlock", "RVQDepthDecoder"] diff --git a/vllm_omni/model_executor/models/minimax_music3/sampling.py b/vllm_omni/model_executor/models/minimax_music3/sampling.py new file mode 100644 index 000000000..226addff5 --- /dev/null +++ b/vllm_omni/model_executor/models/minimax_music3/sampling.py @@ -0,0 +1,223 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Deterministic guided sampling for MiniMax Music 3. + +Two things happen per code, in this order: + +1. **Classifier-free guidance.** Every request decodes twice, once on the real + prompt and once on a prompt whose caption and lyrics have been replaced by + ``<|audio_cfg|>``. Codes are drawn from ``uncond + (cond - uncond) * 1.5``. + For ``c0`` the candidate set is additionally masked to the conditioned + row's own top 50, so the guided draw cannot wander outside what the + conditioned branch already considered plausible. +2. **Seeded top-k.** A batched, order-independent categorical draw: each row + gets its own RNG stream keyed by ``(seed, position)``, so a request's audio + depends only on its own seed and never on who else happens to share the + batch. + +The RNG is a MurmurHash3 x86_32 of ``(seed_lo, seed_hi, position, column)`` +turned into Gumbel noise, and the draw is an argmax over ``logits + gumbel``. +That is mathematically a categorical sample from ``softmax(logits)``, but +unlike ``torch.multinomial`` it needs no per-row generator state, so it is +CUDA-graph safe and reproducible across batch compositions. +""" + +from __future__ import annotations + +import torch +from torch import Tensor + +from .constants import AR_CFG_SCALE, AR_CFG_TOP_K, AR_TOP_K + +_UINT32_MASK = 0xFFFFFFFF +_UINT32_MAX = 4_294_967_295 + +# MurmurHash3 x86_32 constants. +_C1 = 0xCC9E2D51 +_C2 = 0x1B873593 +_R1 = 15 +_R2 = 13 +_M = 5 +_N = 0xE6546B64 + + +def _rotl32(x: Tensor, r: int) -> Tensor: + return ((x << r) | (x >> (32 - r))) & _UINT32_MASK + + +def _murmur3_mix(h: Tensor, k: Tensor) -> Tensor: + """Mix one 32-bit key into the running hash state.""" + k = (k * _C1) & _UINT32_MASK + k = _rotl32(k, _R1) + k = (k * _C2) & _UINT32_MASK + h = h ^ k + h = _rotl32(h, _R2) + return (h * _M + _N) & _UINT32_MASK + + +def _fmix32(h: Tensor) -> Tensor: + """MurmurHash3 finalizer.""" + h = h ^ (h >> 16) + h = (h * 0x85EBCA6B) & _UINT32_MASK + h = h ^ (h >> 13) + h = (h * 0xC2B2AE35) & _UINT32_MASK + return h ^ (h >> 16) + + +# Memoized ``k`` term of the column block's mix. The column block is the same +# ``arange(width)`` on every draw, so its three ops are hoisted out of the per +# draw work and kept per (device, width). Small and bounded: two widths exist, +# the c0 candidate set and the residual codebook. +_COLUMN_MIX_CACHE: dict[tuple[torch.device, int], Tensor] = {} + + +def _column_mix(num_columns: int, device: torch.device) -> Tensor: + """Return the mixed column block, ``[cols]`` int64 holding uint32.""" + key = (device, num_columns) + cached = _COLUMN_MIX_CACHE.get(key) + if cached is None: + columns = torch.arange(num_columns, dtype=torch.int64, device=device) & _UINT32_MASK + k = (columns * _C1) & _UINT32_MASK + k = _rotl32(k, _R1) + cached = (k * _C2) & _UINT32_MASK + _COLUMN_MIX_CACHE[key] = cached + return cached + + +def _absorb_mixed_key(h: Tensor, k: Tensor) -> Tensor: + """Absorb a key whose own three-op transform is already applied.""" + h = h ^ k + h = _rotl32(h, _R2) + return (h * _M + _N) & _UINT32_MASK + + +def row_hash_state(seeds: Tensor, positions: Tensor) -> Tensor: + """Hash state after the seed and position blocks, before the columns. + + Args: + seeds: ``[rows]`` per-request seeds. + positions: ``[rows]`` or ``[rows, draws]`` draw identifiers. The extra + dimension lets every codebook of a frame be hashed in one batch + instead of one dispatch per codebook; the arithmetic is elementwise + integer, so a batched call is bit-identical to the loop it replaces. + + Returns: + A tensor shaped like ``positions``, int64 holding uint32. + """ + seeds = seeds.to(torch.int64) + while seeds.dim() < positions.dim(): + seeds = seeds.unsqueeze(-1) + h = torch.zeros((), dtype=torch.int64, device=seeds.device) + h = _murmur3_mix(h, seeds & _UINT32_MASK) + h = _murmur3_mix(h, (seeds >> 32) & _UINT32_MASK) + return _murmur3_mix(h, positions.to(torch.int64) & _UINT32_MASK) + + +def murmur_hash32(seeds: Tensor, positions: Tensor, columns: Tensor) -> Tensor: + """Hash ``(seed, position, column)`` to a uint32 held in int64. + + The 64-bit seed is consumed as two 32-bit blocks, then the position and + the column, for a hashed length of 16 bytes. + + Args: + seeds: ``[rows]`` per-request seeds. + positions: ``[rows]`` draw identifiers, unique per (frame, codebook). + columns: ``[cols]`` candidate indices, which must be ``arange(cols)``. + + Returns: + ``[rows, cols]`` int64 tensor holding uint32 values. + """ + h = row_hash_state(seeds, positions).unsqueeze(-1) + return _fmix32(_absorb_mixed_key(h, _column_mix(int(columns.shape[0]), h.device)) ^ 16) + + +def gumbel_noise(seeds: Tensor, positions: Tensor, num_columns: int) -> Tensor: + """Return the Gumbel noise for one draw per position, ``[..., cols]``. + + Args: + seeds: ``[rows]`` per-request seeds. + positions: ``[rows]`` or ``[rows, draws]`` draw identifiers. + num_columns: Candidate count, i.e. the width of the logits. + + Returns: + float64 noise shaped ``positions.shape + (num_columns,)``. + """ + h = row_hash_state(seeds, positions).unsqueeze(-1) + hashed = _fmix32(_absorb_mixed_key(h, _column_mix(num_columns, h.device)) ^ 16) + # float64 throughout: the double log is numerically unstable in float32 and + # a single flipped draw changes every frame that follows it. + uniform = hashed.to(torch.float64) / _UINT32_MAX + # Clamp both ends. uniform == 1 gives +inf Gumbel, which turns a -inf + # masked entry into NaN; the upper cap is the hash spacing, so a saturated + # bucket ties with its neighbour instead of always winning. + gumbel = uniform.log().clamp_(min=torch.finfo(torch.float64).min, max=-(2.0**-32)).neg_() + return gumbel.log().neg_() + + +def draw_from_gumbel(logits: Tensor, gumbel: Tensor, *, top_k: int = AR_TOP_K) -> Tensor: + """Argmax of ``top_k``-masked logits plus precomputed Gumbel noise. + + Args: + logits: ``[rows, vocab]`` unnormalized scores. + gumbel: ``[rows, vocab]`` float64 noise from :func:`gumbel_noise`. + top_k: Candidates retained per row before the draw. + + Returns: + ``[rows]`` sampled column indices. + """ + values = torch.nan_to_num(logits.float(), nan=-1e9, posinf=1e9, neginf=-1e9) + k = min(int(top_k), values.shape[-1]) + threshold = torch.topk(values, k, dim=-1).values[..., -1, None] + values = values.masked_fill(values < threshold, -float("inf")) + return (values.to(torch.float64) + gumbel).argmax(dim=-1) + + +def sample_topk_seeded( + logits: Tensor, + seeds: Tensor, + positions: Tensor, + *, + top_k: int = AR_TOP_K, +) -> Tensor: + """Draw one column per row from its own top-k, under its own RNG stream. + + Args: + logits: ``[rows, vocab]`` unnormalized scores. + seeds: ``[rows]`` per-request sampling seeds. + positions: ``[rows]`` draw identifiers, unique per (frame, codebook). + top_k: Candidates retained per row before the draw. + + Returns: + ``[rows]`` sampled column indices. + """ + gumbel = gumbel_noise(seeds, positions, int(logits.shape[-1])) + return draw_from_gumbel(logits, gumbel, top_k=top_k) + + +def apply_c0_cfg(cond: Tensor, uncond: Tensor, *, scale: float = AR_CFG_SCALE) -> Tensor: + """Guide ``cond`` by ``uncond``, restricted to ``cond``'s own top candidates. + + The mask is a second, separate restriction from the top-k the sampler + applies afterwards. Dropping it widens the distribution the model draws + from and audibly loosens how closely the output follows the caption. + """ + guided = uncond + (cond - uncond) * scale + k = min(AR_CFG_TOP_K, cond.shape[-1]) + threshold = torch.topk(cond, k, dim=-1).values[..., -1, None] + return guided.masked_fill(cond < threshold, -float("inf")) + + +def apply_depth_cfg(cond: Tensor, uncond: Tensor, *, scale: float = AR_CFG_SCALE) -> Tensor: + """Guide the depth heads. Unlike ``c0`` these are not additionally masked.""" + return uncond + (cond - uncond) * scale + + +__all__ = [ + "apply_c0_cfg", + "apply_depth_cfg", + "draw_from_gumbel", + "gumbel_noise", + "murmur_hash32", + "row_hash_state", + "sample_topk_seeded", +] diff --git a/vllm_omni/model_executor/models/minimax_music3/staging.py b/vllm_omni/model_executor/models/minimax_music3/staging.py new file mode 100644 index 000000000..ed3eb8a47 --- /dev/null +++ b/vllm_omni/model_executor/models/minimax_music3/staging.py @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Pinned host staging for the small per-step index vectors. + +Building a device tensor from a Python list (``torch.tensor(rows, +device="cuda")``) stages through pageable host memory, which makes the copy +synchronous: the host blocks until the copy retires, and because the copy is +queued behind the step's kernels that means blocking until the whole forward +has finished. On this model that turned four one-line conveniences into the +single largest entry in a CPU profile of the decode loop. + +Writing into a pinned block and pushing it with one non-blocking copy costs +one transfer for the whole step and leaves the host free to keep enqueueing. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import torch + +__all__ = ["PinnedIndexStage"] + + +class PinnedIndexStage: + """A fixed ``[rows, capacity]`` int64 block staged host-to-device. + + The device view returned by :meth:`push` is only populated once the stream + reaches the copy, so it may be consumed by kernels on that stream and by + nothing else. The host side is double-buffered: the next step writes the + slot the device is not reading, so refilling cannot corrupt a copy that is + still in flight. + """ + + def __init__(self, rows: int, capacity: int, device: torch.device) -> None: + self._capacity = capacity + self._host = [torch.zeros((rows, capacity), dtype=torch.int64, pin_memory=True) for _ in range(2)] + self._device = torch.zeros((rows, capacity), dtype=torch.int64, device=device) + self._slot = 0 + + @property + def capacity(self) -> int: + return self._capacity + + def push(self, values: Sequence[Sequence[int]], count: int) -> torch.Tensor: + """Stage ``count`` entries of each row and return the device view. + + Args: + values: One sequence of Python ints per row of the block. + count: How many entries of each row are meaningful this step. + + Returns: + ``[rows, count]`` int64 view of the device block. + + Raises: + ValueError: If ``count`` exceeds the reserved capacity. + """ + if count > self._capacity: + raise ValueError(f"staged {count} entries but only {self._capacity} were reserved") + self._slot ^= 1 + host = self._host[self._slot] + view = host.numpy() + for index, row in enumerate(values): + view[index, :count] = row + self._device.copy_(host, non_blocking=True) + return self._device[:, :count] diff --git a/vllm_omni/model_executor/models/minimax_music3/talker.py b/vllm_omni/model_executor/models/minimax_music3/talker.py new file mode 100644 index 000000000..c4fb876fa --- /dev/null +++ b/vllm_omni/model_executor/models/minimax_music3/talker.py @@ -0,0 +1,990 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Stage-0 talker for MiniMax Music 3. + +A Qwen3 backbone predicts one audio frame per decode step. Each frame is +eight RVQ codebooks deep: the backbone's own ``lm_head`` emits ``c0`` and the +four-layer depth decoder walks ``c1..c7``. The conditioning signal handed to +the acoustic stage is the backbone hidden state concatenated with the seven +depth hidden states, so 8 x 4096 = 32768 dimensions per frame. + +Two things make this model unusual inside vLLM: + +* **Every request decodes as two rows.** Classifier-free guidance is not + optional here. The conditioned row sees the real prompt and the + unconditioned row sees the same prompt with the caption-and-lyrics span + replaced by ``<|audio_cfg|>``. Codes are drawn from the guided blend and + then fed back to *both* rows, so the pair stays token-identical. Rows are + paired by ``cfg_pair_id`` rather than by adjacency, because batch position + is not stable across engine steps. +* **The decode input is not a token embedding.** After prefill, each step's + input is the composed embedding of the previous frame's eight codes. The + sampled ``c0`` token id still flows through vLLM's normal machinery so that + stop handling and accounting work, but the embedding it would produce is + replaced before the backbone runs. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Any + +import torch +from torch import nn +from vllm.config import VllmConfig +from vllm.logger import init_logger +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.models.qwen3 import Qwen3Model +from vllm.model_executor.models.utils import AutoWeightsLoader +from vllm.v1.outputs import SamplerOutput + +from vllm_omni.model_executor.models.output_templates import OmniOutput + +from .chunking import chunk_windows +from .constants import ( + AR_CHUNK_HOP_FRAMES, + AUDIO_VOCAB_SIZE, + BACKBONE_HIDDEN_SIZE, + C0_VOCAB_SIZE, + DEFAULT_MAX_AUDIO_FRAMES, + NUM_CODEBOOKS, +) +from .depth_graph import DepthGraphCache +from .frame_state import RequestARState +from .prompt import AUDIO_CODE_OFFSET, SPECIAL_TOKEN_IDS +from .rvq_decoder import RVQDepthDecoder +from .sampling import ( + apply_c0_cfg, + apply_depth_cfg, + draw_from_gumbel, + gumbel_noise, + sample_topk_seeded, +) +from .staging import PinnedIndexStage +from .weights import load_depth_decoder_weights + +logger = init_logger(__name__) + +__all__ = ["MiniMaxMusic3TalkerForConditionalGeneration"] + +_COND = "cond" +_UNCOND = "uncond" +_STOP_TOKEN = SPECIAL_TOKEN_IDS["<|audio_end|>"] +# Emitted on the step a song ends, to keep the request alive for one more step +# while its terminal span is delivered. It is never fed back as audio: the +# state is already marked finished, so no frame is composed from it. +_HOLD_TOKEN = AUDIO_CODE_OFFSET + +# The unconditioned twin's request id is the conditioned one plus this suffix, +# which is what lets a pair be recognized without a serving-layer injection. +_CFG_UNCOND_SUFFIX = "__cfg_uncond" + + +@dataclass(frozen=True) +class _RowConstants: + """Per-request values that never change once the request is admitted.""" + + request_id: str + role: str + pair_id: str + seed: int + max_frames: int + + +def _request_key(value: Any) -> str: + """Normalize a request id to the form used to key per-request state. + + The engine injects ``global_request_id`` as a single-element list, while + ``on_requests_finished`` receives the bare string. Keying state on the + unwrapped value keeps allocation and release on the same key; getting this + wrong leaks a song's worth of frames per request. + """ + if isinstance(value, (list, tuple)): + return str(value[0]) if value else "" + return "" if value is None else str(value) + + +def _pair_id_from_request(request_id: str) -> str: + """Derive the CFG pair id when the request does not carry one.""" + if request_id.endswith(_CFG_UNCOND_SUFFIX): + return request_id[: -len(_CFG_UNCOND_SUFFIX)] + return request_id + + +def _role_from_request(request_id: str) -> str: + """Derive the CFG role from the companion suffix.""" + return _UNCOND if request_id.endswith(_CFG_UNCOND_SUFFIX) else _COND + + +def _first_int(value: Any, default: int) -> int: + """Read an int that the serving layer wraps in a single-element list.""" + if isinstance(value, (list, tuple)): + value = value[0] if value else None + if value is None: + return default + try: + parsed = int(value) + except (TypeError, ValueError): + return default + return parsed if parsed > 0 else default + + +class MiniMaxMusic3TalkerForConditionalGeneration(nn.Module): + """Qwen3 backbone plus the audio embedding table and RVQ depth decoder.""" + + # Sampling, feedback and window emission are all model-owned. + prefer_model_sampler: bool = True + have_multimodal_outputs: bool = True + has_postprocess: bool = True + skips_model_sampler_output_token_history: bool = True + postprocess_uses_hidden_states: bool = False + postprocess_uses_multimodal_outputs: bool = False + postprocess_uses_req_infos: bool = True + supports_omni_query_start_loc: bool = True + # A row still completing a chunked prefill reaches the sampler but its + # token is discarded; drawing a frame for it would corrupt the song. + requires_request_sample_eligibility: bool = True + # The acoustic stage consumes the composed 32768-d frame, never the raw + # backbone hidden state, so suppress the runner's automatic "hidden" key. + omni_pooler_payload_include_hidden: bool = False + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + self.vllm_config = vllm_config + config = vllm_config.model_config.hf_config + self.config = config + hidden_size = int(getattr(config, "hidden_size", BACKBONE_HIDDEN_SIZE)) + + self.model = Qwen3Model( + vllm_config=vllm_config, + prefix=f"{prefix}.model" if prefix else "model", + ) + if getattr(config, "tie_word_embeddings", False): + self.lm_head = self.model.embed_tokens + else: + self.lm_head = ParallelLMHead( + int(config.vocab_size), + hidden_size, + prefix=f"{prefix}.lm_head" if prefix else "lm_head", + ) + self.logits_processor = LogitsProcessor(int(config.vocab_size)) + + # c1..c7 share one table; codebook i occupies rows [i*1024, (i+1)*1024). + self.audio_embeddings = nn.Embedding(AUDIO_VOCAB_SIZE * (NUM_CODEBOOKS - 1), hidden_size) + self.rvq_decoder = RVQDepthDecoder(hidden_size=hidden_size) + + self.register_buffer( + "audio_embedding_offsets", + (torch.arange(NUM_CODEBOOKS - 1) * AUDIO_VOCAB_SIZE).unsqueeze(0), + persistent=False, + ) + # The only token ids c0 sampling can return, in vocabulary order: + # <|audio_end|> first, then the c0 code range. Column 0 therefore means + # stop and column code+1 means code. + self.register_buffer( + "c0_logit_ids", + torch.cat( + ( + torch.tensor([_STOP_TOKEN], dtype=torch.long), + torch.arange( + AUDIO_CODE_OFFSET, + AUDIO_CODE_OFFSET + C0_VOCAB_SIZE, + dtype=torch.long, + ), + ) + ), + persistent=False, + ) + self.frame_embedding_scale = float(NUM_CODEBOOKS) ** -0.5 + + scheduler_config = getattr(vllm_config, "scheduler_config", None) + # Guidance doubles the rows, so the batch is twice the admitted + # request count. + self._max_rows = max(64, 2 * int(getattr(scheduler_config, "max_num_seqs", 16))) + + # Per-step scratch, rebuilt every step from request-keyed state. + # Nothing here may survive a step: vLLM compacts and reorders the + # persistent batch, so a row index is only meaningful within the step + # that produced it. + self._last_logits_hidden: torch.Tensor | None = None + self._step_engine_ids: list[str] = [] + self._row_request_ids: list[str] = [] + self._row_roles: list[str] = [] + self._row_pair_ids: list[str] = [] + self._row_seeds: list[int] = [] + self._row_max_frames: list[int] = [] + self._row_prompt_tokens: list[int] = [] + self._row_sample_eligible: list[bool] = [] + self._feedback_positions: torch.Tensor | None = None + self._feedback_codes: torch.Tensor | None = None + + # Per-step index vectors are staged through pinned memory rather than + # built with ``torch.tensor(..., device="cuda")``; see + # ``staging.PinnedIndexStage`` for why that matters here. Allocated on + # first use because the device is not known at construction time. + # Rows are (cond_row, uncond_row, seed, position). + self._sample_stage: PinnedIndexStage | None = None + # One row, the flat token offsets of this step's decoding rows. + self._feedback_stage: PinnedIndexStage | None = None + + # Depth-loop draw offsets, ``[1..NUM_CODEBOOKS)``. Held as a buffer so + # the per-step position arithmetic is one broadcast add rather than one + # scalar add per codebook. + self.register_buffer( + "depth_draw_offsets", + torch.arange(1, NUM_CODEBOOKS, dtype=torch.int64), + persistent=False, + ) + + # Frame feedback slab. A captured decode graph replays fixed addresses + # and fixed shapes, so the rows to overwrite cannot be expressed as a + # variable-length ``index_copy`` against a freshly built index tensor. + # Instead every row of the reserved slab carries a codes entry and a + # flag, both written in place before the graph runs, and the forward + # selects with ``torch.where`` over the whole row set. Sized to the + # widest decode batch; wider prefill batches take the eager path in + # ``_apply_frame_feedback``, and prefill is never graph-replayed. + self.register_buffer( + "frame_feedback_codes", + torch.zeros((self._max_rows, NUM_CODEBOOKS), dtype=torch.long), + persistent=False, + ) + self.register_buffer( + "frame_feedback_active", + torch.zeros(self._max_rows, dtype=torch.bool), + persistent=False, + ) + + # Cross-step state. Keyed by CFG pair id, because a guided request is + # two engine rows sharing one song. The engine renames requests to + # internal UUIDs on admission, so the pair id is derived from the + # user-facing id and the engine ids are mapped onto it separately. + self._states: dict[str, RequestARState] = {} + self._pair_of_engine_id: dict[str, str] = {} + # Per-request constants, keyed by engine id. ``forward`` is the only + # hook the engine hands the sampling extra args to, but its Python body + # does not run on a step replayed from a CUDA graph, so what it learns + # is cached here and re-bound to rows in ``prepare_runner_inputs``. + self._row_constants: dict[str, _RowConstants] = {} + self._row_binding_complete = False + self._incomplete_steps = 0 + + # The depth loop is recorded per pair count and replayed after that. + self._depth_graphs = DepthGraphCache(self._depth_decode_eager) + + # ---------------------------------------------------------------- forward + + def prepare_runner_inputs( + self, + *, + req_ids: list[str], + num_computed_tokens: Any, + num_scheduled_tokens: Any, + input_ids: torch.Tensor, + positions: torch.Tensor, + **_: Any, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Bind this step's batch rows to requests, before the forward runs. + + This is the only hook that sees the authoritative row-to-request + mapping for the step about to execute. vLLM compacts and reorders the + persistent batch whenever a request finishes or is preempted, so any + state indexed by row number from a previous step is stale by the time + it is read. Everything the model carries across steps is therefore + keyed by request id and gathered into row order here. + + It also resolves which rows are decoding. A row is decoding when its + prompt is already computed and it is advancing by a single token; a + row still working through a chunked prefill is not, and must keep its + real prompt embeddings. + + Python runs here on every step, including steps whose forward is + replayed from a CUDA graph rather than executed, so this is also where + the row-to-request binding that ``sample`` reads has to be rebuilt. + """ + self._step_engine_ids = [str(rid) for rid in req_ids] + computed = [int(value) for value in num_computed_tokens] + scheduled = [int(value) for value in num_scheduled_tokens] + # Prompt length for a row that is completing its prefill on this step. + # Only meaningful when the row is about to produce its first logit, + # which is the only moment state is created. + self._row_prompt_tokens = [computed[row] + scheduled[row] for row in range(len(req_ids))] + + # Flat token offset of each row's first scheduled token. + offsets: list[int] = [] + running = 0 + for count in scheduled: + offsets.append(running) + running += count + + feedback_rows: list[int] = [] + feedback_codes: list[torch.Tensor] = [] + for row, engine_id in enumerate(self._step_engine_ids): + # Both members of a guided pair are fed the same frame, so state + # is keyed by pair rather than by request. + pair_id = self._pair_of_engine_id.get(engine_id) + state = self._states.get(pair_id) if pair_id else None + if state is None or state.last_codes is None: + continue + # A decode row advances by exactly one token past a complete + # prompt. A row still working through a chunked prefill keeps its + # real prompt embeddings. + if scheduled[row] != 1 or computed[row] < state.prompt_tokens: + continue + feedback_rows.append(offsets[row]) + feedback_codes.append(state.last_codes) + + self._write_frame_feedback(feedback_rows, feedback_codes, input_ids.device) + self._bind_rows(computed, scheduled) + return input_ids, positions + + def _write_frame_feedback( + self, + feedback_rows: list[int], + feedback_codes: list[torch.Tensor], + device: torch.device, + ) -> None: + """Publish this step's frame feedback for the forward to consume. + + Two forms are written, because two forwards can consume it. The slab + is what a captured decode graph reads: fixed address, fixed shape, + selected row by row with a flag. The loose index and code tensors are + the eager fallback for a batch wider than the slab, which is only ever + a prefill batch and therefore never a replayed one. + """ + if self._feedback_stage is None: + self._feedback_stage = PinnedIndexStage(1, self._max_rows, device) + + active = self.frame_feedback_active + if not feedback_rows: + self._feedback_positions = None + self._feedback_codes = None + active.zero_() + return + + codes = torch.cat(feedback_codes, dim=0).to(device) + if max(feedback_rows) < self._max_rows: + positions = self._feedback_stage.push([feedback_rows], len(feedback_rows))[0] + self.frame_feedback_codes.index_copy_(0, positions, codes) + active.zero_() + active.index_fill_(0, positions, True) + else: + # Too wide for the slab. Nothing reads it on this step, but a stale + # flag would corrupt the next graph replay if the batch shrank. + positions = torch.tensor(feedback_rows, dtype=torch.long, device=device) + active.zero_() + self._feedback_positions = positions + self._feedback_codes = codes + + def _bind_rows(self, computed: list[int], scheduled: list[int]) -> None: + """Rebuild the row-order request metadata from the per-request cache. + + ``forward`` learns these values from the engine's own kwargs, but its + Python body is skipped whenever the step is replayed from a CUDA graph, + so the binding is re-established here. A row whose request has not been + seen by ``forward`` yet leaves the binding incomplete; that only + happens on the step a request is admitted, which is a prefill step, and + prefill is never replayed, so ``forward`` fills it in before the + sampler runs. + """ + constants = [self._row_constants.get(engine_id) for engine_id in self._step_engine_ids] + if not constants or any(entry is None for entry in constants): + self._row_binding_complete = False + return + self._row_request_ids = [entry.request_id for entry in constants] + self._row_roles = [entry.role for entry in constants] + self._row_pair_ids = [entry.pair_id for entry in constants] + self._row_seeds = [entry.seed for entry in constants] + self._row_max_frames = [entry.max_frames for entry in constants] + + # A row is eligible to draw a frame once it has computed its whole + # prompt. Mirrors the engine's own rule; the prompt length is taken + # from the pair's state, which exists for every row whose request has + # already produced a logit. + eligible: list[bool] = [] + for row, entry in enumerate(constants): + state = self._states.get(entry.pair_id) + if state is None: + self._row_binding_complete = False + return + eligible.append(computed[row] + scheduled[row] >= state.prompt_tokens) + self._row_sample_eligible = eligible + self._row_binding_complete = True + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: Any | None = None, + inputs_embeds: torch.Tensor | None = None, + **kwargs: Any, + ) -> torch.Tensor: + del intermediate_tensors + self._capture_row_metadata(kwargs) + + if inputs_embeds is None: + hidden_states = self.model.embed_tokens(input_ids) + hidden_states = self._apply_frame_feedback(hidden_states) + else: + hidden_states = inputs_embeds + + residual: torch.Tensor | None = None + for layer in self.model.layers: + hidden_states, residual = layer(positions, hidden_states, residual) + norm_out = self.model.norm(hidden_states, residual) + if isinstance(norm_out, tuple): + norm_out = norm_out[0] + return norm_out + + def _capture_row_metadata(self, kwargs: dict[str, Any]) -> None: + """Read this step's per-row CFG role, pair id and seed. + + Row order here matches ``prepare_runner_inputs``: both are built from + ``input_batch.req_ids``. The request ids resolved there are preferred, + and the info dicts are only a fallback for paths that do not route + through that hook. + """ + if self._row_binding_complete: + # Every row was resolved from the per-request cache before the + # forward, so there is nothing here the sampler still needs. Bail + # out rather than rebuild it: this runs on every eager decode step. + return + infos = kwargs.get("model_intermediate_buffer") + if infos is None: + infos = kwargs.get("runtime_additional_information") + extra_args = kwargs.get("sampling_extra_args") + eligible = kwargs.get("request_sample_eligible") + self._row_sample_eligible = [bool(flag) for flag in eligible] if eligible else [] + if infos is None and extra_args is None: + # Warm-up dummy run: no omni kwargs are supplied. + self._row_pair_ids = [] + return + + infos = infos or [] + extra_args = extra_args or [] + rows = max(len(infos), len(extra_args), len(self._step_engine_ids)) + request_ids: list[str] = [] + roles: list[str] = [] + pair_ids: list[str] = [] + seeds: list[int] = [] + max_frames: list[int] = [] + for index in range(rows): + info = infos[index] if index < len(infos) else {} + info = info if isinstance(info, dict) else {} + extra = extra_args[index] if index < len(extra_args) else {} + extra = extra if isinstance(extra, dict) else {} + # The user-facing id. The engine renames a request to an internal + # UUID on admission, so this is the only id that both the info + # dicts and postprocess agree on, and the only one that carries + # the companion suffix a pair is recognized by. + request_id = _request_key(info.get("global_request_id")) or f"row-{index}" + pair_id = str(extra.get("cfg_pair_id") or _pair_id_from_request(request_id)) + request_ids.append(request_id) + roles.append(str(extra.get("cfg_role") or _role_from_request(request_id))) + pair_ids.append(pair_id) + seeds.append(int(extra.get("tts_local_seed") or 0)) + max_frames.append(_first_int(info.get("max_audio_frames"), DEFAULT_MAX_AUDIO_FRAMES)) + # Remember which pair each engine-side row belongs to, so the next + # step's feedback can be gathered before the info dicts are seen, + # and cache the whole row constant so a graph-replayed step can be + # bound without the info dicts at all. + if index < len(self._step_engine_ids): + engine_id = self._step_engine_ids[index] + self._pair_of_engine_id[engine_id] = pair_id + self._row_constants[engine_id] = _RowConstants( + request_id=request_id, + role=roles[index], + pair_id=pair_id, + seed=seeds[index], + max_frames=max_frames[index], + ) + self._row_request_ids = request_ids + self._row_roles = roles + self._row_pair_ids = pair_ids + self._row_seeds = seeds + self._row_max_frames = max_frames + + def _apply_frame_feedback(self, hidden_states: torch.Tensor) -> torch.Tensor: + """Replace each decoding row's token embedding with its frame embedding. + + The rows to replace and the codes to write were resolved in + ``prepare_runner_inputs`` against this step's own batch, so a mixed + prefill and decode step touches only the decoding rows and a prompt + token is never overwritten. + + Which of the two forms is read depends only on the row count, which is + a constant inside a captured graph: a decode batch fits the reserved + slab and is selected with ``torch.where`` over fixed buffers, while a + prefill batch wider than the slab falls back to a variable-length + ``index_copy``. Prefill is never replayed, so the fallback never has to + be graph-safe. + """ + rows = hidden_states.shape[0] + if rows <= self._max_rows: + frame_embeds = self.embed_frames(self.frame_feedback_codes[:rows]).to(dtype=hidden_states.dtype) + return torch.where( + self.frame_feedback_active[:rows].unsqueeze(1), + frame_embeds, + hidden_states, + ) + positions = self._feedback_positions + codes = self._feedback_codes + if positions is None or codes is None: + return hidden_states + frame_embeds = self.embed_frames(codes).to(dtype=hidden_states.dtype) + return hidden_states.index_copy(0, positions, frame_embeds) + + def embed_frames(self, codes: torch.Tensor) -> torch.Tensor: + """Compose one conditioning embedding per row from all eight codebooks. + + Args: + codes: ``[rows, 8]`` codebook indices, ``c0`` zero-based. + + Returns: + ``[rows, hidden]`` in the embedding table's dtype. + """ + c0 = self.model.embed_tokens(codes[:, 0] + AUDIO_CODE_OFFSET) + extra = self.audio_embeddings(codes[:, 1:] + self.audio_embedding_offsets) + return (c0 + extra.sum(dim=1).to(c0.dtype)) * self.frame_embedding_scale + + def compute_logits(self, hidden_states: torch.Tensor, sampling_metadata: Any = None) -> torch.Tensor: + # sample() has no other route to the backbone hidden state. + self._last_logits_hidden = hidden_states + return self.logits_processor(self.lm_head, hidden_states) + + def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor: + return self.model.embed_tokens(input_ids) + + # ----------------------------------------------------------------- sample + + def sample(self, logits: torch.Tensor, sampling_metadata: Any) -> SamplerOutput | None: + """Draw one frame per conditioned request and feed it back to both rows.""" + del sampling_metadata + hidden = self._last_logits_hidden + self._last_logits_hidden = None + if hidden is None or not self._row_pair_ids: + return None + + rows = int(logits.shape[0]) + pairs = self._resolve_pairs(rows) + if not pairs: + # Nothing to draw this step. Emit a benign audio code so every row + # stays alive until its partner is admitted; returning None would + # hand the rows to the stock sampler, whose token would be read as + # a real code. + held = torch.full((rows, 1), _HOLD_TOKEN, dtype=torch.long, device=logits.device) + return SamplerOutput(sampled_token_ids=held, logprobs_tensors=None) + + states = [self._state_for(self._row_pair_ids[pair[0]], pair[0]) for pair in pairs] + cond_rows, uncond_rows, seeds, positions = self._stage_step_vectors( + [pair[0] for pair in pairs], + [pair[1] for pair in pairs], + [state.seed for state in states], + [state.advance_position() for state in states], + logits.device, + ) + + narrowed = logits.index_select(1, self.c0_logit_ids).float() + c0_logits = apply_c0_cfg(narrowed.index_select(0, cond_rows), narrowed.index_select(0, uncond_rows)) + drawn = sample_topk_seeded(c0_logits, seeds, positions) + stopped = drawn == 0 + c0 = (drawn - 1).clamp_min(0) + + codes, depth_hidden = self._depth_decode( + hidden.index_select(0, cond_rows), + hidden.index_select(0, uncond_rows), + c0, + seeds, + positions, + ) + frame_hidden = torch.cat((hidden.index_select(0, cond_rows), depth_hidden), dim=-1) + + stopped_flags = stopped.tolist() + drain_rows: list[int] = [] + for index, (cond_row, uncond_row) in enumerate(pairs): + state = states[index] + if state.finished: + # The drain step. The terminal span written last step is on + # its way out in this step's payload, so the pair can stop now. + state.pending_stop = False + drain_rows.extend((index, cond_row, uncond_row)) + continue + if stopped_flags[index]: + state.finished = True + state.pending_stop = True + state.finish_reason = "stop" + state.last_codes = None + continue + # Both rows of the pair are fed the same frame next step; the + # codes live on the request, not on a row, because the row this + # request occupies may change before then. + state.last_codes = codes[index : index + 1] + state.frames.append(frame_hidden[index : index + 1]) + state.codes.append(codes[index : index + 1]) + state.generated_frames += 1 + if state.generated_frames >= state.max_frames: + state.finished = True + state.pending_stop = True + state.finish_reason = "length" + del cond_row, uncond_row + + # Rows that resolved into no pair keep this value. A hold token rather + # than id 0, which is an arbitrary text token: a row waiting for its + # partner should wait, not emit something the vocabulary reads as + # prose. It is never fed back as audio, because feedback is gated on + # the pair's own state. + token_ids = torch.full((rows, 1), _HOLD_TOKEN, dtype=torch.long, device=logits.device) + sampled = self.c0_logit_ids[drawn] + token_ids[cond_rows, 0] = sampled + token_ids[uncond_rows, 0] = sampled + + # The stop token is deferred by exactly one step. A pair that ends this + # step must stay alive so that the terminal span ``postprocess`` is + # about to write can ride out on the next step's payload; a pair that + # already published its tail stops now. + # + # This covers the two ways a song ends on its own: the audio-end token, + # and the request's frame budget, which reaches the model as + # ``max_audio_frames``. It does not cover the engine ending a request + # underneath the model, because the payload for a step is fixed before + # the model is told. Hitting ``max_model_len`` mid-song therefore drops + # the final window. In practice the frame budget is reached first; a + # request can only hit the context limit with a prompt long enough that + # the two limits collide, which the deploy config documents. + for index, (cond_row, uncond_row) in enumerate(pairs): + state = states[index] + if state.pending_stop: + token_ids[cond_row, 0] = _HOLD_TOKEN + token_ids[uncond_row, 0] = _HOLD_TOKEN + for _, cond_row, uncond_row in zip(drain_rows[0::3], drain_rows[1::3], drain_rows[2::3], strict=True): + token_ids[cond_row, 0] = _STOP_TOKEN + token_ids[uncond_row, 0] = _STOP_TOKEN + return SamplerOutput(sampled_token_ids=token_ids, logprobs_tensors=None) + + def _stage_step_vectors( + self, + cond_rows: list[int], + uncond_rows: list[int], + seeds: list[int], + positions: list[int], + device: torch.device, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Move this step's four small int64 vectors to the device in one copy. + + The copy is asynchronous, so the four returned views are only valid + once the stream reaches them. Every consumer is a kernel on that same + stream, which is what makes this safe. + """ + if self._sample_stage is None: + self._sample_stage = PinnedIndexStage(4, self._max_rows, device) + block = self._sample_stage.push([cond_rows, uncond_rows, seeds, positions], len(cond_rows)) + return block[0], block[1], block[2], block[3] + + def _resolve_pairs(self, rows: int) -> list[tuple[int, int]]: + """Map ``cfg_pair_id`` to its ``(cond_row, uncond_row)`` batch indices. + + Guidance is mandatory for this checkpoint, so an incomplete pair is a + hard error rather than something to route around. Skipping the step + would leave the conditioned request producing no frames and the client + waiting forever, which is the failure shape this module has been bitten + by before; raising surfaces it to the caller instead. + + Raises: + RuntimeError: If any pair is missing a member in this step. + """ + by_pair: dict[str, dict[str, int]] = {} + limit = min(rows, len(self._row_pair_ids)) + for row in range(limit): + # A row still finishing a chunked prefill is handed to the sampler + # but its token is discarded, so drawing a frame for it would add + # a bogus frame and desync the seeded RNG. + if row < len(self._row_sample_eligible) and not self._row_sample_eligible[row]: + continue + by_pair.setdefault(self._row_pair_ids[row], {})[self._row_roles[row]] = row + + pairs: list[tuple[int, int]] = [] + incomplete: list[str] = [] + for pair_id, roles in by_pair.items(): + if _COND in roles and _UNCOND in roles: + pairs.append((roles[_COND], roles[_UNCOND])) + else: + incomplete.append(f"{pair_id}({'+'.join(sorted(roles))})") + if incomplete: + # A member can be absent for a step while its partner is still + # being admitted. Holding the present row costs one wasted decode + # step; raising would abort the whole engine step for every other + # request in the batch, so the lone row simply does not sample this + # step and waits. A pair that never converges is bounded by the + # request's own token budget rather than hanging forever. + self._incomplete_steps += 1 + if self._incomplete_steps in (1, 64, 512): + logger.warning( + "MiniMax Music 3 waiting on incomplete CFG pair(s) %s " + "(step %d). If this does not resolve, the companion was " + "never enqueued: check that the stage declares " + "extra_args.cfg_role in its deploy config.", + incomplete[:4], + self._incomplete_steps, + ) + return pairs + + def _depth_decode( + self, + cond_hidden: torch.Tensor, + uncond_hidden: torch.Tensor, + c0: torch.Tensor, + seeds: torch.Tensor, + positions: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Walk ``c1..c7``, replaying the loop from a CUDA graph where possible. + + Seven four-layer forwards over at most eight positions is far too + little work per kernel to cover its own launch, so the loop is recorded + once per pair count and replayed after that. The cache falls back to + the eager loop for any shape it has not recorded or could not record. + """ + return self._depth_graphs.run(cond_hidden, uncond_hidden, c0, seeds, positions) + + def _depth_decode_eager( + self, + cond_hidden: torch.Tensor, + uncond_hidden: torch.Tensor, + c0: torch.Tensor, + seeds: torch.Tensor, + positions: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Walk ``c1..c7``, returning the codes and the stacked depth hiddens. + + Both branches run so the depth heads can be guided the same way the + ``c0`` head is; the drawn code is shared, so the two branches stay + aligned. + """ + decoder = self.rvq_decoder + pair_hidden = torch.cat((cond_hidden, uncond_hidden), dim=0) + c0_paired = c0.repeat(2) + c0_embed = self.model.embed_tokens(c0_paired + AUDIO_CODE_OFFSET) + seq = [ + decoder.projection(pair_hidden).unsqueeze(1), + decoder.projection(c0_embed).unsqueeze(1), + ] + n_pairs = int(c0.shape[0]) + codes = [c0] + depth_parts: list[torch.Tensor] = [] + # Every residual draw's noise comes from the same hash of + # ``(seed, position + codebook, candidate)``, so all seven are produced + # in one batch here instead of re-running the twenty-odd integer ops + # per codebook inside the loop. The arithmetic is elementwise, so the + # values are the same ones the per-codebook calls produced. + depth_gumbel = gumbel_noise( + seeds, + positions.unsqueeze(1) + self.depth_draw_offsets, + decoder.audio_vocab_size, + ) + for index in range(1, NUM_CODEBOOKS): + out = decoder(torch.cat(seq, dim=1))[:, -1] + depth_parts.append(out[:n_pairs].detach()) + head_logits = decoder.audio_heads[index - 1](out).float() + guided = apply_depth_cfg(head_logits[:n_pairs], head_logits[n_pairs:]) + code = draw_from_gumbel(guided, depth_gumbel[:, index - 1]) + codes.append(code) + if index < NUM_CODEBOOKS - 1: + embedding = self.audio_embeddings(code.repeat(2) + (index - 1) * AUDIO_VOCAB_SIZE) + seq.append(decoder.projection(embedding).unsqueeze(1)) + return torch.stack(codes, dim=1), torch.cat(depth_parts, dim=-1) + + def _state_for(self, pair_id: str, row: int) -> RequestARState: + """Fetch or create the state for a guided pair. + + Created on the step the pair first produces a logit, which is the step + its prefill completes, so the recorded prompt length is the real one. + """ + state = self._states.get(pair_id) + if state is None: + seed = self._row_seeds[row] if row < len(self._row_seeds) else 0 + state = RequestARState( + seed=seed, + max_frames=self._row_max_frames[row] if row < len(self._row_max_frames) else DEFAULT_MAX_AUDIO_FRAMES, + prompt_tokens=self._row_prompt_tokens[row] if row < len(self._row_prompt_tokens) else 0, + ) + self._states[pair_id] = state + logger.info( + "MiniMax Music 3 AR start pair=%s seed=%d max_frames=%d prompt_tokens=%d", + pair_id, + state.seed, + state.max_frames, + state.prompt_tokens, + ) + return state + + # ------------------------------------------------------------ emit / free + + def postprocess( + self, + hidden_states_slice: torch.Tensor, + multimodal_outputs: Any = None, + **req_infos: Any, + ) -> dict[str, Any]: + """Publish this request's next span of conditioning frames. + + Mid-stream this is one window, released a frame late so that a window + handed downstream is provably not the last. On the final step it is + instead a single contiguous span covering every window not yet + emitted, because one step produces exactly one payload and the tail + cannot be split across two. + """ + del hidden_states_slice, multimodal_outputs + request_id = _request_key(req_infos.get("global_request_id")) + # The pair's two rows share one song, so only the conditioned row + # publishes it. The companion would otherwise emit a duplicate span + # and the acoustic stage would decode the song twice. + if _role_from_request(request_id) == _UNCOND: + return {} + state = self._states.get(_pair_id_from_request(request_id)) + if state is None: + return {} + if state.finished: + return self._terminal_payload(state) + window = state.ready_window() + if window is None: + return {} + chunk = state.take_window(window, is_final=False) + return self._span_payload(state, chunk, start_frame=window.start, is_final=False) + + def _terminal_payload(self, state: RequestARState) -> dict[str, Any]: + """Emit every remaining frame as one span, once.""" + start = state.emitted_windows * AR_CHUNK_HOP_FRAMES + end = state.frames.end_frame + if end <= start: + return {} + chunk = state.frames.slice(start, end) + state.emitted_windows = len(chunk_windows(state.generated_frames)) + state.frames.clear() + return self._span_payload(state, chunk, start_frame=start, is_final=True) + + def _span_payload( + self, + state: RequestARState, + chunk: torch.Tensor, + *, + start_frame: int, + is_final: bool, + ) -> dict[str, Any]: + """Build the cross-stage payload for one contiguous frame span. + + ``num_processed_tokens`` is the span's first global AR frame index, so + the acoustic stage can re-derive window boundaries and therefore which + window is first and which is last. + """ + frames = int(chunk.shape[0]) + codes = state.take_codes(start_frame, start_frame + frames) + payload: dict[str, Any] = { + "latent": chunk.contiguous(), + "meta": { + "num_processed_tokens": start_frame, + "codec_chunk_frames": frames, + "last_chunk": is_final, + "audio_seed": state.seed, + }, + } + if codes is not None: + payload["codes"] = {"audio": codes} + return payload + + def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput, **kwargs: Any) -> OmniOutput: + if isinstance(model_outputs, OmniOutput): + return model_outputs + infos = kwargs.get("model_intermediate_buffer") + if infos is None: + infos = kwargs.get("runtime_additional_information") + if not infos: + return OmniOutput(text_hidden_states=model_outputs, multimodal_outputs=None) + + latents: list[torch.Tensor] = [] + codes: list[torch.Tensor] = [] + metas: list[dict[str, Any]] = [] + any_payload = False + for info in infos: + info = info if isinstance(info, dict) else {} + latent = info.get("latent") + if isinstance(latent, torch.Tensor) and latent.numel(): + latents.append(latent) + sub = info.get("codes") + codes.append(sub.get("audio") if isinstance(sub, dict) else torch.empty(0)) + meta = info.get("meta") + metas.append(dict(meta) if isinstance(meta, dict) else {}) + any_payload = True + # The runner merges postprocess output into this buffer and + # never clears it, so a window left behind would be re-emitted + # on every later step and the acoustic stage would decode the + # same conditioning again and again. Consume it here. + info.pop("latent", None) + info.pop("codes", None) + info.pop("meta", None) + else: + latents.append(torch.empty(0)) + codes.append(torch.empty(0)) + metas.append({}) + if not any_payload: + return OmniOutput(text_hidden_states=model_outputs, multimodal_outputs=None) + payload: dict[str, Any] = {"latent": latents} + if any(isinstance(entry, torch.Tensor) and entry.numel() for entry in codes): + payload["codes"] = {"audio": codes} + # The span's start frame and terminal flag ride along with it: without + # them the acoustic stage cannot tell which window it is looking at, + # and therefore cannot crop it correctly. + # + # Every field is a list with one entry per batch row, in the same order + # as ``latent``. The payload is partitioned per request by indexing + # list values, so a scalar here would be broadcast to the whole batch + # and every request would decode with the last request's seed, window + # offset and terminal flag. + meta_keys = {key for meta in metas for key in meta} + if meta_keys: + payload["meta"] = {key: [meta.get(key) for meta in metas] for key in sorted(meta_keys)} + return OmniOutput(text_hidden_states=model_outputs, multimodal_outputs=payload) + + def on_requests_finished(self, finished_req_ids: Iterable[str]) -> None: + """Release per-request state once the engine has retired the request. + + The key must be normalized the same way it was when the state was + created, or every finished song leaks its frame buffer. + """ + for engine_id in finished_req_ids: + # These are engine-side ids, so translate through the mapping the + # forward pass built. A pair's state is released when its second + # member retires. + self._row_constants.pop(str(engine_id), None) + pair_id = self._pair_of_engine_id.pop(str(engine_id), None) + if pair_id is None or any(pair_id == value for value in self._pair_of_engine_id.values()): + continue + request_id = pair_id + state = self._states.pop(pair_id, None) + if state is not None: + logger.info( + "MiniMax Music 3 AR done request=%s frames=%d reason=%s peak_buffer=%d", + request_id, + state.generated_frames, + state.finish_reason, + state.frames.peak_frames, + ) + state.release() + + # --------------------------------------------------------------- weights + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Load the Qwen3 backbone, then the audio modules from their component. + + The backbone comes from the stage's own checkpoint folder and loads + through vLLM. The audio embedding table and the depth decoder live in + a separate component of the repo and are loaded explicitly. + """ + loader = AutoWeightsLoader(self, skip_prefixes=["audio_embeddings.", "rvq_decoder."]) + loaded = loader.load_weights(weights) + loaded |= load_depth_decoder_weights( + self.vllm_config, + audio_embeddings=self.audio_embeddings, + rvq_decoder=self.rvq_decoder, + ) + return loaded diff --git a/vllm_omni/model_executor/models/minimax_music3/weights.py b/vllm_omni/model_executor/models/minimax_music3/weights.py new file mode 100644 index 000000000..2ab643c65 --- /dev/null +++ b/vllm_omni/model_executor/models/minimax_music3/weights.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Component weight loading for MiniMax Music 3. + +The checkpoint is a multi-component repo. The Qwen3 backbone is a normal +``Qwen3ForCausalLM`` under ``language_model/`` and loads through vLLM. The +audio embedding table and the depth decoder live in ``rvq_depth_decoder/``, +and the acoustic stage's three components live in their own folders. None of +those are things vLLM's loader knows how to map, so they are read directly. + +Every loader here is strict. A silently partial load would produce audio that +sounds plausible but is wrong, which is far worse than a startup failure. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import torch +from torch import nn +from vllm.config import VllmConfig +from vllm.logger import init_logger + +from .constants import AUDIO_VOCAB_SIZE, NUM_CODEBOOKS + +logger = init_logger(__name__) + +_AUDIO_EMBEDDING_KEY = "audio_embeddings.weight" +_COMPONENT_WEIGHT_NAME = "diffusion_pytorch_model.safetensors" + +# Component folders, relative to the repository root. +DEPTH_DECODER_DIR = "rvq_depth_decoder" +CONDITION_ENCODER_DIR = "condition_encoder" +TRANSFORMER_DIR = "transformer" +VOCODER_DIR = "vocoder" + +_ROOT_MARKERS = (DEPTH_DECODER_DIR, TRANSFORMER_DIR, VOCODER_DIR) + + +def resolve_repo_root(model_path: str | os.PathLike[str]) -> Path: + """Find the multi-component repository root from a stage's model path. + + A stage that declares ``model_subdir`` sees the subfolder as its model + path, so the root is usually the parent. Both are checked, and the search + walks up a couple of levels for snapshot layouts. + + Raises: + FileNotFoundError: If no ancestor looks like the repository root. + """ + path = Path(model_path).expanduser().resolve() + candidates = [path, *path.parents[:3]] + for candidate in candidates: + if all((candidate / marker).is_dir() for marker in _ROOT_MARKERS): + return candidate + raise FileNotFoundError( + f"MiniMax Music 3 could not locate the repository root from {path}; expected sibling folders {_ROOT_MARKERS}" + ) + + +def load_component_state(root: Path, component: str, *, device: str = "cpu") -> dict[str, torch.Tensor]: + """Read one component's safetensors, sharded or not. + + Raises: + FileNotFoundError: If the component has no weight files. + """ + from safetensors.torch import load_file + + folder = root / component + single = folder / _COMPONENT_WEIGHT_NAME + if single.exists(): + return load_file(str(single), device=device) + + shards = sorted(folder.glob(f"{Path(_COMPONENT_WEIGHT_NAME).stem}-*.safetensors")) + if not shards: + raise FileNotFoundError(f"MiniMax Music 3 component {component!r} has no weights under {folder}") + state: dict[str, torch.Tensor] = {} + for shard in shards: + state.update(load_file(str(shard), device=device)) + return state + + +def load_depth_decoder_weights( + vllm_config: VllmConfig, + *, + audio_embeddings: nn.Embedding, + rvq_decoder: nn.Module, +) -> set[str]: + """Load the audio embedding table and the RVQ depth decoder in place. + + Args: + vllm_config: The stage's config, used to locate the checkpoint. + audio_embeddings: The ``c1..c7`` embedding table to populate. + rvq_decoder: The depth decoder to populate. + + Returns: + The model-side parameter names that were loaded. + + Raises: + ValueError: If the audio embedding is absent or the wrong shape. + """ + root = resolve_repo_root(vllm_config.model_config.model) + state = load_component_state(root, DEPTH_DECODER_DIR) + + weight = state.pop(_AUDIO_EMBEDDING_KEY, None) + expected = (AUDIO_VOCAB_SIZE * (NUM_CODEBOOKS - 1), audio_embeddings.embedding_dim) + if weight is None or tuple(weight.shape) != expected: + raise ValueError( + f"MiniMax Music 3 audio embedding mismatch: expected {expected}, got " + f"{None if weight is None else tuple(weight.shape)}" + ) + target = audio_embeddings.weight + target.data.copy_(weight.to(device=target.device, dtype=target.dtype)) + + rvq_decoder.load_checkpoint_state(state) + logger.info( + "MiniMax Music 3 loaded audio embedding and depth decoder from %s", + root / DEPTH_DECODER_DIR, + ) + loaded = {"audio_embeddings.weight"} + loaded.update(f"rvq_decoder.{name}" for name, _ in rvq_decoder.named_parameters()) + return loaded + + +__all__ = [ + "CONDITION_ENCODER_DIR", + "DEPTH_DECODER_DIR", + "TRANSFORMER_DIR", + "VOCODER_DIR", + "load_component_state", + "load_depth_decoder_weights", + "resolve_repo_root", +] diff --git a/vllm_omni/model_executor/models/registry.py b/vllm_omni/model_executor/models/registry.py index 3cf888d46..0064f1c4a 100644 --- a/vllm_omni/model_executor/models/registry.py +++ b/vllm_omni/model_executor/models/registry.py @@ -457,6 +457,17 @@ _OMNI_MODELS = { "qwen3_vl", "AuraQwen3VLForConditionalGeneration", ), + ## MiniMax-Music3 (text-to-music; AR talker -> flow-matching acoustic decoder) + "MiniMaxMusic3TalkerForConditionalGeneration": ( + "minimax_music3", + "talker", + "MiniMaxMusic3TalkerForConditionalGeneration", + ), + "MiniMaxMusic3AcousticForConditionalGeneration": ( + "minimax_music3", + "acoustic", + "MiniMaxMusic3AcousticForConditionalGeneration", + ), } diff --git a/vllm_omni/model_executor/stage_input_processors/minimax_music3.py b/vllm_omni/model_executor/stage_input_processors/minimax_music3.py new file mode 100644 index 000000000..44037e4fc --- /dev/null +++ b/vllm_omni/model_executor/stage_input_processors/minimax_music3.py @@ -0,0 +1,544 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Stage-input processors for MiniMax Music 3: AR talker -> acoustic decoder. + +Three entry points, all referenced from +``vllm_omni/model_executor/models/minimax_music3/pipeline.py``: + +* ``expand_cfg_prompts`` (stage 0 ``prompt_expand_func``) builds the + unconditioned twin of a conditioned request. Classifier-free guidance is + mandatory for this checkpoint, so in practice every request expands. +* ``ar2acoustic_async_chunk`` (stage 0 ``async_chunk_process_next_stage_input_func``) + forwards one AR frame span per call to stage 1, and is also the flush point + for the request's tail. +* ``ar2acoustic`` (stage 1 ``sync_process_input_func``) builds the stage-1 + engine input for the non-streaming path, where the whole song arrives at + once after stage 0 finishes. + +Stage-0 emit contract +--------------------- +Per emission the talker publishes a multimodal output shaped like:: + + { + "latent": FloatTensor[1, T, 32768] or FloatTensor[T, 32768], + "codes": {"audio": LongTensor[T, 8]}, + "meta": { + "num_processed_tokens": int, # global AR frame index of frame 0 + "last_chunk": bool, # True only on the terminal emission + "audio_seed": int, # optional + }, + } + +The span is normally one window: ``T == AR_CHUNK_FRAMES``, windows advance by +``AR_CHUNK_HOP_FRAMES``, and window ``k`` starts at AR frame +``k * AR_CHUNK_HOP_FRAMES``. The terminal emission is the exception. A window +cannot be proven non-final until a further frame exists, so the talker holds +the most recently completed window back and, on the step where it knows the +request is ending, publishes the whole remaining frame span at once. That span +covers up to two windows (the held-back complete one plus the truncated final +one), so ``T`` can reach ``AR_CHUNK_FRAMES + AR_CHUNK_HOP_FRAMES``. Stage 1 +re-derives the window boundaries inside a span from +``meta.num_processed_tokens`` and the span length through +``models/minimax_music3/chunking.chunk_windows``. + +``meta.num_processed_tokens`` is required rather than derived: once a terminal +span carries more than one window, a running ``chunk_seq * hop`` count no +longer identifies the span's first frame. The derived value is kept only as a +fallback for pre-terminal emissions. + +``hidden_states.output`` is accepted as an alias for ``latent`` because both +are legal homes for a float payload; ``latent`` is canonical and matches the +stage's ``engine_output_type``. + +Stage-0 -> stage-1 payload +-------------------------- +Every key below is a declared field of ``OmniPayloadStruct`` / +``MetaStruct``, which are ``forbid_unknown_fields=True`` msgspec structs, so +an undeclared key is a construction error rather than silent data loss:: + + latent FloatTensor[T, 32768] span conditioning, bf16 + codes.audio LongTensor[T, 8] RVQ codes for the span + meta.chunk_seq int emission index; is_first == (chunk_seq == 0) + meta.num_processed_tokens int absolute AR frame index of the span start + meta.codec_chunk_frames int T; span end (exclusive) = start + T + meta.last_chunk bool terminal span + meta.stream_finished BoolTensor terminal marker that survives transport + meta.audio_seed int seed for the acoustic stage's noise draw + meta.req_id list[str] single-element external request id + meta.finished BoolTensor overwritten by the chunk adapter + +``codes.audio`` is rank 2 on purpose: the chunk transfer adapter reroutes rank-1 +code tensors onto the token path (``.tolist()``), while rank-2 tensors ride the +payload channel and leave a single ``[0]`` placeholder prompt token per chunk. +``meta.finished`` is stripped before the payload reaches the stage-1 model, so +``meta.stream_finished`` and ``meta.last_chunk`` are what stage 1 reads. + +The overlap crop is deliberately NOT in the payload. Stage 1 recovers it from +the span's global frame range and ``last_chunk`` through +``models/minimax_music3/chunking.crop_sample_bounds``, which is the single +owner of that arithmetic; duplicating it here would let the two copies drift. +""" + +from __future__ import annotations + +from typing import Any + +import torch +from vllm.logger import init_logger + +from vllm_omni.data_entry_keys import CodesStruct, MetaStruct, OmniPayloadStruct +from vllm_omni.inputs.data import OmniTokensPrompt +from vllm_omni.model_executor.models.minimax_music3.constants import ( + AR_CFG_SCALE, + AR_CHUNK_HOP_FRAMES, + AR_HIDDEN_SIZE, + NUM_CODEBOOKS, +) +from vllm_omni.model_executor.models.minimax_music3.prompt import build_cfg_null_token_ids +from vllm_omni.model_executor.stage_input_processors.bagel import ExpandedPrompt + +__all__ = [ + "CFG_UNCOND_SUFFIX", + "ar2acoustic", + "ar2acoustic_async_chunk", + "expand_cfg_prompts", +] + +logger = init_logger(__name__) + +CFG_UNCOND_SUFFIX = "__cfg_uncond" + +_COND = "cond" +_UNCOND = "uncond" + +# Per-request forwarder state, parked on the transfer manager's request map. +_STATE_KEY = "_minimax_music3_async_state" + +# The window conditioning is emitted in bfloat16. Each chunk is its own +# /dev/shm segment and consecutive windows overlap by half, so a float32 +# 200-frame window would move 26 MB of mostly-redundant bytes per hop against +# 13 MB in bfloat16. The backbone runs in bfloat16 anyway, so this discards no +# information the AR stage actually produced. +_LATENT_TRANSPORT_DTYPE = torch.bfloat16 + + +# --------------------------------------------------------------------------- +# Stage 0: classifier-free-guidance prompt expansion +# --------------------------------------------------------------------------- + + +def _prompt_token_ids(prompt: dict[str, Any] | str | Any) -> list[int] | None: + """Return the prompt's token ids, or ``None`` when only text is available.""" + token_ids: Any = None + if isinstance(prompt, dict): + token_ids = prompt.get("prompt_token_ids") + elif isinstance(prompt, (list, tuple)): + token_ids = prompt + if token_ids is None: + return None + if isinstance(token_ids, torch.Tensor): + token_ids = token_ids.reshape(-1).tolist() + if not isinstance(token_ids, (list, tuple)) or not token_ids: + return None + return [int(t) for t in token_ids] + + +def _uncond_extra_args(extra_args: dict[str, Any], cfg_scale: float) -> dict[str, Any]: + """Clone the conditioned row's extra_args for the unconditioned twin. + + Every key is carried through, not only the ``cfg_*`` ones: the stage runs + with ``has_sampling_extra_args`` on, so extra_args reach the model's + ``forward`` per row, and dropping a key here would make the two rows + execute under different settings and decode out of lockstep. + ``cfg_null_prompt`` is the exception: the companion IS the null prompt. + """ + companion = dict(extra_args) + companion.pop("cfg_null_prompt", None) + companion["cfg_role"] = _UNCOND + companion["cfg_scale"] = cfg_scale + return companion + + +def expand_cfg_prompts( + prompt: dict[str, Any] | str, + sampling_params: Any, +) -> list[ExpandedPrompt]: + """Emit the unconditioned companion for a guided MiniMax Music 3 request. + + Classifier-free guidance is not optional for this checkpoint, and the + deploy YAMLs set ``cfg_role: cond`` / ``cfg_scale: 1.5`` as stage-0 + defaults, so an ordinary request expands into exactly one companion and + the pair decodes in lockstep. + + The companion's token ids come from ``build_cfg_null_token_ids``, which + overwrites the caption-and-lyrics span with ``<|audio_cfg|>`` while keeping + the framing tokens. Token ids are preferred over text because the two rows + are only blendable when they have identical length and an identical + ``<|audio_start|>`` position, and only the token-id form guarantees that + without a tokenizer round trip. + + Args: + prompt: The raw user prompt. A dict carrying ``prompt_token_ids`` is + the preferred form; a text-only prompt must supply a pre-built, + length-matched ``cfg_null_prompt`` in ``extra_args``. + sampling_params: Stage-0 sampling params for the conditioned row. + + Returns: + A one-element list with the unconditioned companion, or an empty list + when the request is not CFG-guided. + + Raises: + ValueError: If the request is guided but neither prompt token ids nor + a pre-built ``cfg_null_prompt`` is available. + """ + extra_args = getattr(sampling_params, "extra_args", None) or {} + if extra_args.get("cfg_role") != _COND: + # Not guided, or this IS the companion. Either way, nothing to expand. + return [] + try: + cfg_scale = float(extra_args.get("cfg_scale", AR_CFG_SCALE)) + except (TypeError, ValueError): + logger.warning( + "MiniMax Music 3: ignoring non-numeric cfg_scale %r; request decodes unguided", + extra_args.get("cfg_scale"), + ) + return [] + if cfg_scale <= 1.0: + return [] + + token_ids = _prompt_token_ids(prompt) + companion_prompt: dict[str, Any] | str + if token_ids is not None: + null_ids = build_cfg_null_token_ids(token_ids) + if isinstance(prompt, dict): + companion_prompt = {**prompt, "prompt_token_ids": null_ids} + # Drop the text form so the input processor does not re-tokenize + # and hand back a row of a different length. + companion_prompt.pop("prompt", None) + companion_prompt.pop("prompt_embeds", None) + else: + companion_prompt = {"prompt_token_ids": null_ids} + else: + null_prompt = extra_args.get("cfg_null_prompt") + if not isinstance(null_prompt, str) or not null_prompt: + raise ValueError( + "MiniMax Music 3 CFG request carries neither prompt_token_ids nor a " + "pre-built cfg_null_prompt in extra_args. Submit the prompt as token " + "ids (preferred; the null twin is then built by " + "models.minimax_music3.prompt.build_cfg_null_token_ids) or attach a " + "length-matched cfg_null_prompt string." + ) + if isinstance(prompt, dict): + companion_prompt = {**prompt, "prompt": null_prompt} + companion_prompt.pop("prompt_token_ids", None) + companion_prompt.pop("prompt_embeds", None) + else: + companion_prompt = null_prompt + + return [ + ExpandedPrompt( + prompt=companion_prompt, + role=_UNCOND, + request_id_suffix=CFG_UNCOND_SUFFIX, + sampling_params_override={"extra_args": _uncond_extra_args(extra_args, cfg_scale)}, + ) + ] + + +# --------------------------------------------------------------------------- +# Stage 0 -> stage 1: async-chunk window forwarder +# --------------------------------------------------------------------------- + + +def _payload_section(payload: Any, key: str) -> dict[str, Any]: + """Return a nested payload section as a dict (empty when absent).""" + if not isinstance(payload, dict): + return {} + section = payload.get(key) + return section if isinstance(section, dict) else {} + + +def _window_latent(payload: Any) -> torch.Tensor | None: + """Return this window's conditioning as ``[T, 32768]``, or ``None``. + + ``latent`` is the canonical key; ``hidden_states.output`` is accepted for + talkers that publish through the hidden-state channel instead. A leading + singleton batch dimension is squeezed so stage 1 sees a stable rank. + """ + if not isinstance(payload, dict): + return None + tensor = payload.get("latent") + if not isinstance(tensor, torch.Tensor): + tensor = _payload_section(payload, "hidden_states").get("output") + if not isinstance(tensor, torch.Tensor) or tensor.numel() == 0: + return None + if tensor.ndim == 3: + if tensor.shape[0] != 1: + raise ValueError( + f"MiniMax Music 3 window latent must have a singleton batch dim; got {tuple(tensor.shape)}" + ) + tensor = tensor[0] + if tensor.ndim != 2: + raise ValueError(f"MiniMax Music 3 window latent must be rank 2 or 3; got {tuple(tensor.shape)}") + if tensor.shape[1] != AR_HIDDEN_SIZE: + raise ValueError(f"MiniMax Music 3 window latent has {tensor.shape[1]} features; expected {AR_HIDDEN_SIZE}") + return tensor + + +def _window_codes(payload: Any) -> torch.Tensor | None: + """Return this window's RVQ codes as ``[T, 8]``, or ``None``.""" + codes = _payload_section(payload, "codes").get("audio") + if not isinstance(codes, torch.Tensor) or codes.numel() == 0: + return None + if codes.ndim == 1: + if codes.numel() % NUM_CODEBOOKS != 0: + raise ValueError( + f"MiniMax Music 3 flat code payload of length {codes.numel()} is not a " + f"multiple of {NUM_CODEBOOKS} codebooks" + ) + codes = codes.reshape(-1, NUM_CODEBOOKS) + if codes.ndim != 2 or codes.shape[1] != NUM_CODEBOOKS: + raise ValueError(f"MiniMax Music 3 codes must be [T, {NUM_CODEBOOKS}]; got {tuple(codes.shape)}") + return codes.to(torch.long) + + +def _request_audio_seed(request: Any, payload: Any) -> int | None: + """Resolve the seed the acoustic stage draws its noise from. + + A seed published by the talker wins, because a talker that already + consumed the request seed knows which value it used. Otherwise fall back + to the stage-0 request's own sampling seed. + """ + published = _payload_section(payload, "meta").get("audio_seed") + if isinstance(published, int): + return published + seed = getattr(getattr(request, "sampling_params", None), "seed", None) + return int(seed) if isinstance(seed, int) else None + + +def _forwarder_state(transfer_manager: Any, request_id: str) -> dict[str, Any]: + """Return (creating if needed) the per-request forwarder state.""" + request_state = transfer_manager.request_payload.get(request_id) + if not isinstance(request_state, dict) or _STATE_KEY not in request_state: + request_state = {_STATE_KEY: {"chunk_seq": 0, "terminal_sent": False}} + transfer_manager.request_payload[request_id] = request_state + return request_state[_STATE_KEY] + + +def ar2acoustic_async_chunk( + transfer_manager: Any, + multimodal_output: dict[str, Any] | None, + request: Any, + is_finished: bool = False, +) -> OmniPayloadStruct | None: + """Forward one AR frame span to the acoustic stage. + + The talker publishes whole spans, so this is a relay plus bookkeeping + rather than an accumulator: it stamps the emission index, the span's global + start frame, its length and the terminal flags, then hands the tensors to + the connector. + + This hook is the flush point for the request's tail. The AR scheduler calls + ``save_async`` on the step that stops the request, including when that + step produced no inter-stage output, so this function is guaranteed one + call with ``is_finished=True`` (omni_ar_scheduler.py:551, base + chunk_transfer_adapter.py:341). Whatever the talker published on that step + therefore reaches stage 1 in the same hop. + + Returns ``None`` on steps where the talker released no span and the request + is still running. + """ + request_id = request.external_req_id + finished = bool(is_finished or request.is_finished()) + + state = _forwarder_state(transfer_manager, request_id) + + latent = _window_latent(multimodal_output) + codes = _window_codes(multimodal_output) + published_meta = _payload_section(multimodal_output, "meta") + published_last = published_meta.get("last_chunk") is True + + if state["terminal_sent"]: + if latent is not None: + logger.error( + "MiniMax Music 3 talker emitted %d more frames for request %s after the " + "terminal span; the downstream stream is already closed and they are dropped", + int(latent.shape[0]), + request_id, + ) + return None + + if latent is None: + if not finished: + return None + # Terminal step with nothing left to send: still deliver the marker so + # stage 1 can flush and complete instead of polling until timeout. + state["terminal_sent"] = True + # The unconditioned twin never publishes a span: the pair shares one + # song and the conditioned row emits it. Its silence is the contract, + # not a failure. + if state["chunk_seq"] == 0 and not request_id.endswith(CFG_UNCOND_SUFFIX): + logger.error("MiniMax Music 3 talker produced no audio for request %s", request_id) + return OmniPayloadStruct( + meta=MetaStruct( + chunk_seq=state["chunk_seq"], + last_chunk=True, + stream_finished=torch.tensor(True, dtype=torch.bool), + finished=torch.tensor(True, dtype=torch.bool), + req_id=[request_id], + ) + ) + + span_frames = int(latent.shape[0]) + if codes is not None and codes.shape[0] != span_frames: + raise ValueError( + f"MiniMax Music 3 span mismatch for request {request_id}: latent has " + f"{span_frames} frames but codes have {codes.shape[0]}" + ) + + chunk_seq = int(state["chunk_seq"]) + state["chunk_seq"] = chunk_seq + 1 + + # A span is terminal when the engine stopped the request or the talker said + # so. Only the engine's verdict closes the stream, so a talker that flags + # a span last while generation continues is reported rather than obeyed. + terminal = bool(finished or published_last) + if published_last and not finished: + logger.warning( + "MiniMax Music 3 talker flagged a terminal span for request %s while the " + "engine still has the request running", + request_id, + ) + state["terminal_sent"] = terminal + + span_start = published_meta.get("num_processed_tokens") + if not isinstance(span_start, int): + # Fallback for talkers that publish one hop-aligned window per step. + # It is only sound before the terminal span, which may cover more than + # one window and therefore break the running count. + span_start = chunk_seq * AR_CHUNK_HOP_FRAMES + if terminal and span_frames > AR_CHUNK_HOP_FRAMES: + logger.warning( + "MiniMax Music 3 terminal span for request %s carries %d frames but no " + "meta.num_processed_tokens; the derived start frame %d assumes a single " + "hop-aligned window and may misplace the tail", + request_id, + span_frames, + span_start, + ) + + finished_flag = torch.tensor(terminal, dtype=torch.bool) + meta = MetaStruct( + chunk_seq=chunk_seq, + num_processed_tokens=int(span_start), + codec_chunk_frames=span_frames, + last_chunk=terminal, + # meta.finished is stripped before the payload reaches the stage-1 + # model; stream_finished is what survives and drives the final flush. + stream_finished=finished_flag, + finished=finished_flag, + req_id=[request_id], + ) + audio_seed = _request_audio_seed(request, multimodal_output) + if audio_seed is not None: + meta.audio_seed = audio_seed + + payload = OmniPayloadStruct( + latent=latent.detach().to(_LATENT_TRANSPORT_DTYPE).contiguous(), + meta=meta, + ) + if codes is not None: + payload.codes = CodesStruct(audio=codes.detach().contiguous()) + return payload + + +# --------------------------------------------------------------------------- +# Stage 1: non-streaming engine-input builder +# --------------------------------------------------------------------------- + + +def _empty_acoustic_prompt() -> OmniTokensPrompt: + """Zero-length stage-1 request. + + Stage 1 decodes it to empty audio, which the serving layer's no-audio + guard turns into a request-level error. That is preferable to raising + here, where the failure would surface as a stalled request. + """ + return OmniTokensPrompt( + prompt_token_ids=[], + multi_modal_data=None, + mm_processor_kwargs=None, + additional_information=None, + ) + + +def ar2acoustic( + source_outputs: list[Any], + prompt: Any = None, + _requires_multimodal_data: bool = False, +) -> list[OmniTokensPrompt]: + """Build stage-1 engine inputs for the non-streaming path. + + With ``async_chunk: false`` the whole song arrives once stage 0 has + finished, so the song is handed over as a single logical window and stage + 1 does its own windowing through ``chunking.chunk_windows``. The payload + layout matches the streaming one key for key, so the acoustic stage reads + one contract in both modes. + + The prompt is ``[0] * num_windows``: the generation scheduler sizes the + request from ``prompt_token_ids``, and one placeholder per acoustic window + keeps that budget proportional to the work without shipping the + conditioning through the token path. + """ + del prompt, _requires_multimodal_data + + acoustic_inputs: list[OmniTokensPrompt] = [] + for ar_output in source_outputs: + if not ar_output.finished: + continue + output = ar_output.outputs[0] + multimodal_output = getattr(output, "multimodal_output", None) + + try: + latent = _window_latent(multimodal_output) + codes = _window_codes(multimodal_output) + except ValueError as exc: + logger.warning("Skipping malformed MiniMax Music 3 stage-0 output: %s", exc) + acoustic_inputs.append(_empty_acoustic_prompt()) + continue + + if latent is None: + logger.error("MiniMax Music 3 talker produced no audio for request %s", ar_output.request_id) + acoustic_inputs.append(_empty_acoustic_prompt()) + continue + + total_frames = int(latent.shape[0]) + meta: dict[str, Any] = { + "chunk_seq": 0, + "num_processed_tokens": 0, + "codec_chunk_frames": total_frames, + "last_chunk": True, + "stream_finished": torch.tensor(True, dtype=torch.bool), + "req_id": [ar_output.request_id], + } + published_seed = _payload_section(multimodal_output, "meta").get("audio_seed") + if isinstance(published_seed, int): + meta["audio_seed"] = published_seed + + additional_information: dict[str, Any] = { + "latent": latent.detach().to(_LATENT_TRANSPORT_DTYPE).contiguous(), + "meta": meta, + } + if codes is not None: + additional_information["codes"] = {"audio": codes.detach().contiguous()} + + num_windows = max(1, -(-total_frames // AR_CHUNK_HOP_FRAMES)) + acoustic_inputs.append( + OmniTokensPrompt( + prompt_token_ids=[0] * num_windows, + multi_modal_data=None, + mm_processor_kwargs=None, + additional_information=additional_information, + ) + ) + return acoustic_inputs diff --git a/vllm_omni/transformers_utils/configs/__init__.py b/vllm_omni/transformers_utils/configs/__init__.py index cdf105d69..2a31b461c 100644 --- a/vllm_omni/transformers_utils/configs/__init__.py +++ b/vllm_omni/transformers_utils/configs/__init__.py @@ -23,6 +23,7 @@ _CLASS_TO_MODULE: dict[str, str] = { "DotsTTSConfig": "vllm_omni.transformers_utils.configs.dots_tts", "VoxtralTTSConfig": "vllm_omni.transformers_utils.configs.voxtral_tts", "CosyVoice3Config": "vllm_omni.transformers_utils.configs.cosyvoice3", + "MiniMaxMusic3Config": "vllm_omni.transformers_utils.configs.minimax_music3", "OmniVoiceConfig": "vllm_omni.transformers_utils.configs.omnivoice", "BailingMoeV2Config": "vllm_omni.transformers_utils.configs.ming_flash_omni", "BailingMM2Config": "vllm_omni.transformers_utils.configs.ming_flash_omni", @@ -46,6 +47,7 @@ __all__ = [ "DotsTTSConfig", "VoxtralTTSConfig", "CosyVoice3Config", + "MiniMaxMusic3Config", "OmniVoiceConfig", "BailingMoeV2Config", "BailingMM2Config", @@ -78,6 +80,7 @@ from vllm_omni.transformers_utils.configs import glm_tts as _glm_tts # noqa: F4 from vllm_omni.transformers_utils.configs import higgs_audio_v3 as _higgs_audio_v3 # noqa: F401, E402 from vllm_omni.transformers_utils.configs import mammoth_moda2 as _mammoth_moda2 # noqa: F401, E402 from vllm_omni.transformers_utils.configs import ming_flash_omni as _ming_flash_omni # noqa: F401, E402 +from vllm_omni.transformers_utils.configs import minimax_music3 as _minimax_music3 # noqa: F401, E402 from vllm_omni.transformers_utils.configs import omnivoice as _omnivoice # noqa: F401, E402 from vllm_omni.transformers_utils.configs import voxcpm2 as _voxcpm2 # noqa: F401, E402 from vllm_omni.transformers_utils.configs import voxtral_tts as _voxtral_tts # noqa: F401, E402 diff --git a/vllm_omni/transformers_utils/configs/minimax_music3.py b/vllm_omni/transformers_utils/configs/minimax_music3.py new file mode 100644 index 000000000..be951055d --- /dev/null +++ b/vllm_omni/transformers_utils/configs/minimax_music3.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""HuggingFace config for the MiniMax Music 3 composite repository. + +The checkpoint root carries only:: + + {"architectures": ["MiniMaxMusic3ForConditionalGeneration"], + "model_type": "minimax_music3"} + +and ships no modelling code, so ``trust_remote_code`` has nothing to load and +``AutoConfig`` rejects the repository outright. Stage 0 sidesteps this by +declaring ``model_subdir="language_model"`` and reading the plain +``Qwen3ForCausalLM`` config, but the acoustic stage must open the root to reach +``condition_encoder/``, ``transformer/`` and ``vocoder/``. Registering this +class is what lets that stage build a ``ModelConfig`` at all. + +The generic transformer fields describe the AR backbone, because the composite +root has no transformer of its own: they are metadata for vLLM's config +plumbing, not a description of the acoustic stage. The acoustic stage holds no +vLLM attention layers, so no KV cache is derived from them; it reads its real +geometry from ``models/minimax_music3/constants.py`` and its weights from the +component folders. +""" + +from __future__ import annotations + +from typing import Any + +from transformers import AutoConfig +from transformers.configuration_utils import PretrainedConfig + + +class MiniMaxMusic3Config(PretrainedConfig): + """Root config for the multi-component MiniMax Music 3 repository.""" + + model_type = "minimax_music3" + + def __init__(self, **kwargs: Any) -> None: + # AR backbone geometry (language_model/config.json). Kept in sync with + # the checkpoint so max_model_len validation sees the real context. + kwargs.setdefault("hidden_size", 4096) + kwargs.setdefault("num_hidden_layers", 36) + kwargs.setdefault("num_attention_heads", 32) + kwargs.setdefault("num_key_value_heads", 8) + kwargs.setdefault("head_dim", 128) + kwargs.setdefault("intermediate_size", 12288) + kwargs.setdefault("vocab_size", 200000) + kwargs.setdefault("max_position_embeddings", 10240) + kwargs.setdefault("tie_word_embeddings", False) + super().__init__(**kwargs) + + # Audio contract, mirroring models/minimax_music3/constants.py. These + # are fixed by the checkpoint; they are exposed here only so a caller + # inspecting the config sees the same numbers the model uses. + self.audio_frame_rate = 25 + self.num_codebooks = 8 + self.audio_vocab_size = 1024 + self.c0_vocab_size = 16384 + self.sample_rate = 32000 + self.audio_channels = 2 + + +AutoConfig.register("minimax_music3", MiniMaxMusic3Config) + +__all__ = ["MiniMaxMusic3Config"] From 0a1846aa458283a13c1a7b5a96ab585937f9726c Mon Sep 17 00:00:00 2001 From: Yukim1 <121286183+zwhzzz0821@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:00:24 +0800 Subject: [PATCH 054/267] [Diffusion] Add native KV cache initialization and Scheduler-managed block allocation (#6094) Signed-off-by: zwhzzz0821 Co-authored-by: zwhzzz0821 --- docs/configuration/stage_configs.md | 7 +- .../diffusion_kv/test_initialization.py | 297 ++++++++++++++++++ tests/diffusion/diffusion_kv/test_manager.py | 211 +++++++++++++ tests/diffusion/diffusion_kv/test_request.py | 7 +- .../diffusion_kv/test_spec_discovery.py | 259 +++++++++++++++ .../diffusion_kv/test_worker_contract.py | 69 +++- .../test_image_kv_cache_manager.py | 8 + .../test_diffusion_config_propagation.py | 15 + .../test_diffusion_engine_cleanup.py | 74 +++++ .../test_diffusion_engine_dummy_run.py | 156 +++++++++ .../test_diffusion_engine_rpc_routing.py | 32 ++ .../diffusion/test_diffusion_model_runner.py | 79 +++++ tests/diffusion/test_diffusion_request.py | 1 + tests/diffusion/test_diffusion_scheduler.py | 281 ++++++++++++++++- tests/diffusion/test_diffusion_worker.py | 80 +---- .../test_multiproc_engine_concurrency.py | 20 ++ vllm_omni/config/omni_config.py | 4 + .../diffusion/attention/backends/abstract.py | 13 + vllm_omni/diffusion/attention/layer.py | 32 ++ vllm_omni/diffusion/data.py | 16 + vllm_omni/diffusion/diffusion_engine.py | 193 ++++++++++-- .../diffusion/diffusion_kv/initialization.py | 82 +++++ vllm_omni/diffusion/diffusion_kv/manager.py | 191 +++++++++++ vllm_omni/diffusion/diffusion_kv/request.py | 2 +- vllm_omni/diffusion/executor/abstract.py | 35 +++ .../diffusion/executor/multiproc_executor.py | 11 +- vllm_omni/diffusion/io_support.py | 7 + .../hunyuan_image3_transformer.py | 9 +- vllm_omni/diffusion/request.py | 5 +- vllm_omni/diffusion/sched/base_scheduler.py | 95 +++++- vllm_omni/diffusion/vllm_config.py | 180 +++++++++++ .../worker/diffusion_model_runner.py | 145 ++++++++- .../diffusion/worker/diffusion_worker.py | 258 +++++++++------ 33 files changed, 2609 insertions(+), 265 deletions(-) create mode 100644 tests/diffusion/diffusion_kv/test_initialization.py create mode 100644 tests/diffusion/diffusion_kv/test_manager.py create mode 100644 tests/diffusion/diffusion_kv/test_spec_discovery.py create mode 100644 vllm_omni/diffusion/diffusion_kv/initialization.py create mode 100644 vllm_omni/diffusion/diffusion_kv/manager.py create mode 100644 vllm_omni/diffusion/vllm_config.py diff --git a/docs/configuration/stage_configs.md b/docs/configuration/stage_configs.md index 290c15b94..38eceb634 100644 --- a/docs/configuration/stage_configs.md +++ b/docs/configuration/stage_configs.md @@ -38,11 +38,12 @@ Each entry under `stages:` accepts any `StageDeployConfig` field directly (no ne |-------|------|----------|---------|-------------| | `stage_id` | int | required | — | Stage identity; matched against `PipelineConfig.stages[*].stage_id`. | | `max_num_seqs` | int | optional | `64` | Max concurrent sequences per stage. | -| `gpu_memory_utilization` | float | optional | `0.9` | Per-stage memory budget. | +| `gpu_memory_utilization` | float | optional | `0.9` | Per-stage total memory target; used for automatic KV-cache sizing. | +| `kv_cache_memory_bytes` | int \| null | optional | `null` | Explicit per-rank KV-cache budget in bytes; overrides automatic sizing. | | `tensor_parallel_size` | int | optional | `1` | TP degree for this stage. | | `enforce_eager` | bool | optional | `false` | Disable CUDA graphs. | -| `max_num_batched_tokens` | int | optional | `32768` | Prefill budget. | -| `max_model_len` | int \| null | optional | `null` | Per-stage context length (auto-sets `VLLM_ALLOW_LONG_MAX_MODEL_LEN=1` when larger than HF default). | +| `max_num_batched_tokens` | int | optional | `32768` | Per-stage prefill/token budget; also contributes to the native maximum in-flight token limit. | +| `max_model_len` | int \| null | optional | `null` | Per-sequence context or KV length; `-1` enables native cache-capacity auto-fitting, while values above the HF default auto-set `VLLM_ALLOW_LONG_MAX_MODEL_LEN=1`. | | `async_scheduling` | bool \| null | optional | `null` | Per-stage async scheduling toggle. | | `devices` | str | optional | `"0"` | `CUDA_VISIBLE_DEVICES`-style device list. | | `output_connectors` | dict \| null | optional | `null` | Keyed by `to_stage_`; values are names registered under top-level `connectors:`. | diff --git a/tests/diffusion/diffusion_kv/test_initialization.py b/tests/diffusion/diffusion_kv/test_initialization.py new file mode 100644 index 000000000..126b25d18 --- /dev/null +++ b/tests/diffusion/diffusion_kv/test_initialization.py @@ -0,0 +1,297 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch +from vllm.v1.kv_cache_interface import FullAttentionSpec + +import vllm_omni.diffusion.diffusion_kv.initialization as initialization +import vllm_omni.diffusion.vllm_config as diffusion_vllm_config +from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode +from vllm_omni.diffusion.sched.request_scheduler import RequestScheduler +from vllm_omni.quantization import build_quant_config + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + + +class _Executor: + def __init__(self) -> None: + self.specs = [{"layer0": object()}, {"layer0": object()}] + self.memory = [1024, 1024] + self.configs = None + self.profile_requests = None + + def get_kv_cache_specs(self): + return self.specs + + def determine_available_kv_memory(self, profile_requests): + self.profile_requests = profile_requests + return self.memory + + def set_kv_cache_configs(self, configs) -> None: + self.configs = configs + + +def _od_config(**overrides): + values = dict( + model="test-diffusion", + diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER, + dtype=torch.bfloat16, + max_model_len=64, + quantization_config=None, + tf_model_config=None, + enforce_eager=True, + is_moe=False, + additional_config={}, + profiler_config=None, + parallel_config=SimpleNamespace( + tensor_parallel_size=1, + pipeline_parallel_size=1, + data_parallel_size=1, + enable_expert_parallel=False, + ), + gpu_memory_utilization=0.9, + kv_cache_memory_bytes=None, + max_num_batched_tokens=64, + num_gpus=1, + ) + values.update(overrides) + return SimpleNamespace(**values) + + +def test_control_plane_builds_worker_and_scheduler_configs(monkeypatch) -> None: + executor = _Executor() + od_config = _od_config(num_gpus=2) + vllm_config = SimpleNamespace( + model_config=SimpleNamespace(max_model_len=4096), + max_in_flight_tokens=128, + ) + worker_configs = [object(), object()] + scheduler_kv_cache_config = object() + profile_requests = [object()] + monkeypatch.setattr(initialization, "create_diffusion_vllm_config", lambda *args, **kwargs: vllm_config) + build = Mock(return_value=(worker_configs, scheduler_kv_cache_config)) + monkeypatch.setattr( + initialization, + "build_native_kv_cache_configs", + build, + ) + monkeypatch.setattr(initialization, "resolve_kv_cache_block_sizes", lambda *args: (16, 16)) + + result = initialization.initialize_diffusion_kv_control_plane( + executor, + od_config, + profile_requests=profile_requests, + device=torch.device("cpu"), + ) + + build.assert_called_once_with(vllm_config, executor.specs, executor.memory) + assert result == (scheduler_kv_cache_config, 16, 16, vllm_config) + assert executor.configs == worker_configs + assert executor.profile_requests is profile_requests + + +def test_dense_mode_skips_worker_kv_initialization(monkeypatch) -> None: + executor = _Executor() + od_config = _od_config(diffusion_kv_mode=DiffusionKVCacheMode.DENSE_LEGACY) + get_device = Mock(side_effect=AssertionError("dense mode must not resolve a KV device")) + monkeypatch.setattr(initialization.current_omni_platform, "get_torch_device", get_device) + + result = initialization.initialize_diffusion_kv_control_plane(executor, od_config) + + assert result is None + assert executor.configs is None + get_device.assert_not_called() + + +def test_control_plane_rejects_rank_count_mismatch(monkeypatch) -> None: + executor = _Executor() + executor.memory = [1024] + od_config = _od_config(num_gpus=2) + monkeypatch.setattr(initialization, "create_diffusion_vllm_config", lambda *args, **kwargs: object()) + + with pytest.raises(ValueError, match="rank count mismatch"): + initialization.initialize_diffusion_kv_control_plane( + executor, + od_config, + profile_requests=[object()], + device=torch.device("cpu"), + ) + + +def test_paged_control_plane_requires_prepared_profile_request() -> None: + with pytest.raises(ValueError, match="prepared memory profile requests"): + initialization.initialize_diffusion_kv_control_plane( + _Executor(), + _od_config(), + device=torch.device("cpu"), + ) + + +def test_native_config_builder_produces_matching_worker_and_scheduler_pools() -> None: + od_config = _od_config() + vllm_config = diffusion_vllm_config.create_diffusion_vllm_config(torch.device("cpu"), od_config) + spec = FullAttentionSpec( + block_size=vllm_config.cache_config.block_size, + num_kv_heads=2, + head_size=8, + dtype=torch.bfloat16, + non_causal=True, + ) + num_blocks = 32 + + worker_configs, scheduler_kv_cache_config = initialization.build_native_kv_cache_configs( + vllm_config, + [{"layer0": spec}], + [spec.page_size_bytes * num_blocks], + ) + + assert len(worker_configs) == 1 + assert worker_configs[0].num_blocks == num_blocks + assert scheduler_kv_cache_config.num_blocks == num_blocks + assert worker_configs[0].kv_cache_groups == scheduler_kv_cache_config.kv_cache_groups + + +def test_native_control_plane_config_initializes_scheduler() -> None: + od_config = _od_config() + vllm_config = diffusion_vllm_config.create_diffusion_vllm_config(torch.device("cpu"), od_config) + spec = FullAttentionSpec( + block_size=vllm_config.cache_config.block_size, + num_kv_heads=2, + head_size=8, + dtype=torch.bfloat16, + non_causal=True, + ) + _, scheduler_kv_cache_config = initialization.build_native_kv_cache_configs( + vllm_config, + [{"layer0": spec}], + [spec.page_size_bytes * 32], + ) + + scheduler = RequestScheduler() + scheduler.initialize( + od_config, + kv_cache_config=scheduler_kv_cache_config, + scheduler_block_size=spec.block_size, + hash_block_size=spec.block_size, + kv_vllm_config=vllm_config, + ) + + assert scheduler._diffusion_kv_manager is not None + scheduler.close() + + +def test_dense_config_keeps_new_paged_sizing_inputs_inactive() -> None: + od_config = _od_config( + diffusion_kv_mode=DiffusionKVCacheMode.DENSE_LEGACY, + parallel_config=SimpleNamespace( + tensor_parallel_size=1, + pipeline_parallel_size=4, + data_parallel_size=1, + enable_expert_parallel=False, + ), + gpu_memory_utilization=0.5, + kv_cache_memory_bytes=4096, + max_num_batched_tokens=64, + ) + vllm_config = diffusion_vllm_config.create_base_diffusion_vllm_config(torch.device("cpu"), od_config) + original_pp_size = vllm_config.parallel_config.pipeline_parallel_size + original_memory_utilization = vllm_config.cache_config.gpu_memory_utilization + original_memory_bytes = vllm_config.cache_config.kv_cache_memory_bytes + original_token_budget = vllm_config.scheduler_config.max_num_batched_tokens + + diffusion_vllm_config.configure_diffusion_vllm_config(vllm_config, od_config) + + assert vllm_config.parallel_config.pipeline_parallel_size == original_pp_size + assert vllm_config.cache_config.gpu_memory_utilization == original_memory_utilization + assert vllm_config.cache_config.kv_cache_memory_bytes == original_memory_bytes + assert vllm_config.scheduler_config.max_num_batched_tokens == original_token_budget + + +def test_paged_config_forwards_gpu_memory_utilization_to_native_cache_config() -> None: + vllm_config = diffusion_vllm_config.create_diffusion_vllm_config( + torch.device("cpu"), + _od_config(gpu_memory_utilization=0.42), + ) + + assert vllm_config.cache_config.gpu_memory_utilization == 0.42 + + +def test_diffusion_vllm_model_config_supplies_dtype_for_quant_methods() -> None: + quantization_config = build_quant_config( + { + "quant_method": "modelopt", + "quant_algo": "FP8", + "ignore": [], + } + ) + model_config = diffusion_vllm_config._make_diffusion_vllm_model_config( + _od_config( + diffusion_kv_mode=DiffusionKVCacheMode.DENSE_LEGACY, + quantization_config=quantization_config, + ) + ) + + assert model_config.dtype is torch.bfloat16 + assert model_config.quantization == "modelopt" + assert model_config.quantization_config is quantization_config + assert model_config.is_quantized + assert model_config.is_moe is False + assert model_config.is_nvfp4_quantized() is False + assert model_config.original_max_model_len == model_config.max_model_len + + unquantized_model_config = diffusion_vllm_config._make_diffusion_vllm_model_config( + _od_config( + diffusion_kv_mode=DiffusionKVCacheMode.DENSE_LEGACY, + quantization_config=None, + ) + ) + assert not unquantized_model_config.is_quantized + + +def test_minus_one_max_model_len_uses_model_limit() -> None: + od_config = SimpleNamespace( + max_model_len=-1, + tf_model_config=SimpleNamespace(max_position_embeddings=4096), + diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER, + ) + + assert diffusion_vllm_config.resolve_diffusion_max_model_len(od_config) == 4096 + + model_config = diffusion_vllm_config._make_diffusion_vllm_model_config( + _od_config( + max_model_len=-1, + tf_model_config=SimpleNamespace(max_position_embeddings=4096), + ) + ) + assert model_config.max_model_len == 4096 + assert model_config.original_max_model_len == -1 + + +def test_minus_one_max_model_len_is_auto_fitted_by_native_cache_sizing() -> None: + vllm_config = diffusion_vllm_config.create_diffusion_vllm_config( + torch.device("cpu"), + _od_config( + max_model_len=-1, + tf_model_config=SimpleNamespace(max_position_embeddings=4096), + ), + ) + spec = FullAttentionSpec( + block_size=vllm_config.cache_config.block_size, + num_kv_heads=2, + head_size=8, + dtype=torch.bfloat16, + non_causal=True, + ) + + initialization.build_native_kv_cache_configs( + vllm_config, + [{"layer0": spec}], + [spec.page_size_bytes * 16], + ) + + assert vllm_config.model_config.original_max_model_len == -1 + assert vllm_config.model_config.max_model_len == 256 diff --git a/tests/diffusion/diffusion_kv/test_manager.py b/tests/diffusion/diffusion_kv/test_manager.py new file mode 100644 index 000000000..5152c2a67 --- /dev/null +++ b/tests/diffusion/diffusion_kv/test_manager.py @@ -0,0 +1,211 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch +import vllm.v1.core.single_type_kv_cache_manager as native_kv_managers +from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, KVCacheTensor + +from vllm_omni.diffusion.diffusion_kv.manager import DiffusionKVAdmissionError, DiffusionKVCacheManager +from vllm_omni.diffusion.diffusion_kv.request import DiffusionKVContext, DiffusionKVRequest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + +BLOCK_SIZE = 4 + + +def _config(num_blocks: int) -> KVCacheConfig: + native_kv_managers.register_all_kvcache_specs(None) + spec = FullAttentionSpec( + block_size=BLOCK_SIZE, + num_kv_heads=2, + head_size=8, + dtype=torch.bfloat16, + ) + return KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=[KVCacheTensor(size=spec.page_size_bytes * num_blocks, shared_by=["layer0"])], + kv_cache_groups=[KVCacheGroupSpec(layer_names=["layer0"], kv_cache_spec=spec)], + ) + + +def _request(public_id: str, sequence_id: int, *, seq_len: int = 8, kv_contexts=()) -> DiffusionKVRequest: + return DiffusionKVRequest( + f"{public_id}/diffusion-kv/{sequence_id}", + sequence_id=sequence_id, + prefix_len=4, + target_len=4, + seq_len=seq_len, + kv_contexts=kv_contexts, + ) + + +def _manager(num_blocks: int, *, max_model_len: int = 64) -> DiffusionKVCacheManager: + return DiffusionKVCacheManager( + _config(num_blocks), + max_model_len=max_model_len, + scheduler_block_size=BLOCK_SIZE, + hash_block_size=BLOCK_SIZE, + ) + + +def test_reserve_and_free_multi_cfg_request() -> None: + manager = _manager(8) + free_before = manager.native_manager.block_pool.get_num_free_blocks() + requests = (_request("public", 0), _request("public", 1)) + + metadata = manager.reserve_request("public", requests) + + assert metadata is not None + assert metadata.request_id == "public" + assert metadata.allocation_generation == 1 + assert [sequence.sequence_id for sequence in metadata.sequences] == [0, 1] + assert [sequence.prefix_len for sequence in metadata.sequences] == [4, 4] + assert [sequence.target_len for sequence in metadata.sequences] == [4, 4] + assert [sequence.seq_len for sequence in metadata.sequences] == [8, 8] + assert [len(sequence.block_ids[0]) for sequence in metadata.sequences] == [2, 2] + assert all(sequence.context_ids == () for sequence in metadata.sequences) + assert metadata.contexts == () + assert metadata == manager.get_metadata("public") + assert manager.native_manager.block_pool.get_num_free_blocks() == free_before - 4 + + manager.free_request("public") + assert manager.native_manager.block_pool.get_num_free_blocks() == free_before + + +def test_impossible_cfg_allocation_rolls_back_and_fails_fast() -> None: + manager = _manager(3) + free_before = manager.native_manager.block_pool.get_num_free_blocks() + + with pytest.raises(DiffusionKVAdmissionError, match="cannot fit even when the block pool is empty"): + manager.reserve_request( + "public", + (_request("public", 0), _request("public", 1)), + ) + + assert manager.has_request("public") is False + assert manager.native_manager.block_pool.get_num_free_blocks() == free_before + + +def test_impossible_cfg_allocation_fails_fast_while_pool_is_busy() -> None: + manager = _manager(5) + assert manager.reserve_request("running", (_request("running", 0),)) is not None + free_before = manager.native_manager.block_pool.get_num_free_blocks() + + with pytest.raises(DiffusionKVAdmissionError, match="required_blocks=6, available_blocks=4"): + manager.reserve_request( + "impossible", + tuple(_request("impossible", sequence_id) for sequence_id in range(3)), + ) + + assert manager.has_request("impossible") is False + assert manager.native_manager.block_pool.get_num_free_blocks() == free_before + + +def test_temporary_capacity_pressure_returns_none() -> None: + manager = _manager(3) + assert manager.reserve_request("running", (_request("running", 0),)) is not None + + allocation = manager.reserve_request("waiting", (_request("waiting", 0),)) + + assert allocation is None + assert manager.has_request("waiting") is False + + +def test_partial_cfg_rollback_can_retry_after_capacity_is_released() -> None: + manager = _manager(5) + assert manager.reserve_request("running", (_request("running", 0),)) is not None + waiting = (_request("waiting", 0), _request("waiting", 1)) + + assert manager.reserve_request("waiting", waiting) is None + assert manager.has_request("waiting") is False + + manager.free_request("running") + metadata = manager.reserve_request("waiting", waiting) + + assert metadata is not None + assert [len(sequence.block_ids[0]) for sequence in metadata.sequences] == [2, 2] + + +def test_reallocated_request_gets_a_new_allocation_generation() -> None: + manager = _manager(3) + request = (_request("public", 0),) + + first = manager.reserve_request("public", request) + assert first is not None + manager.free_request("public") + second = manager.reserve_request("public", request) + + assert second is not None + assert second.allocation_generation == first.allocation_generation + 1 + + +def test_cfg_allocation_exception_rolls_back(monkeypatch) -> None: + manager = _manager(8) + free_before = manager.native_manager.block_pool.get_num_free_blocks() + native_allocate = manager.native_manager.allocate_slots + num_calls = 0 + + def fail_second_allocation(*args, **kwargs): + nonlocal num_calls + num_calls += 1 + if num_calls == 2: + raise RuntimeError("injected allocation failure") + return native_allocate(*args, **kwargs) + + monkeypatch.setattr(manager.native_manager, "allocate_slots", fail_second_allocation) + + with pytest.raises(RuntimeError, match="injected allocation failure"): + manager.reserve_request( + "public", + (_request("public", 0), _request("public", 1)), + ) + + assert manager.has_request("public") is False + assert manager.native_manager.block_pool.get_num_free_blocks() == free_before + + +def test_rejects_independent_context_until_role_routing_is_implemented() -> None: + manager = _manager(8) + context = DiffusionKVContext(context_id="text", cache_role="cross.text", num_tokens=8) + + with pytest.raises(DiffusionKVAdmissionError, match="DiffusionKVContext"): + manager.reserve_request( + "public", + (_request("public", 0, kv_contexts=(context,)),), + ) + + +def test_rejects_sequence_longer_than_admission_bound() -> None: + manager = _manager(8, max_model_len=8) + + with pytest.raises(DiffusionKVAdmissionError, match="exceeds max_model_len"): + manager.reserve_request("public", (_request("public", 0, seq_len=9),)) + + +@pytest.mark.parametrize( + "requests", + [ + (_request("public", 1),), + (_request("public", 1), _request("public", 0)), + (_request("public", 0), _request("public", 2)), + ], +) +def test_rejects_ambiguous_sequence_order(requests) -> None: + manager = _manager(8) + + with pytest.raises(ValueError, match="ordered by contiguous sequence_id"): + manager.reserve_request("public", requests) + + +def test_close_releases_every_public_request() -> None: + manager = _manager(8) + free_before = manager.native_manager.block_pool.get_num_free_blocks() + assert manager.reserve_request("first", (_request("first", 0),)) is not None + assert manager.reserve_request("second", (_request("second", 0),)) is not None + + manager.close() + + assert manager.native_manager.block_pool.get_num_free_blocks() == free_before + assert manager.has_request("first") is False + assert manager.has_request("second") is False diff --git a/tests/diffusion/diffusion_kv/test_request.py b/tests/diffusion/diffusion_kv/test_request.py index 43734ca31..ae5609b2a 100644 --- a/tests/diffusion/diffusion_kv/test_request.py +++ b/tests/diffusion/diffusion_kv/test_request.py @@ -107,11 +107,8 @@ def test_request_rejects_non_context_values() -> None: def _manager(*, enable_caching: bool) -> KVCacheManager: # vLLM 0.26 lazily registers its built-in KV specs. Another test may have # already registered an out-of-tree spec, making the registry non-empty - # before FullAttentionSpec is registered. Initialize the native mappings - # explicitly; vLLM 0.25 has no registration hook and needs no setup. - register_all = getattr(native_kv_managers, "register_all_kvcache_specs", None) - if register_all is not None: - register_all(None) + # before FullAttentionSpec is registered. Initialize the native mappings. + native_kv_managers.register_all_kvcache_specs(None) spec = FullAttentionSpec( block_size=BLOCK_SIZE, diff --git a/tests/diffusion/diffusion_kv/test_spec_discovery.py b/tests/diffusion/diffusion_kv/test_spec_discovery.py new file mode 100644 index 000000000..1d1d6c8d9 --- /dev/null +++ b/tests/diffusion/diffusion_kv/test_spec_discovery.py @@ -0,0 +1,259 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch +from torch import nn +from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, KVCacheTensor + +import vllm_omni.diffusion.worker.diffusion_worker as diffusion_worker_module +from vllm_omni.diffusion.attention.layer import Attention +from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode +from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner +from vllm_omni.diffusion.worker.diffusion_worker import DiffusionWorker + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + + +class _Backend: + @classmethod + def indexes_kv_by_block_stride(cls) -> bool: + return True + + +def _attention(*, enabled: bool) -> Attention: + attention = Attention.__new__(Attention) + nn.Module.__init__(attention) + attention.paged_kv_cache_role = "primary" if enabled else None + attention.paged_kv_cache_dtype = torch.bfloat16 + attention.num_kv_heads = 2 + attention.head_size = 8 + attention.causal = False + attention.attn_backend = _Backend + return attention + + +def _runner(attention: Attention) -> DiffusionModelRunner: + runner = object.__new__(DiffusionModelRunner) + runner.od_config = SimpleNamespace(diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER) + runner.vllm_config = SimpleNamespace( + cache_config=SimpleNamespace(block_size=4), + model_config=SimpleNamespace(dtype=torch.float16), + ) + pipeline = nn.Module() + pipeline.image_attention = attention + runner.pipeline = pipeline + return runner + + +def test_runner_discovers_native_spec_from_loaded_attention() -> None: + specs = _runner(_attention(enabled=True)).get_kv_cache_spec() + + assert set(specs) == {"image_attention"} + spec = specs["image_attention"] + assert isinstance(spec, FullAttentionSpec) + assert spec.block_size == 4 + assert spec.num_kv_heads == 2 + assert spec.head_size == 8 + assert spec.dtype is torch.bfloat16 + assert spec.indexes_kv_by_block_stride is True + assert spec.non_causal is True + + +def test_runner_rejects_paged_mode_without_cache_enabled_attention() -> None: + runner = _runner(_attention(enabled=False)) + + with pytest.raises(RuntimeError, match="no cache-enabled Attention"): + runner.get_kv_cache_spec() + + +def test_runner_retains_matching_rank_local_config() -> None: + runner = _runner(_attention(enabled=True)) + spec = runner.get_kv_cache_spec()["image_attention"] + config = KVCacheConfig( + num_blocks=8, + kv_cache_tensors=[KVCacheTensor(size=spec.page_size_bytes * 8, shared_by=["image_attention"])], + kv_cache_groups=[KVCacheGroupSpec(layer_names=["image_attention"], kv_cache_spec=spec)], + ) + + runner.set_kv_cache_config(config) + + assert runner.kv_cache_config is config + + +def test_runner_rejects_rank_local_config_for_different_layers() -> None: + runner = _runner(_attention(enabled=True)) + spec = runner.get_kv_cache_spec()["image_attention"] + config = KVCacheConfig( + num_blocks=8, + kv_cache_tensors=[KVCacheTensor(size=spec.page_size_bytes * 8, shared_by=["other_attention"])], + kv_cache_groups=[KVCacheGroupSpec(layer_names=["other_attention"], kv_cache_spec=spec)], + ) + + with pytest.raises(ValueError, match="layer mismatch"): + runner.set_kv_cache_config(config) + + +def test_worker_selects_its_rank_local_config() -> None: + worker = object.__new__(DiffusionWorker) + worker.rank = 1 + worker.od_config = SimpleNamespace(num_gpus=2) + worker.model_runner = SimpleNamespace(set_kv_cache_config=lambda config: setattr(worker, "installed", config)) + configs = [object(), object()] + + worker.set_kv_cache_configs(configs) + + assert worker.installed is configs[1] + with pytest.raises(ValueError, match="rank count mismatch"): + worker.set_kv_cache_configs(configs[:1]) + + +def test_worker_honors_explicit_kv_memory_budget(monkeypatch) -> None: + worker = object.__new__(DiffusionWorker) + worker.rank = 0 + worker.vllm_config = SimpleNamespace(cache_config=SimpleNamespace(kv_cache_memory_bytes=4096)) + worker.init_snapshot = SimpleNamespace(free_memory=16384) + worker.requested_memory = 8192 + worker.model_runner = SimpleNamespace(profile_run=Mock(), model_memory_usage=1024) + monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", lambda value: [value]) + profile_request = object() + + assert worker.determine_available_kv_memory(profile_request) == [4096] + worker.model_runner.profile_run.assert_called_once_with(profile_request) + + +def test_worker_treats_zero_kv_memory_budget_as_profiled_auto_sizing(monkeypatch) -> None: + worker = object.__new__(DiffusionWorker) + worker.rank = 0 + worker.init_snapshot = SimpleNamespace(free_memory=900) + worker.requested_memory = 750 + worker.vllm_config = SimpleNamespace( + cache_config=SimpleNamespace( + kv_cache_memory_bytes=0, + gpu_memory_utilization=0.75, + ) + ) + worker.model_runner = SimpleNamespace(profile_run=Mock(), model_memory_usage=100) + monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", lambda value: [value]) + profile_result = SimpleNamespace( + non_kv_cache_memory=400, + after_profile=SimpleNamespace(free_memory=500), + ) + monkeypatch.setattr( + diffusion_worker_module, + "memory_profiling", + lambda snapshot, weights_memory: nullcontext(profile_result), + ) + profile_request = object() + + assert worker.determine_available_kv_memory(profile_request) == [350] + worker.model_runner.profile_run.assert_called_once_with(profile_request) + + +@pytest.mark.parametrize("non_kv_cache_memory", [750, 800]) +def test_worker_rejects_non_positive_profiled_kv_memory(monkeypatch, non_kv_cache_memory: int) -> None: + worker = object.__new__(DiffusionWorker) + worker.rank = 0 + worker.init_snapshot = SimpleNamespace(free_memory=900) + worker.requested_memory = 750 + worker.vllm_config = SimpleNamespace( + cache_config=SimpleNamespace( + kv_cache_memory_bytes=None, + gpu_memory_utilization=0.75, + ) + ) + worker.model_runner = SimpleNamespace(profile_run=Mock(), model_memory_usage=100) + monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", lambda value: [value]) + profile_result = SimpleNamespace( + non_kv_cache_memory=non_kv_cache_memory, + after_profile=SimpleNamespace(free_memory=100), + ) + monkeypatch.setattr( + diffusion_worker_module, + "memory_profiling", + lambda snapshot, weights_memory: nullcontext(profile_result), + ) + profile_request = object() + + with pytest.raises( + RuntimeError, + match=( + "rank 0: RuntimeError: No memory remains for Diffusion KV cache after profiling: " + f"requested_memory=750 bytes, non_kv_cache_memory={non_kv_cache_memory} bytes" + ), + ): + worker.determine_available_kv_memory(profile_request) + + worker.model_runner.profile_run.assert_called_once_with(profile_request) + + +def test_worker_profiles_activation_headroom_instead_of_current_residency(monkeypatch) -> None: + worker = object.__new__(DiffusionWorker) + worker.rank = 0 + worker.init_snapshot = SimpleNamespace(free_memory=1200) + worker.requested_memory = 1000 + worker.vllm_config = SimpleNamespace( + cache_config=SimpleNamespace( + kv_cache_memory_bytes=None, + gpu_memory_utilization=0.8, + ) + ) + worker.model_runner = SimpleNamespace(profile_run=Mock(), model_memory_usage=250) + monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", lambda value: [value]) + profile_result = SimpleNamespace( + non_kv_cache_memory=650, + after_profile=SimpleNamespace(free_memory=550), + ) + profiling_calls = [] + + def profile(snapshot, *, weights_memory): + profiling_calls.append((snapshot, weights_memory)) + return nullcontext(profile_result) + + monkeypatch.setattr( + diffusion_worker_module, + "memory_profiling", + profile, + ) + profile_request = object() + + assert worker.determine_available_kv_memory(profile_request) == [350] + assert profiling_calls == [(worker.init_snapshot, 250)] + worker.model_runner.profile_run.assert_called_once_with(profile_request) + + +def test_worker_rejects_profile_without_preload_snapshot(monkeypatch) -> None: + worker = object.__new__(DiffusionWorker) + worker.init_snapshot = None + worker.requested_memory = None + worker.vllm_config = SimpleNamespace(cache_config=SimpleNamespace(kv_cache_memory_bytes=None)) + worker.model_runner = SimpleNamespace() + monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", lambda value: [value]) + + with pytest.raises(RuntimeError, match="snapshot was not captured before model loading"): + worker.determine_available_kv_memory(object()) + + +def test_rank_probe_gathers_local_failure_before_raising(monkeypatch) -> None: + gathered = [] + + def gather(local_result): + gathered.append(local_result) + return [local_result, (True, "peer-result")] + + def fail_local_probe(): + raise ValueError("local failure") + + monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", gather) + + with pytest.raises(RuntimeError, match="rank 0: ValueError: local failure"): + diffusion_worker_module._run_and_gather_rank_values( + "test probe", + fail_local_probe, + ) + + assert gathered == [(False, "ValueError: local failure")] diff --git a/tests/diffusion/diffusion_kv/test_worker_contract.py b/tests/diffusion/diffusion_kv/test_worker_contract.py index 0c78abbaf..e0f428fb6 100644 --- a/tests/diffusion/diffusion_kv/test_worker_contract.py +++ b/tests/diffusion/diffusion_kv/test_worker_contract.py @@ -85,19 +85,15 @@ def test_new_request_data_carries_scheduler_allocation_atomically() -> None: assert new_req.diffusion_kv_metadata is metadata -def test_paged_request_without_metadata_reaches_request_forward() -> None: +def test_paged_request_without_metadata_fails_before_request_forward() -> None: runner = make_runner(DiffusionKVCacheMode.PAGED_SCHEDULER) - expected_output = DiffusionOutput(output=None) - runner._execute_request_list = Mock( - return_value=SimpleNamespace( - runner_outputs=[SimpleNamespace(result=expected_output)], - ) - ) + runner._execute_request_list = Mock() req = SimpleNamespace(request_id="req-0") - assert runner.execute_model(req) is expected_output + with pytest.raises(ValueError, match="requires Diffusion KV metadata"): + runner.execute_model(req) - runner._execute_request_list.assert_called_once() + runner._execute_request_list.assert_not_called() def test_dense_request_rejects_metadata_before_request_forward() -> None: @@ -141,18 +137,18 @@ def test_request_rpc_rejects_envelope_request_identity_mismatch() -> None: assert calls == [] -def test_batch_path_allows_missing_metadata_before_forward() -> None: +def test_batch_path_rejects_missing_metadata_before_forward() -> None: runner = make_runner(DiffusionKVCacheMode.PAGED_SCHEDULER) - expected_output = object() - runner._execute_request_list = Mock(return_value=expected_output) + runner._execute_request_list = Mock() new_req = NewRequestData( request_id="req-0", req=SimpleNamespace(request_id="req-0"), ) - assert runner.execute_model_batch(make_scheduler_output(new_req), runner.od_config) is expected_output + with pytest.raises(ValueError, match="requires Diffusion KV metadata"): + runner.execute_model_batch(make_scheduler_output(new_req), runner.od_config) - runner._execute_request_list.assert_called_once() + runner._execute_request_list.assert_not_called() def test_batch_path_rejects_envelope_request_identity_mismatch() -> None: @@ -170,7 +166,7 @@ def test_batch_path_rejects_envelope_request_identity_mismatch() -> None: runner._execute_request_list.assert_not_called() -def test_step_path_allows_missing_metadata_before_step_support_check() -> None: +def test_step_path_rejects_missing_metadata_before_step_support_check() -> None: runner = make_runner(DiffusionKVCacheMode.PAGED_SCHEDULER) runner.pipeline = object() runner._supports_step_mode = Mock(return_value=False) @@ -179,10 +175,10 @@ def test_step_path_allows_missing_metadata_before_step_support_check() -> None: req=SimpleNamespace(request_id="req-0"), ) - with pytest.raises(ValueError, match="does not support step execution"): + with pytest.raises(ValueError, match="requires Diffusion KV metadata"): runner.execute_stepwise(make_scheduler_output(new_req)) - runner._supports_step_mode.assert_called_once() + runner._supports_step_mode.assert_not_called() def test_step_path_rejects_envelope_request_identity_mismatch() -> None: @@ -288,3 +284,42 @@ def test_dense_worker_call_does_not_extend_model_runner_signature() -> None: worker.execute_model(req, SimpleNamespace()) assert calls == [(req, None)] + + +def test_dlo_worker_selects_request_and_metadata_from_same_envelope(monkeypatch) -> None: + calls: list[tuple] = [] + + class ModelRunner: + def execute_model(self, req, **kwargs): + calls.append((req, kwargs)) + return DiffusionOutput(output=None) + + worker = object.__new__(DiffusionWorker) + worker.model_runner = ModelRunner() + worker.lora_manager = None + worker._get_profiler = lambda: None + monkeypatch.setattr( + "vllm_omni.diffusion.distributed.parallel_state.get_data_parallel_rank", + lambda: 1, + ) + req_0 = SimpleNamespace(request_id="req-0") + req_1 = SimpleNamespace(request_id="req-1") + metadata_0 = make_metadata("req-0") + metadata_1 = make_metadata("req-1") + envelopes = [ + NewRequestData(request_id="req-0", req=req_0, diffusion_kv_metadata=metadata_0), + NewRequestData(request_id="req-1", req=req_1, diffusion_kv_metadata=metadata_1), + ] + + result = worker.execute_model(envelopes, SimpleNamespace()) + + assert calls == [ + ( + req_1, + { + "kv_prefetch_job": None, + "diffusion_kv_metadata": metadata_1, + }, + ) + ] + assert result["dp_rank"] == 1 diff --git a/tests/diffusion/models/hunyuan_image3/test_image_kv_cache_manager.py b/tests/diffusion/models/hunyuan_image3/test_image_kv_cache_manager.py index 543b93a25..fed56af2d 100644 --- a/tests/diffusion/models/hunyuan_image3/test_image_kv_cache_manager.py +++ b/tests/diffusion/models/hunyuan_image3/test_image_kv_cache_manager.py @@ -118,6 +118,14 @@ def _call_mgr( ) +def test_cache_manager_registers_attention_without_adding_dense_state() -> None: + mgr = _make_cache_mgr() + + assert isinstance(mgr, nn.Module) + assert dict(mgr.named_modules())["attn"] is mgr.attn + assert mgr.state_dict() == {} + + # ============================================================ # Test 1: No AR KV — basic cache → reuse # ============================================================ diff --git a/tests/diffusion/test_diffusion_config_propagation.py b/tests/diffusion/test_diffusion_config_propagation.py index e25d6497c..e456e302b 100644 --- a/tests/diffusion/test_diffusion_config_propagation.py +++ b/tests/diffusion/test_diffusion_config_propagation.py @@ -130,6 +130,21 @@ class TestCreateDefaultDiffusion: assert od.diffusion_kv_mode is DiffusionKVCacheMode.PAGED_SCHEDULER + def test_diffusion_kv_sizing_fields_roundtrip(self, monkeypatch): + monkeypatch.setattr(OmniDiffusionConfig, "_resolve_master_port", lambda self: 29500) + od = _roundtrip_diffusion_config( + model="x", + kv_cache_memory_bytes=4096, + gpu_memory_utilization=0.75, + max_num_batched_tokens=2048, + max_model_len=4096, + ) + + assert od.kv_cache_memory_bytes == 4096 + assert od.gpu_memory_utilization == 0.75 + assert od.max_num_batched_tokens == 2048 + assert od.max_model_len == 4096 + def test_qwen_image_edit_plus_sets_generic_multimodal_limit(): od_config = OmniDiffusionConfig(model="Qwen/Qwen-Image-Edit-2511", model_class_name="QwenImageEditPlusPipeline") diff --git a/tests/diffusion/test_diffusion_engine_cleanup.py b/tests/diffusion/test_diffusion_engine_cleanup.py index fb8858a48..b626f8ac6 100644 --- a/tests/diffusion/test_diffusion_engine_cleanup.py +++ b/tests/diffusion/test_diffusion_engine_cleanup.py @@ -161,9 +161,83 @@ def test_scheduler_initialization_failure_closes_scheduler_and_executor(monkeypa assert exc_info.value is initialization_error custom_scheduler.initialize.assert_called_once_with(od_config) custom_scheduler.close.assert_called_once_with() + + +def test_init_shuts_down_executor_when_kv_control_plane_initialization_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_executor = SimpleNamespace(shutdown=Mock()) + monkeypatch.setattr(DiffusionEngine, "_init_process_hooks", lambda self, config: None) + monkeypatch.setattr( + DiffusionEngine, + "_resolve_execution_mode", + lambda self, config: DiffusionExecutionMode.REQUEST_BATCH, + ) + monkeypatch.setattr( + DiffusionEngine, "_init_executor", lambda self, config: setattr(self, "executor", fake_executor) + ) + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.initialize_diffusion_kv_control_plane", + Mock(side_effect=RuntimeError("KV initialization failed")), + ) + + with pytest.raises(RuntimeError, match="KV initialization failed"): + DiffusionEngine(SimpleNamespace()) + fake_executor.shutdown.assert_called_once_with() +def test_paged_init_profiles_before_scheduler_initialization(monkeypatch: pytest.MonkeyPatch) -> None: + events = [] + expected_profile_requests = [object()] + fake_executor = SimpleNamespace(shutdown=Mock()) + kv_cache_config = object() + kv_vllm_config = object() + od_config = SimpleNamespace(diffusion_kv_mode="paged_scheduler") + + monkeypatch.setattr(DiffusionEngine, "_init_process_hooks", lambda self, config: None) + monkeypatch.setattr( + DiffusionEngine, + "_resolve_execution_mode", + lambda self, config: DiffusionExecutionMode.REQUEST_BATCH, + ) + monkeypatch.setattr( + DiffusionEngine, + "_init_executor", + lambda self, config: setattr(self, "executor", fake_executor), + ) + monkeypatch.setattr( + DiffusionEngine, + "_prepare_diffusion_kv_profile_requests", + lambda self: events.append("prepare-profile") or expected_profile_requests, + ) + + def initialize(executor, config, *, profile_requests): + events.append("profile-workers") + assert executor is fake_executor + assert config is od_config + assert profile_requests is expected_profile_requests + return kv_cache_config, 16, 16, kv_vllm_config + + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.initialize_diffusion_kv_control_plane", + initialize, + ) + + def init_scheduler(self, config, scheduler, *args, **kwargs): + events.append("initialize-scheduler") + self.scheduler = SimpleNamespace(close=Mock()) + + monkeypatch.setattr(DiffusionEngine, "_init_scheduler", init_scheduler) + monkeypatch.setattr(DiffusionEngine, "_init_runtime_state", lambda self: None) + monkeypatch.setattr(DiffusionEngine, "_init_execute_fn", lambda self: None) + monkeypatch.setattr(DiffusionEngine, "_log_execution_mode", lambda self, config: None) + + DiffusionEngine(od_config) + + assert events == ["prepare-profile", "profile-workers", "initialize-scheduler"] + + @pytest.mark.asyncio async def test_step_compatibility_wrapper_returns_final_batch() -> None: engine = _make_engine() diff --git a/tests/diffusion/test_diffusion_engine_dummy_run.py b/tests/diffusion/test_diffusion_engine_dummy_run.py index 50dde3083..247fee029 100644 --- a/tests/diffusion/test_diffusion_engine_dummy_run.py +++ b/tests/diffusion/test_diffusion_engine_dummy_run.py @@ -1,9 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace +from unittest.mock import Mock + import pytest from vllm_omni.diffusion import io_support +from vllm_omni.diffusion.diffusion_engine import DiffusionEngine, DiffusionExecutionMode +from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode +from vllm_omni.diffusion.diffusion_kv.request import DiffusionKVRequest +from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.inputs.data import OmniDiffusionSamplingParams pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] @@ -55,3 +63,151 @@ def test_dummy_run_num_frames_uses_audio_input_fallback(monkeypatch: pytest.Monk ) assert io_support.get_dummy_run_num_frames("unknown", supports_audio_input=True) == 2 + + +def test_dummy_run_image_count_resolves_hunyuan_architecture_alias() -> None: + assert io_support.get_dummy_run_num_image_inputs("HunyuanImage3ForCausalMM") == 3 + assert io_support.get_dummy_run_num_image_inputs("unknown") == 1 + + +def test_dense_mode_does_not_build_kv_profile_request() -> None: + engine = object.__new__(DiffusionEngine) + engine.od_config = SimpleNamespace(diffusion_kv_mode=DiffusionKVCacheMode.DENSE_LEGACY) + engine._make_dummy_request = Mock(side_effect=AssertionError("dense mode must not profile")) + + assert engine._prepare_diffusion_kv_profile_requests() is None + engine._make_dummy_request.assert_not_called() + + +@pytest.mark.parametrize( + ("execution_mode", "uses_dlo_dp", "max_num_seqs", "expected_profile_requests"), + [ + (DiffusionExecutionMode.STEP_BATCH, False, 2, 2), + (DiffusionExecutionMode.REQUEST_BATCH, True, 4, 1), + (DiffusionExecutionMode.STEP_BATCH, True, 3, 3), + ], +) +def test_paged_kv_profile_requests_match_per_rank_batch( + execution_mode: DiffusionExecutionMode, + uses_dlo_dp: bool, + max_num_seqs: int, + expected_profile_requests: int, +) -> None: + engine = object.__new__(DiffusionEngine) + engine.execution_mode = execution_mode + engine.od_config = SimpleNamespace( + diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER, + model_class_name="HunyuanImage3ForCausalMM", + max_num_seqs=max_num_seqs, + parallel_config=SimpleNamespace(data_parallel_size=max_num_seqs if uses_dlo_dp else 1), + enable_distributed_layerwise_offload=uses_dlo_dp, + dlo_use_allgather=True, + ) + request = OmniDiffusionRequest( + prompt="profile", + request_id="profile-request", + sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1), + ) + engine._make_dummy_request = Mock(return_value=request) + prepared_layout = object() + scheduler_kv_state = ( + DiffusionKVRequest( + "profile-request/diffusion-kv/0", + sequence_id=0, + prefix_len=3, + target_len=8, + seq_len=12, + ), + DiffusionKVRequest( + "profile-request/diffusion-kv/1", + sequence_id=1, + prefix_len=4, + target_len=8, + seq_len=13, + ), + ) + + def preprocess(req: OmniDiffusionRequest) -> OmniDiffusionRequest: + req.prepared_layout = prepared_layout + req.diffusion_kv_requests = scheduler_kv_state + return req + + engine._prepare_request_for_admission = Mock(side_effect=preprocess) + + result = engine._prepare_diffusion_kv_profile_requests() + + assert result is not None + assert len(result) == expected_profile_requests + assert len({profile_request.request_id for profile_request in result}) == expected_profile_requests + assert all(profile_request.is_dummy_run() for profile_request in result) + assert all(profile_request.prepared_layout is prepared_layout for profile_request in result) + assert all(profile_request.diffusion_kv_requests is None for profile_request in result) + assert len({id(profile_request.sampling_params) for profile_request in result}) == expected_profile_requests + assert engine._diffusion_kv_profile_limits == (2, 13, 8) + assert len(result) * engine._diffusion_kv_profile_limits[0] == expected_profile_requests * 2 + engine._make_dummy_request.assert_called_once_with( + height=1024, + width=1024, + guidance_scale=5.0, + num_image_inputs=3, + ) + engine._prepare_request_for_admission.assert_called_once_with(request) + + +@pytest.mark.parametrize( + ("num_sequences", "seq_len", "target_len"), + [ + (3, 16, 8), + (1, 17, 8), + (1, 16, 9), + ], +) +def test_paged_kv_admission_rejects_shape_beyond_profile_envelope( + num_sequences: int, + seq_len: int, + target_len: int, +) -> None: + engine = object.__new__(DiffusionEngine) + engine._diffusion_kv_profile_limits = (2, 16, 8) + engine.pre_process_func = lambda request: request + request = OmniDiffusionRequest( + prompt="too large", + request_id="too-large", + sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1), + diffusion_kv_requests=tuple( + DiffusionKVRequest( + f"too-large/diffusion-kv/{sequence_id}", + sequence_id=sequence_id, + prefix_len=0, + target_len=target_len, + seq_len=seq_len, + ) + for sequence_id in range(num_sequences) + ), + ) + + with pytest.raises(ValueError, match="exceeds the startup memory-profile envelope"): + engine._prepare_request_for_admission(request) + + +def test_paged_kv_admission_accepts_shape_at_profile_envelope() -> None: + engine = object.__new__(DiffusionEngine) + engine._diffusion_kv_profile_limits = (2, 16, 8) + engine.pre_process_func = lambda request: request + request = OmniDiffusionRequest( + prompt="fits", + request_id="fits", + sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1), + diffusion_kv_requests=tuple( + DiffusionKVRequest( + f"fits/diffusion-kv/{sequence_id}", + sequence_id=sequence_id, + prefix_len=8, + target_len=8, + seq_len=16, + ) + for sequence_id in range(2) + ), + ) + + assert engine._prepare_request_for_admission(request) is request diff --git a/tests/diffusion/test_diffusion_engine_rpc_routing.py b/tests/diffusion/test_diffusion_engine_rpc_routing.py index 5aef831aa..119939ae2 100644 --- a/tests/diffusion/test_diffusion_engine_rpc_routing.py +++ b/tests/diffusion/test_diffusion_engine_rpc_routing.py @@ -473,6 +473,38 @@ async def test_cancelled_future_does_not_kill_busy_loop(): _stop_engine(engine) +@pytest.mark.asyncio +async def test_native_kv_reservation_error_wakes_stream_without_killing_busy_loop(): + loop = asyncio.get_running_loop() + engine = _make_engine_with_loop(loop) + + def fail_reservation(*args, **kwargs): + raise RuntimeError("native allocation bug") + + engine.scheduler._diffusion_kv_manager = SimpleNamespace( + has_request=lambda request_id: False, + reserve_request=fail_reservation, + free_request=lambda request_id: None, + ) + request = _make_request("kv-error") + request.diffusion_kv_requests = (object(),) + try: + response_stream = engine.async_add_req_and_stream_response(request) + output_queue = engine._out_streams[request.request_id] + for _ in range(300): + if not output_queue.empty(): + break + await asyncio.sleep(0.01) + assert not output_queue.empty(), "terminal KV allocation error was not delivered to the request stream" + + output = await anext(response_stream) + assert output.error == "native allocation bug" + assert output.finished + assert engine.worker_thread.is_alive() + finally: + _stop_engine(engine) + + # ─────────────────────────── _RpcTask basics ─────────────────────────────── diff --git a/tests/diffusion/test_diffusion_model_runner.py b/tests/diffusion/test_diffusion_model_runner.py index b60cec89c..66da14fd5 100644 --- a/tests/diffusion/test_diffusion_model_runner.py +++ b/tests/diffusion/test_diffusion_model_runner.py @@ -3,6 +3,7 @@ from contextlib import contextmanager from types import SimpleNamespace +from unittest.mock import Mock import pytest import torch @@ -512,6 +513,84 @@ def test_execute_model_accepts_bare_diffusion_output_from_single_request_pipelin assert runner.pipeline.last_req.num_reqs == 1 +@pytest.mark.core_model +@pytest.mark.cpu +def test_profile_run_executes_forward_without_scheduler_kv_validation(monkeypatch): + runner = _make_runner(cache_backend=None, cache_backend_name="none") + request = _make_request() + runner._validate_diffusion_kv_metadata = Mock(side_effect=AssertionError("profile must bypass admission")) + record_names = [] + original_execute = runner._execute_request_list + + def execute(*args, **kwargs): + record_names.append(kwargs["record_name"]) + return original_execute(*args, **kwargs) + + runner._execute_request_list = execute + monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context) + reset_peak_memory_stats = Mock() + monkeypatch.setattr( + model_runner_module.current_omni_platform, + "reset_peak_memory_stats", + reset_peak_memory_stats, + ) + monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_reserved", lambda: 0) + monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_allocated", lambda: 0) + synchronize = Mock() + monkeypatch.setattr(model_runner_module.current_omni_platform, "synchronize", synchronize) + + runner.profile_run([request]) + + assert runner.pipeline.forward_calls == 1 + assert record_names == ["pipeline_memory_profile"] + runner._validate_diffusion_kv_metadata.assert_not_called() + reset_peak_memory_stats.assert_not_called() + synchronize.assert_called_once_with() + + +@pytest.mark.core_model +@pytest.mark.cpu +def test_profile_run_executes_maximum_step_batch_without_resetting_peak(monkeypatch): + runner = _make_runner(cache_backend=None, cache_backend_name=None) + runner.pipeline = _FinalOnlyStepPipeline() + runner.od_config.step_execution = True + requests = [_make_request(), _make_request()] + requests[0].request_id = "profile-0" + requests[1].request_id = "profile-1" + for request in requests: + request.sampling_params.num_inference_steps = 1 + + observed_batch_rows = [] + original_denoise_step = runner.pipeline.denoise_step + + def denoise_step(input_batch, states): + observed_batch_rows.append((len(states), int(input_batch.latents.shape[0]))) + return original_denoise_step(input_batch, states) + + runner.pipeline.denoise_step = denoise_step + runner._validate_diffusion_kv_metadata = Mock(side_effect=AssertionError("profile must bypass admission")) + monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context) + reset_peak_memory_stats = Mock() + monkeypatch.setattr( + model_runner_module.current_omni_platform, + "reset_peak_memory_stats", + reset_peak_memory_stats, + ) + monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_reserved", lambda: 0) + monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_allocated", lambda: 0) + synchronize = Mock() + monkeypatch.setattr(model_runner_module.current_omni_platform, "synchronize", synchronize) + + runner.profile_run(requests) + + assert observed_batch_rows == [(2, 2)] + assert runner.state_cache == {} + assert runner.input_batch is None + runner._validate_diffusion_kv_metadata.assert_not_called() + reset_peak_memory_stats.assert_not_called() + synchronize.assert_called_once_with() + + class _BatchPipeline: """Pipeline returning a configurable list of outputs from forward().""" diff --git a/tests/diffusion/test_diffusion_request.py b/tests/diffusion/test_diffusion_request.py index 259492c1e..7476ec9c1 100644 --- a/tests/diffusion/test_diffusion_request.py +++ b/tests/diffusion/test_diffusion_request.py @@ -52,6 +52,7 @@ def test_dummy_run_request_is_identified_by_reserved_request_id(): assert req.is_dummy_run() assert OmniDiffusionRequest.is_dummy_run_request_id(DUMMY_DIFFUSION_REQUEST_ID) + assert OmniDiffusionRequest.is_dummy_run_request_id(f"{DUMMY_DIFFUSION_REQUEST_ID}/kv-profile-1") def test_non_dummy_request_is_not_identified_as_dummy_run(): diff --git a/tests/diffusion/test_diffusion_scheduler.py b/tests/diffusion/test_diffusion_scheduler.py index 46ec0cbe6..56fc95912 100644 --- a/tests/diffusion/test_diffusion_scheduler.py +++ b/tests/diffusion/test_diffusion_scheduler.py @@ -8,12 +8,14 @@ from types import SimpleNamespace import pytest import torch +import vllm.v1.core.single_type_kv_cache_manager as native_kv_managers from pytest_mock import MockerFixture +from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, KVCacheTensor from vllm_omni.diffusion.data import DiffusionOutput, DiffusionRequestAbortedError from vllm_omni.diffusion.diffusion_engine import DiffusionEngine, DiffusionExecutionMode from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode -from vllm_omni.diffusion.diffusion_kv.request import DiffusionKVContext, DiffusionKVRequest +from vllm_omni.diffusion.diffusion_kv.request import DiffusionKVRequest from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.sched import ( BaseScheduler, @@ -87,6 +89,52 @@ def _cached_ids(sched_output) -> list[str]: return list(sched_output.scheduled_cached_reqs.request_ids) +def _initialize_paged_scheduler( + scheduler: BaseScheduler, + *, + num_blocks: int = 64, + max_num_seqs: int = 1, +) -> None: + native_kv_managers.register_all_kvcache_specs(None) + spec = FullAttentionSpec( + block_size=4, + num_kv_heads=2, + head_size=8, + dtype=torch.bfloat16, + ) + config = KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=[KVCacheTensor(size=spec.page_size_bytes * num_blocks, shared_by=["layer0"])], + kv_cache_groups=[KVCacheGroupSpec(layer_names=["layer0"], kv_cache_spec=spec)], + ) + scheduler.initialize( + SimpleNamespace( + diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER, + max_model_len=64, + max_num_seqs=max_num_seqs, + ), + kv_cache_config=config, + scheduler_block_size=4, + hash_block_size=4, + kv_vllm_config=SimpleNamespace( + model_config=SimpleNamespace(max_model_len=64), + max_in_flight_tokens=64, + ), + ) + + +def _attach_diffusion_kv(request: OmniDiffusionRequest, *, seq_len: int = 8) -> None: + request.diffusion_kv_requests = ( + DiffusionKVRequest( + f"{request.request_id}/diffusion-kv/0", + sequence_id=0, + prefix_len=4, + target_len=4, + seq_len=seq_len, + ), + ) + + class _StubScheduler: def __init__(self, request: OmniDiffusionRequest, output) -> None: self._request = request @@ -400,18 +448,16 @@ class TestRequestScheduler: assert self.scheduler.has_requests() is False def test_diffusion_kv_request_moves_to_scheduler_state(self) -> None: - self.scheduler.initialize(SimpleNamespace(diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER)) + _initialize_paged_scheduler(self.scheduler) request = _make_request("diffusion-kv") prepared_layout = object() request.prepared_layout = prepared_layout - text_context = DiffusionKVContext(context_id="text", cache_role="cross.text", num_tokens=8) kv_request = DiffusionKVRequest( "diffusion-kv/diffusion-kv/0", sequence_id=0, prefix_len=4, target_len=8, seq_len=16, - kv_contexts=(text_context,), ) request.diffusion_kv_requests = (kv_request,) @@ -421,12 +467,179 @@ class TestRequestScheduler: assert state is not None assert state.diffusion_kv_requests == (kv_request,) - assert state.diffusion_kv_requests[0].kv_contexts == (text_context,) assert request.diffusion_kv_requests is None assert scheduler_output.scheduled_new_reqs[0].req.prepared_layout is prepared_layout + metadata = scheduler_output.scheduled_new_reqs[0].diffusion_kv_metadata + assert metadata is not None + assert metadata.request_id == request.request_id + assert len(metadata.sequences[0].block_ids[0]) == 4 + + def test_diffusion_kv_capacity_backpressures_fifo_until_blocks_are_freed(self) -> None: + _initialize_paged_scheduler(self.scheduler, num_blocks=3, max_num_seqs=2) + first_request = _make_request("first") + second_request = _make_request("second") + _attach_diffusion_kv(first_request) + _attach_diffusion_kv(second_request) + self.scheduler.add_request(first_request) + self.scheduler.add_request(second_request) + + first_output = self.scheduler.schedule() + + assert _new_ids(first_output) == ["first"] + assert first_output.num_waiting_reqs == 1 + + self.scheduler.update_from_output(first_output, _make_request_output("first")) + second_output = self.scheduler.schedule() + + assert _new_ids(second_output) == ["second"] + assert second_output.num_waiting_reqs == 0 + # The same output carries both the release and the newly admitted block + # table; the future Worker data plane must process finished ids first. + assert second_output.finished_req_ids == {"first"} + assert second_output.scheduled_new_reqs[0].diffusion_kv_metadata is not None + + def test_impossible_diffusion_kv_capacity_finishes_only_that_request(self) -> None: + _initialize_paged_scheduler(self.scheduler, num_blocks=2, max_num_seqs=2) + impossible = _make_request("impossible") + schedulable = _make_request("schedulable") + _attach_diffusion_kv(impossible, seq_len=8) + schedulable.diffusion_kv_requests = ( + DiffusionKVRequest( + "schedulable/diffusion-kv/0", + sequence_id=0, + prefix_len=0, + target_len=4, + seq_len=4, + ), + ) + self.scheduler.add_request(impossible) + self.scheduler.add_request(schedulable) + + sched_output = self.scheduler.schedule() + + failed_state = self.scheduler.get_request_state("impossible") + assert failed_state is not None + assert failed_state.status == DiffusionRequestStatus.FINISHED_ERROR + assert failed_state.error is not None + assert "cannot fit even when the block pool is empty" in failed_state.error + assert sched_output.finished_req_ids == {"impossible"} + assert _new_ids(sched_output) == ["schedulable"] + + finished = self.scheduler.update_from_output( + sched_output, + _make_request_output("schedulable"), + ) + + assert finished == {"impossible", "schedulable"} + + def test_impossible_diffusion_kv_capacity_does_not_block_waiters_under_load(self) -> None: + _initialize_paged_scheduler(self.scheduler, num_blocks=4, max_num_seqs=2) + running = _make_request("running") + _attach_diffusion_kv(running) + self.scheduler.add_request(running) + assert _new_ids(self.scheduler.schedule()) == ["running"] + + impossible = _make_request("impossible") + impossible.diffusion_kv_requests = tuple( + DiffusionKVRequest( + f"impossible/diffusion-kv/{sequence_id}", + sequence_id=sequence_id, + prefix_len=0, + target_len=4, + seq_len=4, + ) + for sequence_id in range(4) + ) + schedulable = _make_request("schedulable") + schedulable.diffusion_kv_requests = ( + DiffusionKVRequest( + "schedulable/diffusion-kv/0", + sequence_id=0, + prefix_len=0, + target_len=4, + seq_len=4, + ), + ) + self.scheduler.add_request(impossible) + self.scheduler.add_request(schedulable) + + sched_output = self.scheduler.schedule() + + failed_state = self.scheduler.get_request_state("impossible") + assert failed_state is not None + assert failed_state.status == DiffusionRequestStatus.FINISHED_ERROR + assert failed_state.error is not None + assert "required_blocks=4, available_blocks=3" in failed_state.error + assert sched_output.finished_req_ids == {"impossible"} + assert _new_ids(sched_output) == ["schedulable"] + + def test_diffusion_kv_internal_allocation_error_finishes_request(self, monkeypatch) -> None: + _initialize_paged_scheduler(self.scheduler) + request = _make_request("native-error") + _attach_diffusion_kv(request) + self.scheduler.add_request(request) + manager = self.scheduler._diffusion_kv_manager + assert manager is not None + + def raise_native_error(*args, **kwargs): + raise ValueError("native allocation bug") + + monkeypatch.setattr( + manager, + "reserve_request", + raise_native_error, + ) + + sched_output = self.scheduler.schedule() + + state = self.scheduler.get_request_state("native-error") + assert state is not None + assert state.status == DiffusionRequestStatus.FINISHED_ERROR + assert state.error == "native allocation bug" + assert sched_output.finished_req_ids == {"native-error"} + assert sched_output.scheduled_request_ids == [] + + def test_diffusion_kv_preemption_retains_allocation(self) -> None: + _initialize_paged_scheduler(self.scheduler, num_blocks=3) + request = _make_request("preempted") + _attach_diffusion_kv(request) + self.scheduler.add_request(request) + self.scheduler.schedule() + manager = self.scheduler._diffusion_kv_manager + assert manager is not None + metadata = manager.get_metadata("preempted") + + assert self.scheduler.preempt_request("preempted") is True + resumed_output = self.scheduler.schedule() + + assert resumed_output.scheduled_new_reqs == [] + assert _cached_ids(resumed_output) == ["preempted"] + assert manager.get_metadata("preempted") == metadata + + @pytest.mark.parametrize( + "status", + [ + DiffusionRequestStatus.FINISHED_COMPLETED, + DiffusionRequestStatus.FINISHED_ABORTED, + DiffusionRequestStatus.FINISHED_ERROR, + ], + ) + def test_diffusion_kv_terminal_status_releases_allocation(self, status: DiffusionRequestStatus) -> None: + _initialize_paged_scheduler(self.scheduler, num_blocks=3) + request = _make_request("terminal") + _attach_diffusion_kv(request) + self.scheduler.add_request(request) + self.scheduler.schedule() + manager = self.scheduler._diffusion_kv_manager + assert manager is not None + assert manager.has_request("terminal") is True + + self.scheduler.finish_requests("terminal", status) + + assert manager.has_request("terminal") is False def test_paged_scheduler_rejects_missing_kv_request(self) -> None: - self.scheduler.initialize(SimpleNamespace(diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER)) + _initialize_paged_scheduler(self.scheduler) request = _make_request("diffusion-kv") with pytest.raises(ValueError, match="did not produce DiffusionKVRequest"): @@ -443,7 +656,7 @@ class TestRequestScheduler: ], ) def test_diffusion_kv_request_rejects_legacy_dense_kv(self, owner_name: str, field_name: str) -> None: - self.scheduler.initialize(SimpleNamespace(diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER)) + _initialize_paged_scheduler(self.scheduler) request = _make_request("diffusion-kv") owner = request if owner_name == "request" else request.sampling_params setattr(owner, field_name, object()) @@ -921,6 +1134,45 @@ class TestDiffusionEngine: assert engine.scheduler.max_num_running_reqs == 1 fake_executor_cls.assert_called_once_with(od_config) + def test_initializes_paged_scheduler_from_native_control_plane( + self, + monkeypatch: pytest.MonkeyPatch, + mocker: MockerFixture, + ) -> None: + od_config = SimpleNamespace(model_class_name="mock_model", streaming_output=False) + fake_executor_cls = mocker.Mock(return_value=mocker.Mock()) + scheduler = mocker.Mock() + kv_cache_config = object() + kv_vllm_config = object() + + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.get_diffusion_post_process_func", + lambda *args, **kwargs: None, + ) + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.get_diffusion_pre_process_func", + lambda *args, **kwargs: None, + ) + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.DiffusionExecutor.get_class", + lambda *args, **kwargs: fake_executor_cls, + ) + monkeypatch.setattr( + "vllm_omni.diffusion.diffusion_engine.initialize_diffusion_kv_control_plane", + lambda *args, **kwargs: (kv_cache_config, 16, 16, kv_vllm_config), + ) + + engine = DiffusionEngine(od_config, scheduler=scheduler) + + assert engine.scheduler is scheduler + scheduler.initialize.assert_called_once_with( + od_config, + kv_cache_config=kv_cache_config, + scheduler_block_size=16, + hash_block_size=16, + kv_vllm_config=kv_vllm_config, + ) + def test_scheduler_alias_keeps_default_request_scheduler(self) -> None: scheduler = Scheduler() scheduler.initialize(SimpleNamespace()) @@ -987,6 +1239,21 @@ class TestDiffusionEngine: assert output.aborted is True assert output.abort_message == "Request req-finalize aborted." + def test_finalize_finished_request_returns_scheduler_admission_error(self) -> None: + engine = DiffusionEngine.__new__(DiffusionEngine) + engine.scheduler = StepScheduler() + engine.scheduler.initialize(SimpleNamespace()) + + req_id = engine.scheduler.add_request(_make_request("req-error")) + engine.scheduler._finish_requests( + {req_id: DiffusionRequestStatus.FINISHED_ERROR}, + {req_id: "KV request cannot fit"}, + ) + + output = engine._finalize_finished_request(req_id) + + assert output.error == "KV request cannot fit" + @pytest.mark.asyncio async def test_streaming_runner_output_notifies_each_chunk(self) -> None: engine = DiffusionEngine.__new__(DiffusionEngine) diff --git a/tests/diffusion/test_diffusion_worker.py b/tests/diffusion/test_diffusion_worker.py index 6a49d0909..0d8d5e92a 100644 --- a/tests/diffusion/test_diffusion_worker.py +++ b/tests/diffusion/test_diffusion_worker.py @@ -13,11 +13,7 @@ import pytest import torch from pytest_mock import MockerFixture -from vllm_omni.diffusion.worker.diffusion_worker import ( - DiffusionWorker, - _create_diffusion_worker_vllm_config, - _make_diffusion_vllm_model_config, -) +from vllm_omni.diffusion.worker.diffusion_worker import DiffusionWorker pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.gpu] @@ -58,34 +54,6 @@ def mock_gpu_worker(mocker: MockerFixture, mock_od_config): return worker -def test_diffusion_vllm_model_config_supplies_dtype_for_quant_methods(): - from types import SimpleNamespace - - from vllm_omni.quantization import build_quant_config - - od_config = SimpleNamespace( - model="dummy", - dtype=torch.bfloat16, - quantization_config=build_quant_config( - { - "quant_method": "modelopt", - "quant_algo": "FP8", - "ignore": [], - } - ), - tf_model_config=SimpleNamespace(), - enforce_eager=True, - is_moe=False, - ) - - model_config = _make_diffusion_vllm_model_config(od_config) - - assert model_config.dtype is torch.bfloat16 - assert model_config.quantization == "modelopt" - assert model_config.quantization_config is od_config.quantization_config - assert model_config.is_quantized() - - class TestDiffusionWorkerSleep: """Test DiffusionWorker.sleep method.""" @@ -329,49 +297,3 @@ class TestDiffusionWorkerWakeUp: mock_buffer2.data.copy_.assert_not_called() assert result is True - - -class TestWorkerVllmConfigAdditionalConfig: - """Test worker-side VllmConfig construction with additional_config.""" - - def test_create_diffusion_worker_vllm_config_passes_additional_config(self, mocker: MockerFixture): - mock_vllm_config = mocker.Mock() - mock_vllm_cls = mocker.patch( - "vllm_omni.diffusion.worker.diffusion_worker.VllmConfig", - return_value=mock_vllm_config, - ) - od_config = mocker.Mock(additional_config={"torchair_graph_config": {"enabled": True}}) - - result = _create_diffusion_worker_vllm_config(torch.device("cpu"), od_config) - - assert result is mock_vllm_config - assert mock_vllm_cls.call_args.kwargs["additional_config"] == od_config.additional_config - - def test_create_diffusion_worker_vllm_config_falls_back_when_constructor_rejects_additional_config( - self, mocker: MockerFixture - ): - mock_vllm_config = mocker.Mock() - mock_vllm_cls = mocker.patch( - "vllm_omni.diffusion.worker.diffusion_worker.VllmConfig", - side_effect=[ - TypeError("VllmConfig.__init__() got an unexpected keyword argument 'additional_config'"), - mock_vllm_config, - ], - ) - od_config = mocker.Mock(additional_config={"ascend_scheduler_config": {"foo": "bar"}}) - - result = _create_diffusion_worker_vllm_config(torch.device("cpu"), od_config) - - assert result is mock_vllm_config - assert mock_vllm_cls.call_count == 2 - assert getattr(mock_vllm_config, "additional_config") == od_config.additional_config - - def test_create_diffusion_worker_vllm_config_reraises_other_type_errors(self, mocker: MockerFixture): - mocker.patch( - "vllm_omni.diffusion.worker.diffusion_worker.VllmConfig", - side_effect=TypeError("additional_config values must be hashable"), - ) - od_config = mocker.Mock(additional_config={"ascend_scheduler_config": {"foo": "bar"}}) - - with pytest.raises(TypeError, match="hashable"): - _create_diffusion_worker_vllm_config(torch.device("cpu"), od_config) diff --git a/tests/diffusion/test_multiproc_engine_concurrency.py b/tests/diffusion/test_multiproc_engine_concurrency.py index aa40d0770..d5c36a093 100644 --- a/tests/diffusion/test_multiproc_engine_concurrency.py +++ b/tests/diffusion/test_multiproc_engine_concurrency.py @@ -413,6 +413,26 @@ class TestRequestModeDispatch: assert [output.result.error for output in result.runner_outputs] == ["A", "B"] executor.collective_rpc.assert_called_once() + def test_dlo_dp_forwards_request_metadata_envelopes_as_one_wave(self): + executor, _, _ = _make_executor(num_gpus=2) + executor.od_config = SimpleNamespace( + step_execution=False, + parallel_config=SimpleNamespace(data_parallel_size=2), + enable_distributed_layerwise_offload=True, + dlo_use_allgather=True, + ) + executor.collective_rpc = Mock(return_value=[_tagged_output("A"), _tagged_output("B")]) + scheduler_output = _make_sched_output("A", "B") + for new_req in scheduler_output.scheduled_new_reqs: + new_req.diffusion_kv_metadata = SimpleNamespace(request_id=new_req.request_id) + + result = executor.execute_request(scheduler_output) + + assert [output.result.error for output in result.runner_outputs] == ["A", "B"] + forwarded_envelopes = executor.collective_rpc.call_args.kwargs["args"][0] + assert forwarded_envelopes is scheduler_output.scheduled_new_reqs + assert [envelope.diffusion_kv_metadata.request_id for envelope in forwarded_envelopes] == ["A", "B"] + @pytest.mark.parametrize( ("field", "value"), [ diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index 8c4a933e2..4a8a54eca 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -814,6 +814,10 @@ _DIFFUSION_MOVED_SHARED_FIELDS = frozenset( "omni_kv_config", "cfg_kv_collect_func", "max_num_seqs", + "kv_cache_memory_bytes", + "gpu_memory_utilization", + "max_num_batched_tokens", + "max_model_len", "enable_sleep_mode", "enforce_eager", "enable_multithread_weight_load", diff --git a/vllm_omni/diffusion/attention/backends/abstract.py b/vllm_omni/diffusion/attention/backends/abstract.py index f3a5f99ef..4a34aa765 100644 --- a/vllm_omni/diffusion/attention/backends/abstract.py +++ b/vllm_omni/diffusion/attention/backends/abstract.py @@ -81,6 +81,19 @@ class AttentionBackend(ABC): supported_head_sizes = cls.get_supported_head_sizes() return (not supported_head_sizes) or head_size in supported_head_sizes + @classmethod + def indexes_kv_by_block_stride(cls) -> bool: + """Whether this backend reads K/V pages by the runtime block stride. + + Returning ``True`` means the physical cache layout has ``num_blocks`` + as its outer stride, so native vLLM may safely use page-size padding + when it unifies cache layouts across layers. Dense diffusion backends + conservatively keep the default ``False``; a paged backend should + override this only when its kernel actually follows that layout. + """ + + return False + @dataclass(frozen=True, slots=True) class QueryRange: diff --git a/vllm_omni/diffusion/attention/layer.py b/vllm_omni/diffusion/attention/layer.py index 7eb74a941..ad9fe2b36 100644 --- a/vllm_omni/diffusion/attention/layer.py +++ b/vllm_omni/diffusion/attention/layer.py @@ -11,8 +11,10 @@ from dataclasses import replace import torch import torch.nn as nn +from vllm.config import VllmConfig, set_current_vllm_config from vllm.logger import init_logger from vllm.model_executor.models.utils import extract_layer_index +from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheSpec from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend @@ -61,12 +63,23 @@ class Attention(nn.Module): # perf for this layer (e.g. Wan2.2 cross-attn has short sequences and # block-FP8 quant offers no win). Default False = follow global config. disable_kv_quant: bool = False, + # Opt-in marker for Scheduler-managed paged KV. Unmarked diffusion + # attention remains dense and contributes no native KVCacheSpec. + paged_kv_cache_role: str | None = None, + paged_kv_cache_dtype: torch.dtype | None = None, ): super().__init__() self.role = role self.role_category = role_category self.qkv_layout = qkv_layout + if paged_kv_cache_role == "": + raise ValueError("paged_kv_cache_role must be non-empty when provided") + self.paged_kv_cache_role = paged_kv_cache_role + self.paged_kv_cache_dtype = paged_kv_cache_dtype + self.num_heads = num_heads + self.num_kv_heads = num_kv_heads if num_kv_heads is not None else num_heads + self.head_size = head_size # Resolve backend via role-aware config. # The global diffusion config is set during model init via @@ -171,6 +184,25 @@ class Attention(nn.Module): self._disable_kv_quant: bool = disable_kv_quant self._init_kv_cache_quantization(config) + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None: + """Return native rank-local geometry for an opted-in paged cache.""" + + if self.paged_kv_cache_role is None: + return None + dtype = self.paged_kv_cache_dtype or vllm_config.model_config.dtype + # Keep backend layout discovery under the same config context used by + # upstream vLLM's attention-spec collector. + with set_current_vllm_config(vllm_config): + indexes_kv_by_block_stride = self.attn_backend.indexes_kv_by_block_stride() + return FullAttentionSpec( + block_size=vllm_config.cache_config.block_size, + num_kv_heads=self.num_kv_heads, + head_size=self.head_size, + dtype=dtype, + indexes_kv_by_block_stride=indexes_kv_by_block_stride, + non_causal=not self.causal, + ) + def _get_active_parallel_strategy(self): """Get the parallel strategy based on current SP active state. diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index 0ea85405b..34c8ccb96 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -851,6 +851,14 @@ class OmniDiffusionConfig: # Maximum number of sequences to generate in a batch max_num_seqs: int = 1 + # Native vLLM KV-cache sizing inputs used only by paged_scheduler mode. + # These mirror the structured stage cache/scheduler config until all + # diffusion runtime construction consumes VllmOmniConfig directly. + kv_cache_memory_bytes: int | None = None + gpu_memory_utilization: float = 0.9 + max_num_batched_tokens: int | None = None + max_model_len: int | None = None + # Request-mode batch admission: wait briefly for compatible requests to # accumulate in the scheduler waiting queue before the first schedule() of # a wave. Improves fused forward batch sizes under bursty HTTP ingress. @@ -936,6 +944,14 @@ class OmniDiffusionConfig: if not isinstance(self.diffusion_compile_dynamic, bool): raise TypeError(f"diffusion_compile_dynamic must be a bool, got {type(self.diffusion_compile_dynamic)!r}") self.diffusion_kv_mode = parse_diffusion_kv_cache_mode(self.diffusion_kv_mode) + if self.kv_cache_memory_bytes is not None and self.kv_cache_memory_bytes < 0: + raise ValueError("kv_cache_memory_bytes must be non-negative") + if not 0.0 < self.gpu_memory_utilization <= 1.0: + raise ValueError("gpu_memory_utilization must be in (0, 1]") + if self.max_num_batched_tokens is not None and self.max_num_batched_tokens <= 0: + raise ValueError("max_num_batched_tokens must be positive") + if self.max_model_len is not None and self.max_model_len != -1 and self.max_model_len <= 0: + raise ValueError("max_model_len must be positive or -1") if self.omni_kv_config is None: self.omni_kv_config = {} diff --git a/vllm_omni/diffusion/diffusion_engine.py b/vllm_omni/diffusion/diffusion_engine.py index fe9ed5c8c..4812c8925 100644 --- a/vllm_omni/diffusion/diffusion_engine.py +++ b/vllm_omni/diffusion/diffusion_engine.py @@ -5,6 +5,7 @@ from __future__ import annotations import asyncio import concurrent.futures +import copy import inspect import queue import threading @@ -17,18 +18,23 @@ from typing import TYPE_CHECKING, Any import numpy as np import PIL.Image import torch +from vllm.config import VllmConfig from vllm.logger import init_logger from vllm.utils.import_utils import resolve_obj_by_qualname from vllm.v1.engine.exceptions import EngineDeadError +from vllm.v1.kv_cache_interface import KVCacheConfig from vllm_omni.diffusion.data import ( DiffusionOutput, DiffusionRequestAbortedError, OmniDiffusionConfig, ) +from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode +from vllm_omni.diffusion.diffusion_kv.initialization import initialize_diffusion_kv_control_plane from vllm_omni.diffusion.executor.abstract import DiffusionExecutor from vllm_omni.diffusion.io_support import ( get_dummy_run_num_frames, + get_dummy_run_num_image_inputs, image_color_format, supports_audio_output, supports_multimodal_input, @@ -184,12 +190,39 @@ class DiffusionEngine: from the resolved execution mode. """ self.od_config = od_config + # Set after the paged-KV profile request has gone through model-owned + # preprocessing. Real requests are admitted only within this measured + # activation envelope: (max execution sequences, max seq_len, + # max target_len). + self._diffusion_kv_profile_limits: tuple[int, int, int] | None = None self._init_process_hooks(od_config) self.execution_mode = self._resolve_execution_mode(od_config) self._init_executor(od_config) try: - self._init_scheduler(od_config, scheduler) + profile_requests = self._prepare_diffusion_kv_profile_requests() + kv_control_plane = initialize_diffusion_kv_control_plane( + self.executor, + od_config, + profile_requests=profile_requests, + ) + if kv_control_plane is None: + self._init_scheduler(od_config, scheduler) + else: + ( + kv_cache_config, + scheduler_block_size, + hash_block_size, + kv_vllm_config, + ) = kv_control_plane + self._init_scheduler( + od_config, + scheduler, + kv_cache_config, + scheduler_block_size=scheduler_block_size, + hash_block_size=hash_block_size, + kv_vllm_config=kv_vllm_config, + ) self._init_runtime_state() self._init_execute_fn() self._log_execution_mode(od_config) @@ -242,6 +275,11 @@ class DiffusionEngine: self, od_config: OmniDiffusionConfig, scheduler: BaseScheduler | None = None, + kv_cache_config: KVCacheConfig | None = None, + *, + scheduler_block_size: int | None = None, + hash_block_size: int | None = None, + kv_vllm_config: VllmConfig | None = None, ) -> None: if scheduler is not None: self.scheduler = scheduler @@ -249,7 +287,16 @@ class DiffusionEngine: self.scheduler = StepScheduler() else: self.scheduler = RequestScheduler() - self.scheduler.initialize(od_config) + if kv_cache_config is None: + self.scheduler.initialize(od_config) + else: + self.scheduler.initialize( + od_config, + kv_cache_config=kv_cache_config, + scheduler_block_size=scheduler_block_size, + hash_block_size=hash_block_size, + kv_vllm_config=kv_vllm_config, + ) def _init_runtime_state(self) -> None: # DP multi-concurrency: allow batching dp_size requests so each @@ -703,9 +750,36 @@ class DiffusionEngine: """Run model-owned preprocessing once, before entering Engine locks.""" pre_process_func = getattr(self, "pre_process_func", None) - if pre_process_func is None: - return request - return pre_process_func(request) + if pre_process_func is not None: + request = pre_process_func(request) + self._validate_diffusion_kv_profile_limits(request) + return request + + def _validate_diffusion_kv_profile_limits(self, request: OmniDiffusionRequest) -> None: + """Keep admitted paged-KV requests within the profiled activation shape.""" + + profile_limits = getattr(self, "_diffusion_kv_profile_limits", None) + if profile_limits is None: + return + kv_requests = request.diffusion_kv_requests + if not kv_requests: + return + + request_limits = ( + len(kv_requests), + max(kv_request.seq_len for kv_request in kv_requests), + max(kv_request.target_len for kv_request in kv_requests), + ) + if any(actual > profiled for actual, profiled in zip(request_limits, profile_limits, strict=True)): + request_sequences, request_seq_len, request_target_len = request_limits + profile_sequences, profile_seq_len, profile_target_len = profile_limits + raise ValueError( + f"Diffusion KV request {request.request_id!r} exceeds the startup memory-profile envelope: " + f"sequences={request_sequences} (profiled={profile_sequences}), " + f"max_seq_len={request_seq_len} (profiled={profile_seq_len}), " + f"max_target_len={request_target_len} (profiled={profile_target_len}). " + "Reduce the request shape or extend the model's paged-KV profile recipe." + ) def _add_prepared_request(self, request: OmniDiffusionRequest) -> str: """Admit a request whose model-owned preprocessing is complete.""" @@ -869,47 +943,111 @@ class DiffusionEngine: self.close() raise e - def _dummy_run(self): - """A dummy run to warm up the model.""" - num_inference_steps = 1 - height = 512 - width = 512 - prompt: OmniTextPrompt = {"prompt": "dummy run"} + def _make_dummy_request( + self, + *, + height: int, + width: int, + guidance_scale: float, + num_image_inputs: int = 1, + ) -> OmniDiffusionRequest | None: + """Build a one-step model request for startup profiling or warmup.""" + prompt: OmniTextPrompt = {"prompt": "dummy run"} supports_image_input, supports_audio_input = supports_multimodal_input(self.od_config) if supports_image_input: - # Provide a dummy image input if the model supports it color_format = image_color_format(self.od_config.model_class_name) - dummy_image = PIL.Image.new(color_format, (width, height)) - prompt.setdefault("multi_modal_data", {})["image"] = dummy_image + images = [PIL.Image.new(color_format, (width, height)) for _ in range(num_image_inputs)] + prompt.setdefault("multi_modal_data", {})["image"] = images[0] if len(images) == 1 else images if supports_audio_input: audio_sr = 16000 - dummy_audio = np.random.randn(audio_sr * 2).astype(np.float32) - prompt.setdefault("multi_modal_data", {})["audio"] = dummy_audio + prompt.setdefault("multi_modal_data", {})["audio"] = np.random.randn(audio_sr * 2).astype(np.float32) num_frames = get_dummy_run_num_frames(self.od_config.model_class_name, supports_audio_input) if num_frames <= 0: - logger.info("Skipping dummy warmup run (num_frames=0)") - return - req = OmniDiffusionRequest( + return None + return OmniDiffusionRequest( prompt=prompt, request_id=DUMMY_DIFFUSION_REQUEST_ID, sampling_params=OmniDiffusionSamplingParams( height=height, width=width, - num_inference_steps=num_inference_steps, + num_inference_steps=1, num_frames=num_frames, - # Keep warmup path minimal and robust across text encoders. - # Some models may fail when warmup implicitly triggers - # classifier-free guidance with an empty negative prompt. - guidance_scale=0.0, + guidance_scale=guidance_scale, num_outputs_per_prompt=1, - # Disable CFG for warmup to avoid triggering CFG parallel - # validation when cfg_parallel_size > 1. extra_args={"cfg_text_scale": 1.0, "cfg_img_scale": 1.0}, ), ) + + def _prepare_diffusion_kv_profile_requests(self) -> list[OmniDiffusionRequest] | None: + """Prepare the per-rank request batch used to profile paged-KV headroom. + + The profile executes directly on each Worker before the Scheduler and + its KV manager exist. It uses the maximum number of requests that one + rank can execute together. DLO+DP is the exception because each rank + executes one request from the collective wave. + + Hunyuan is currently the only model integrated with + ``paged_scheduler``; 1024x1024, enabled CFG, and the maximum advertised + reference-image count exercise its first-step activation peak. + Admission compares each preprocessed request's CFG count and tokenized + sequence/target shape with the resulting per-request profile envelope. + Future paged model integrations must extend this recipe for their + serving limits (for example, video frame count) rather than reusing it + silently. + """ + + if ( + getattr(self.od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY) + is not DiffusionKVCacheMode.PAGED_SCHEDULER + ): + return None + request = self._make_dummy_request( + height=1024, + width=1024, + guidance_scale=5.0, + num_image_inputs=get_dummy_run_num_image_inputs(self.od_config.model_class_name), + ) + if request is None: + raise RuntimeError("paged_scheduler requires a runnable Diffusion KV memory profile request") + request = self._prepare_request_for_admission(request) + kv_requests = request.diffusion_kv_requests + if not kv_requests: + raise RuntimeError("paged_scheduler profile preprocessing must produce Diffusion KV requests") + self._diffusion_kv_profile_limits = ( + len(kv_requests), + max(kv_request.seq_len for kv_request in kv_requests), + max(kv_request.target_len for kv_request in kv_requests), + ) + dlo_dp_request_mode = self.execution_mode is DiffusionExecutionMode.REQUEST_BATCH and _uses_dlo_dp_concurrency( + self.od_config + ) + profile_batch_size = 1 if dlo_dp_request_mode else _max_num_seqs(self.od_config) + profile_requests: list[OmniDiffusionRequest] = [] + for index in range(profile_batch_size): + profile_request = copy.copy(request) + profile_request.request_id = f"{DUMMY_DIFFUSION_REQUEST_ID}/kv-profile-{index}" + profile_request.sampling_params = copy.deepcopy(request.sampling_params) + # Preprocessing owns request geometry and prepared model inputs, + # but native KV requests remain Scheduler-only mutable state. The + # profile bypasses Scheduler admission and must not send them to + # the Worker. + profile_request.diffusion_kv_requests = None + profile_requests.append(profile_request) + return profile_requests + + def _dummy_run(self): + """A dummy run to warm up the model.""" + req = self._make_dummy_request( + height=512, + width=512, + guidance_scale=0.0, + ) + if req is None: + logger.info("Skipping dummy warmup run (num_frames=0)") + return logger.info("dummy run to warm up the model") output = self.add_req_and_wait_for_response(req) if output.error: @@ -1127,4 +1265,7 @@ class DiffusionEngine: if runner_output is not None and runner_output.async_output_id is not None: return DiffusionOutput(async_output_id=runner_output.async_output_id) + if state.status == DiffusionRequestStatus.FINISHED_ERROR and state.error: + return DiffusionOutput(error=state.error) + return DiffusionOutput(error=missing_result_error) diff --git a/vllm_omni/diffusion/diffusion_kv/initialization.py b/vllm_omni/diffusion/diffusion_kv/initialization.py new file mode 100644 index 000000000..854f3aaa5 --- /dev/null +++ b/vllm_omni/diffusion/diffusion_kv/initialization.py @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch +from vllm.config import VllmConfig +from vllm.v1.core.kv_cache_utils import ( + generate_scheduler_kv_cache_config, + get_kv_cache_configs, + resolve_kv_cache_block_sizes, +) +from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec + +from vllm_omni.diffusion.data import OmniDiffusionConfig +from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode +from vllm_omni.diffusion.vllm_config import create_diffusion_vllm_config +from vllm_omni.platforms import current_omni_platform + +if TYPE_CHECKING: + from vllm_omni.diffusion.executor.abstract import DiffusionExecutor + from vllm_omni.diffusion.request import OmniDiffusionRequest + + +def build_native_kv_cache_configs( + vllm_config: VllmConfig, + worker_specs: list[dict[str, KVCacheSpec]], + available_memory: list[int], +) -> tuple[list[KVCacheConfig], KVCacheConfig]: + """Build rank-local and Scheduler-native configs using vLLM utilities.""" + + from vllm.v1.core import single_type_kv_cache_manager + + single_type_kv_cache_manager.register_all_kvcache_specs(vllm_config) + worker_configs = get_kv_cache_configs(vllm_config, worker_specs, available_memory) + return worker_configs, generate_scheduler_kv_cache_config(worker_configs) + + +def initialize_diffusion_kv_control_plane( + executor: DiffusionExecutor, + od_config: OmniDiffusionConfig, + *, + profile_requests: list[OmniDiffusionRequest] | None = None, + device: torch.device | None = None, +) -> tuple[KVCacheConfig, int, int, VllmConfig] | None: + """Run the native Worker-spec to Scheduler-config initialization chain.""" + + if ( + getattr(od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY) + is not DiffusionKVCacheMode.PAGED_SCHEDULER + ): + return None + if not profile_requests: + raise ValueError("paged_scheduler Diffusion KV initialization requires prepared memory profile requests") + if device is None: + device = current_omni_platform.get_torch_device(0) + vllm_config = create_diffusion_vllm_config(device, od_config) + worker_specs = executor.get_kv_cache_specs() + available_memory = executor.determine_available_kv_memory(profile_requests) + if len(worker_specs) != od_config.num_gpus or len(available_memory) != od_config.num_gpus: + raise ValueError( + "Diffusion KV initialization rank count mismatch: " + f"expected={od_config.num_gpus}, specs={len(worker_specs)}, memory={len(available_memory)}" + ) + worker_configs, scheduler_kv_cache_config = build_native_kv_cache_configs( + vllm_config, + worker_specs, + available_memory, + ) + scheduler_block_size, hash_block_size = resolve_kv_cache_block_sizes( + scheduler_kv_cache_config, + vllm_config, + ) + executor.set_kv_cache_configs(worker_configs) + return ( + scheduler_kv_cache_config, + scheduler_block_size, + hash_block_size, + vllm_config, + ) diff --git a/vllm_omni/diffusion/diffusion_kv/manager.py b/vllm_omni/diffusion/diffusion_kv/manager.py new file mode 100644 index 000000000..eee51b864 --- /dev/null +++ b/vllm_omni/diffusion/diffusion_kv/manager.py @@ -0,0 +1,191 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from collections.abc import Sequence + +from vllm.v1.core.kv_cache_manager import KVCacheManager +from vllm.v1.kv_cache_interface import KVCacheConfig + +from vllm_omni.diffusion.diffusion_kv.metadata import ( + DiffusionKVMetadata, + DiffusionKVSequenceMetadata, +) +from vllm_omni.diffusion.diffusion_kv.request import DiffusionKVRequest + + +class DiffusionKVAdmissionError(ValueError): + """A request-specific KV admission failure safe to return to the caller.""" + + +class DiffusionKVCacheManager: + """Atomic public-request facade over native vLLM KVCacheManager.""" + + def __init__( + self, + kv_cache_config: KVCacheConfig, + *, + max_model_len: int, + scheduler_block_size: int, + hash_block_size: int, + max_in_flight_tokens: int | None = None, + ) -> None: + if max_model_len <= 0: + raise ValueError(f"max_model_len must be positive, got {max_model_len}") + if kv_cache_config.num_blocks <= 0 or not kv_cache_config.kv_cache_groups: + raise ValueError("Diffusion KVCacheConfig must contain a positive block pool and at least one group") + if scheduler_block_size <= 0 or hash_block_size <= 0: + raise ValueError("scheduler_block_size and hash_block_size must be positive") + if max_in_flight_tokens is not None and max_in_flight_tokens <= 0: + raise ValueError(f"max_in_flight_tokens must be positive when provided, got {max_in_flight_tokens}") + self.native_manager = KVCacheManager( + kv_cache_config=kv_cache_config, + max_model_len=max_model_len, + scheduler_block_size=scheduler_block_size, + hash_block_size=hash_block_size, + max_in_flight_tokens=max_in_flight_tokens, + enable_caching=False, + ) + self.max_model_len = max_model_len + # Native vLLM may reserve a null block, so an idle BlockPool does not + # necessarily report ``kv_cache_config.num_blocks`` free blocks. + self._empty_pool_num_free_blocks = self.native_manager.block_pool.get_num_free_blocks() + self._requests: dict[str, tuple[DiffusionKVRequest, ...]] = {} + self._metadata: dict[str, DiffusionKVMetadata] = {} + self._internal_request_ids: set[str] = set() + self._next_allocation_generation = 1 + + def _get_empty_pool_required_blocks(self, requests: Sequence[DiffusionKVRequest]) -> int: + """Return the native full-sequence reservation for one public request. + + vLLM performs this calculation inside ``allocate_slots`` when + ``full_sequence_must_fit`` is enabled. Diffusion CFG adds one semantic + requirement that native requests do not have: every execution sequence + must fit atomically. Sum the native per-sequence result so a request + that can never fit is rejected independently of current pool usage. + """ + + coordinator = self.native_manager.coordinator + empty_blocks = self.native_manager.empty_kv_cache_blocks.blocks + return sum( + coordinator.get_num_blocks_to_allocate( + request_id=request.request_id, + num_tokens=request.num_tokens, + new_computed_blocks=empty_blocks, + num_encoder_tokens=0, + total_computed_tokens=0, + num_local_computed_tokens=0, + num_tokens_main_model=request.num_tokens, + apply_admission_cap=True, + ) + for request in requests + ) + + def has_request(self, public_request_id: str) -> bool: + return public_request_id in self._requests + + def reserve_request( + self, + public_request_id: str, + kv_requests: Sequence[DiffusionKVRequest], + ) -> DiffusionKVMetadata | None: + """Reserve all CFG sequences or roll back the complete public request.""" + + if not public_request_id: + raise ValueError("public_request_id must be non-empty") + if public_request_id in self._requests: + raise ValueError(f"Diffusion KV request {public_request_id!r} is already allocated") + requests = tuple(kv_requests) + if not requests: + raise ValueError("Diffusion KV allocation requires at least one sequence") + + sequence_ids = [request.sequence_id for request in requests] + internal_ids = [request.request_id for request in requests] + expected_sequence_ids = list(range(len(requests))) + if sequence_ids != expected_sequence_ids: + raise ValueError( + "Diffusion KV requests must be ordered by contiguous sequence_id values: " + f"expected={expected_sequence_ids}, got={sequence_ids}" + ) + if len(internal_ids) != len(set(internal_ids)): + raise ValueError(f"Diffusion KV internal request IDs must be unique, got {internal_ids}") + conflicts = self._internal_request_ids.intersection(internal_ids) + if conflicts: + raise ValueError(f"Diffusion KV internal request IDs are already allocated: {sorted(conflicts)}") + for request in requests: + if request.kv_contexts: + raise DiffusionKVAdmissionError("independent DiffusionKVContext allocation is not implemented yet") + if request.seq_len > self.max_model_len: + raise DiffusionKVAdmissionError( + f"Diffusion KV sequence {request.request_id!r} exceeds max_model_len: " + f"seq_len={request.seq_len}, max_model_len={self.max_model_len}" + ) + + required_blocks = self._get_empty_pool_required_blocks(requests) + if required_blocks > self._empty_pool_num_free_blocks: + raise DiffusionKVAdmissionError( + f"Diffusion KV request {public_request_id!r} cannot fit even when the block pool is empty: " + f"required_blocks={required_blocks}, available_blocks={self._empty_pool_num_free_blocks}; " + "increase KV cache capacity or reduce the request sequence count/length" + ) + + allocated: list[DiffusionKVRequest] = [] + sequence_metadata: list[DiffusionKVSequenceMetadata] = [] + try: + for request in requests: + blocks = self.native_manager.allocate_slots( + request, + num_new_tokens=request.seq_len, + delay_cache_blocks=True, + full_sequence_must_fit=True, + ) + if blocks is None: + self._rollback(allocated) + allocated.clear() + return None + allocated.append(request) + sequence_metadata.append( + DiffusionKVSequenceMetadata( + sequence_id=request.sequence_id, + prefix_len=request.prefix_len, + target_len=request.target_len, + seq_len=request.seq_len, + block_ids=blocks.get_block_ids(), + ) + ) + except Exception: + self._rollback(allocated) + raise + + metadata = DiffusionKVMetadata( + request_id=public_request_id, + allocation_generation=self._next_allocation_generation, + sequences=tuple(sequence_metadata), + ) + self._next_allocation_generation += 1 + self._requests[public_request_id] = requests + self._metadata[public_request_id] = metadata + self._internal_request_ids.update(internal_ids) + return metadata + + def get_metadata(self, public_request_id: str) -> DiffusionKVMetadata: + try: + return self._metadata[public_request_id] + except KeyError as exc: + raise KeyError(f"Diffusion KV request {public_request_id!r} is not allocated") from exc + + def free_request(self, public_request_id: str) -> None: + requests = self._requests.pop(public_request_id, ()) + self._metadata.pop(public_request_id, None) + for request in reversed(requests): + self.native_manager.free(request) + self._internal_request_ids.discard(request.request_id) + + def close(self) -> None: + for public_request_id in tuple(self._requests): + self.free_request(public_request_id) + + def _rollback(self, requests: Sequence[DiffusionKVRequest]) -> None: + for request in reversed(requests): + self.native_manager.free(request) diff --git a/vllm_omni/diffusion/diffusion_kv/request.py b/vllm_omni/diffusion/diffusion_kv/request.py index 4db9701a1..fd4740099 100644 --- a/vllm_omni/diffusion/diffusion_kv/request.py +++ b/vllm_omni/diffusion/diffusion_kv/request.py @@ -105,7 +105,7 @@ class DiffusionKVRequest: self.status = RequestStatus.WAITING self.num_preemptions = 0 self.num_in_flight_tokens = 0 - # vLLM 0.26 uses this optional boundary when publishing cached blocks; + # vLLM 0.26+ uses this optional boundary when publishing cached blocks; # zero means that no sparse-retention boundary is active. self.shared_prefix_boundary = 0 diff --git a/vllm_omni/diffusion/executor/abstract.py b/vllm_omni/diffusion/executor/abstract.py index ec6b3486a..e653dc7ce 100644 --- a/vllm_omni/diffusion/executor/abstract.py +++ b/vllm_omni/diffusion/executor/abstract.py @@ -5,10 +5,12 @@ from collections.abc import Callable from typing import TYPE_CHECKING, Any from vllm.utils.import_utils import resolve_obj_by_qualname +from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec from vllm_omni.diffusion.data import OmniDiffusionConfig if TYPE_CHECKING: + from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.sched.interface import DiffusionSchedulerOutput from vllm_omni.diffusion.worker.utils import BaseRunnerOutput @@ -115,6 +117,39 @@ class DiffusionExecutor(ABC): """ return None + def get_kv_cache_specs(self) -> list[dict[str, KVCacheSpec]]: + """Collect rank-local native specs after every Worker loads its model.""" + + result = self.collective_rpc( + "get_kv_cache_specs", + unique_reply_rank=0, + exec_all_ranks=True, + ) + if not isinstance(result, list): + raise TypeError(f"get_kv_cache_specs returned {type(result).__name__}, expected list") + return result + + def determine_available_kv_memory(self, profile_requests: list[OmniDiffusionRequest]) -> list[int]: + """Profile and collect the KV memory budget on every Worker rank.""" + + result = self.collective_rpc( + "determine_available_kv_memory", + args=(profile_requests,), + unique_reply_rank=0, + exec_all_ranks=True, + ) + if not isinstance(result, list) or not all(isinstance(value, int) for value in result): + raise TypeError("determine_available_kv_memory must return list[int]") + return result + + def set_kv_cache_configs(self, kv_cache_configs: list[KVCacheConfig]) -> None: + """Send each native rank-local config back to all Workers.""" + + # The default control-plane RPC mode executes on every rank and has + # rank 0 return the gathered rank statuses, so failures on nonzero + # ranks are not silently dropped. + self.collective_rpc("set_kv_cache_configs", args=(kv_cache_configs,)) + @abstractmethod def shutdown(self) -> None: """Shutdown the executor and release resources.""" diff --git a/vllm_omni/diffusion/executor/multiproc_executor.py b/vllm_omni/diffusion/executor/multiproc_executor.py index 6f0917e32..ff9253962 100644 --- a/vllm_omni/diffusion/executor/multiproc_executor.py +++ b/vllm_omni/diffusion/executor/multiproc_executor.py @@ -465,16 +465,14 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): for new_req in new_reqs: validate_new_request_data_identity(new_req) - has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs) - # DP multi-concurrency: when DLO+AllGather is active and multiple - # requests are scheduled, send ALL requests in one broadcast RPC. - # Each rank picks req[rank % len(reqs)] and computes independently. + # requests are scheduled, send every complete NewRequestData envelope + # in one broadcast RPC. Each rank picks one envelope, keeping its + # request and Diffusion KV metadata inseparable. # All ranks reply (unique_reply_rank=None) so we collect dp_size # responses and match by dp_rank. if ( len(new_reqs) > 1 - and not has_diffusion_kv_metadata and getattr(self.od_config, "enable_distributed_layerwise_offload", False) and getattr(self.od_config, "dlo_use_allgather", True) ): @@ -507,12 +505,11 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): f"empty prompt request IDs: {empty_prompt_ids}." ) - reqs_list = [nr.req for nr in new_reqs] try: results = self.collective_rpc( "execute_model", timeout=_DLO_DP_WAVE_TIMEOUT_S, - args=(reqs_list, self.od_config, scheduler_output.kv_prefetch_job), + args=(new_reqs, self.od_config, scheduler_output.kv_prefetch_job), unique_reply_rank=None, exec_all_ranks=True, ) diff --git a/vllm_omni/diffusion/io_support.py b/vllm_omni/diffusion/io_support.py index 8d1405a6c..368577a0a 100644 --- a/vllm_omni/diffusion/io_support.py +++ b/vllm_omni/diffusion/io_support.py @@ -6,6 +6,7 @@ from __future__ import annotations import inspect from vllm_omni.diffusion.data import OmniDiffusionConfig +from vllm_omni.diffusion.model_metadata import get_diffusion_model_metadata from vllm_omni.diffusion.registry import DiffusionModelRegistry @@ -47,3 +48,9 @@ def get_dummy_run_num_frames(model_class_name: str, supports_audio_input: bool) if model_cls is not None and hasattr(model_cls, "dummy_run_num_frames"): return int(getattr(model_cls, "dummy_run_num_frames")) return 2 if supports_audio_input or supports_audio_output(model_class_name) else 1 + + +def get_dummy_run_num_image_inputs(model_class_name: str) -> int: + """Return the maximum advertised image-input count for profiling.""" + + return get_diffusion_model_metadata(model_class_name).max_multimodal_image_inputs or 1 diff --git a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py b/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py index 877fd1c3f..45a94d76d 100644 --- a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py +++ b/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py @@ -913,7 +913,7 @@ class HunYuanRotary2DEmbedder: return q, k -class ImageKVCacheManager: +class ImageKVCacheManager(nn.Module): """ Manages specialized caching and updating of KV-Cache for image tokens in multimodal models. """ @@ -932,6 +932,7 @@ class ImageKVCacheManager: image_token_len: Number of tokens per image (including special placeholders), default 4097 (timestamp + 4096 image tokens). """ + super().__init__() # attention related self.num_heads = num_heads self.num_kv_heads = num_kv_heads @@ -953,6 +954,10 @@ class ImageKVCacheManager: softmax_scale=self.scaling, num_kv_heads=self.num_kv_heads, prefix=f"{prefix}.attn" if prefix else "", + paged_kv_cache_role="primary", + # Hunyuan image RoPE produces bfloat16 K and denoising runs under + # bfloat16 autocast, independent of the weight-loading dtype. + paged_kv_cache_dtype=torch.bfloat16, ) @staticmethod @@ -1095,7 +1100,7 @@ class ImageKVCacheManager: ) return key, value - def __call__( + def forward( self, query: torch.Tensor, key: torch.Tensor, diff --git a/vllm_omni/diffusion/request.py b/vllm_omni/diffusion/request.py index ce097a2d9..82ff7ebe2 100644 --- a/vllm_omni/diffusion/request.py +++ b/vllm_omni/diffusion/request.py @@ -14,6 +14,7 @@ if TYPE_CHECKING: from vllm_omni.diffusion.diffusion_kv.request import DiffusionKVRequest DUMMY_DIFFUSION_REQUEST_ID = "dummy_req_id" +_DUMMY_DIFFUSION_REQUEST_ID_PREFIX = f"{DUMMY_DIFFUSION_REQUEST_ID}/" @dataclass @@ -90,4 +91,6 @@ class OmniDiffusionRequest: @classmethod def is_dummy_run_request_id(cls, request_id: str | None) -> bool: - return request_id == DUMMY_DIFFUSION_REQUEST_ID + return request_id == DUMMY_DIFFUSION_REQUEST_ID or ( + isinstance(request_id, str) and request_id.startswith(_DUMMY_DIFFUSION_REQUEST_ID_PREFIX) + ) diff --git a/vllm_omni/diffusion/sched/base_scheduler.py b/vllm_omni/diffusion/sched/base_scheduler.py index b558c1ee6..17291db37 100644 --- a/vllm_omni/diffusion/sched/base_scheduler.py +++ b/vllm_omni/diffusion/sched/base_scheduler.py @@ -7,10 +7,14 @@ from abc import ABC, abstractmethod from collections import deque from dataclasses import fields +from vllm.config import VllmConfig from vllm.logger import init_logger +from vllm.v1.kv_cache_interface import KVCacheConfig from vllm_omni.diffusion.data import OmniDiffusionConfig from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode +from vllm_omni.diffusion.diffusion_kv.manager import DiffusionKVAdmissionError, DiffusionKVCacheManager +from vllm_omni.diffusion.diffusion_kv.metadata import DiffusionKVMetadata from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.sched.interface import ( CachedRequestData, @@ -49,9 +53,17 @@ class BaseScheduler(ABC): self._finished_req_ids: set[str] = set() self.max_num_running_reqs: int = 1 self._prefetch_enabled: bool = False - self._diffusion_kv_enabled = False + self._diffusion_kv_manager: DiffusionKVCacheManager | None = None - def initialize(self, od_config: OmniDiffusionConfig) -> None: + def initialize( + self, + od_config: OmniDiffusionConfig, + *, + kv_cache_config: KVCacheConfig | None = None, + scheduler_block_size: int | None = None, + hash_block_size: int | None = None, + kv_vllm_config: VllmConfig | None = None, + ) -> None: self.od_config = od_config self._request_states.clear() self._step_id = 0 @@ -66,10 +78,36 @@ class BaseScheduler(ABC): self.max_num_running_reqs = 1 omni_kv = getattr(od_config, "omni_kv_config", None) or {} self._prefetch_enabled = bool(omni_kv.get("enable_kv_async_prefetch", False)) - self._diffusion_kv_enabled = ( + diffusion_kv_enabled = ( getattr(od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY) is DiffusionKVCacheMode.PAGED_SCHEDULER ) + if diffusion_kv_enabled: + if kv_cache_config is None: + raise ValueError("paged_scheduler Diffusion KV requires a native Scheduler KVCacheConfig") + if scheduler_block_size is None or hash_block_size is None: + raise ValueError("paged_scheduler Diffusion KV requires native scheduler/hash block sizes") + if kv_vllm_config is None: + raise ValueError("paged_scheduler Diffusion KV requires the native VllmConfig used for cache sizing") + self._diffusion_kv_manager = DiffusionKVCacheManager( + kv_cache_config, + max_model_len=kv_vllm_config.model_config.max_model_len, + scheduler_block_size=scheduler_block_size, + hash_block_size=hash_block_size, + max_in_flight_tokens=kv_vllm_config.max_in_flight_tokens, + ) + else: + if any( + value is not None + for value in ( + kv_cache_config, + scheduler_block_size, + hash_block_size, + kv_vllm_config, + ) + ): + raise ValueError("dense_legacy Scheduler received unexpected Diffusion KV cache initialization state") + self._diffusion_kv_manager = None self._reset_scheduler_state() def add_request(self, request: OmniDiffusionRequest) -> str: @@ -104,6 +142,35 @@ class BaseScheduler(ABC): if not self._can_schedule_waiting(state): break + diffusion_kv_metadata: DiffusionKVMetadata | None = None + if self._diffusion_kv_manager is not None: + if self._diffusion_kv_manager.has_request(request_id): + diffusion_kv_metadata = self._diffusion_kv_manager.get_metadata(request_id) + else: + try: + allocation = self._diffusion_kv_manager.reserve_request( + request_id, + state.diffusion_kv_requests, + ) + except Exception as exc: + # schedule() runs outside the Engine execution-error + # wrapper. Convert both expected admission failures and + # unexpected native allocator failures into a terminal + # request error so the busy loop can wake its stream. + if not isinstance(exc, DiffusionKVAdmissionError): + logger.exception( + "Unexpected Diffusion KV allocation failure for request %s", + request_id, + ) + self._finish_requests( + {request_id: DiffusionRequestStatus.FINISHED_ERROR}, + {request_id: str(exc)}, + ) + continue + if allocation is None: + break + diffusion_kv_metadata = allocation + self._waiting.popleft() was_new_request = state.status == DiffusionRequestStatus.WAITING if not self._running: @@ -111,7 +178,12 @@ class BaseScheduler(ABC): state.status = DiffusionRequestStatus.RUNNING self._running.append(request_id) if was_new_request: - scheduled_new_reqs.append(NewRequestData.from_state(state)) + scheduled_new_reqs.append( + NewRequestData.from_state( + state, + diffusion_kv_metadata=diffusion_kv_metadata, + ) + ) else: scheduled_cached_request_ids.append(request_id) @@ -184,6 +256,8 @@ class BaseScheduler(ABC): return self._request_states.get(request_id) def pop_request_state(self, request_id: str) -> SchedulerRequestState | None: + if self._diffusion_kv_manager is not None: + self._diffusion_kv_manager.free_request(request_id) self._pop_extra_request_state(request_id) return self._request_states.pop(request_id, None) @@ -206,6 +280,9 @@ class BaseScheduler(ABC): self._finish_requests({request_id: status for request_id in request_ids}) def close(self) -> None: + if self._diffusion_kv_manager is not None: + self._diffusion_kv_manager.close() + self._diffusion_kv_manager = None self._request_states.clear() self._waiting.clear() self._running.clear() @@ -245,6 +322,8 @@ class BaseScheduler(ABC): self._waiting = deque(request_id for request_id in self._waiting if request_id not in waiting_to_remove) for request_id in finished_req_ids: + if self._diffusion_kv_manager is not None: + self._diffusion_kv_manager.free_request(request_id) state = self._request_states[request_id] status = statuses[request_id] state.status = status @@ -266,7 +345,13 @@ class BaseScheduler(ABC): # update_from_output() processes the runner output. It is already # marked finished at that point, but we still need to surface its id # in this update so the engine can observe the terminal state. + # Also surface admission failures recorded while schedule() built this + # output. Older finished ids retained only for Worker cleanup have + # already been popped by the Engine and are deliberately ignored. finished_req_ids = { + request_id for request_id in sched_output.finished_req_ids if request_id in self._request_states + } + finished_req_ids |= { request_id for request_id in sched_output.scheduled_request_ids if request_id in self._finished_req_ids } finished_req_ids |= self._finish_requests(statuses, errors) @@ -280,7 +365,7 @@ class BaseScheduler(ABC): def _make_request_state(self, request_id: str, request: OmniDiffusionRequest) -> SchedulerRequestState: kv_requests = request.diffusion_kv_requests or () - if self._diffusion_kv_enabled: + if self._diffusion_kv_manager is not None: self._reject_legacy_dense_kv(request) if not kv_requests: raise ValueError("paged_scheduler request preprocessing did not produce DiffusionKVRequest state") diff --git a/vllm_omni/diffusion/vllm_config.py b/vllm_omni/diffusion/vllm_config.py new file mode 100644 index 000000000..816739de4 --- /dev/null +++ b/vllm_omni/diffusion/vllm_config.py @@ -0,0 +1,180 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Build native vLLM configuration views for the Omni diffusion runtime.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch +from vllm.config import CompilationConfig, DeviceConfig, VllmConfig +from vllm.transformers_utils.config import get_hf_text_config + +from vllm_omni.diffusion.data import OmniDiffusionConfig +from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode + + +def resolve_diffusion_max_model_len(od_config: OmniDiffusionConfig) -> int: + """Resolve the native KV admission bound for a diffusion stage.""" + + configured = getattr(od_config, "max_model_len", None) + if configured is not None: + configured = int(configured) + if configured == -1: + configured = None + elif configured <= 0: + raise ValueError(f"max_model_len must be positive for Scheduler-managed Diffusion KV, got {configured}") + else: + return configured + + # ``max_model_len`` keeps the native vLLM meaning: the per-sequence KV + # admission ceiling consumed by ``KVCacheManager``. HunyuanImage3 exposes + # that ceiling directly as ``max_position_embeddings`` because its text, + # reference-image, and generated-image tokens share one Transformer + # sequence. Other diffusion models commonly express their serving limits + # through resolution, duration, or frame-count constraints instead. When + # paged KV support expands to those models, the model integration must + # derive ``max_model_len`` from its serving limits, while model-owned + # preprocessing validates each request and derives its matching + # ``seq_len``. Text-encoder length fields must not be mistaken for the DiT + # KV limit. + hf_config = getattr(od_config, "tf_model_config", None) + for field_name in ("max_position_embeddings", "max_sequence_length", "seq_length"): + value = getattr(hf_config, field_name, None) + if value is None and isinstance(hf_config, dict): + value = hf_config.get(field_name) + if value is not None and int(value) > 0: + return int(value) + + mode = getattr(getattr(od_config, "diffusion_kv_mode", None), "value", None) + if mode == "paged_scheduler": + raise ValueError( + "paged_scheduler Diffusion KV requires max_model_len or a model config " + "with max_position_embeddings/max_sequence_length/seq_length" + ) + # Dense execution does not consume the native KV admission bound, but the + # shared Worker VllmConfig still requires a positive placeholder. + return 1 + + +@dataclass +class _DiffusionVllmModelConfig: + """Private adapter for the native ``VllmConfig.model_config`` surface.""" + + model: str | None + dtype: torch.dtype + max_model_len: int + original_max_model_len: int | None + runner_type: str = "generate" + quantization: str | None = None + quantization_config: Any | None = None + hf_config: Any | None = None + hf_text_config: Any | None = None + multimodal_config: Any | None = None + enforce_eager: bool = False + disable_cascade_attn: bool = False + enable_return_routed_experts: bool = False + enable_prompt_embeds: bool = False + logits_processors: tuple[str, ...] = () + use_mla: bool = False + is_moe: bool = False + + @property + def is_quantized(self) -> bool: + return self.quantization is not None + + def is_nvfp4_quantized(self) -> bool: + return self.quantization == "modelopt_fp4" + + @property + def is_diffusion(self) -> bool: + # This config is also installed as the current VllmConfig for the + # existing Omni diffusion runtime, which does not use vLLM's native + # diffusion runner. Preserve that established backend-selection flag. + return False + + +def _make_diffusion_vllm_model_config(od_config: OmniDiffusionConfig) -> _DiffusionVllmModelConfig: + quant_config = getattr(od_config, "quantization_config", None) + quantization = quant_config.get_name() if quant_config is not None and hasattr(quant_config, "get_name") else None + hf_config = getattr(od_config, "tf_model_config", None) + hf_text_config = get_hf_text_config(hf_config) if hasattr(hf_config, "get_text_config") else hf_config + configured_max_model_len = getattr(od_config, "max_model_len", None) + max_model_len = resolve_diffusion_max_model_len(od_config) + paged_scheduler = ( + getattr(od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY) + is DiffusionKVCacheMode.PAGED_SCHEDULER + ) + return _DiffusionVllmModelConfig( + model=od_config.model, + dtype=od_config.dtype, + max_model_len=max_model_len, + # Preserve the user's -1 sentinel only for the native paged-cache + # sizing flow. Dense keeps its established resolved model-config view. + original_max_model_len=configured_max_model_len if paged_scheduler else max_model_len, + quantization=quantization, + quantization_config=quant_config, + hf_config=hf_config, + hf_text_config=hf_text_config, + enforce_eager=getattr(od_config, "enforce_eager", False), + is_moe=bool(getattr(od_config, "is_moe", False)), + ) + + +def create_base_diffusion_vllm_config( + device: torch.device, + od_config: OmniDiffusionConfig, +) -> VllmConfig: + """Create the native vLLM 0.27 config used by diffusion.""" + + return VllmConfig( + compilation_config=CompilationConfig(), + device_config=DeviceConfig(device=device), + additional_config=od_config.additional_config, + ) + + +def configure_diffusion_vllm_config(vllm_config: VllmConfig, od_config: OmniDiffusionConfig) -> VllmConfig: + """Populate native model, parallel, cache, and Scheduler configuration.""" + + parallel_config = od_config.parallel_config + vllm_config.parallel_config.tensor_parallel_size = parallel_config.tensor_parallel_size + vllm_config.parallel_config.data_parallel_size = parallel_config.data_parallel_size + if parallel_config.enable_expert_parallel and od_config.is_moe: + vllm_config.parallel_config.data_parallel_size = ( + parallel_config.data_parallel_size * parallel_config.cfg_parallel_size + ) + vllm_config.parallel_config.prefill_context_parallel_size = parallel_config.sequence_parallel_size + vllm_config.parallel_config.enable_expert_parallel = parallel_config.enable_expert_parallel + + vllm_config.model_config = _make_diffusion_vllm_model_config(od_config) # type: ignore[assignment] + vllm_config.quant_config = od_config.quantization_config + vllm_config.profiler_config = od_config.profiler_config + if ( + getattr(od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY) + is DiffusionKVCacheMode.PAGED_SCHEDULER + ): + vllm_config.parallel_config.pipeline_parallel_size = parallel_config.pipeline_parallel_size + vllm_config.cache_config.gpu_memory_utilization = float(getattr(od_config, "gpu_memory_utilization", 0.9)) + vllm_config.cache_config.kv_cache_memory_bytes = getattr(od_config, "kv_cache_memory_bytes", None) + # Prefix hashes/publication are intentionally deferred to the next PR. + vllm_config.cache_config.enable_prefix_caching = False + max_num_batched_tokens = getattr(od_config, "max_num_batched_tokens", None) + if max_num_batched_tokens is not None: + vllm_config.scheduler_config.max_num_batched_tokens = int(max_num_batched_tokens) + return vllm_config + + +def create_diffusion_vllm_config( + device: torch.device, + od_config: OmniDiffusionConfig, +) -> VllmConfig: + """Build the same fully populated native config in Engine and Workers.""" + + vllm_config = create_base_diffusion_vllm_config( + device, + od_config, + ) + return configure_diffusion_vllm_config(vllm_config, od_config) diff --git a/vllm_omni/diffusion/worker/diffusion_model_runner.py b/vllm_omni/diffusion/worker/diffusion_model_runner.py index 4a763d651..44993fd2c 100644 --- a/vllm_omni/diffusion/worker/diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/diffusion_model_runner.py @@ -11,6 +11,7 @@ model-related operations. from __future__ import annotations import copy +import gc import time from collections.abc import Callable from contextlib import AbstractContextManager, nullcontext @@ -21,6 +22,7 @@ from torch.profiler import record_function from vllm.config import LoadConfig, VllmConfig from vllm.logger import init_logger from vllm.utils.mem_utils import DeviceMemoryProfiler, GiB_bytes +from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec from vllm_omni.diffusion.cache.cachedit import cache_summary from vllm_omni.diffusion.cache.prompt_embed_cache import ( @@ -45,8 +47,10 @@ from vllm_omni.diffusion.offloader import get_offload_backend from vllm_omni.diffusion.registry import _NO_CACHE_ACCELERATION from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.sched.interface import ( + CachedRequestData, DiffusionSchedulerOutput, KVPrefetchJob, + NewRequestData, validate_new_request_data_identity, ) from vllm_omni.diffusion.worker.input_batch import InputBatch, scatter_latents @@ -139,6 +143,11 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): self.offload_backend: Any | None = None self.prompt_embed_cache: Any | None = None self.input_batch: InputBatch | None = None + self.model_memory_usage = 0 + # Rank-local native config generated by the Engine. Physical tensors + # and BlockTable binding remain the responsibility of the paged + # data-plane initialization path. + self.kv_cache_config: KVCacheConfig | None = None # Cache for per-request stepwise state. self.state_cache: dict[str, StepRequestState] = {} @@ -167,8 +176,8 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): metadata: DiffusionKVMetadata | None, ) -> None: cache_mode = getattr(self.od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY) - # W0 does not wire the Scheduler allocator or Worker installer yet, so - # validate an allocation snapshot when present without requiring one. + if cache_mode is DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is None: + raise ValueError(f"paged_scheduler request {request_id!r} requires Diffusion KV metadata") if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None: raise ValueError(f"{cache_mode.value} request {request_id!r} must not carry Diffusion KV metadata") @@ -273,6 +282,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): m.consumed_memory / GiB_bytes, time_after_load - time_before_load, ) + self.model_memory_usage = int(m.consumed_memory) logger.info("Model runner: Model loaded successfully.") if self.od_config.streaming_output and not getattr(self.od_config, "step_execution", False): @@ -368,6 +378,45 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): logger.info("Model runner: Initialization complete.") + def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]: + """Collect native specs from cache-enabled loaded attention modules.""" + + from vllm_omni.diffusion.attention.layer import Attention + from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode + + if self.od_config.diffusion_kv_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER: + return {} + if self.pipeline is None: + raise RuntimeError("Model must be loaded before collecting Diffusion KV cache specs") + + specs: dict[str, KVCacheSpec] = {} + for layer_name, module in self.pipeline.named_modules(): + if not isinstance(module, Attention): + continue + spec = module.get_kv_cache_spec(self.vllm_config) + if spec is not None: + specs[layer_name] = spec + if not specs: + raise RuntimeError( + "paged_scheduler Diffusion KV found no cache-enabled Attention modules " + f"in {type(self.pipeline).__name__}" + ) + return specs + + def set_kv_cache_config(self, kv_cache_config: KVCacheConfig) -> None: + """Retain the Engine-generated rank-local config for the data plane.""" + + configured_layers = { + layer_name for group in kv_cache_config.kv_cache_groups for layer_name in group.layer_names + } + expected_layers = set(self.get_kv_cache_spec()) + if configured_layers != expected_layers: + raise ValueError( + "Rank-local Diffusion KVCacheConfig layer mismatch: " + f"expected={sorted(expected_layers)}, configured={sorted(configured_layers)}" + ) + self.kv_cache_config = kv_cache_config + def clear_prompt_embed_cache(self) -> None: """Evict all cached text-encoder outputs (e.g. between training epochs). @@ -516,6 +565,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): require_request_batch_support: bool, kv_prefetch_job: KVPrefetchJob | None = None, record_name: str, + record_output_peak_memory: bool = True, ) -> BatchRunnerOutput: assert self.pipeline is not None, "Model not loaded. Call load_model() first." if not reqs: @@ -545,7 +595,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): batch = DiffusionRequestBatch(requests=reqs) is_primary = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 - if is_primary: + if is_primary and record_output_peak_memory: current_omni_platform.reset_peak_memory_stats() with set_forward_context(vllm_config=self.vllm_config, omni_diffusion_config=od_config): @@ -558,7 +608,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): pipeline_name=type(self.pipeline).__name__, ) - if is_primary and outputs: + if is_primary and outputs and record_output_peak_memory: batch_peak_memory_mb = self._sample_peak_memory_mb() for output in outputs: output.peak_memory_mb = max(output.peak_memory_mb, batch_peak_memory_mb) @@ -626,6 +676,59 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): assert output is not None return output + def profile_run(self, requests: list[OmniDiffusionRequest]) -> None: + """Run the maximum per-rank request batch for memory profiling. + + This deliberately bypasses Scheduler admission and Diffusion KV + metadata validation because cache capacity has not been sized yet. + It otherwise uses the normal execution-mode path so model inputs, + collective communication, backend workspaces, denoising, and decode + allocations contribute to the observed peak. Step execution profiles + one fused ``InputBatch`` instead of sequential single-request forwards. + """ + + if not requests: + raise ValueError("Diffusion memory profiling requires at least one request.") + + runner_output: BatchRunnerOutput | None = None + request_ids = [request.request_id for request in requests] + try: + if getattr(self.od_config, "step_execution", False): + scheduler_output = DiffusionSchedulerOutput( + step_id=0, + scheduled_new_reqs=[ + NewRequestData(request_id=request.request_id, req=request) for request in requests + ], + scheduled_cached_reqs=CachedRequestData.make_empty(), + finished_req_ids=set(), + num_running_reqs=len(requests), + num_waiting_reqs=0, + ) + runner_output = self._execute_stepwise( + scheduler_output, + validate_kv_metadata=False, + record_output_peak_memory=False, + ) + else: + runner_output = self._execute_request_list( + requests, + od_config=self.od_config, + allow_single_output=len(requests) == 1, + require_request_batch_support=len(requests) > 1, + record_name="pipeline_memory_profile", + # The enclosing native memory_profiling context owns the + # peak counters. Resetting them here would discard request + # preparation allocations and understate the budget. + record_output_peak_memory=False, + ) + current_omni_platform.synchronize() + finally: + for request_id in request_ids: + self.state_cache.pop(request_id, None) + self.input_batch = None + del runner_output + gc.collect() + def execute_model_batch( self, scheduler_output: DiffusionSchedulerOutput, @@ -750,13 +853,29 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): def execute_stepwise(self, scheduler_output: DiffusionSchedulerOutput) -> BatchRunnerOutput: """Execute one step for one scheduled request and return runner output.""" + return self._execute_stepwise( + scheduler_output, + validate_kv_metadata=True, + record_output_peak_memory=True, + ) + + def _execute_stepwise( + self, + scheduler_output: DiffusionSchedulerOutput, + *, + validate_kv_metadata: bool, + record_output_peak_memory: bool, + ) -> BatchRunnerOutput: + """Execute one step with explicit validation and profiling policy.""" + assert self.pipeline is not None, "Model not loaded. Call load_model() first." for new_req in scheduler_output.scheduled_new_reqs: validate_new_request_data_identity(new_req) - self._validate_diffusion_kv_metadata( - request_id=new_req.req.request_id, - metadata=new_req.diffusion_kv_metadata, - ) + if validate_kv_metadata: + self._validate_diffusion_kv_metadata( + request_id=new_req.req.request_id, + metadata=new_req.diffusion_kv_metadata, + ) if not self._supports_step_mode(): raise ValueError("Current pipeline does not support step execution.") # Stepwise mode only supports the basic state-driven denoise path for now. @@ -771,7 +890,13 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): had_active_states = bool(self.state_cache) states, new_request_ids = self._update_states(scheduler_output) is_primary = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 - if new_request_ids and not had_active_states and is_primary and current_omni_platform.is_available(): + if ( + record_output_peak_memory + and new_request_ids + and not had_active_states + and is_primary + and current_omni_platform.is_available() + ): current_omni_platform.reset_peak_memory_stats() input_batch = self._prepare_batch_inputs(states, new_request_ids) attn_metadata = {} @@ -864,7 +989,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): f"but batched noise_pred has {noise_pred.shape[0]} rows." ) - if is_primary: + if is_primary and record_output_peak_memory: batch_peak_memory_mb = self._sample_peak_memory_mb() states_by_id = {state.request_id: state for state in states} for state in states: diff --git a/vllm_omni/diffusion/worker/diffusion_worker.py b/vllm_omni/diffusion/worker/diffusion_worker.py index 436fcca9a..d9030e891 100644 --- a/vllm_omni/diffusion/worker/diffusion_worker.py +++ b/vllm_omni/diffusion/worker/diffusion_worker.py @@ -16,20 +16,21 @@ import signal import threading import traceback import uuid -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import AbstractContextManager, contextmanager, nullcontext -from dataclasses import dataclass from typing import Any import torch +import torch.distributed as dist import zmq -from vllm.config import CompilationConfig, DeviceConfig, VllmConfig, set_current_vllm_config +from vllm.config import VllmConfig, set_current_vllm_config from vllm.distributed.device_communicators.shm_broadcast import MessageQueue from vllm.logger import init_logger from vllm.profiler.wrapper import CudaProfilerWrapper, WorkerProfiler -from vllm.transformers_utils.config import get_hf_text_config from vllm.utils.import_utils import resolve_obj_by_qualname -from vllm.utils.mem_utils import GiB_bytes +from vllm.utils.mem_utils import GiB_bytes, MemorySnapshot, format_gib, memory_profiling +from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec +from vllm.v1.worker.utils import request_memory from vllm.v1.worker.workspace import init_workspace_manager from vllm_omni.diffusion.data import ( @@ -41,6 +42,7 @@ from vllm_omni.diffusion.data import ( OmniSleepTask, OmniWakeTask, ) +from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode from vllm_omni.diffusion.diffusion_kv.metadata import DiffusionKVMetadata from vllm_omni.diffusion.distributed.parallel_state import ( destroy_distributed_env, @@ -52,7 +54,13 @@ from vllm_omni.diffusion.ipc import DIFFUSION_RPC_RESULT_ENVELOPE, pack_diffusio from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager, LoRABackend from vllm_omni.diffusion.registry import get_diffusion_ir_op_priority_func from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.diffusion.sched.interface import DiffusionSchedulerOutput, KVPrefetchJob +from vllm_omni.diffusion.sched.interface import ( + DiffusionSchedulerOutput, + KVPrefetchJob, + NewRequestData, + validate_new_request_data_identity, +) +from vllm_omni.diffusion.vllm_config import create_diffusion_vllm_config from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner from vllm_omni.diffusion.worker.utils import BaseRunnerOutput, BatchRunnerOutput from vllm_omni.engine.stage_init_utils import set_death_signal @@ -76,49 +84,28 @@ _ASYNC_OUTPUT_DRAIN_TIMEOUT_S = 10.0 _MEMORY_RELEASING_METHODS = frozenset({"sleep", "handle_sleep_task"}) -@dataclass -class _DiffusionVllmModelConfig: - model: str - dtype: torch.dtype - quantization: str | None = None - quantization_config: Any | None = None - hf_config: Any | None = None - hf_text_config: Any | None = None - multimodal_config: Any | None = None - enforce_eager: bool = False - disable_cascade_attn: bool = False - enable_return_routed_experts: bool = False - is_moe: bool = False - - def is_quantized(self) -> bool: - return self.quantization is not None - - def is_model_moe(self) -> bool: - return self.is_moe - - def is_nvfp4_quantized(self) -> bool: - return self.quantization == "modelopt_fp4" - - @property - def is_diffusion(self) -> bool: - return False +def _all_gather_rank_values(value: Any) -> list[Any]: + if not dist.is_available() or not dist.is_initialized(): + return [value] + values: list[Any] = [None] * dist.get_world_size() + dist.all_gather_object(values, value) + return values -def _make_diffusion_vllm_model_config(od_config: OmniDiffusionConfig) -> _DiffusionVllmModelConfig: - quant_config = getattr(od_config, "quantization_config", None) - quantization = quant_config.get_name() if quant_config is not None and hasattr(quant_config, "get_name") else None - hf_config = getattr(od_config, "tf_model_config", None) - hf_text_config = get_hf_text_config(hf_config) if hasattr(hf_config, "get_text_config") else hf_config - return _DiffusionVllmModelConfig( - model=od_config.model, - dtype=od_config.dtype, - quantization=quantization, - quantization_config=quant_config, - hf_config=hf_config, - hf_text_config=hf_text_config, - enforce_eager=getattr(od_config, "enforce_eager", False), - is_moe=bool(getattr(od_config, "is_moe", False)), - ) +def _run_and_gather_rank_values(operation: str, func: Callable[[], Any]) -> list[Any]: + """Run one rank-local probe without stranding peers on local failure.""" + + try: + local_result = (True, func()) + except Exception as exc: + logger.exception("%s failed on this Worker rank", operation) + local_result = (False, f"{type(exc).__name__}: {exc}") + + rank_results = _all_gather_rank_values(local_result) + failures = [f"rank {rank}: {result}" for rank, (ok, result) in enumerate(rank_results) if not ok] + if failures: + raise RuntimeError(f"{operation} failed on " + "; ".join(failures)) + return [result for _, result in rank_results] @contextmanager @@ -152,37 +139,6 @@ def _force_cutlass_fp8_linear_kernel(quant_config: object | None) -> Iterator[No yield -def _is_unexpected_additional_config_type_error(exc: TypeError) -> bool: - """Return True only for constructor rejections of the additional_config kwarg.""" - message = str(exc) - return "unexpected keyword argument" in message and "additional_config" in message - - -def _create_diffusion_worker_vllm_config(device: torch.device, od_config: OmniDiffusionConfig) -> VllmConfig: - """Create a worker-local VllmConfig while preserving additional_config when supported.""" - config_kwargs: dict[str, Any] = { - "compilation_config": CompilationConfig(), - "device_config": DeviceConfig(device=device), - } - if od_config.additional_config: - config_kwargs["additional_config"] = od_config.additional_config - - try: - return VllmConfig(**config_kwargs) - except TypeError as exc: - if not _is_unexpected_additional_config_type_error(exc): - raise - - logger.debug("Worker-local VllmConfig does not accept additional_config in constructor: %s", exc) - config_kwargs.pop("additional_config", None) - vllm_config = VllmConfig(**config_kwargs) - try: - setattr(vllm_config, "additional_config", dict(od_config.additional_config)) - except Exception as set_exc: # pragma: no cover - defensive for older vLLM builds - logger.warning("Failed to attach additional_config to worker VllmConfig: %s", set_exc) - return vllm_config - - def _get_cumem_allocator_class() -> type: from vllm.device_allocator.cumem import CuMemAllocator @@ -223,6 +179,8 @@ class DiffusionWorker: self.device: torch.device | None = None self.vllm_config: VllmConfig | None = None self.model_runner: DiffusionModelRunner | None = None + self.init_snapshot: MemorySnapshot | None = None + self.requested_memory: int | None = None self._sleep_saved_buffers: dict[str, torch.Tensor] = {} self.lora_manager: DiffusionLoRAManager | None = None # Worker-side cache of (lora_request, lora_scale) per scheduled @@ -287,22 +245,7 @@ class DiffusionWorker: # Create vllm_config for parallel configuration. Pass explicit device_config # so DeviceConfig does not rely on current_platform in worker subprocesses. - vllm_config = _create_diffusion_worker_vllm_config(self.device, self.od_config) - parallel_config = self.od_config.parallel_config - vllm_config.parallel_config.tensor_parallel_size = parallel_config.tensor_parallel_size - vllm_config.parallel_config.data_parallel_size = parallel_config.data_parallel_size - if parallel_config.enable_expert_parallel and self.od_config.is_moe: - # Diffusion uses its own DP/CFG/SP groups normally. vLLM groups are - # only remapped for expert-parallel runtimes that consume vLLM's - # FusedMoE/EP semantics. - vllm_config.parallel_config.data_parallel_size = ( - parallel_config.data_parallel_size * parallel_config.cfg_parallel_size - ) - vllm_config.parallel_config.prefill_context_parallel_size = parallel_config.sequence_parallel_size - vllm_config.parallel_config.enable_expert_parallel = parallel_config.enable_expert_parallel - vllm_config.profiler_config = self.od_config.profiler_config - vllm_config.model_config = _make_diffusion_vllm_model_config(self.od_config) # type: ignore[assignment] - vllm_config.quant_config = self.od_config.quantization_config + vllm_config = create_diffusion_vllm_config(self.device, self.od_config) # Since vLLM v0.20.0, IR wraps GPU ops. Set IR op priority preference to enforce GPU op fusion during wrapping. # Also need to log, because vLLM internally logs another line in VllmConfig.__post_init__. Avoid confusion. vllm_config.kernel_config.ir_op_priority = _resolve_ir_op_priority(self.od_config, vllm_config) @@ -340,6 +283,23 @@ class DiffusionWorker: hsdp_replicate_size=parallel_config.hsdp_replicate_size if parallel_config.use_hsdp else 1, enable_expert_parallel=parallel_config.enable_expert_parallel, ) + if ( + getattr(self.od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY) + is DiffusionKVCacheMode.PAGED_SCHEDULER + ): + gc.collect() + current_omni_platform.empty_cache() + self.init_snapshot = MemorySnapshot(device=self.device) + self.requested_memory = request_memory( + self.init_snapshot, + vllm_config.cache_config, + ) + logger.debug( + "Worker %d: Diffusion KV initial memory snapshot: %r; requested=%s GiB", + self.rank, + self.init_snapshot, + format_gib(self.requested_memory), + ) init_workspace_manager(self.device) def _create_profiler(self) -> WorkerProfiler | None: @@ -393,6 +353,96 @@ class DiffusionWorker: if load_format != "dummy": assert self.model_runner.pipeline is not None + def get_kv_cache_specs(self) -> list[dict[str, KVCacheSpec]]: + """Return native rank-local specs for every diffusion Worker.""" + + assert self.model_runner is not None + return _run_and_gather_rank_values( + "Diffusion KV cache spec discovery", + self.model_runner.get_kv_cache_spec, + ) + + def determine_available_kv_memory(self, profile_requests: list[OmniDiffusionRequest]) -> list[int]: + """Profile and return each rank's safe Diffusion KV memory budget.""" + + def determine_local_memory() -> int: + assert self.vllm_config is not None + assert self.model_runner is not None + if self.init_snapshot is None or self.requested_memory is None: + raise RuntimeError("Diffusion KV memory snapshot was not captured before model loading") + override = self.vllm_config.cache_config.kv_cache_memory_bytes + if override: + # Match native vLLM: an explicit cache budget skips automatic + # capacity derivation, but still runs the maximum-shape model + # request so lazy kernels and communication buffers initialize. + self.model_runner.profile_run(profile_requests) + logger.info( + "Worker %d: Initial free memory %s GiB, reserved %s GiB memory for " + "Diffusion KV Cache as specified by kv_cache_memory_bytes config and " + "skipped automatic memory profiling. This does not respect the " + "gpu_memory_utilization config. A profile warmup was still executed.", + self.rank, + format_gib(self.init_snapshot.free_memory), + format_gib(int(override)), + ) + return int(override) + + with memory_profiling( + self.init_snapshot, + weights_memory=self.model_runner.model_memory_usage, + ) as profile_result: + self.model_runner.profile_run(profile_requests) + + available_memory = self.requested_memory - profile_result.non_kv_cache_memory + if available_memory <= 0: + raise RuntimeError( + "No memory remains for Diffusion KV cache after profiling: " + f"requested_memory={self.requested_memory} bytes, " + f"non_kv_cache_memory={profile_result.non_kv_cache_memory} bytes. " + "Increase gpu_memory_utilization or reduce the maximum profile request shape." + ) + free_gpu_memory = profile_result.after_profile.free_memory + unrequested_memory = self.init_snapshot.free_memory - self.requested_memory + logger.debug( + "Worker %d: Initial free memory: %s GiB; Requested memory: %f (util), %s GiB", + self.rank, + format_gib(self.init_snapshot.free_memory), + self.vllm_config.cache_config.gpu_memory_utilization, + format_gib(self.requested_memory), + ) + logger.debug( + "Worker %d: Free memory after profiling: %s GiB (total), %s GiB (within requested)", + self.rank, + format_gib(free_gpu_memory), + format_gib(free_gpu_memory - unrequested_memory), + ) + logger.debug("Worker %d: %r", self.rank, profile_result) + logger.info( + "Worker %d: Available Diffusion KV cache memory: %s GiB", + self.rank, + format_gib(available_memory), + ) + return int(available_memory) + + return [ + int(value) + for value in _run_and_gather_rank_values( + "Diffusion KV memory discovery", + determine_local_memory, + ) + ] + + def set_kv_cache_configs(self, kv_cache_configs: list[KVCacheConfig]) -> None: + """Install this rank's native config without allocating tensors yet.""" + + assert self.model_runner is not None + if len(kv_cache_configs) != self.od_config.num_gpus: + raise ValueError( + "Diffusion KVCacheConfig rank count mismatch: " + f"expected={self.od_config.num_gpus}, got={len(kv_cache_configs)}" + ) + self.model_runner.set_kv_cache_config(kv_cache_configs[self.rank]) + def init_lora_manager(self) -> None: """Initialize the LoRA manager for this worker.""" if self.model_runner.pipeline is None: @@ -465,18 +515,19 @@ class DiffusionWorker: def execute_model( self, - req: OmniDiffusionRequest | list[OmniDiffusionRequest], + req: OmniDiffusionRequest | list[NewRequestData], od_config: OmniDiffusionConfig, kv_prefetch_job: KVPrefetchJob | None = None, diffusion_kv_metadata: DiffusionKVMetadata | None = None, ) -> DiffusionOutput: """Execute a forward pass by delegating to the model runner. - If *req* is a list (DP multi-concurrency), each rank picks one - request based on its distributed rank. AllGather in the layerwise - offload only gathers weight shards (request-independent), so all - ranks stay synchronised at each AllGather call while computing - different activations. + If *req* is a list (DP multi-concurrency), each rank picks one complete + NewRequestData envelope based on its distributed rank. AllGather in + the layerwise offload only gathers weight shards (request-independent), + so all ranks stay synchronised at each AllGather call while computing + different activations. Selecting the envelope keeps Scheduler-issued + KV metadata bound to the request that owns its block tables. Each rank returns its OWN DiffusionOutput (no gather). The executor collects N responses via the per-worker result queues. @@ -492,7 +543,10 @@ class DiffusionWorker: dp_rank = get_data_parallel_rank() idx = dp_rank % len(req) - req = req[idx] + new_req = req[idx] + validate_new_request_data_identity(new_req) + req = new_req.req + diffusion_kv_metadata = new_req.diffusion_kv_metadata if self.lora_manager is not None: try: @@ -1443,7 +1497,7 @@ class WorkerWrapperBase: def execute_model( self, - req: OmniDiffusionRequest, + req: OmniDiffusionRequest | list[NewRequestData], od_config: OmniDiffusionConfig, kv_prefetch_job: KVPrefetchJob | None = None, diffusion_kv_metadata: DiffusionKVMetadata | None = None, From cd9ae66b3a67b47e13092abcfe648b4ada86042e Mon Sep 17 00:00:00 2001 From: zhumingjue138 Date: Tue, 18 Aug 2026 17:36:08 +0800 Subject: [PATCH 055/267] [Bugfix][Test] fix Qwen3-omni OOM reliability and drop voxcpm2 stability/reliability test cases (#6299) Signed-off-by: zhumingjue --- .buildkite/cuda/test-weekly.yml | 8 - docs/contributing/ci/test_execution_guide.md | 3 - docs/contributing/ci/test_writing_guide.md | 9 +- .../fault_injection_reliability_matrix.md | 2 +- tests/dfx/reliability/README.md | 2 - .../test_reliability_qwen3_omni.py | 2 +- .../reliability/test_reliability_voxcpm2.py | 508 ------------------ .../scripts/test_stability_voxcpm2.py | 69 --- tools/nightly/run_nightly_jobs.sh | 3 +- 9 files changed, 5 insertions(+), 601 deletions(-) delete mode 100644 tests/dfx/reliability/test_reliability_voxcpm2.py delete mode 100644 tests/dfx/stability/scripts/test_stability_voxcpm2.py diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml index 5ddbf22ce..ed20e78dc 100644 --- a/.buildkite/cuda/test-weekly.yml +++ b/.buildkite/cuda/test-weekly.yml @@ -30,14 +30,6 @@ steps: - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow" mirror_hardwares: h100_4 - - label: "Reliability Test - voxcpm2" - timeout_in_minutes: 180 - depends_on: upload-weekly-pipeline - if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" - commands: - - pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow" - mirror_hardwares: l4_1 - - group: "Reliability Test - Invalid parameters Test" depends_on: upload-weekly-pipeline if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" diff --git a/docs/contributing/ci/test_execution_guide.md b/docs/contributing/ci/test_execution_guide.md index d1c2c560f..ff53cbbf7 100644 --- a/docs/contributing/ci/test_execution_guide.md +++ b/docs/contributing/ci/test_execution_guide.md @@ -184,9 +184,6 @@ Failed jobs: 1/2 # Reliability: HunyuanImage DiT (H100 × 4) pytest -s -v dfx/reliability/test_reliability_hunyuan_image.py -m slow - # Reliability: VoxCPM2 (L4) - pytest -s -v dfx/reliability/test_reliability_voxcpm2.py -m slow - ``` The latest L5 CI jobs (reliability + invalid-parameter weekly steps) are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-weekly.yml). diff --git a/docs/contributing/ci/test_writing_guide.md b/docs/contributing/ci/test_writing_guide.md index 397f5e83c..ea80ea21e 100644 --- a/docs/contributing/ci/test_writing_guide.md +++ b/docs/contributing/ci/test_writing_guide.md @@ -562,7 +562,6 @@ L5 level testing focuses on the performance of model services under ***long-runn - `test_reliability_qwen3_omni.py` — Qwen3-Omni chat / multimodal reliability (GPU OOM, process kill, recovery, error contract under `--async-chunk` vs default). - `test_reliability_wan22.py` — Wan2.2 I2V video API reliability (`/v1/videos` under OOM and process kill, recovery). - `test_reliability_hunyuan_image.py` — HunyuanImage-3.0-Instruct DiT-only reliability (`/v1/images/generations` under OOM and process kill; deploy `hunyuan_image3_dit.yaml`, H100 × 4). - - `test_reliability_voxcpm2.py` — VoxCPM2 TTS reliability (`/v1/audio/speech` under OOM and process kill; deploy `voxcpm2.yaml`, L4 × 1). - `helpers.py` — Shared primitives used by current suites: raw HTTP probes for `/v1/chat/completions` and `/health`, OpenAI-style error parsing, GPU OOM sidecar (`inject_gpu_oom` / `stop_gpu_oom_hogs`), and `pgrep`-based process-kill injector construction (`make_process_kill_fault_injector`). - `conftest.py` — `fault_injector` and `omni_server_after_fault` / `omni_server_after_fault_function` fixtures to run a callable **after** the server is ready. - `README.md` — Short local run commands for this directory. @@ -662,13 +661,12 @@ Reliability tests are **short fault-injection** integration runs (L5 **(b)** in | `test_reliability_qwen3_omni.py` | Qwen3-Omni: OOM vs **text vs text+audio** error contract, large multimodal chat under OOM, concurrent pressure, **SIGKILL** on `VLLM::Worker`, `/health` → 503 + fast-fail + concurrent chat; optional OOM recovery scenario (may be skipped while tracked in issues). | | `test_reliability_wan22.py` | Wan2.2 I2V: large `/v1/videos` under OOM, **SIGKILL** on `multiprocessing.spawn` chain, health / fast-fail / concurrent video requests; optional recovery test (may be skipped). | | `test_reliability_hunyuan_image.py` | HunyuanImage DiT-only: large `/v1/images/generations` under OOM, **SIGKILL** on `vLLM-Omni::` workers and serve/tree targets, health / fast-fail / concurrent image requests; some OOM/recovery cases may be skipped while tracked in issues. | -| `test_reliability_voxcpm2.py` | VoxCPM2: `/v1/audio/speech` under OOM (error contract), **SIGKILL** on `VLLM::` workers and serve/tree targets, health / fast-fail / concurrent speech requests; some OOM cases may be skipped while tracked in issues. | | `README.md` | Minimal run / collect examples. | ##### Parametrization and markers - Each test module defines a **`RELIABILITY_SCENARIOS`** list (`test_name`, `server_params`: model, `stage_config_name` or diffusion `server_args`, etc.). **`create_reliability_omni_server_params()`** in `tests/dfx/conftest.py` resolves stage paths (including XPU substitutions where applicable) and builds **`OmniServerParams`** lists consumed by **`@pytest.mark.parametrize(..., indirect=True)`** on `omni_server` or `omni_server_function`. -- Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** or **`@hardware_test(res={"cuda": "L4"}, num_cards=1)`** (Qwen3-Omni **2**× H100; Wan2.2 **1**× H100; HunyuanImage DiT **4**× H100; VoxCPM2 **1**× L4). +- Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** (Qwen3-Omni **2**× H100; Wan2.2 **1**× H100; HunyuanImage DiT **4**× H100). - **Process-kill** tests use **`@pytest.mark.skipif(os.name == "nt", ...)`** because injection uses POSIX **`pgrep` / `kill`**. ##### CI trigger @@ -680,13 +678,11 @@ Weekly Buildkite (`.buildkite/cuda/test-weekly.yml`) runs one step per model sui | Reliability Test - qwen3-omni | `test_reliability_qwen3_omni.py` | H100 × 2 (`mithril-h100-pool`) | | Reliability Test - wan22 | `test_reliability_wan22.py` | H100 × 2 (`mithril-h100-pool`) | | Reliability Test - hunyuan-image | `test_reliability_hunyuan_image.py` | H100 × 4 (`mithril-h100-pool`) | -| Reliability Test - voxcpm2 | `test_reliability_voxcpm2.py` | L4 × 1 (`gpu_1_queue`) | ```bash pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow" pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow" pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow" -pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow" ``` ##### Local commands @@ -696,7 +692,6 @@ pytest --collect-only tests/dfx/reliability pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow -pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow ``` ##### Adding a new model suite @@ -709,4 +704,4 @@ pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow - - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_.py` alongside a matching JSON config) - - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`, `voxcpm2`; add `test_reliability_.py` and a matching step in `.buildkite/cuda/test-weekly.yml` for new models) + - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`; add `test_reliability_.py` and a matching step in `.buildkite/cuda/test-weekly.yml` for new models) diff --git a/docs/user_guide/fault_injection_reliability_matrix.md b/docs/user_guide/fault_injection_reliability_matrix.md index 28271affa..4a2fa0a16 100644 --- a/docs/user_guide/fault_injection_reliability_matrix.md +++ b/docs/user_guide/fault_injection_reliability_matrix.md @@ -65,7 +65,7 @@ More failure modes (for example, network jitter and network interruption) will b ## Source of Conclusions -The behaviors and conclusions above are summarized from current fault injection validation results on **`Qwen3-Omni`**, **`Wan2.2`**, **`HunyuanImage-3.0-Instruct`** (DiT-only, `/v1/images/generations`), and **`VoxCPM2`** (`/v1/audio/speech`). Automated coverage lives under `tests/dfx/reliability/` (`test_reliability_qwen3_omni.py`, `test_reliability_wan22.py`, `test_reliability_hunyuan_image.py`, `test_reliability_voxcpm2.py`) and runs weekly via `.buildkite/cuda/test-weekly.yml`. +The behaviors and conclusions above are summarized from current fault injection validation results on **`Qwen3-Omni`**, **`Wan2.2`**, and **`HunyuanImage-3.0-Instruct`** (DiT-only, `/v1/images/generations`). Automated coverage lives under `tests/dfx/reliability/` (`test_reliability_qwen3_omni.py`, `test_reliability_wan22.py`, `test_reliability_hunyuan_image.py`) and runs weekly via `.buildkite/cuda/test-weekly.yml`. ### Example: `SIGTERM` fault injection log (Qwen3-Omni) diff --git a/tests/dfx/reliability/README.md b/tests/dfx/reliability/README.md index 8dc69dc55..79d862025 100644 --- a/tests/dfx/reliability/README.md +++ b/tests/dfx/reliability/README.md @@ -7,12 +7,10 @@ The detailed RFC document is maintained outside this repository (local/internal - Qwen3 tests: `test_reliability_qwen3_omni.py` - Wan2.2 tests: `test_reliability_wan22.py` - HunyuanImage tests: `test_reliability_hunyuan_image.py` -- VoxCPM2 tests: `test_reliability_voxcpm2.py` ```bash pytest --collect-only tests/dfx/reliability pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow -pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow ``` diff --git a/tests/dfx/reliability/test_reliability_qwen3_omni.py b/tests/dfx/reliability/test_reliability_qwen3_omni.py index 32c660954..d12a11ba1 100644 --- a/tests/dfx/reliability/test_reliability_qwen3_omni.py +++ b/tests/dfx/reliability/test_reliability_qwen3_omni.py @@ -71,7 +71,7 @@ def _default_oom_device_spec() -> str: OOM_INJECTION_CONFIG = { "device": _default_oom_device_spec(), - "target_mem_ratio": 0.92, + "target_mem_ratio": 0.95, "hold_seconds": 0, "startup_timeout_sec": 20, "strict": False, diff --git a/tests/dfx/reliability/test_reliability_voxcpm2.py b/tests/dfx/reliability/test_reliability_voxcpm2.py deleted file mode 100644 index 632c51081..000000000 --- a/tests/dfx/reliability/test_reliability_voxcpm2.py +++ /dev/null @@ -1,508 +0,0 @@ -""" -VoxCPM2 reliability integration tests. - -Scenarios follow ``docs/user_guide/fault_injection_reliability_matrix.md``. -""" - -from __future__ import annotations - -import concurrent.futures -import os -import time -from pathlib import Path -from typing import Any - -import pytest - -from tests.dfx.conftest import ( - assert_fault_exception, - create_reliability_omni_server_params, - resolve_oom_device_spec, -) -from tests.dfx.reliability.helpers import ( - FaultInjector, - assert_no_server_tree_process_residual_and_gpu_release, - extract_openai_error_contract_from_bytes, - get_health_raw, - inject_gpu_oom, - make_process_kill_fault_injector, - make_server_root_kill_fault_injector, - make_server_tree_kill_fault_injector, - post_json_raw, - run_fault_injection_with_rate_load, - stop_gpu_oom_hogs, - worker_residual_timeout_after_kill_signal, -) -from tests.helpers.mark import hardware_test -from tests.helpers.media import load_test_audio_data_url -from tests.helpers.runtime import OpenAIClientHandler - -RELIABILITY_SCENARIOS: list[dict[str, Any]] = [ - { - "test_name": "voxcpm2_reliability_default", - "server_params": { - "model": "openbmb/VoxCPM2", - "stage_config_name": "voxcpm2.yaml", - # VoxCPM2 uses a custom HF tokenizer; CLI default trust_remote_code=False - # would otherwise override deploy YAML settings during server startup. - "server_args": ["--trust-remote-code", "--disable-log-stats"], - }, - }, -] - -DEPLOY_CONFIGS_DIR = Path(__file__).resolve().parent.parent.parent.parent / "vllm_omni" / "deploy" -OOM_INJECTION_CONFIG = { - "target_mem_ratio": 0.96, - "hold_seconds": 0, - "startup_timeout_sec": 20, - "strict": False, -} -FAULT_ERROR_KEYWORDS = ( - "the request failed", - "oom", - "out of memory", - "cuda", - "orchestrator", - "timeout", - "connection", - "500", - "503", -) -PROCESS_KILL_ERROR_KEYWORDS = ( - "timeout", - "did not complete within", - "connection", - "engine", - "orchestrator", - "dead", - "internal", - "500", - "503", -) -RUNTIME_WORKER_PATTERN = "VLLM::" -SERVE_SIGNAL_PARAMS = [ - pytest.param("SIGTERM", id="sigterm"), - pytest.param("SIGINT", id="sigint"), - pytest.param("SIGKILL", id="sigkill"), -] -SERVE_WITH_LOAD_SIGNAL_PARAMS = [ - pytest.param("SIGTERM", id="sigterm"), - pytest.param("SIGINT", id="sigint"), - pytest.param("SIGKILL", id="sigkill"), -] -TREE_SIGNAL_PARAMS = [ - pytest.param("SIGTERM", id="sigterm"), - pytest.param("SIGKILL", id="sigkill"), -] -TREE_WITH_LOAD_SIGNAL_PARAMS = [ - pytest.param("SIGTERM", id="sigterm"), - pytest.param("SIGKILL", id="sigkill"), -] -WORKER_SIGNAL_FAULT_PARAMS = [ - pytest.param( - make_process_kill_fault_injector( - grep_patterns=RUNTIME_WORKER_PATTERN, - signal_name="SIGTERM", - limit=1, - post_kill_wait_seconds=2.0, - ), - id="runtime_process_chain_sigterm", - ), - pytest.param( - make_process_kill_fault_injector( - grep_patterns=RUNTIME_WORKER_PATTERN, - signal_name="SIGKILL", - limit=1, - post_kill_wait_seconds=2.0, - ), - id="runtime_process_chain_sigkill", - ), -] - -VOXCPM2_PARAMS = create_reliability_omni_server_params(RELIABILITY_SCENARIOS, DEPLOY_CONFIGS_DIR) -INFLIGHT_INJECTION_REQUEST_RATE = 0.3 -INFLIGHT_INJECTION_REQUEST_COUNT = 10 -_SPEECH_FF_HTTP_TIMEOUT_SEC = 30 -_SPEECH_FF_MAX_ELAPSED_SEC = 30 - - -def _speech_request_config(omni_server: Any) -> dict[str, Any]: - return { - "model": omni_server.model, - "input": "The weather is nice today, perfect for a walk in the park.", - "stream": False, - "timeout": 120.0, - "response_format": "wav", - "voice": "default", - } - - -def _submit_speech_request(openai_client: OpenAIClientHandler, omni_server: Any) -> None: - openai_client.send_audio_speech_request(_speech_request_config(omni_server), request_num=1) - - -def _oom_speech_request_config(omni_server: Any) -> dict[str, Any]: - base = "The weather is nice today, perfect for a walk in the park. " - return { - "model": omni_server.model, - "input": base * 80, - "max_new_tokens": 4096, - "ref_audio": load_test_audio_data_url("qwen3_tts/clone_2.wav"), - "stream": False, - "timeout": 300.0, - "response_format": "wav", - "voice": "default", - } - - -def _submit_oom_speech_request(openai_client: OpenAIClientHandler, omni_server: Any) -> None: - openai_client.send_audio_speech_request(_oom_speech_request_config(omni_server), request_num=1) - - -def _assert_post_fault_speech_fast_fail(host: str, port: int, *, model: str, scenario: str) -> None: - payload = { - "model": model, - "input": "post-fault fast-fail check", - "voice": "default", - } - start = time.monotonic() - try: - status, body = post_json_raw(host, port, "/v1/audio/speech", payload, timeout_sec=20) - elapsed = time.monotonic() - start - assert elapsed < 15, f"[{scenario} fast_fail] /v1/audio/speech did not fail fast: {elapsed:.2f}s" - assert status >= 500, ( - f"[{scenario} fast_fail] expected server-side error after fault, got status={status}, body={body[:200]!r}" - ) - except Exception: # noqa: BLE001 - elapsed = time.monotonic() - start - assert elapsed < 15, f"[{scenario} fast_fail] exception was too slow after fault: {elapsed:.2f}s" - - -def _assert_post_fault_health_terminal(host: str, port: int, *, scenario: str) -> None: - deadline = time.monotonic() + 20.0 - last_observation = "" - while time.monotonic() < deadline: - try: - status, body = get_health_raw(host, port, timeout_sec=5) - last_observation = f"http={status}, body={body[:200]!r}" - if status == 503: - return - except Exception as exc: # noqa: BLE001 - last_observation = f"exception={exc!r}" - return - time.sleep(0.5) - pytest.fail(f"[{scenario} health] no terminal post-fault health observed: {last_observation}") - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.skip(reason="issue#4285") -@hardware_test(res={"cuda": "L4"}, num_cards=1) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_gpu_oom_speech_request_failure( - omni_server_function, - openai_client_function, -) -> None: - stage_config_path = getattr(omni_server_function, "stage_config_path", None) - device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path) - handle = inject_gpu_oom( - device=device_spec, - target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"], - hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"], - startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"], - strict=OOM_INJECTION_CONFIG["strict"], - ) - try: - try: - _submit_oom_speech_request(openai_client_function, omni_server_function) - except Exception as exc: - assert_fault_exception(exc, FAULT_ERROR_KEYWORDS) - else: - pytest.fail("expected /v1/audio/speech request failure during GPU OOM injection") - finally: - stop_gpu_oom_hogs(handle) - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.skip(reason="issue#4285") -@hardware_test(res={"cuda": "L4"}, num_cards=1) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_gpu_oom_speech_error_contract( - omni_server_function, -) -> None: - stage_config_path = getattr(omni_server_function, "stage_config_path", None) - device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path) - handle = inject_gpu_oom( - device=device_spec, - target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"], - hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"], - startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"], - strict=OOM_INJECTION_CONFIG["strict"], - ) - host = omni_server_function.host - port = omni_server_function.port - oom_config = _oom_speech_request_config(omni_server_function) - payload = { - "model": oom_config["model"], - "input": oom_config["input"], - "voice": oom_config["voice"], - "max_new_tokens": oom_config["max_new_tokens"], - "ref_audio": oom_config["ref_audio"], - } - try: - status, body = post_json_raw(host, port, "/v1/audio/speech", payload, timeout_sec=300) - finally: - stop_gpu_oom_hogs(handle) - - assert status >= 500, f"expected speech runtime error under OOM, got status={status}" - speech_error = extract_openai_error_contract_from_bytes(body) - assert speech_error is not None, f"speech error payload not OpenAI-compatible: {body[:300]!r}" - assert "code" in speech_error, f"speech error lacks code field: {speech_error!r}" - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") -@pytest.mark.parametrize( - "fault_injector", - WORKER_SIGNAL_FAULT_PARAMS, - indirect=True, -) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_process_kill_request_failure( - omni_server_after_fault_function, - openai_client_function, -) -> None: - try: - _submit_speech_request(openai_client_function, omni_server_after_fault_function) - except Exception as exc: - assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS) - else: - pytest.fail("expected /v1/audio/speech request failure after process-kill injection") - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") -@pytest.mark.parametrize( - "fault_injector", - WORKER_SIGNAL_FAULT_PARAMS, - indirect=True, -) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_process_kill_fast_fail_and_concurrent( - omni_server_after_fault_function, - openai_client_function, -) -> None: - host = omni_server_after_fault_function.host - port = omni_server_after_fault_function.port - model = omni_server_after_fault_function.model - payload = { - "model": model, - "input": "Say hello in one short sentence.", - "voice": "default", - } - - start = time.monotonic() - try: - status, body = post_json_raw( - host, - port, - "/v1/audio/speech", - payload, - timeout_sec=_SPEECH_FF_HTTP_TIMEOUT_SEC, - ) - elapsed = time.monotonic() - start - assert elapsed <= _SPEECH_FF_MAX_ELAPSED_SEC + 1.0, ( - f"[process_kill fast_fail] /v1/audio/speech did not fail fast after fault: {elapsed:.2f}s" - ) - assert status >= 500, ( - "[process_kill fast_fail] expected server-side failure after fault, " - f"got status={status}, body={body[:200]!r}" - ) - except Exception: - elapsed = time.monotonic() - start - assert elapsed <= _SPEECH_FF_MAX_ELAPSED_SEC + 1.0, ( - f"[process_kill fast_fail] /v1/audio/speech exception was too slow after fault: {elapsed:.2f}s" - ) - - start = time.monotonic() - with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor: - futures = [ - executor.submit(_submit_speech_request, openai_client_function, omni_server_after_fault_function) - for _ in range(3) - ] - done, pending = concurrent.futures.wait( - futures, - timeout=30, - return_when=concurrent.futures.ALL_COMPLETED, - ) - - elapsed = time.monotonic() - start - assert not pending, f"[process_kill concurrent] some fault-time speech requests hung: pending={len(pending)}" - assert elapsed < 30, f"[process_kill concurrent] fault-time speech request convergence is too slow: {elapsed:.2f}s" - - fault_observed = False - for future in done: - try: - future.result() - except Exception as exc: - fault_observed = True - assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS) - assert fault_observed, ( - "[process_kill concurrent] expected at least one /v1/audio/speech request to fail after fault" - ) - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") -@pytest.mark.parametrize( - "fault_injector", - WORKER_SIGNAL_FAULT_PARAMS, - indirect=True, -) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_process_kill_worker_with_load_request_failure( - omni_server_function, - openai_client_function, - fault_injector: FaultInjector, -) -> None: - scenario = "kill_worker_with_load" - load_result = run_fault_injection_with_rate_load( - submit_request=lambda: _submit_speech_request(openai_client_function, omni_server_function), - inject_fault=lambda: fault_injector(omni_server_function), - num_requests=INFLIGHT_INJECTION_REQUEST_COUNT, - request_rate=INFLIGHT_INJECTION_REQUEST_RATE, - completion_timeout_sec=120.0, - ) - assert load_result["failure_observed"], ( - f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}" - ) - host = omni_server_function.host - port = omni_server_function.port - _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) - _assert_post_fault_health_terminal(host, port, scenario=scenario) - - -@pytest.mark.slow -@pytest.mark.tts -@hardware_test(res={"cuda": "L4"}, num_cards=1) -@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") -@pytest.mark.parametrize("signal_name", SERVE_SIGNAL_PARAMS) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_process_kill_serve_root_no_load_fast_fail_and_cleanup( - omni_server_function, - signal_name: str, -) -> None: - scenario = f"kill_serve_root_no_load_{signal_name.lower()}" - injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0) - injector(omni_server_function) - host = omni_server_function.host - port = omni_server_function.port - _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) - _assert_post_fault_health_terminal(host, port, scenario=scenario) - assert_no_server_tree_process_residual_and_gpu_release( - omni_server_function, - scenario=scenario, - timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), - ) - - -@pytest.mark.slow -@pytest.mark.tts -@hardware_test(res={"cuda": "L4"}, num_cards=1) -@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") -@pytest.mark.parametrize("signal_name", SERVE_WITH_LOAD_SIGNAL_PARAMS) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_process_kill_serve_root_with_load_fast_fail_and_cleanup( - omni_server_function, - openai_client_function, - signal_name: str, -) -> None: - scenario = f"kill_serve_root_with_load_{signal_name.lower()}" - injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0) - load_result = run_fault_injection_with_rate_load( - submit_request=lambda: _submit_speech_request(openai_client_function, omni_server_function), - inject_fault=lambda: injector(omni_server_function), - num_requests=INFLIGHT_INJECTION_REQUEST_COUNT, - request_rate=INFLIGHT_INJECTION_REQUEST_RATE, - completion_timeout_sec=300.0, - ) - assert load_result["failure_observed"], ( - f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}" - ) - host = omni_server_function.host - port = omni_server_function.port - _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) - _assert_post_fault_health_terminal(host, port, scenario=scenario) - assert_no_server_tree_process_residual_and_gpu_release( - omni_server_function, - scenario=scenario, - timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), - ) - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") -@pytest.mark.parametrize("signal_name", TREE_SIGNAL_PARAMS) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_process_kill_tree_no_load_fast_fail_and_cleanup( - omni_server_function, - signal_name: str, -) -> None: - scenario = f"kill_serve_tree_no_load_{signal_name.lower()}" - injector = make_server_tree_kill_fault_injector( - signal_name=signal_name, - post_kill_wait_seconds=2.0, - inter_kill_wait_seconds=0.1, - ) - injector(omni_server_function) - host = omni_server_function.host - port = omni_server_function.port - _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) - _assert_post_fault_health_terminal(host, port, scenario=scenario) - assert_no_server_tree_process_residual_and_gpu_release( - omni_server_function, - scenario=scenario, - timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), - ) - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") -@pytest.mark.parametrize("signal_name", TREE_WITH_LOAD_SIGNAL_PARAMS) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup( - omni_server_function, - openai_client_function, - signal_name: str, -) -> None: - scenario = f"kill_serve_tree_with_load_{signal_name.lower()}" - injector = make_server_tree_kill_fault_injector( - signal_name=signal_name, - post_kill_wait_seconds=2.0, - inter_kill_wait_seconds=0.1, - ) - load_result = run_fault_injection_with_rate_load( - submit_request=lambda: _submit_speech_request(openai_client_function, omni_server_function), - inject_fault=lambda: injector(omni_server_function), - num_requests=INFLIGHT_INJECTION_REQUEST_COUNT, - request_rate=INFLIGHT_INJECTION_REQUEST_RATE, - completion_timeout_sec=120.0, - ) - assert load_result["failure_observed"], ( - f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}" - ) - host = omni_server_function.host - port = omni_server_function.port - _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) - _assert_post_fault_health_terminal(host, port, scenario=scenario) - assert_no_server_tree_process_residual_and_gpu_release( - omni_server_function, - scenario=scenario, - timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), - ) diff --git a/tests/dfx/stability/scripts/test_stability_voxcpm2.py b/tests/dfx/stability/scripts/test_stability_voxcpm2.py deleted file mode 100644 index e26a2050c..000000000 --- a/tests/dfx/stability/scripts/test_stability_voxcpm2.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -VoxCPM2 stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration. - -Configuration: ``tests/dfx/stability/tests/test_voxcpm2.json``. -""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from tests.dfx.conftest import ( - create_benchmark_indices, - create_test_parameter_mapping, - create_unique_server_params, - load_configs, -) -from tests.dfx.stability.helpers import _run_one_vllm_bench_batch, run_stability_benchmark_loop - -STABILITY_DIR = Path(__file__).resolve().parent.parent -DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy" -CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_voxcpm2.json") -DEFAULT_NUM_PROMPTS_PER_BATCH = 20 -STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "600"] - -try: - BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH) -except FileNotFoundError: - BENCHMARK_CONFIGS = [] - -test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] -server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} -benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.parametrize("omni_server", test_params, indirect=True) -@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) -def test_stability_voxcpm2(omni_server, stability_benchmark_params): - test_name = stability_benchmark_params["test_name"] - params = stability_benchmark_params["params"] - duration_sec = params.get("duration_sec", 300) - num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH) - request_rate = params.get("request_rate") - max_concurrency = params.get("max_concurrency") - - bench_params = { - k: v - for k, v in params.items() - if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch") - } - - result = run_stability_benchmark_loop( - host=omni_server.host, - port=omni_server.port, - model=omni_server.model, - duration_sec=duration_sec, - params=bench_params, - request_rate=request_rate, - max_concurrency=max_concurrency, - result_dir=str(STABILITY_DIR), - num_prompts_per_batch=num_prompts_per_batch, - run_one_batch=_run_one_vllm_bench_batch, - ) - - assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}" - assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed" diff --git a/tools/nightly/run_nightly_jobs.sh b/tools/nightly/run_nightly_jobs.sh index a95eb8876..501f6a605 100644 --- a/tools/nightly/run_nightly_jobs.sh +++ b/tools/nightly/run_nightly_jobs.sh @@ -43,7 +43,7 @@ # # stability (when included in TEST_TYPE): # From repo root: pytest -s -v --run-level full_model tests/dfx/stability/scripts/test_stability_*.py -# model_type: omni → qwen3_omni; tts → qwen3_tts + voxcpm2; diffusion → qwen_image + wan22 + hunyuan_image; all → all six +# model_type: omni → qwen3_omni; tts → qwen3_tts; diffusion → qwen_image + wan22 + hunyuan_image; all → all five # LABEL_SUBSTR: if set, script path / job key / filename must contain it # # Requirements: bash, python3, PyYAML (pip install pyyaml) @@ -588,7 +588,6 @@ def _write_job_timeouts_manifest(jobs_dir: Path, job_timeouts: dict[str, int]) - STABILITY_CASES: list[tuple[str, str, tuple[str, ...]]] = [ ("stability_qwen3_omni", "tests/dfx/stability/scripts/test_stability_qwen3_omni.py", ("omni",)), ("stability_qwen3_tts", "tests/dfx/stability/scripts/test_stability_qwen3_tts.py", ("tts",)), - ("stability_voxcpm2", "tests/dfx/stability/scripts/test_stability_voxcpm2.py", ("tts",)), ("stability_qwen_image", "tests/dfx/stability/scripts/test_stability_qwen_image.py", ("diffusion",)), ("stability_wan22", "tests/dfx/stability/scripts/test_stability_wan22.py", ("diffusion",)), ("stability_hunyuan_image", "tests/dfx/stability/scripts/test_stability_hunyuan_image.py", ("diffusion",)), From 02069a5bbf95b4f88de1c1e1e9c0da7564e697ad Mon Sep 17 00:00:00 2001 From: TJian Date: Tue, 18 Aug 2026 17:45:05 +0800 Subject: [PATCH 056/267] [ROCm] [CI] Fix circular import for rocm due to patch (#6287) Signed-off-by: tjtanaa --- vllm_omni/diffusion/lora/manager.py | 3 +-- vllm_omni/diffusion/lora/utils.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/vllm_omni/diffusion/lora/manager.py b/vllm_omni/diffusion/lora/manager.py index d11b1b9a2..33094bc49 100644 --- a/vllm_omni/diffusion/lora/manager.py +++ b/vllm_omni/diffusion/lora/manager.py @@ -9,7 +9,7 @@ from typing import get_args import torch import torch.nn as nn -from vllm.config.lora import MaxLoRARanks +from vllm.config.lora import LoRAConfig, MaxLoRARanks from vllm.logger import init_logger from vllm.lora.layers import BaseLayerWithLoRA from vllm.lora.lora_model import LoRAModel @@ -23,7 +23,6 @@ from vllm.lora.utils import ( ) from vllm.model_executor.layers.linear import MergedColumnParallelLinear, QKVParallelLinear -from vllm_omni.config.lora import LoRAConfig from vllm_omni.diffusion.lora.utils import ( _expand_expected_modules_for_packed_layers, _match_target_modules, diff --git a/vllm_omni/diffusion/lora/utils.py b/vllm_omni/diffusion/lora/utils.py index 465edc99c..a8fdc2776 100644 --- a/vllm_omni/diffusion/lora/utils.py +++ b/vllm_omni/diffusion/lora/utils.py @@ -5,8 +5,8 @@ from __future__ import annotations import torch.nn as nn from transformers import PretrainedConfig +from vllm.config.lora import LoRAConfig -from vllm_omni.config.lora import LoRAConfig from vllm_omni.diffusion.lora.layers import ( DiffusionColumnParallelLinearWithLoRA, DiffusionMergedColumnParallelLinearWithLoRA, From e1c5f59aab091b5aed864808d118903ceb77d6ac Mon Sep 17 00:00:00 2001 From: Wang fuyin Date: Tue, 18 Aug 2026 21:27:58 +0800 Subject: [PATCH 057/267] [Config] Reuse vLLM configs in VllmOmniConfig (#6050) Signed-off-by: Acerak01-fy Co-authored-by: Hongsheng Liu --- docs/design/module/vllm_omni_config.md | 61 ++++- tests/config/test_omni_config.py | 203 +++++++++++--- tests/engine/test_stage_engine_args.py | 142 +++++++++- vllm_omni/config/omni_config.py | 356 +++++++++++++++++++++---- vllm_omni/engine/stage_init_utils.py | 60 ++++- 5 files changed, 734 insertions(+), 88 deletions(-) diff --git a/docs/design/module/vllm_omni_config.md b/docs/design/module/vllm_omni_config.md index 085e4cee6..eabb9908f 100644 --- a/docs/design/module/vllm_omni_config.md +++ b/docs/design/module/vllm_omni_config.md @@ -18,8 +18,8 @@ validation_paths: - tests/config/** upstream_refs: - vllm.config -last_reviewed: 2026-08-07 -last_verified_commit: 3d7fc3b9ba3cac88d579d4dc35b78b0b641675fc +last_reviewed: 2026-08-11 +last_verified_commit: e356708da4cd39992405686dacfc32b18fcccc7f --- # vLLM-Omni configuration @@ -39,11 +39,66 @@ The PR review proposal to capture environment-derived values in configuration objects at initialization is an open design question, not a current invariant. +## Current structured-stage reuse behavior + +The current RFC #4021 implementation uses upstream vLLM config classes as the +schema source for AR and generation stages: + +- `OmniStageLoadConfig` inherits `vllm.config.LoadConfig`. +- `OmniStageCacheConfig` inherits `vllm.config.CacheConfig`. +- `OmniStageSchedulerConfig` inherits `vllm.config.SchedulerConfig`. +- `OmniStageParallelConfig` inherits `vllm.config.ParallelConfig`. + +This inheritance preserves the existing load, cache, scheduler, and parallel +concern boundaries while reducing duplicate downstream declarations. It does +not, by itself, make every inherited field an effective vLLM-Omni runtime +option. Construction and runtime consumption are separate surfaces: + +| Surface | Current behavior | +| --- | --- | +| Structured schema | Upstream dataclass fields are inherited. Their defaults, default factories, and applicable Pydantic validation can participate when the structured object is constructed. | +| Omni input ownership | Pipeline/deploy/CLI construction continues to accept only fields with an existing structured owner. Direct construction of an inherited sub-config can expose a wider upstream schema. | +| Engine projection | For AR and generation stages, reusable fields are discovered by intersecting each upstream config dataclass with upstream `EngineArgs`. Only constructor-explicit values are emitted; inherited defaults remain deferred to terminal materialization. | +| Terminal materialization | The engine-owning process constructs the final upstream `VllmConfig` and performs model-, platform-, rank-, port-, and backend-dependent initialization. | + +The generated projection maps include known upstream naming differences such +as `cache_dtype` to `kv_cache_dtype`, `policy` to `scheduling_policy`, and +`data_parallel_master_ip` to `data_parallel_address`. A field added to both an +upstream concern config and `EngineArgs` therefore enters the AR/generation +projection without requiring a second downstream allowlist. Explicit inherited +fields that have no projection raise an error rather than being silently +dropped. + +Ownership exclusions remain deliberate. Stage topology owns `scheduler_cls`, +cache owns `disable_hybrid_kv_cache_manager`, and Omni runtime owns +`distributed_executor_backend` and `worker_cls`; vLLM's private API-process +fields are terminal internals. These exclusions prevent one input from being +constructed or projected through two config concerns. Effective-engine-argument +tests cover both the dynamic mapping and this exclusion boundary. + +`CompilationConfig` and `ProfilerConfig` are direct structured-stage fields: +`stage.compilation_config` and `stage.profiler_config`. They accept mapping/YAML +inputs and complete upstream value objects. Mapping inputs are validated and +materialized as their concrete upstream types when the structured config is +constructed, so downstream consumers see one resolved representation. +Prebuilt upstream objects retain their type across the structured projection +boundary. + +Quantization remains on its existing Omni-owned transport contract. This +reuse step does not adopt upstream `QuantizationConfigArgs`, because the +current engine-specific quantization split needs to be resolved separately. + +Diffusion keeps its existing engine-field ownership and projection surface. +Shared Python inheritance must not be treated as evidence that an LLM-only +parallel input is effective for a diffusion stage. + ## Safe-change guide while deferred Changes should trace every affected producer to its runtime consumer and test the structured, legacy, CLI, and deployment paths that are actually supported. -Do not infer a stable contract from this placeholder page. +For inherited vLLM fields, tests should separately cover constructor behavior +and effective `EngineArgs` projection. Do not infer a stable contract from +this placeholder page. ## Promotion gate diff --git a/tests/config/test_omni_config.py b/tests/config/test_omni_config.py index cc08ed397..108d997a5 100644 --- a/tests/config/test_omni_config.py +++ b/tests/config/test_omni_config.py @@ -5,11 +5,20 @@ from __future__ import annotations from dataclasses import fields +from inspect import Parameter, signature from pathlib import Path +import msgspec import pytest from pydantic import ValidationError +from pydantic.fields import FieldInfo from transformers import Qwen3OmniMoeConfig +from vllm.config import CacheConfig as VllmCacheConfig +from vllm.config import CompilationConfig as VllmCompilationConfig +from vllm.config import LoadConfig as VllmLoadConfig +from vllm.config import ParallelConfig as VllmParallelConfig +from vllm.config import ProfilerConfig as VllmProfilerConfig +from vllm.config import SchedulerConfig as VllmSchedulerConfig from tests.helpers.stage_config import get_deploy_config_path from vllm_omni.config import omni_config as omni_config_module @@ -40,6 +49,7 @@ from vllm_omni.config.stage_config import ( merge_pipeline_deploy, ) from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode +from vllm_omni.engine.stage_engine_startup import _serialize_stage_config from vllm_omni.engine.stage_init_utils import build_legacy_engine_args_dict pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -243,9 +253,29 @@ def test_from_pipeline_config_rejects_unowned_deploy_engine_extras(engine_extras ], ids=["global", "ar-stage", "generation-stage"], ) -def test_from_pipeline_config_rejects_diffusion_only_cli_fields_for_llm_stages(cli_overrides, stage_id): - with pytest.raises(ValueError, match=rf"Stage {stage_id} .*no structured config owner: kv_cache_dtype"): - _from_pipeline_key("qwen3_tts", cli_overrides=cli_overrides) +def test_from_pipeline_config_accepts_upstream_cache_dtype_for_llm_stages(cli_overrides, stage_id): + stage = _from_pipeline_key("qwen3_tts", cli_overrides=cli_overrides).stage_by_id(stage_id) + + assert stage.cache_config.cache_dtype == "fp8" + assert "cache_dtype" in stage.cache_config._omni_explicit_fields + + +@pytest.mark.parametrize("field_name", ["data_parallel_master_ip", "data_parallel_address"]) +def test_from_pipeline_config_normalizes_nested_parallel_config_aliases(field_name): + pipeline = _resolve_pipeline_or_skip("qwen3_tts") + deploy = DeployConfig( + stages=[ + StageDeployConfig( + stage_id=0, + engine_extras={"parallel_config": {field_name: "10.0.0.1"}}, + ) + ] + ) + + stage = VllmOmniConfig.from_pipeline_config(pipeline, user_deploy_config=deploy).stage_by_id(0) + + assert stage.parallel_config.data_parallel_master_ip == "10.0.0.1" + assert "data_parallel_master_ip" in stage.parallel_config._omni_explicit_fields def test_from_pipeline_config_accepts_diffusion_only_cli_fields_for_diffusion_stage(): @@ -437,6 +467,8 @@ def test_vllm_omni_stage_config_public_fields_use_typed_stage_realizations(): "connector_config", "runtime_config", "parallel_config", + "compilation_config", + "profiler_config", "quantization_config", } assert "diffusion_config" not in public_fields @@ -455,10 +487,47 @@ def test_runtime_config_fields_match_structured_runtime_scope(): "num_gpus", "log_level", "log_stats", - "profiler_config", } +def test_upstream_config_compatible_fields_materialize_mapping_inputs(): + compilation_config = {"backend": "eager"} + profiler_config = {"profiler": "cuda"} + + stage_config = VllmOmniARStageConfig( + stage_pipeline_config=_resolve_pipeline_or_skip("qwen3_tts").stages[0], + compilation_config=compilation_config, + profiler_config=profiler_config, + ) + + assert isinstance(stage_config.compilation_config, VllmCompilationConfig) + assert stage_config.compilation_config.backend == "eager" + assert isinstance(stage_config.profiler_config, VllmProfilerConfig) + assert stage_config.profiler_config.profiler == "cuda" + + +def test_upstream_config_compatible_fields_validate_mapping_inputs(): + with pytest.raises(ValidationError): + VllmOmniARStageConfig( + stage_pipeline_config=_resolve_pipeline_or_skip("qwen3_tts").stages[0], + profiler_config={"delay_iterations": -1}, + ) + + +def test_upstream_config_compatible_fields_keep_prebuilt_config_types(): + compilation_config = VllmCompilationConfig(backend="eager") + profiler_config = VllmProfilerConfig(profiler="cuda") + + stage_config = VllmOmniARStageConfig( + stage_pipeline_config=_resolve_pipeline_or_skip("qwen3_tts").stages[0], + compilation_config=compilation_config, + profiler_config=profiler_config, + ) + + assert stage_config.compilation_config is compilation_config + assert stage_config.profiler_config is profiler_config + + def test_sub_config_fields_match_structured_scopes(): assert {f.name for f in fields(OmniStageModelConfig)} == { "model", @@ -489,7 +558,6 @@ def test_sub_config_fields_match_structured_scopes(): "enforce_eager", "max_cudagraph_capture_size", "enable_flashinfer_autotune", - "compilation_config", "enable_multithread_weight_load", "num_weight_load_threads", "disable_autocast", @@ -499,29 +567,25 @@ def test_sub_config_fields_match_structured_scopes(): "model_subdir", "tokenizer_subdir", } - assert {f.name for f in fields(OmniStageLoadConfig)} == { + vllm_load_fields = {f.name for f in fields(VllmLoadConfig)} + assert issubclass(OmniStageLoadConfig, VllmLoadConfig) + assert {f.name for f in fields(OmniStageLoadConfig)} == vllm_load_fields | { "tokenizer", - "download_dir", "skip_tokenizer_init", - "load_format", "tokenizer_mode", "config_format", "skip_mm_profiling", } - assert {f.name for f in fields(OmniStageCacheConfig)} == { - "kv_cache_memory_bytes", - "gpu_memory_utilization", - "enable_prefix_caching", + assert OmniStageLoadConfig(load_format="PT").load_format == "pt" + assert issubclass(OmniStageCacheConfig, VllmCacheConfig) + assert {f.name for f in fields(OmniStageCacheConfig)} == {f.name for f in fields(VllmCacheConfig)} | { "disable_hybrid_kv_cache_manager", "mm_processor_cache_gb", "mamba_ssm_cache_dtype", } - assert {f.name for f in fields(OmniStageSchedulerConfig)} == { - "max_num_seqs", - "max_num_batched_tokens", + assert issubclass(OmniStageSchedulerConfig, VllmSchedulerConfig) + assert {f.name for f in fields(OmniStageSchedulerConfig)} == {f.name for f in fields(VllmSchedulerConfig)} | { "max_model_len", - "enable_chunked_prefill", - "async_scheduling", } assert {f.name for f in fields(OmniStageConnectorConfig)} == { "async_chunk", @@ -530,17 +594,10 @@ def test_sub_config_fields_match_structured_scopes(): "output_connectors", "input_connectors", } - assert {f.name for f in fields(OmniStageParallelConfig)} == { - "pipeline_parallel_size", - "data_parallel_size", - "tensor_parallel_size", - "enable_expert_parallel", - "world_size", - } - assert {f.name for f in fields(OmniStageDiffusionParallelConfig)} == { - "pipeline_parallel_size", - "data_parallel_size", - "tensor_parallel_size", + vllm_parallel_fields = {f.name for f in fields(VllmParallelConfig)} + assert issubclass(OmniStageParallelConfig, VllmParallelConfig) + assert {f.name for f in fields(OmniStageParallelConfig)} == vllm_parallel_fields + assert {f.name for f in fields(OmniStageDiffusionParallelConfig)} == vllm_parallel_fields | { "sequence_parallel_size", "ulysses_degree", "ring_degree", @@ -554,20 +611,86 @@ def test_sub_config_fields_match_structured_scopes(): "mask_sp_padding", "hsdp_shard_size", "hsdp_replicate_size", - "enable_expert_parallel", - "world_size", } +def test_inherited_sub_configs_initialize_transport_safe_derived_fields(): + scheduler_config = OmniStageSchedulerConfig(max_num_batched_tokens=4096) + assert scheduler_config.max_num_encoder_input_tokens == 4096 + assert scheduler_config.encoder_cache_size == 4096 + + unresolved_scheduler_config = OmniStageSchedulerConfig() + assert unresolved_scheduler_config.max_num_encoder_input_tokens is None + assert unresolved_scheduler_config.encoder_cache_size is None + + parallel_config = OmniStageParallelConfig(data_parallel_size=3, data_parallel_rank=2) + assert parallel_config.data_parallel_index == 2 + + diffusion_parallel_config = OmniStageDiffusionParallelConfig(data_parallel_size=3, data_parallel_rank=2) + assert diffusion_parallel_config.data_parallel_index == 2 + + +@pytest.mark.parametrize( + "config_cls", + [ + OmniStageLoadConfig, + OmniStageCacheConfig, + OmniStageSchedulerConfig, + OmniStageParallelConfig, + OmniStageDiffusionParallelConfig, + ], +) +def test_inherited_sub_configs_are_keyword_only(config_cls): + assert all(parameter.kind is Parameter.KEYWORD_ONLY for parameter in signature(config_cls).parameters.values()) + + with pytest.raises(TypeError): + config_cls(1) + + +@pytest.mark.parametrize( + "config_cls", + [ + OmniStageLoadConfig, + OmniStageCacheConfig, + OmniStageSchedulerConfig, + OmniStageParallelConfig, + OmniStageDiffusionParallelConfig, + ], +) +def test_inherited_sub_configs_remain_msgpack_transport_safe(config_cls): + config = config_cls() + + uninitialized_fields = [ + config_field.name + for config_field in fields(config) + if isinstance(getattr(config, config_field.name), FieldInfo) + ] + assert uninitialized_fields == [] + + msgspec.msgpack.encode(_serialize_stage_config(config)) + + +def test_structured_llm_stage_registration_payloads_remain_msgpack_transport_safe(): + omni_config = _from_pipeline_key("qwen3_tts") + + for stage_config in omni_config.stage_configs: + payload = { + "stage_id": stage_config.stage_id, + "stage_config": _serialize_stage_config(stage_config), + } + msgspec.msgpack.encode(payload) + + def test_diffusion_parallel_config_fields_cover_legacy_surface(): from vllm_omni.diffusion.data import DiffusionParallelConfig legacy_fields = {f.name for f in fields(DiffusionParallelConfig)} structured_fields = {f.name for f in fields(OmniStageDiffusionParallelConfig)} expected_upstream_fields = {"mask_sp_padding"} + vllm_parallel_fields = {f.name for f in fields(VllmParallelConfig)} assert legacy_fields | expected_upstream_fields <= structured_fields - assert structured_fields - legacy_fields - expected_upstream_fields == {"world_size"} + assert structured_fields - legacy_fields - expected_upstream_fields - vllm_parallel_fields == set() def test_diffusion_parallel_config_keeps_current_diffusion_parallel_surface(): @@ -1078,3 +1201,21 @@ def test_diffusion_config_projection_keeps_mapping_quantization_config_serializa cfg = omni_config_module._DiffusionConfigProjection.from_kwargs(quantization_config=quantization_config) assert cfg.quantization_config == quantization_config + + +def test_diffusion_quantization_mapping_reaches_terminal_config(monkeypatch): + from vllm_omni.diffusion.data import OmniDiffusionConfig + + quantization_config = {"method": "int8", "activation_scheme": "dynamic"} + cfg = omni_config_module._DiffusionConfigProjection.from_kwargs( + quantization_config=quantization_config, + ) + + # Exercise the terminal construction performed by the future typed startup + # path without probing ports or loading remote model metadata. + monkeypatch.setattr(OmniDiffusionConfig, "_resolve_master_port", lambda _self: 29500) + monkeypatch.setattr(OmniDiffusionConfig, "enrich_config", lambda _self: None) + cfg.enrich_config() + + assert cfg.quantization_config is not None + assert cfg.quantization_config.get_name() == "int8" diff --git a/tests/engine/test_stage_engine_args.py b/tests/engine/test_stage_engine_args.py index 55ae43603..e7c9b2311 100644 --- a/tests/engine/test_stage_engine_args.py +++ b/tests/engine/test_stage_engine_args.py @@ -8,8 +8,23 @@ from pathlib import Path import pytest from pydantic.fields import FieldInfo +from vllm.config import CacheConfig as VllmCacheConfig +from vllm.config import CompilationConfig as VllmCompilationConfig +from vllm.config import LoadConfig as VllmLoadConfig +from vllm.config import ParallelConfig as VllmParallelConfig +from vllm.config import ProfilerConfig as VllmProfilerConfig +from vllm.config import SchedulerConfig as VllmSchedulerConfig +from vllm.engine.arg_utils import EngineArgs -from vllm_omni.config.omni_config import VllmOmniConfig +from vllm_omni.config.omni_config import ( + _LLM_STAGE_ENGINE_FIELDS, + OmniStageCacheConfig, + OmniStageLoadConfig, + OmniStageParallelConfig, + OmniStageSchedulerConfig, + VllmOmniARStageConfig, + VllmOmniConfig, +) from vllm_omni.config.pipeline_registry import OMNI_PIPELINES, resolve_pipeline_config from vllm_omni.config.stage_config import ( DeployConfig, @@ -53,6 +68,30 @@ def _effective_backend_values(config_cls: type, engine_args: dict) -> dict[str, _LLM_BACKEND_FIELDS = frozenset(field.name for field in fields(OmniEngineArgs)) _DIFFUSION_BACKEND_FIELDS = frozenset(field.name for field in fields(OmniDiffusionConfig)) +_OMNI_ONLY_LLM_STAGE_ENGINE_FIELDS = frozenset( + { + "active_stream_window", + "codec_frame_rate_hz", + "custom_voice_dir", + "devices", + "disable_autocast", + "enable_multithread_weight_load", + "env", + "has_sampling_extra_args", + "log_level", + "log_stats", + "model_arch", + "model_subdir", + "num_gpus", + "num_replicas", + "num_weight_load_threads", + "omni_kv_config", + "parallel_config", + "subtalker_sampling_params", + "task_type", + "tokenizer_subdir", + } +) @pytest.fixture(autouse=True) @@ -197,6 +236,100 @@ def _legacy_and_typed_stages( return legacy_stages, omni_config +def test_llm_stage_engine_field_schema_tracks_upstream_engine_args(): + upstream_engine_fields = frozenset(field.name for field in fields(EngineArgs)) + + assert _LLM_STAGE_ENGINE_FIELDS - upstream_engine_fields == _OMNI_ONLY_LLM_STAGE_ENGINE_FIELDS + + +def test_typed_llm_projection_discovers_explicit_upstream_config_fields(): + stage_config = VllmOmniARStageConfig( + stage_pipeline_config=StagePipelineConfig(stage_id=0, model_stage="test"), + load_config=OmniStageLoadConfig(ignore_patterns=["*.bin"]), + cache_config=OmniStageCacheConfig(block_size=32, cache_dtype="fp8"), + scheduler_config=OmniStageSchedulerConfig( + prefill_schedule_interval=2, + policy="priority", + ), + parallel_config=OmniStageParallelConfig( + data_parallel_master_ip="10.0.0.1", + enable_dbo=True, + ), + ) + + engine_args = stage_init_utils._project_omni_stage_engine_args(stage_config) + + assert engine_args["ignore_patterns"] == ["*.bin"] + assert engine_args["block_size"] == 32 + assert engine_args["kv_cache_dtype"] == "fp8" + assert engine_args["prefill_schedule_interval"] == 2 + assert engine_args["scheduling_policy"] == "priority" + assert engine_args["data_parallel_address"] == "10.0.0.1" + assert engine_args["enable_dbo"] is True + + +def test_typed_llm_projection_does_not_emit_inherited_upstream_defaults(): + stage_config = VllmOmniARStageConfig( + stage_pipeline_config=StagePipelineConfig(stage_id=0, model_stage="test"), + ) + + engine_args = stage_init_utils._project_omni_stage_engine_args(stage_config) + + inherited_defaults = { + "ignore_patterns", + "block_size", + "kv_cache_dtype", + "prefill_schedule_interval", + "scheduling_policy", + "data_parallel_address", + "data_parallel_rank", + "data_parallel_size_local", + "data_parallel_rpc_port", + "enable_dbo", + } + assert inherited_defaults.isdisjoint(engine_args) + + +def test_typed_llm_projection_rejects_explicit_fields_owned_by_another_boundary(): + stage_config = VllmOmniARStageConfig( + stage_pipeline_config=StagePipelineConfig(stage_id=0, model_stage="test"), + scheduler_config=OmniStageSchedulerConfig(scheduler_cls="test.CustomScheduler"), + ) + + with pytest.raises(ValueError, match=r"no EngineArgs projection: scheduler_cls"): + stage_init_utils._project_omni_stage_engine_args(stage_config) + + +@pytest.mark.parametrize("stage_id", [0, 1], ids=["ar", "generation"]) +def test_typed_llm_engine_args_preserve_upstream_config_objects(tmp_path, stage_id): + pipeline, deploy, model = _engine_arg_inputs(tmp_path) + compilation_config = VllmCompilationConfig(backend="eager") + profiler_config = VllmProfilerConfig(profiler="cuda") + stage_prefix = f"stage_{stage_id}_" + omni_config = VllmOmniConfig.from_pipeline_config( + pipeline, + user_deploy_config=copy.deepcopy(deploy), + cli_overrides={ + "model": model, + f"{stage_prefix}compilation_config": compilation_config, + f"{stage_prefix}profiler_config": profiler_config, + }, + ) + + stage_config = omni_config.stage_by_id(stage_id) + typed_args = build_engine_args_dict_from_omni_stage_config(stage_config, model) + + assert isinstance(stage_config.compilation_config, VllmCompilationConfig) + assert isinstance(typed_args["compilation_config"], VllmCompilationConfig) + assert typed_args["compilation_config"] is not stage_config.compilation_config + assert typed_args["compilation_config"].backend == "eager" + + assert isinstance(stage_config.profiler_config, VllmProfilerConfig) + assert isinstance(typed_args["profiler_config"], VllmProfilerConfig) + assert typed_args["profiler_config"] is not stage_config.profiler_config + assert typed_args["profiler_config"].profiler == "cuda" + + def test_typed_llm_engine_args_preserve_legacy_adapter_behavior(tmp_path): pipeline, deploy, model = _engine_arg_inputs(tmp_path) legacy_stages, omni_config = _legacy_and_typed_stages(pipeline, deploy, model) @@ -222,6 +355,13 @@ def test_typed_llm_engine_args_preserve_legacy_adapter_behavior(tmp_path): assert {name: typed_args[name] for name in legacy_args} == legacy_args thinker_args = typed_args_by_stage[0] + inherited_vllm_fields = { + field.name + for config_cls in (VllmLoadConfig, VllmCacheConfig, VllmSchedulerConfig, VllmParallelConfig) + for field in fields(config_cls) + } + topology_projected_fields = {"scheduler_cls"} + assert (inherited_vllm_fields - _LLM_STAGE_ENGINE_FIELDS - topology_projected_fields).isdisjoint(thinker_args) assert thinker_args["model"] == str(tmp_path / "stage-model" / "ar-model") assert thinker_args["tokenizer"] == str(tmp_path / "stage-model" / "ar-tokenizer") assert thinker_args["stage_id"] == 0 diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index 4a8a54eca..9cd7ed518 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -12,12 +12,22 @@ from __future__ import annotations import copy from collections.abc import Mapping -from dataclasses import dataclass, field, fields +from dataclasses import InitVar, dataclass, field, fields +from functools import wraps +from inspect import Parameter, signature from pathlib import Path from typing import Any, Literal, TypeAlias, TypedDict, cast -from pydantic import ConfigDict, Field +from pydantic import ConfigDict, Field, model_validator +from typing_extensions import Self +from vllm.config import CacheConfig as VllmCacheConfig +from vllm.config import CompilationConfig as VllmCompilationConfig +from vllm.config import LoadConfig as VllmLoadConfig +from vllm.config import ParallelConfig as VllmParallelConfig +from vllm.config import ProfilerConfig as VllmProfilerConfig +from vllm.config import SchedulerConfig as VllmSchedulerConfig from vllm.config.utils import config +from vllm.engine.arg_utils import EngineArgs as VllmEngineArgs from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm_omni.config.stage_config import ( @@ -77,6 +87,49 @@ class _QuantizationEngineOverrides(TypedDict, total=False): quantization: str +class _TrackExplicitConfigFields: + """Record constructor inputs without adding a serialized config field.""" + + @model_validator(mode="wrap") + @classmethod + def _record_explicit_config_fields(cls, value: Any, handler: Any) -> Any: + result = handler(value) + kwargs = getattr(value, "kwargs", None) + if kwargs is not None: + explicit_fields = frozenset(kwargs) + elif isinstance(value, cls): + explicit_fields = getattr(value, "_omni_explicit_fields", frozenset()) + else: + explicit_fields = frozenset() + object.__setattr__(result, "_omni_explicit_fields", explicit_fields) + return result + + +def _enforce_keyword_only_init(cls: type[Any]) -> type[Any]: + """Make inherited Pydantic dataclass fields keyword-only as well.""" + generated_init = cls.__init__ + generated_signature = signature(cls) + keyword_only_signature = generated_signature.replace( + parameters=[ + parameter.replace(kind=Parameter.KEYWORD_ONLY) + if parameter.kind in {Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD} + else parameter + for parameter in generated_signature.parameters.values() + ] + ) + + @wraps(generated_init) + def keyword_only_init(self: Any, *args: Any, **kwargs: Any) -> None: + if args: + raise TypeError(f"{cls.__name__}() accepts keyword arguments only; got {len(args)} positional argument(s)") + generated_init(self, **kwargs) + + cls.__init__ = keyword_only_init + cls.__signature__ = keyword_only_signature + cls.__match_args__ = () + return cls + + class _ModelEngineOverrides(TypedDict, total=False): model: str model_arch: str @@ -106,7 +159,6 @@ class _ModelEngineOverrides(TypedDict, total=False): enforce_eager: bool max_cudagraph_capture_size: int enable_flashinfer_autotune: bool - compilation_config: dict[str, Any] enable_multithread_weight_load: bool num_weight_load_threads: int disable_autocast: bool @@ -148,7 +200,6 @@ class _RuntimeEngineOverrides(TypedDict, total=False): num_gpus: int log_level: str log_stats: bool - profiler_config: dict[str, Any] class _ParallelConfigEngineOverrides(TypedDict, total=False): @@ -192,6 +243,8 @@ class _StageEngineValues: runtime: _RuntimeEngineOverrides parallel: _ParallelEngineOverrides diffusion: _DiffusionEngineOverrides + compilation_config: Mapping[str, Any] | VllmCompilationConfig | None + profiler_config: Mapping[str, Any] | VllmProfilerConfig | None @dataclass(frozen=True) @@ -341,7 +394,6 @@ class OmniStageModelConfig: enforce_eager: bool = False max_cudagraph_capture_size: int | None = Field(default=None, ge=0) enable_flashinfer_autotune: bool | None = None - compilation_config: dict[str, Any] | None = None enable_multithread_weight_load: bool = True num_weight_load_threads: int = Field(default=4, ge=1) disable_autocast: bool = False @@ -352,21 +404,21 @@ class OmniStageModelConfig: tokenizer_subdir: str | None = None -@config -class OmniStageLoadConfig: - """Per-stage loading behavior and resolved tokenizer input.""" +@_enforce_keyword_only_init +@config(kw_only=True) +class OmniStageLoadConfig(_TrackExplicitConfigFields, VllmLoadConfig): + """vLLM loading behavior plus Omni stage-specific tokenizer inputs.""" tokenizer: str | None = None - download_dir: str | None = None skip_tokenizer_init: bool = False - load_format: str = "auto" tokenizer_mode: str = "auto" config_format: str | None = None skip_mm_profiling: bool | None = None -@config -class OmniStageCacheConfig: +@_enforce_keyword_only_init +@config(kw_only=True) +class OmniStageCacheConfig(_TrackExplicitConfigFields, VllmCacheConfig): """Per-stage engine cache and memory behavior. This is separate from ``_DiffusionConfigProjection.cache_config``, which configures @@ -383,17 +435,26 @@ class OmniStageCacheConfig: mamba_ssm_cache_dtype: str | None = None -@config -class OmniStageSchedulerConfig: +@_enforce_keyword_only_init +@config(kw_only=True) +class OmniStageSchedulerConfig(_TrackExplicitConfigFields, VllmSchedulerConfig): """Per-stage request scheduling behavior.""" + # Upstream receives max_model_len only while materializing SchedulerConfig. + # Omni retains it as unresolved stage input until the owning engine process. max_num_seqs: int | None = Field(default=None, ge=1) max_num_batched_tokens: int | None = Field(default=None, ge=1) max_model_len: int | None = Field(default=None, ge=-1) + is_encoder_decoder: InitVar[bool] = False enable_chunked_prefill: bool | None = None async_scheduling: bool | None = None - def __post_init__(self) -> None: + def __post_init__(self, is_encoder_decoder: bool = False) -> None: + # Upstream initializes these derived fields in its terminal post-init. + # Keep them serializable here without running model-dependent checks. + self.max_num_encoder_input_tokens = self.max_num_batched_tokens + self.encoder_cache_size = self.max_num_batched_tokens + if ( self.max_num_batched_tokens is not None and self.max_num_seqs is not None @@ -432,24 +493,73 @@ class OmniStageRuntimeConfig: num_gpus: int = Field(default=1, ge=1) log_level: str = "info" log_stats: bool = False - profiler_config: dict[str, Any] | None = None -@config -class OmniStageParallelConfig: +@_enforce_keyword_only_init +@config(kw_only=True) +class OmniStageParallelConfig(_TrackExplicitConfigFields, VllmParallelConfig): """Common per-stage distributed parallelism behavior.""" - pipeline_parallel_size: int = Field(default=1, ge=1) - data_parallel_size: int = Field(default=1, ge=1) - tensor_parallel_size: int = Field(default=1, ge=1) - enable_expert_parallel: bool = False - world_size: int = Field(default=1, ge=1, init=False) + # EngineArgs intentionally leaves these unresolved. The upstream terminal + # ParallelConfig defaults (rank 0, local size 1, port 29550, worker + # ``"auto"``) must not turn into explicit engine inputs in the head + # process merely because this transport class inherits ParallelConfig. + data_parallel_size_local: int | None = Field(default=None, ge=0) + data_parallel_rank: int | None = Field(default=None, ge=0) + data_parallel_rpc_port: int | None = None + worker_cls: str | None = None + + @model_validator(mode="after") + def _validate_parallel_config(self) -> Self: + """Run upstream validation without resolving deferred engine inputs.""" + deferred_values = ( + self.data_parallel_size_local, + self.data_parallel_rank, + self.data_parallel_rpc_port, + self.worker_cls, + self.all2all_backend, + self.disable_custom_all_reduce, + ) + if self.data_parallel_size_local is not None and self.data_parallel_size_local > self.data_parallel_size: + raise ValueError( + f"data_parallel_size_local ({self.data_parallel_size_local}) " + f"must be <= data_parallel_size ({self.data_parallel_size})" + ) + if self.data_parallel_rank is not None and not 0 <= self.data_parallel_rank < self.data_parallel_size: + raise ValueError( + f"data_parallel_rank ({self.data_parallel_rank}) must be in the range [0, {self.data_parallel_size})" + ) + self.data_parallel_size_local = self.data_parallel_size + self.data_parallel_rank = 0 + self.data_parallel_rpc_port = VllmParallelConfig.data_parallel_rpc_port + self.worker_cls = VllmParallelConfig.worker_cls + try: + VllmParallelConfig._validate_parallel_config(self) + finally: + ( + self.data_parallel_size_local, + self.data_parallel_rank, + self.data_parallel_rpc_port, + self.worker_cls, + self.all2all_backend, + self.disable_custom_all_reduce, + ) = deferred_values + return self def __post_init__(self) -> None: + # Keep config construction transport-safe. Upstream runtime backend, + # rank, port, and platform resolution happens in the owning process. + self.data_parallel_index = self.data_parallel_rank self.world_size = self.pipeline_parallel_size * self.data_parallel_size * self.tensor_parallel_size + @property + def world_size_across_dp(self) -> int: + # Omni's public world_size has historically included DP. + return self.world_size -@config + +@_enforce_keyword_only_init +@config(kw_only=True) class OmniStageDiffusionParallelConfig(OmniStageParallelConfig): """Diffusion-stage distributed parallelism behavior.""" @@ -468,6 +578,7 @@ class OmniStageDiffusionParallelConfig(OmniStageParallelConfig): hsdp_replicate_size: int = Field(default=1, ge=1) def __post_init__(self) -> None: + self.data_parallel_index = self.data_parallel_rank self.sequence_parallel_size = ( self.allgather_degree if self.allgather_degree > 1 else self.ulysses_degree * self.ring_degree ) @@ -843,6 +954,58 @@ _DIFFUSION_STAGE_ENGINE_FIELDS = (_DIFFUSION_CONFIG_FIELDS | _DIFFUSION_BACKCOMP "stage_id", } + +def _upstream_engine_field_map( + config_cls: type[Any], + *, + aliases: Mapping[str, str] = {}, + exclude: frozenset[str] = frozenset(), +) -> dict[str, str]: + """Map reusable upstream config fields to their EngineArgs inputs.""" + engine_fields = frozenset(config_field.name for config_field in fields(VllmEngineArgs)) + return { + config_field.name: engine_name + for config_field in fields(config_cls) + if config_field.init + and config_field.name not in exclude + and (engine_name := aliases.get(config_field.name, config_field.name)) in engine_fields + } + + +_LOAD_CONFIG_ENGINE_FIELD_MAP = _upstream_engine_field_map(VllmLoadConfig) +_CACHE_CONFIG_ENGINE_FIELD_MAP = _upstream_engine_field_map( + VllmCacheConfig, + aliases={"cache_dtype": "kv_cache_dtype"}, +) +_SCHEDULER_CONFIG_ENGINE_FIELD_MAP = _upstream_engine_field_map( + VllmSchedulerConfig, + aliases={"policy": "scheduling_policy"}, + # ``scheduler_cls`` is selected by the immutable stage topology, while + # ``disable_hybrid_kv_cache_manager`` belongs to the cache concern in the + # Omni schema. Keep both out of the dynamic SchedulerConfig projection so + # one input cannot acquire two owners. + exclude=frozenset( + { + "scheduler_cls", + "disable_hybrid_kv_cache_manager", + } + ), +) +_PARALLEL_CONFIG_ENGINE_FIELD_MAP = _upstream_engine_field_map( + VllmParallelConfig, + aliases={"data_parallel_master_ip": "data_parallel_address"}, + # Runtime owns worker selection. The private API-process fields are + # terminal vLLM internals rather than per-stage user inputs. + exclude=frozenset( + { + "distributed_executor_backend", + "worker_cls", + "_api_process_count", + "_api_process_rank", + } + ), +) + _QUANTIZATION_ENGINE_FIELDS = frozenset(_QuantizationEngineOverrides.__annotations__) _MODEL_ENGINE_FIELDS = frozenset(_ModelEngineOverrides.__annotations__) _LOAD_ENGINE_FIELDS = frozenset(_LoadEngineOverrides.__annotations__) @@ -850,7 +1013,17 @@ _CACHE_ENGINE_FIELDS = frozenset(_CacheEngineOverrides.__annotations__) _SCHEDULER_ENGINE_FIELDS = frozenset(_SchedulerEngineOverrides.__annotations__) _CONNECTOR_ENGINE_FIELDS = frozenset(_ConnectorEngineOverrides.__annotations__) _RUNTIME_ENGINE_FIELDS = frozenset(_RuntimeEngineOverrides.__annotations__) -_PARALLEL_CONFIG_ENGINE_FIELDS = frozenset(_ParallelConfigEngineOverrides.__annotations__) +_DIRECT_VLLM_CONFIG_ENGINE_FIELDS = frozenset({"compilation_config", "profiler_config"}) +_LLM_LOAD_ENGINE_FIELDS = _LOAD_ENGINE_FIELDS | frozenset(_LOAD_CONFIG_ENGINE_FIELD_MAP.values()) +_LLM_CACHE_ENGINE_FIELDS = _CACHE_ENGINE_FIELDS | frozenset(_CACHE_CONFIG_ENGINE_FIELD_MAP.values()) +_LLM_SCHEDULER_ENGINE_FIELDS = _SCHEDULER_ENGINE_FIELDS | frozenset(_SCHEDULER_CONFIG_ENGINE_FIELD_MAP.values()) +_LLM_PARALLEL_CONFIG_ENGINE_FIELDS = frozenset(_PARALLEL_CONFIG_ENGINE_FIELD_MAP.values()) +_DIFFUSION_PARALLEL_CONFIG_ENGINE_FIELDS = frozenset( + f.name for f in fields(OmniStageDiffusionParallelConfig) +) & frozenset(_ParallelConfigEngineOverrides.__annotations__) +_DIFFUSION_PARALLEL_CONFIG_FIELD_MAP = {name: name for name in _DIFFUSION_PARALLEL_CONFIG_ENGINE_FIELDS} +_LLM_PARALLEL_CONFIG_FIELDS = frozenset(_PARALLEL_CONFIG_ENGINE_FIELD_MAP) +_PARALLEL_CONFIG_ENGINE_FIELDS = _LLM_PARALLEL_CONFIG_ENGINE_FIELDS | _DIFFUSION_PARALLEL_CONFIG_ENGINE_FIELDS _PARALLEL_ENGINE_FIELDS = _PARALLEL_CONFIG_ENGINE_FIELDS | {"parallel_config"} _COMMON_STAGE_ENGINE_FIELDS = ( _QUANTIZATION_ENGINE_FIELDS @@ -860,12 +1033,16 @@ _COMMON_STAGE_ENGINE_FIELDS = ( | _SCHEDULER_ENGINE_FIELDS | _CONNECTOR_ENGINE_FIELDS | _RUNTIME_ENGINE_FIELDS + | _DIRECT_VLLM_CONFIG_ENGINE_FIELDS +) +_LLM_STAGE_ENGINE_FIELDS = ( + _COMMON_STAGE_ENGINE_FIELDS + | _LLM_LOAD_ENGINE_FIELDS + | _LLM_CACHE_ENGINE_FIELDS + | _LLM_SCHEDULER_ENGINE_FIELDS + | _LLM_PARALLEL_CONFIG_ENGINE_FIELDS + | {"parallel_config"} ) -_LLM_PARALLEL_CONFIG_ENGINE_FIELDS = frozenset(f.name for f in fields(OmniStageParallelConfig)) - {"world_size"} -_DIFFUSION_PARALLEL_CONFIG_ENGINE_FIELDS = frozenset(f.name for f in fields(OmniStageDiffusionParallelConfig)) - { - "world_size" -} -_LLM_STAGE_ENGINE_FIELDS = _COMMON_STAGE_ENGINE_FIELDS | _LLM_PARALLEL_CONFIG_ENGINE_FIELDS | {"parallel_config"} _DIFFUSION_OWNED_STAGE_ENGINE_FIELDS = ( _COMMON_STAGE_ENGINE_FIELDS | _DIFFUSION_PARALLEL_CONFIG_ENGINE_FIELDS @@ -886,6 +1063,27 @@ _PARALLEL_CONFIG_ENGINE_FIELDS_BY_EXECUTION_TYPE = { StageExecutionType.LLM_GENERATION: _LLM_PARALLEL_CONFIG_ENGINE_FIELDS, StageExecutionType.DIFFUSION: _DIFFUSION_PARALLEL_CONFIG_ENGINE_FIELDS, } +_PARALLEL_CONFIG_FIELDS_BY_EXECUTION_TYPE = { + StageExecutionType.LLM_AR: _LLM_PARALLEL_CONFIG_FIELDS, + StageExecutionType.LLM_GENERATION: _LLM_PARALLEL_CONFIG_FIELDS, + StageExecutionType.DIFFUSION: frozenset(_DIFFUSION_PARALLEL_CONFIG_FIELD_MAP), +} + +_LOAD_STAGE_ENGINE_FIELD_MAP = { + **{name: name for name in _LOAD_ENGINE_FIELDS}, + **_LOAD_CONFIG_ENGINE_FIELD_MAP, +} +_CACHE_STAGE_ENGINE_FIELD_MAP = { + **{name: name for name in _CACHE_ENGINE_FIELDS}, + **_CACHE_CONFIG_ENGINE_FIELD_MAP, +} +_SCHEDULER_STAGE_ENGINE_FIELD_MAP = { + **{name: name for name in _SCHEDULER_ENGINE_FIELDS}, + **_SCHEDULER_CONFIG_ENGINE_FIELD_MAP, +} +_DIFFUSION_LOAD_STAGE_ENGINE_FIELD_MAP = {name: name for name in _LOAD_ENGINE_FIELDS} +_DIFFUSION_CACHE_STAGE_ENGINE_FIELD_MAP = {name: name for name in _CACHE_ENGINE_FIELDS} +_DIFFUSION_SCHEDULER_STAGE_ENGINE_FIELD_MAP = {name: name for name in _SCHEDULER_ENGINE_FIELDS} def _validate_stage_engine_override_ownership( @@ -896,13 +1094,19 @@ def _validate_stage_engine_override_ownership( try: owner_fields = _STAGE_ENGINE_FIELDS_BY_EXECUTION_TYPE[execution_type] parallel_owner_fields = _PARALLEL_CONFIG_ENGINE_FIELDS_BY_EXECUTION_TYPE[execution_type] + parallel_config_fields = _PARALLEL_CONFIG_FIELDS_BY_EXECUTION_TYPE[execution_type] except KeyError as exc: raise ValueError(f"Unsupported stage execution type: {execution_type!r}") from exc unowned_fields = set(overrides) - owner_fields parallel_config = overrides.get("parallel_config") if isinstance(parallel_config, Mapping): - unowned_fields.update(f"parallel_config.{name}" for name in set(parallel_config) - parallel_owner_fields) + # Nested values traditionally use upstream config names, while flat + # EngineArgs uses aliases such as ``data_parallel_address``. Accept + # either spelling at the boundary; the builder canonicalizes aliases + # before instantiating the inherited config. + accepted_parallel_names = parallel_config_fields | parallel_owner_fields + unowned_fields.update(f"parallel_config.{name}" for name in set(parallel_config) - accepted_parallel_names) if unowned_fields: names = ", ".join(sorted(unowned_fields)) raise ValueError( @@ -973,17 +1177,28 @@ def _stage_engine_values( topology.execution_type, engine, ) + if topology.execution_type in { + StageExecutionType.LLM_AR, + StageExecutionType.LLM_GENERATION, + }: + load_engine_fields = _LLM_LOAD_ENGINE_FIELDS + cache_engine_fields = _LLM_CACHE_ENGINE_FIELDS + scheduler_engine_fields = _LLM_SCHEDULER_ENGINE_FIELDS + else: + load_engine_fields = _LOAD_ENGINE_FIELDS + cache_engine_fields = _CACHE_ENGINE_FIELDS + scheduler_engine_fields = _SCHEDULER_ENGINE_FIELDS return _StageEngineValues( quantization=cast( _QuantizationEngineOverrides, _select_engine_overrides(engine, _QUANTIZATION_ENGINE_FIELDS), ), model=cast(_ModelEngineOverrides, _select_engine_overrides(engine, _MODEL_ENGINE_FIELDS)), - load=cast(_LoadEngineOverrides, _select_engine_overrides(engine, _LOAD_ENGINE_FIELDS)), - cache=cast(_CacheEngineOverrides, _select_engine_overrides(engine, _CACHE_ENGINE_FIELDS)), + load=cast(_LoadEngineOverrides, _select_engine_overrides(engine, load_engine_fields)), + cache=cast(_CacheEngineOverrides, _select_engine_overrides(engine, cache_engine_fields)), scheduler=cast( _SchedulerEngineOverrides, - _select_engine_overrides(engine, _SCHEDULER_ENGINE_FIELDS), + _select_engine_overrides(engine, scheduler_engine_fields), ), connector=cast( _ConnectorEngineOverrides, @@ -992,6 +1207,8 @@ def _stage_engine_values( runtime=cast(_RuntimeEngineOverrides, _select_engine_overrides(engine, _RUNTIME_ENGINE_FIELDS)), parallel=cast(_ParallelEngineOverrides, _select_engine_overrides(engine, _PARALLEL_ENGINE_FIELDS)), diffusion=_DiffusionEngineOverrides.from_engine(engine), + compilation_config=_copy_value(engine.get("compilation_config")), + profiler_config=_copy_value(engine.get("profiler_config")), ) @@ -1046,6 +1263,8 @@ class BaseVllmOmniStageConfig: connector_config: OmniStageConnectorConfig = field(default_factory=OmniStageConnectorConfig) runtime_config: OmniStageRuntimeConfig = field(default_factory=OmniStageRuntimeConfig) parallel_config: OmniStageParallelConfig = field(default_factory=OmniStageParallelConfig) + compilation_config: VllmCompilationConfig | None = None + profiler_config: VllmProfilerConfig | None = None quantization_config: _QuantizationConfigType = None @property @@ -1197,6 +1416,8 @@ def _build_common_stage_config_kwargs( parallel_config, ), "parallel_config": parallel_config, + "compilation_config": _copy_value(engine.compilation_config), + "profiler_config": _copy_value(engine.profiler_config), "quantization_config": _copy_value(quantization_config), }, input_proc, @@ -1391,12 +1612,51 @@ def _build_load_config( return OmniStageLoadConfig(**kwargs) +def _config_kwargs_from_engine_args( + engine: Mapping[str, Any], + field_map: Mapping[str, str], +) -> dict[str, Any]: + """Convert EngineArgs names back to their upstream config field names.""" + return { + config_name: _copy_value(engine[engine_name]) + for config_name, engine_name in field_map.items() + if engine_name in engine and engine[engine_name] is not None + } + + +def _normalize_config_mapping( + values: Mapping[str, Any], + field_map: Mapping[str, str], +) -> dict[str, Any]: + """Canonicalize config and EngineArgs spellings to config field names.""" + engine_to_config = {engine_name: config_name for config_name, engine_name in field_map.items()} + normalized: dict[str, Any] = {} + source_names: dict[str, str] = {} + for name, value in values.items(): + if value is None: + continue + config_name = name if name in field_map else engine_to_config.get(name, name) + if config_name in normalized: + raise ValueError( + f"Config mapping specifies both {source_names[config_name]!r} and {name!r} " + f"for upstream field {config_name!r}" + ) + normalized[config_name] = _copy_value(value) + source_names[config_name] = name + return normalized + + +def _omni_config_kwargs(engine: Mapping[str, Any], fields_: frozenset[str]) -> dict[str, Any]: + return {name: _copy_value(engine[name]) for name in fields_ if name in engine and engine[name] is not None} + + def _build_cache_config( deploy: DeployConfig, engine: _CacheEngineOverrides, execution_type: StageExecutionType, ) -> OmniStageCacheConfig: - kwargs = _config_kwargs(engine) + kwargs = _config_kwargs_from_engine_args(engine, _CACHE_CONFIG_ENGINE_FIELD_MAP) + kwargs.update(_omni_config_kwargs(engine, _CACHE_ENGINE_FIELDS)) if "enable_prefix_caching" not in kwargs and deploy.enable_prefix_caching is not None: kwargs["enable_prefix_caching"] = _copy_value(deploy.enable_prefix_caching) if "disable_hybrid_kv_cache_manager" not in kwargs and execution_type == StageExecutionType.LLM_GENERATION: @@ -1410,7 +1670,8 @@ def _build_scheduler_config( engine: _SchedulerEngineOverrides, execution_type: StageExecutionType, ) -> OmniStageSchedulerConfig: - kwargs = _config_kwargs(engine) + kwargs = _config_kwargs_from_engine_args(engine, _SCHEDULER_CONFIG_ENGINE_FIELD_MAP) + kwargs.update(_omni_config_kwargs(engine, _SCHEDULER_ENGINE_FIELDS)) if "enable_chunked_prefill" not in kwargs and deploy.enable_chunked_prefill is not None: kwargs["enable_chunked_prefill"] = _copy_value(deploy.enable_chunked_prefill) if "async_scheduling" not in kwargs and execution_type == StageExecutionType.LLM_AR: @@ -1458,16 +1719,21 @@ def _build_parallel_config( config_cls: type[OmniStageParallelConfig] = OmniStageParallelConfig, ) -> OmniStageParallelConfig: parallel_config = _mapping_or_empty(engine.get("parallel_config")) - config_fields = { - config_field.name - for config_field in fields(config_cls) - if getattr(config_field.default, "init", None) is not False - } - kwargs = { - name: _copy_value(value) for name in _PARALLEL_CONFIG_ENGINE_FIELDS if (value := engine.get(name)) is not None - } + config_fields = frozenset(signature(config_cls).parameters) + is_diffusion = issubclass(config_cls, OmniStageDiffusionParallelConfig) + field_map = _DIFFUSION_PARALLEL_CONFIG_FIELD_MAP if is_diffusion else _PARALLEL_CONFIG_ENGINE_FIELD_MAP + owned_parallel_fields = frozenset(field_map) + kwargs = _config_kwargs(engine) if is_diffusion else _config_kwargs_from_engine_args(engine, field_map) + kwargs = {name: value for name, value in kwargs.items() if name in owned_parallel_fields} kwargs = {name: value for name, value in kwargs.items() if name in config_fields} - kwargs.update({name: value for name, value in _config_kwargs(parallel_config).items() if name in config_fields}) + nested_kwargs = _normalize_config_mapping(parallel_config, field_map) + kwargs.update( + { + name: value + for name, value in nested_kwargs.items() + if name in owned_parallel_fields and name in config_fields + } + ) if "pipeline_parallel_size" not in kwargs and deploy.pipeline_parallel_size is not None: kwargs["pipeline_parallel_size"] = _copy_value(deploy.pipeline_parallel_size) if "data_parallel_size" not in kwargs and deploy.data_parallel_size is not None: diff --git a/vllm_omni/engine/stage_init_utils.py b/vllm_omni/engine/stage_init_utils.py index 9da6f2cfa..683726d28 100644 --- a/vllm_omni/engine/stage_init_utils.py +++ b/vllm_omni/engine/stage_init_utils.py @@ -14,7 +14,7 @@ import importlib import multiprocessing as mp import os import time -from collections.abc import Callable, Generator, Sequence +from collections.abc import Callable, Generator, Mapping, Sequence from contextlib import contextmanager from dataclasses import dataclass, fields, replace from typing import Any, Literal, cast @@ -29,6 +29,14 @@ from vllm.v1.engine.input_processor import InputProcessor from vllm.v1.executor import Executor from vllm_omni.config.omni_config import ( + _CACHE_STAGE_ENGINE_FIELD_MAP, + _DIFFUSION_CACHE_STAGE_ENGINE_FIELD_MAP, + _DIFFUSION_LOAD_STAGE_ENGINE_FIELD_MAP, + _DIFFUSION_PARALLEL_CONFIG_ENGINE_FIELDS, + _DIFFUSION_SCHEDULER_STAGE_ENGINE_FIELD_MAP, + _LOAD_STAGE_ENGINE_FIELD_MAP, + _PARALLEL_CONFIG_ENGINE_FIELD_MAP, + _SCHEDULER_STAGE_ENGINE_FIELD_MAP, BaseVllmOmniStageConfig, VllmOmniDiffusionStageConfig, ) @@ -781,20 +789,38 @@ def stage_runtime_env(stage_id: int, runtime_cfg: Any) -> Generator[None, None, def _project_omni_config_fields( config: Any, *, + field_map: Mapping[str, str] | None = None, exclude: frozenset[str] = frozenset(), ) -> dict[str, Any]: """Copy defined typed config fields into backend adapter kwargs.""" projected: dict[str, Any] = {} for config_field in fields(config): name = config_field.name - if name in exclude: + if name in exclude or (field_map is not None and name not in field_map): continue value = getattr(config, name) if value is not None: - projected[name] = copy.deepcopy(value) + projected[field_map.get(name, name) if field_map is not None else name] = copy.deepcopy(value) return projected +def _project_upstream_config_fields( + config: Any, + field_map: Mapping[str, str], +) -> dict[str, Any]: + """Project every explicit upstream input, including newly added fields.""" + explicit_fields = getattr(config, "_omni_explicit_fields", frozenset()) + unprojected_fields = explicit_fields - frozenset(field_map) + if unprojected_fields: + names = ", ".join(sorted(unprojected_fields)) + raise ValueError(f"{type(config).__name__} has explicit field(s) with no EngineArgs projection: {names}") + return _project_omni_config_fields( + config, + field_map=field_map, + exclude=frozenset(field_map) - explicit_fields, + ) + + def _project_omni_stage_engine_args( stage_config: BaseVllmOmniStageConfig, ) -> dict[str, Any]: @@ -810,9 +836,6 @@ def _project_omni_stage_engine_args( stage_config.model_config, frozenset({"default_sampling_params", "has_sampling_extra_args"}), ), - (stage_config.load_config, frozenset()), - (stage_config.cache_config, frozenset()), - (stage_config.scheduler_config, frozenset()), ( stage_config.runtime_config, frozenset({"devices", "num_replicas", "env", "num_gpus"}), @@ -825,6 +848,26 @@ def _project_omni_stage_engine_args( ) ) + if is_diffusion: + for config, field_map in ( + (stage_config.load_config, _DIFFUSION_LOAD_STAGE_ENGINE_FIELD_MAP), + (stage_config.cache_config, _DIFFUSION_CACHE_STAGE_ENGINE_FIELD_MAP), + (stage_config.scheduler_config, _DIFFUSION_SCHEDULER_STAGE_ENGINE_FIELD_MAP), + ): + engine_args.update(_project_omni_config_fields(config, field_map=field_map)) + else: + for config, field_map in ( + (stage_config.load_config, _LOAD_STAGE_ENGINE_FIELD_MAP), + (stage_config.cache_config, _CACHE_STAGE_ENGINE_FIELD_MAP), + (stage_config.scheduler_config, _SCHEDULER_STAGE_ENGINE_FIELD_MAP), + ): + engine_args.update(_project_upstream_config_fields(config, field_map)) + + for name in ("compilation_config", "profiler_config"): + value = getattr(stage_config, name) + if value is not None: + engine_args[name] = copy.deepcopy(value) + # The legacy builder always emits this key, including for pipelines such # as Audex that intentionally defer architecture discovery to HF config. engine_args["model_arch"] = copy.deepcopy(stage_config.model_config.model_arch) @@ -851,13 +894,14 @@ def _project_omni_stage_engine_args( if is_diffusion: engine_args["parallel_config"] = _project_omni_config_fields( stage_config.parallel_config, + field_map={name: name for name in _DIFFUSION_PARALLEL_CONFIG_ENGINE_FIELDS}, exclude=frozenset({"world_size"}), ) else: engine_args.update( - _project_omni_config_fields( + _project_upstream_config_fields( stage_config.parallel_config, - exclude=frozenset({"world_size"}), + _PARALLEL_CONFIG_ENGINE_FIELD_MAP, ) ) From 4cd667875f983819414ca6a19cf612b58e6c138e Mon Sep 17 00:00:00 2001 From: Joshna-Medisetty Date: Tue, 18 Aug 2026 07:03:31 -0700 Subject: [PATCH 058/267] [XPU][SDXL] Fix text encoder input device under CPU offload (#6125) Signed-off-by: Joshna Medisetty Co-authored-by: Hongsheng Liu Co-authored-by: Cursor Co-authored-by: Nick Cao --- vllm_omni/diffusion/models/sdxl/pipeline_sdxl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vllm_omni/diffusion/models/sdxl/pipeline_sdxl.py b/vllm_omni/diffusion/models/sdxl/pipeline_sdxl.py index 0bb8a48cd..ac7231fbc 100644 --- a/vllm_omni/diffusion/models/sdxl/pipeline_sdxl.py +++ b/vllm_omni/diffusion/models/sdxl/pipeline_sdxl.py @@ -98,7 +98,7 @@ class StableDiffusionXLPipeline( prefetch_list=sdxl_subfolders, local_files_only=local_files_only, torch_dtype=dtype, - ) + ).to(self.device) self.text_encoder_2 = from_pretrained_with_prefetch( CLIPTextModelWithProjection.from_pretrained, model, @@ -106,7 +106,7 @@ class StableDiffusionXLPipeline( prefetch_list=sdxl_subfolders, local_files_only=local_files_only, torch_dtype=dtype, - ) + ).to(self.device) self.unet = SDXLUNet2DConditionModel(od_config=od_config) self.vae = from_pretrained_with_prefetch( DistributedAutoencoderKL.from_pretrained, @@ -146,7 +146,7 @@ class StableDiffusionXLPipeline( ) text_input_ids = text_inputs.input_ids - text_encoder_device = next(text_encoder.parameters()).device + text_encoder_device = self.device outputs = text_encoder(text_input_ids.to(text_encoder_device), output_hidden_states=True) prompt_embeds = outputs.hidden_states[-2].to(dtype=self.od_config.dtype, device=self.device) From c3de04e12777708cfb994958c614fc8708cf5225 Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Tue, 18 Aug 2026 23:42:03 +0800 Subject: [PATCH 059/267] [Bugfix] Avoid eager pi0 runtime import in pipeline registry (#6322) Signed-off-by: Hongsheng Liu --- tests/config/test_config_import_cycle.py | 24 ++++++++++++++ vllm_omni/config/pipeline_registry.py | 2 +- .../diffusion/models/pi0/pipeline_pi0.py | 32 +------------------ .../diffusion/models/pi0_pipeline_config.py | 30 +++++++++++++++++ 4 files changed, 56 insertions(+), 32 deletions(-) create mode 100644 vllm_omni/diffusion/models/pi0_pipeline_config.py diff --git a/tests/config/test_config_import_cycle.py b/tests/config/test_config_import_cycle.py index d73f6265e..8a28b7afb 100644 --- a/tests/config/test_config_import_cycle.py +++ b/tests/config/test_config_import_cycle.py @@ -93,3 +93,27 @@ if DiffusionOutput is None or LoRAConfig is None: raise SystemExit("expected LoRAConfig and DiffusionOutput") """ ) + + +def test_pipeline_registry_does_not_import_pi0_runtime() -> None: + """Resolving the pi0 topology must not import its runtime pipeline. + + ``pipeline_pi0`` imports ``diffusion.data``. Importing it while + ``diffusion.data`` is only partially initialized closes the Ascend plugin's + circular-import loop, so the registry must depend only on a lightweight + pipeline-config module. + """ + _assert_isolated_import_succeeds( + """ +import sys + +from vllm_omni.config.pipeline_registry import resolve_pipeline_config + +if "vllm_omni.diffusion.models.pi0.pipeline_pi0" in sys.modules: + raise SystemExit("pipeline_registry imported the pi0 runtime pipeline") + +pipeline = resolve_pipeline_config("pi0") +if pipeline is None or pipeline.model_type != "pi0": + raise SystemExit("expected the registered pi0 pipeline config") +""" + ) diff --git a/vllm_omni/config/pipeline_registry.py b/vllm_omni/config/pipeline_registry.py index f498b7bde..fd6d700a9 100644 --- a/vllm_omni/config/pipeline_registry.py +++ b/vllm_omni/config/pipeline_registry.py @@ -33,7 +33,7 @@ from vllm.logger import init_logger from vllm_omni.config.stage_config import ( PipelineConfig, ) -from vllm_omni.diffusion.models.pi0.pipeline_pi0 import PI0_PIPELINE +from vllm_omni.diffusion.models.pi0_pipeline_config import PI0_PIPELINE from vllm_omni.model_executor.models.audex.pipeline import ( AUDEX_S2S_PIPELINE, AUDEX_THINKER_ONLY_PIPELINE, diff --git a/vllm_omni/diffusion/models/pi0/pipeline_pi0.py b/vllm_omni/diffusion/models/pi0/pipeline_pi0.py index 0877df4d6..1eeca2969 100644 --- a/vllm_omni/diffusion/models/pi0/pipeline_pi0.py +++ b/vllm_omni/diffusion/models/pi0/pipeline_pi0.py @@ -22,15 +22,11 @@ import torch from torch import nn from vllm.logger import init_logger -from vllm_omni.config.stage_config import ( - PipelineConfig, - StageExecutionType, - StagePipelineConfig, -) from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig from vllm_omni.diffusion.models.pi0.config import Pi0Config from vllm_omni.diffusion.models.pi0.modeling_pi0 import Pi0ForActionPrediction from vllm_omni.diffusion.models.pi0.processor_pi0 import build_model_inputs +from vllm_omni.diffusion.models.pi0_pipeline_config import PI0_PIPELINE as PI0_PIPELINE from vllm_omni.diffusion.request import OmniDiffusionRequest logger = init_logger(__name__) @@ -54,32 +50,6 @@ def get_pi0_post_process_func(od_config: OmniDiffusionConfig): return _pi0_post_process -# Single-stage diffusion topology for online OpenPI serving. -# -# π0 is one diffusion stage (robot observation -> action chunk via flow -# matching). It is registered in ``OMNI_PIPELINES`` (rather than relying on the -# default single-stage fallback) because π0 is *online-served* via -# ``vllm serve --deploy-config pi0.yaml``; that path -# (``OmniStagePipelineFactory._create_from_registry``) requires a registered -# pipeline so the deploy yaml's ``policy_server_config`` reaches the OpenPI -# websocket layer. Offline-only models (e.g. ``internvla_a1``) need no topology. -PI0_PIPELINE = PipelineConfig( - model_type="pi0", - model_arch="Pi0Pipeline", - stages=( - StagePipelineConfig( - stage_id=0, - model_stage="diffusion", - execution_type=StageExecutionType.DIFFUSION, - input_sources=(), - final_output=True, - final_output_type="action", - model_arch="Pi0Pipeline", - ), - ), -) - - class Pi0Pipeline(nn.Module): """π0 VLA pipeline: raw robot obs → continuous action chunk. diff --git a/vllm_omni/diffusion/models/pi0_pipeline_config.py b/vllm_omni/diffusion/models/pi0_pipeline_config.py new file mode 100644 index 000000000..d778ca81a --- /dev/null +++ b/vllm_omni/diffusion/models/pi0_pipeline_config.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Lightweight pipeline topology for π0 (Pi-Zero).""" + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +# π0 is one diffusion stage (robot observation -> action chunk via flow +# matching). It is registered in ``OMNI_PIPELINES`` because π0 is online-served +# via ``vllm serve --deploy-config pi0.yaml``. Keep this topology outside the +# ``pi0`` package: importing a package submodule executes ``pi0.__init__``, +# which imports the runtime pipeline and ``diffusion.data``. +PI0_PIPELINE = PipelineConfig( + model_type="pi0", + model_arch="Pi0Pipeline", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="diffusion", + execution_type=StageExecutionType.DIFFUSION, + input_sources=(), + final_output=True, + final_output_type="action", + model_arch="Pi0Pipeline", + ), + ), +) From 284e05c88b7b46be9fae6d822bf22075840cbfbb Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Tue, 18 Aug 2026 23:48:07 +0800 Subject: [PATCH 060/267] [Core] Support online FP8 with DLO AllGather (#6279) Signed-off-by: Hongsheng Liu --- .../distributed_layerwise_offload.md | 33 ++++++- docs/design/feature/quantization.md | 15 ++-- .../distributed_layerwise_offload.md | 7 +- docs/user_guide/quantization/fp8.md | 18 ++++ recipes/MiniMaxAI/MiniMax-H3.md | 12 +-- recipes/cosmos3/Cosmos3-DistOffload.md | 6 +- .../model_loader/test_diffusers_loader.py | 85 ++++++++++++++++++- .../test_distributed_layerwise_backend.py | 73 ++++++++++++++++ .../model_loader/diffusers_loader.py | 40 +++++++-- 9 files changed, 265 insertions(+), 24 deletions(-) diff --git a/docs/design/feature/offloader/distributed_layerwise_offload.md b/docs/design/feature/offloader/distributed_layerwise_offload.md index b38f0443d..e254e3a43 100644 --- a/docs/design/feature/offloader/distributed_layerwise_offload.md +++ b/docs/design/feature/offloader/distributed_layerwise_offload.md @@ -10,6 +10,30 @@ For user-facing commands, see the [distributed layerwise offloading guide](../../../user_guide/diffusion/offloader/distributed_layerwise_offload.md) and the [Cosmos3 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md). +## Feature compatibility + +Host-storage optimization and runtime compatibility are separate decisions. +When direct checkpoint mmap is unavailable, DLO can still use tensors produced +by the ordinary loader. "Compatibility path" below means that fallback is +implemented but has less end-to-end coverage than the primary path. + +Legend: ✅ supported, ⚠️ compatibility path or limited validation, ❌ unsupported. + +| Feature | DLO + AllGather | DLO without AllGather | +|---|---|---| +| **DP** | ✅ Primary path; host weights are sharded across the DP group. | ✅ Each DP rank streams complete rank-local blocks. | +| **SP** | ✅ When DP=1, DLO uses the SP group for weight sharding. | ✅ SP remains active without a DLO weight collective. | +| **TP > 1** | ⚠️ Ordinary TP-aware loader only; no direct checkpoint mmap. | ⚠️ Ordinary TP-aware loader only; no direct checkpoint mmap. | +| **HSDP** | ❌ Rejected to avoid double-sharding parameters. | ⚠️ Limited end-to-end coverage. | +| **Per-tensor online FP8 linears** | ✅ Ordinary loader finalizes weights and scales before DLO sharding. | ✅ Ordinary loader retains complete rank-local tensors. | +| **Other online quantization methods** | ❌ Rejected until runtime packing and scale layouts are validated. | ⚠️ Allowed through the ordinary loader; validation is method-specific. | +| **Model-level or standard layerwise CPU offload** | ❌ Disabled because DLO takes priority. | ❌ Disabled because DLO takes priority. | +| **Resident leading layers** | ❌ Rejected. | ✅ Requires eligible resident paths in the model's `OffloadPlan`. | + +See [Parallelism compatibility](#parallelism-compatibility) and +[Request and loading constraints](#request-and-loading-constraints) for the +detailed contracts and validation boundaries. + ## Status DLO is implemented for multi-device diffusion execution. The default @@ -181,8 +205,11 @@ requirements. Direct checkpoint mmap can back either transfer path. It is currently limited to proven TP1, non-HSDP, non-online-quantized layouts. Other layouts use the -ordinary loader. Online quantization remains incompatible with DLO AllGather; -use `--dlo-no-use-allgather` or disable online quantization. +ordinary loader. Per-tensor online FP8 linears can use DLO AllGather after the +ordinary loader finalizes their runtime weights and scales; DLO then shards and +reconstructs those tensors with their recorded layouts. Other online methods +must use `--dlo-no-use-allgather` or disable online quantization until their +runtime layouts are validated. A normalized runtime mmap cache, built through the ordinary loader, is the proposed general mechanism for sharing transformed TP or quantized layouts. @@ -197,6 +224,8 @@ Current source-level validation includes: - HSDP + DLO without AllGather acceptance at configuration level; - loader preflight fallback for TP, HSDP, online quantization, unknown custom loaders, missing keys, and shape/dtype mismatches; +- ordinary-loader fallback for per-tensor online FP8 linears followed by DLO + sharding of finalized weights and scales; - exact loader-to-backend plan transfer and ordinary-loader fallback; - rank-local mmap source retention, bounded two-slot staging, and adapter transforms without parameter-side flags; diff --git a/docs/design/feature/quantization.md b/docs/design/feature/quantization.md index 8d6cd09c3..84c9641b3 100644 --- a/docs/design/feature/quantization.md +++ b/docs/design/feature/quantization.md @@ -190,12 +190,15 @@ configured parallelism strategy: - `WeightsMapper` transformations must be applied before matching ignored-layer lists or component prefixes. -Distributed layerwise offload has an additional boundary: its sharded mmap -AllGather path is not compatible with online quantization because the path -expects already materialized checkpoint weights. Use the rank-local -`--dlo-no-use-allgather` path or a compatible pre-quantized checkpoint when -combining these features. See the [DLO design document](offloader/distributed_layerwise_offload.md) -for the full compatibility matrix. +Distributed layerwise offload has an additional boundary: direct checkpoint +mmap cannot prepare online-quantized tensors because it bypasses the ordinary +loader that creates their runtime weights and scales. Per-tensor online FP8 +linears can still use DLO AllGather through the ordinary loader: each rank first +finalizes the FP8 tensors, then DLO shards those runtime tensors for +reconstruction during prefetch. Other online methods remain limited to +`--dlo-no-use-allgather` until their packing and scale layouts are validated. +See the [DLO design document](offloader/distributed_layerwise_offload.md) for +the full compatibility matrix. ## Adding a quantization backend diff --git a/docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md b/docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md index 6157c866b..49139e695 100644 --- a/docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md +++ b/docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md @@ -138,8 +138,11 @@ must enter each collective. but it provides no shared-mmap host-memory benefit or guarantee. - HSDP plus AllGather is rejected to avoid double sharding. HSDP without AllGather has limited end-to-end validation. -- Online quantization uses the ordinary loader with no-AllGather. It remains - incompatible with DLO AllGather. +- Per-tensor online FP8 linears use the ordinary loader and can run with either + DLO transfer path. With AllGather, every rank temporarily materializes the + complete FP8 model in host memory before DLO retains only its shard. Other + online quantization methods require no-AllGather until their runtime layouts + are validated. - Resident leading layers require `--dlo-no-use-allgather` and a model `OffloadPlan` that declares eligible `resident_dit_paths`. - DP concurrency requires an explicit, identical inference-step count. diff --git a/docs/user_guide/quantization/fp8.md b/docs/user_guide/quantization/fp8.md index 397a412e8..c57642587 100644 --- a/docs/user_guide/quantization/fp8.md +++ b/docs/user_guide/quantization/fp8.md @@ -179,6 +179,24 @@ python text_to_image.py --model --quantization fp8 --ignored-layers vllm serve --omni --quantization fp8 ``` +### Distributed layerwise offload + +Per-tensor online FP8 linears can be combined with DLO's default AllGather path: + +```bash +vllm serve \ + --omni \ + --quantization fp8 \ + --enable-distributed-layerwise-offload +``` + +Direct checkpoint mmap is not used for this combination. Each rank runs the +ordinary loader to create the final FP8 weights and scales before DLO shards +them, so startup temporarily materializes the complete FP8 model in host memory +on every rank. Runtime DLO host residency is sharded normally. Use +`--dlo-no-use-allgather` when the synchronized AllGather request-wave contract +does not fit the workload. + ## Parameters | Parameter | Type | Default | Description | diff --git a/recipes/MiniMaxAI/MiniMax-H3.md b/recipes/MiniMaxAI/MiniMax-H3.md index f6e5dd43b..e8c66ca37 100644 --- a/recipes/MiniMaxAI/MiniMax-H3.md +++ b/recipes/MiniMaxAI/MiniMax-H3.md @@ -416,9 +416,10 @@ For example, keep the first main block's attention projections in BF16 with: ``` The structured option replaces `--quantization fp8`. Online FP8 can be used -with H3 layerwise offload and with distributed layerwise offload's full-weight -per-rank path (`--dlo-no-use-allgather`). The sharded DLO AllGather path is not -supported for runtime-created FP8 weights. +with H3 layerwise offload and with both DLO transfer paths. The default +AllGather path uses the ordinary loader to finalize FP8 weights and scales +before sharding them across ranks. `--dlo-no-use-allgather` instead retains +complete rank-local tensors and avoids the synchronized request-wave contract. ## AMD ROCm (gfx942 / gfx950) @@ -870,8 +871,9 @@ vllm serve "${MODEL_ROOT}/FL2VA" \ H3 native `tile` mode only. - A U2 x Ring2 hybrid currently fails with an attention-mask length mismatch; use pure Ulysses. -- Runtime-created FP8 weights do not support the sharded DLO AllGather path; - use `--dlo-no-use-allgather` when combining online FP8 with DLO. +- Online FP8 with DLO AllGather temporarily materializes the complete FP8 model + in host memory on every rank during startup before retaining only each rank's + shard. Size startup host memory for that transient peak. - TeaCache and Cache-DiT cannot be enabled on the same server. - Image+audio Ref2VA accepts exactly one image and one audio reference. - Video Ref2VA accepts one or more video files, but not an additional standalone diff --git a/recipes/cosmos3/Cosmos3-DistOffload.md b/recipes/cosmos3/Cosmos3-DistOffload.md index 8eeea5cc2..2866e7a0c 100644 --- a/recipes/cosmos3/Cosmos3-DistOffload.md +++ b/recipes/cosmos3/Cosmos3-DistOffload.md @@ -189,8 +189,10 @@ wait - `--dlo-use-allgather` (default) requires all concurrent requests to have the same `num_inference_steps` (AllGather is a collective that needs all ranks synchronized) -- Online quantization (FP8) is incompatible with the DLO+AllGather mmap path; - use `--dlo-no-use-allgather` or disable online quantization +- Online per-tensor FP8 linears with DLO AllGather use the ordinary loader + instead of direct checkpoint mmap, then shard the finalized FP8 weights and + scales. Size host memory for a transient complete FP8 model on every rank at + startup - Tensor Parallel > 1 is rejected in the DLO+AllGather mmap path. TP with `--dlo-no-use-allgather` retains the standard TP loader and should be treated as an experimental compatibility path diff --git a/tests/diffusion/model_loader/test_diffusers_loader.py b/tests/diffusion/model_loader/test_diffusers_loader.py index aa7651bb6..527942961 100644 --- a/tests/diffusion/model_loader/test_diffusers_loader.py +++ b/tests/diffusion/model_loader/test_diffusers_loader.py @@ -14,7 +14,7 @@ from huggingface_hub import snapshot_download from vllm.config.load import LoadConfig from vllm_omni.diffusion.config import get_current_diffusion_config, get_current_diffusion_config_or_none -from vllm_omni.diffusion.data import OmniDiffusionConfig +from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.model_loader.host_weight_plan import ( HostWeightPlan, @@ -91,6 +91,17 @@ def _make_loader_with_weights(weight_names: list[str]) -> DiffusersPipelineLoade return loader +def _make_dlo_online_quant_config() -> OmniDiffusionConfig: + return OmniDiffusionConfig( + model="", + dtype=torch.float32, + quantization_config="fp8", + parallel_config=DiffusionParallelConfig(data_parallel_size=2), + enable_distributed_layerwise_offload=True, + dlo_use_allgather=True, + ) + + def test_strict_check_only_validates_source_prefix_parameters(): model = _DummyPipelineModel(source_prefix="transformer.") loader = _make_loader_with_weights(["transformer.weight"]) @@ -394,6 +405,78 @@ def test_dlo_plan_fallback_runs_ordinary_loader(monkeypatch): assert loader.take_host_weight_plan() is None +def test_dlo_allgather_online_fp8_uses_ordinary_loader(monkeypatch): + from vllm.model_executor.layers.quantization.online.fp8 import ( + Fp8PerTensorOnlineLinearMethod, + ) + + import vllm_omni.diffusion.model_loader.diffusers_loader as loader_mod + + od_config = _make_dlo_online_quant_config() + loader = DiffusersPipelineLoader(LoadConfig(), od_config) + model = nn.Module() + model.transformer = nn.Linear(2, 2, bias=False) + model.transformer.quant_method = object.__new__(Fp8PerTensorOnlineLinearMethod) + model.transformer.quant_method.uses_meta_device = True + calls: list[object] = [] + allowlist_models: list[nn.Module] = [] + + original_allowlist_check = loader._unsupported_dlo_allgather_online_quant_methods + + def check_allowlist(candidate: nn.Module) -> tuple[str, ...]: + allowlist_models.append(candidate) + return original_allowlist_check(candidate) + + loader._init_from_load_format = lambda *_args, **_kwargs: model # type: ignore[method-assign] + monkeypatch.setattr(loader, "_unsupported_dlo_allgather_online_quant_methods", check_allowlist) + loader._request_offload_after_quant = lambda _model: 1 # type: ignore[method-assign] + loader.load_weights = ( # type: ignore[method-assign] + lambda _model, *, stream_online_quant_to_cpu=False: calls.append(("load", stream_online_quant_to_cpu)) + ) + loader._process_weights_after_loading = lambda *_args: calls.append("process") # type: ignore[method-assign] + loader._apply_skip_softmax_calibration = lambda _model: None # type: ignore[method-assign] + monkeypatch.setattr( + loader_mod, + "build_checkpoint_mmap_plan", + lambda *_args, **_kwargs: HostWeightPlanResult( + None, + "online quantization requires the ordinary loader", + ), + ) + + assert loader.load_model(load_device="cpu", device=torch.device("cpu")) is model + assert allowlist_models == [model] + assert calls == [("load", True), "process"] + assert loader.take_host_weight_plan() is None + + +def test_dlo_allgather_rejects_unvalidated_online_quant_method(monkeypatch): + import vllm_omni.diffusion.model_loader.diffusers_loader as loader_mod + + class UnsupportedOnlineMethod: + uses_meta_device = True + + od_config = _make_dlo_online_quant_config() + loader = DiffusersPipelineLoader(LoadConfig(), od_config) + model = nn.Module() + model.transformer = nn.Linear(2, 2, bias=False) + model.transformer.quant_method = UnsupportedOnlineMethod() + + loader._init_from_load_format = lambda *_args, **_kwargs: model # type: ignore[method-assign] + loader._apply_skip_softmax_calibration = lambda _model: None # type: ignore[method-assign] + monkeypatch.setattr( + loader_mod, + "build_checkpoint_mmap_plan", + lambda *_args, **_kwargs: HostWeightPlanResult( + None, + "online quantization requires the ordinary loader", + ), + ) + + with pytest.raises(ValueError, match="per-tensor FP8 linears"): + loader.load_model(load_device="cpu") + + def test_hsdp_processes_quantized_weights_before_sharding(mocker): import vllm_omni.diffusion.model_loader.diffusers_loader as loader_mod from vllm_omni.diffusion.offloader.module_collector import PipelineModules diff --git a/tests/diffusion/offloader/test_distributed_layerwise_backend.py b/tests/diffusion/offloader/test_distributed_layerwise_backend.py index 6c8248391..c67a2d45b 100644 --- a/tests/diffusion/offloader/test_distributed_layerwise_backend.py +++ b/tests/diffusion/offloader/test_distributed_layerwise_backend.py @@ -185,6 +185,79 @@ class TestDistributedLayerwiseOffloadHook: assert next_block.weight.stride() == expected_stride assert torch.equal(next_block.weight, expected) + def test_allgather_reconstructs_online_fp8_weight_and_scale( + self, + patched_offload_runtime, + monkeypatch, + ): + """DLO must reconstruct finalized online-FP8 tensors byte-for-byte.""" + + class OnlineFp8Block(nn.Module): + def __init__(self): + super().__init__() + logical_weight = torch.tensor( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ], + dtype=torch.float32, + ).to(torch.float8_e4m3fn) + self.weight = nn.Parameter(logical_weight.t(), requires_grad=False) + self.weight_scale = nn.Parameter(torch.tensor([0.25]), requires_grad=False) + + current_block = OnlineFp8Block() + rank0_block = OnlineFp8Block() + rank1_block = OnlineFp8Block() + expected_weight = rank0_block.weight.detach().clone() + expected_scale = rank0_block.weight_scale.detach().clone() + expected_stride = rank0_block.weight.stride() + + rank0_hook = DistributedLayerwiseOffloadHook( + next_block=rank0_block, + device=torch.device("cpu"), + dp_group=object(), + dp_size=2, + rank=0, + copy_stream=DummyStream(), + comm_stream=DummyStream(), + pin_memory=False, + ) + rank1_hook = DistributedLayerwiseOffloadHook( + next_block=rank1_block, + device=torch.device("cpu"), + dp_group=object(), + dp_size=2, + rank=1, + copy_stream=DummyStream(), + comm_stream=DummyStream(), + pin_memory=False, + ) + rank0_hook.initialize_hook(current_block) + rank1_hook.initialize_hook(OnlineFp8Block()) + rank0_hook.gpu_shard_buffers = [ + {dtype: torch.empty_like(shard) for dtype, shard in rank0_hook.cpu_shards.items()} for _ in range(2) + ] + + def fake_allgather(output, local_shard, *, group): + del group + remote_shard = rank1_hook.cpu_shards[local_shard.dtype] + shard_size = local_shard.numel() + output[:shard_size].copy_(local_shard) + output[shard_size : 2 * shard_size].copy_(remote_shard) + + monkeypatch.setattr(torch.distributed, "all_gather_into_tensor", fake_allgather) + rank0_hook.prefetch_layer(slot=0, non_blocking=False) + + assert rank0_block.weight.stride() == expected_stride + torch.testing.assert_close( + rank0_block.weight.float(), + expected_weight.float(), + rtol=0, + atol=0, + ) + torch.testing.assert_close(rank0_block.weight_scale, expected_scale) + def test_dtensor_wrapper_preserved_across_prefetch_and_offload(self, dist_group, patched_offload_runtime): """DTensor wrapper should be preserved through prefetch/offload cycle.""" current_block = TinyBlock(_make_values(1.0)) diff --git a/vllm_omni/diffusion/model_loader/diffusers_loader.py b/vllm_omni/diffusion/model_loader/diffusers_loader.py index 5e05bd547..62bc0525f 100644 --- a/vllm_omni/diffusion/model_loader/diffusers_loader.py +++ b/vllm_omni/diffusion/model_loader/diffusers_loader.py @@ -482,12 +482,17 @@ class DiffusersPipelineLoader: ) else: if _dist_offload and _use_ag and _has_online_quant: - raise ValueError( - "Online quantization is incompatible with DLO+AllGather: " - "the sharding + AllGather mechanism flattens weights by " - "dtype, which breaks quantized weight/scale layouts. " - "Please use --dlo-no-use-allgather or disable online " - "quantization." + unsupported_methods = self._unsupported_dlo_allgather_online_quant_methods(model) + if unsupported_methods: + raise ValueError( + "DLO+AllGather supports online quantization only for " + "per-tensor FP8 linears; unsupported online methods: " + f"{', '.join(unsupported_methods)}. Please use " + "--dlo-no-use-allgather or disable online quantization." + ) + logger.info( + "Online per-tensor FP8 with DLO+AllGather: using the " + "ordinary loader before sharding finalized weights and scales" ) if _dist_offload and plan_result is not None: logger.info( @@ -548,6 +553,29 @@ class DiffusersPipelineLoader: ``uses_meta_device=True``, e.g. online FP8).""" return has_online_quantization(model) + @staticmethod + def _unsupported_dlo_allgather_online_quant_methods(model: nn.Module) -> tuple[str, ...]: + """Return unsupported online-quant methods for DLO AllGather. + + Per-tensor online FP8 is safe after the ordinary loader has finalized + its weight and scale parameters. DLO shards those runtime tensors by + dtype and reconstructs their recorded shapes and strides before the + kernel consumes them. Other online methods may create different scale, + packing, or aliasing layouts and remain fail-closed until validated. + """ + from vllm.model_executor.layers.quantization.online.fp8 import ( + Fp8PerTensorOnlineLinearMethod, + ) + + unsupported: set[str] = set() + for module in model.modules(): + quant_method = getattr(module, "quant_method", None) + if not getattr(quant_method, "uses_meta_device", False): + continue + if not isinstance(quant_method, Fp8PerTensorOnlineLinearMethod): + unsupported.add(type(quant_method).__name__) + return tuple(sorted(unsupported)) + def _apply_skip_softmax_calibration(self, model: nn.Module) -> None: from vllm_omni.diffusion.attention.backends.trtllm_calibration import ( apply_skip_softmax_calibration, From f208f907bc351dbb863e353ab5d996707bd4c339 Mon Sep 17 00:00:00 2001 From: Weiming Liao Date: Wed, 19 Aug 2026 00:06:29 +0800 Subject: [PATCH 061/267] [BugFix][NPU] Wan2.2-S2V RoPE: replace complex64 advanced indexing with index_select (#6320) Signed-off-by: Weiming Liao --- vllm_omni/diffusion/layers/rope.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/vllm_omni/diffusion/layers/rope.py b/vllm_omni/diffusion/layers/rope.py index fae1a3414..e78574b04 100644 --- a/vllm_omni/diffusion/layers/rope.py +++ b/vllm_omni/diffusion/layers/rope.py @@ -423,16 +423,16 @@ class WanS2VRotaryPosEmbed(torch.nn.Module): h_sam = torch.linspace(int(h_o), int(t_h + h_o) - 1, seq_h_int, device=device).long() w_sam = torch.linspace(int(w_o), int(t_w + w_o) - 1, seq_w_int, device=device).long() - freqs_0 = freqs_split[0][f_sam] if f_o >= 0 else freqs_split[0][f_sam].conj() + freqs_0 = torch.index_select(freqs_split[0] if f_o >= 0 else freqs_split[0].conj(), 0, f_sam) freqs_0 = freqs_0.view(seq_f_int, 1, 1, -1) freqs_i = torch.cat( [ freqs_0.expand(seq_f_int, seq_h_int, seq_w_int, -1), - freqs_split[1][h_sam] + torch.index_select(freqs_split[1], 0, h_sam) .view(1, seq_h_int, 1, -1) .expand(seq_f_int, seq_h_int, seq_w_int, -1), - freqs_split[2][w_sam] + torch.index_select(freqs_split[2], 0, w_sam) .view(1, 1, seq_w_int, -1) .expand(seq_f_int, seq_h_int, seq_w_int, -1), ], @@ -523,14 +523,18 @@ class RotaryEmbeddingS2VGrid(torch.nn.Module): h_sam = torch.linspace(int(h_o), int(t_h + h_o) - 1, seq_h_int, device=device).long() w_sam = torch.linspace(int(w_o), int(t_w + w_o) - 1, seq_w_int, device=device).long() - freqs_0 = freqs[0][f_sam] if f_o >= 0 else freqs[0][f_sam].conj() + freqs_0 = torch.index_select(freqs[0] if f_o >= 0 else freqs[0].conj(), 0, f_sam) freqs_0 = freqs_0.view(seq_f_int, 1, 1, -1) freqs_i = torch.cat( [ freqs_0.expand(seq_f_int, seq_h_int, seq_w_int, -1), - freqs[1][h_sam].view(1, seq_h_int, 1, -1).expand(seq_f_int, seq_h_int, seq_w_int, -1), - freqs[2][w_sam].view(1, 1, seq_w_int, -1).expand(seq_f_int, seq_h_int, seq_w_int, -1), + torch.index_select(freqs[1], 0, h_sam) + .view(1, seq_h_int, 1, -1) + .expand(seq_f_int, seq_h_int, seq_w_int, -1), + torch.index_select(freqs[2], 0, w_sam) + .view(1, 1, seq_w_int, -1) + .expand(seq_f_int, seq_h_int, seq_w_int, -1), ], dim=-1, ).reshape(seg_len, 1, -1) From b4b907e7b60086c6428145f8dbab687314971dea Mon Sep 17 00:00:00 2001 From: Alex Brooks Date: Tue, 18 Aug 2026 10:43:05 -0600 Subject: [PATCH 062/267] [Bugfix | Model] Fix SenseNova & Use Well-defined Model Configs (#5877) Signed-off-by: Alex Brooks --- .../test_sensenova_u1_logits_processor.py | 72 ++++++++++ .../sensenova_u1/pipeline_sensenova_u1.py | 113 ++++----------- vllm_omni/diffusion/vllm_config.py | 3 + .../transformers_utils/configs/__init__.py | 3 + .../configs/sensenova_u1.py | 129 ++++++++++++++++++ 5 files changed, 231 insertions(+), 89 deletions(-) create mode 100644 tests/diffusion/models/sensenova_u1/test_sensenova_u1_logits_processor.py create mode 100644 vllm_omni/transformers_utils/configs/sensenova_u1.py diff --git a/tests/diffusion/models/sensenova_u1/test_sensenova_u1_logits_processor.py b/tests/diffusion/models/sensenova_u1/test_sensenova_u1_logits_processor.py new file mode 100644 index 000000000..26af9a259 --- /dev/null +++ b/tests/diffusion/models/sensenova_u1/test_sensenova_u1_logits_processor.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression test: SenseNovaU1ForCausalLM instantiation under the diffusion +config shim must not crash on missing ``head_dtype`` and the resulting +LogitsProcessor must have ``head_dtype is None`` (= use model dtype).""" + +import os + +import pytest +import torch +from vllm.config import DeviceConfig, VllmConfig, set_current_vllm_config +from vllm.distributed.parallel_state import ( + cleanup_dist_env_and_memory, + init_distributed_environment, + initialize_model_parallel, +) + +from vllm_omni.diffusion.models.sensenova_u1.sensenova_u1_transformer import ( + SenseNovaU1ForCausalLM, +) +from vllm_omni.diffusion.vllm_config import _DiffusionVllmModelConfig +from vllm_omni.transformers_utils.configs.sensenova_u1 import ( + SenseNovaU1Config, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + +def _build_tiny_sensenova_u1(quant_config=None, prefix="model"): + """Build a minimal SenseNovaU1ForCausalLM for unit tests.""" + config = SenseNovaU1Config( + llm_config={ + "hidden_size": 64, + "num_attention_heads": 2, + "num_hidden_layers": 1, + "intermediate_size": 128, + "vocab_size": 32, + "max_position_embeddings": 128, + "max_position_embeddings_hw": 128, + }, + ) + return SenseNovaU1ForCausalLM(config.llm_config, quant_config=quant_config, prefix=prefix) + + +def test_logits_processor_head_dtype_under_diffusion_shim(): + # Initialize the distributed environment + os.environ.setdefault("MASTER_ADDR", "localhost") + os.environ.setdefault("MASTER_PORT", "29543") + init_distributed_environment(world_size=1, rank=0, local_rank=0, distributed_init_method="env://") + initialize_model_parallel() + + # Create a vLLM config with a wrapped diffusion config + fake_diff_config = _DiffusionVllmModelConfig( + model="sensenova-u1-test", + dtype=torch.bfloat16, + max_model_len=8192, + original_max_model_len=8192, + ) + vllm_config = VllmConfig( + device_config=DeviceConfig(device="cpu"), + ) + vllm_config.modelconfig = fake_diff_config # type: ignore[assignment] + + # Build a tiny version of the Causal LM component + with set_current_vllm_config(vllm_config): + model = _build_tiny_sensenova_u1() + + # Ensure that we have a set head_dtype attribute. Currently, the head_dtype + # is set to None + head_dtype = model.logits_processor.head_dtype + assert head_dtype is None + cleanup_dist_env_and_memory() diff --git a/vllm_omni/diffusion/models/sensenova_u1/pipeline_sensenova_u1.py b/vllm_omni/diffusion/models/sensenova_u1/pipeline_sensenova_u1.py index 67e141e6a..3a828fa69 100644 --- a/vllm_omni/diffusion/models/sensenova_u1/pipeline_sensenova_u1.py +++ b/vllm_omni/diffusion/models/sensenova_u1/pipeline_sensenova_u1.py @@ -17,7 +17,6 @@ Key integration points: from __future__ import annotations -import json import math import os from collections.abc import Iterable @@ -28,6 +27,7 @@ import numpy as np import torch import torch.nn as nn import torchvision.transforms as T +from huggingface_hub import snapshot_download from PIL import Image from transformers import AutoTokenizer from vllm.logger import init_logger @@ -40,6 +40,9 @@ from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.transformers_utils.configs.sensenova_u1 import ( + SenseNovaU1Config, +) from .sensenova_u1_transformer import ( SenseNovaU1ForCausalLM, @@ -86,16 +89,12 @@ SYSTEM_MESSAGE_FOR_GEN = ( # --------------------------------------------------------------------------- -# Config parsed from config.json +# Config # --------------------------------------------------------------------------- - - def _resolve_model_path(model_path: str) -> str: """Resolve a HuggingFace model ID or local path to a local directory.""" if os.path.isdir(model_path): return model_path - from huggingface_hub import snapshot_download - return snapshot_download(model_path) @@ -182,73 +181,6 @@ def _load_image_native( return flatten_pv, grid_hw -def _load_sensenova_u1_config(model_path: str): - """Parse config.json and return (top_config, llm_config, vision_config) namespaces.""" - local_path = _resolve_model_path(model_path) - cfg_path = os.path.join(local_path, "config.json") - with open(cfg_path, encoding="utf-8") as f: - raw = json.load(f) - - llm_raw = raw.get("llm_config", {}) - vis_raw = raw.get("vision_config", {}) - - # LLM config needs these extra fields for 3D RoPE - llm_cfg = SimpleNamespace( - hidden_size=llm_raw.get("hidden_size", 4096), - intermediate_size=llm_raw.get("intermediate_size", 11008), - num_hidden_layers=llm_raw.get("num_hidden_layers", 32), - num_attention_heads=llm_raw.get("num_attention_heads", 32), - num_key_value_heads=llm_raw.get("num_key_value_heads", llm_raw.get("num_attention_heads", 32)), - head_dim=llm_raw.get("head_dim", llm_raw.get("hidden_size", 4096) // llm_raw.get("num_attention_heads", 32)), - hidden_act=llm_raw.get("hidden_act", "silu"), - rms_norm_eps=llm_raw.get("rms_norm_eps", 1e-6), - vocab_size=llm_raw.get("vocab_size", 152064), - rope_theta=llm_raw.get("rope_theta", 1000000.0), - rope_theta_hw=llm_raw.get("rope_theta_hw", 10000.0), - max_position_embeddings=llm_raw.get("max_position_embeddings", 40960), - max_position_embeddings_hw=llm_raw.get("max_position_embeddings_hw", 10000), - attention_bias=llm_raw.get("attention_bias", True), - layer_types=llm_raw.get("layer_types", ["full_attention"] * llm_raw.get("num_hidden_layers", 32)), - tie_word_embeddings=llm_raw.get("tie_word_embeddings", False), - ) - - vis_cfg = SimpleNamespace( - num_channels=vis_raw.get("num_channels", 3), - patch_size=vis_raw.get("patch_size", 16), - hidden_size=vis_raw.get("hidden_size", 1024), - llm_hidden_size=vis_raw.get("llm_hidden_size", [llm_cfg.hidden_size]), - downsample_ratio=vis_raw.get("downsample_ratio", [0.5]), - rope_theta_vision=vis_raw.get("rope_theta_vision", 10000.0), - max_position_embeddings_vision=vis_raw.get("max_position_embeddings_vision", 10000), - output_hidden_states=vis_raw.get("output_hidden_states", False), - use_return_dict=vis_raw.get("use_return_dict", True), - ) - - top_cfg = SimpleNamespace( - downsample_ratio=raw.get("downsample_ratio", 0.5), - template=raw.get("template", "neo1_0"), - fm_head_layers=raw.get("fm_head_layers", 2), - fm_head_dim=raw.get("fm_head_dim", 4096), - fm_head_mlp_ratio=raw.get("fm_head_mlp_ratio", 1.0), - use_pixel_head=raw.get("use_pixel_head", False), - noise_scale=raw.get("noise_scale", 1.0), - noise_scale_mode=raw.get("noise_scale_mode", "none"), - noise_scale_base_image_seq_len=raw.get("noise_scale_base_image_seq_len", 256), - noise_scale_max_value=raw.get("noise_scale_max_value", 10.0), - add_noise_scale_embedding=raw.get("add_noise_scale_embedding", False), - time_schedule=raw.get("time_schedule", "standard"), - time_shift_type=raw.get("time_shift_type", "exponential"), - base_shift=raw.get("base_shift", 0.5), - max_shift=raw.get("max_shift", 1.15), - base_image_seq_len=raw.get("base_image_seq_len", 256), - max_image_seq_len=raw.get("max_image_seq_len", 4096), - concat_time_token_num=raw.get("concat_time_token_num", 0), - t_eps=raw.get("t_eps", 0.02), - ) - - return top_cfg, llm_cfg, vis_cfg - - # --------------------------------------------------------------------------- # Vision Embedding — 2D RoPE + Conv patch embed (no transformer blocks) # --------------------------------------------------------------------------- @@ -549,7 +481,9 @@ class SenseNovaU1Pipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipeli model_path = od_config.model self.local_model_path = _resolve_model_path(model_path) - self.top_cfg, self.llm_cfg, self.vis_cfg = _load_sensenova_u1_config(model_path) + self.model_cfg = SenseNovaU1Config.from_pretrained(self.local_model_path) + self.llm_cfg = self.model_cfg.llm_config + self.vis_cfg = self.model_cfg.vision_config self.tokenizer = AutoTokenizer.from_pretrained(self.local_model_path) self.img_context_token_id = self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN) @@ -573,13 +507,13 @@ class SenseNovaU1Pipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipeli # FM modules llm_hidden_size = self.llm_cfg.hidden_size patch_size = self.vis_cfg.patch_size - merge_size = int(1 / self.top_cfg.downsample_ratio) + merge_size = int(1 / self.model_cfg.downsample_ratio) output_dim = 3 * (patch_size * merge_size) ** 2 vision_model_mot_gen = NEOVisionModel(self.vis_cfg) timestep_embedder = TimestepEmbedder(llm_hidden_size) - if self.top_cfg.use_pixel_head: + if self.model_cfg.use_pixel_head: fm_head = ConvDecoder(llm_hidden_size) else: # The reference implementation uses a fixed 2-layer GELU MLP with a 4096-d @@ -601,13 +535,13 @@ class SenseNovaU1Pipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipeli } ) - if self.top_cfg.add_noise_scale_embedding: + if self.model_cfg.add_noise_scale_embedding: self.fm_modules["noise_scale_embedder"] = TimestepEmbedder(llm_hidden_size) # Config shortcuts self.patch_size = patch_size self.merge_size = merge_size - self.downsample_ratio = self.top_cfg.downsample_ratio + self.downsample_ratio = self.model_cfg.downsample_ratio # Weight sources for diffusers_loader self.weights_sources = [ @@ -727,6 +661,7 @@ class SenseNovaU1Pipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipeli t, z, image_token_num, + t_eps: float, image_size=None, cache_dit_skip=False, **_kw, @@ -748,7 +683,7 @@ class SenseNovaU1Pipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipeli ) last_hidden_state = outputs.hidden_states - if self.top_cfg.use_pixel_head: + if self.model_cfg.use_pixel_head: merge_size = self.merge_size token_h = image_size[1] // (self.patch_size * merge_size) token_w = image_size[0] // (self.patch_size * merge_size) @@ -765,7 +700,7 @@ class SenseNovaU1Pipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipeli else: x_pred = self.fm_modules["fm_head"](last_hidden_state[:, -image_token_num:].view(B, L, -1)).view(B, L, -1) - v_pred = (x_pred - z) / (1 - t).clamp_min(self.top_cfg.t_eps) + v_pred = (x_pred - z) / (1 - t).clamp_min(t_eps) return v_pred def _apply_time_schedule(self, t, image_seq_len, timestep_shift): @@ -1055,14 +990,14 @@ class SenseNovaU1Pipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipeli token_w = p.image_size[0] // (self.patch_size * merge_size) grid_hw = torch.tensor([[grid_h, grid_w]] * p.batch_size, device=self.device) - noise_scale = self.top_cfg.noise_scale - if self.top_cfg.noise_scale_mode in ("resolution", "dynamic", "dynamic_sqrt"): - base = float(self.top_cfg.noise_scale_base_image_seq_len) + noise_scale = self.model_cfg.noise_scale + if self.model_cfg.noise_scale_mode in ("resolution", "dynamic", "dynamic_sqrt"): + base = float(self.model_cfg.noise_scale_base_image_seq_len) scale = math.sqrt((grid_h * grid_w) / (merge_size**2) / base) - noise_scale = scale * float(self.top_cfg.noise_scale) - if self.top_cfg.noise_scale_mode == "dynamic_sqrt": + noise_scale = scale * float(self.model_cfg.noise_scale) + if self.model_cfg.noise_scale_mode == "dynamic_sqrt": noise_scale = math.sqrt(noise_scale) - noise_scale = min(noise_scale, self.top_cfg.noise_scale_max_value) + noise_scale = min(noise_scale, self.model_cfg.noise_scale_max_value) generator = torch.Generator(self.device).manual_seed(p.seed) image_prediction = noise_scale * torch.randn( @@ -1101,6 +1036,7 @@ class SenseNovaU1Pipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipeli z=z, image_token_num=ns.token_h * ns.token_w, image_size=p.image_size, + t_eps=p.t_eps, ) if cache_dit_skip: kwargs["cache_dit_skip"] = True @@ -1241,7 +1177,6 @@ class SenseNovaU1Pipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipeli @torch.inference_mode() def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput: p = self._parse_request(req) - self.top_cfg.t_eps = p.t_eps input_images = self._extract_input_images(p.first_prompt) modalities = p.first_prompt.get("modalities", []) if isinstance(p.first_prompt, dict) else [] @@ -1438,8 +1373,8 @@ class SenseNovaU1Pipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipeli ns.token_h * ns.token_w, -1, ) - if self.top_cfg.add_noise_scale_embedding: - ns_tensor = torch.full_like(t_expanded, ns.noise_scale / self.top_cfg.noise_scale_max_value) + if self.model_cfg.add_noise_scale_embedding: + ns_tensor = torch.full_like(t_expanded, ns.noise_scale / self.model_cfg.noise_scale_max_value) ns_emb = self.fm_modules["noise_scale_embedder"](ns_tensor).view( p.batch_size, ns.token_h * ns.token_w, diff --git a/vllm_omni/diffusion/vllm_config.py b/vllm_omni/diffusion/vllm_config.py index 816739de4..b56109de1 100644 --- a/vllm_omni/diffusion/vllm_config.py +++ b/vllm_omni/diffusion/vllm_config.py @@ -81,6 +81,9 @@ class _DiffusionVllmModelConfig: use_mla: bool = False is_moe: bool = False + # Needed for models that bundle things like LogitsProcessors, e.g., SenseNova + head_dtype: torch.dtype | None = None + @property def is_quantized(self) -> bool: return self.quantization is not None diff --git a/vllm_omni/transformers_utils/configs/__init__.py b/vllm_omni/transformers_utils/configs/__init__.py index 2a31b461c..42d6d8d45 100644 --- a/vllm_omni/transformers_utils/configs/__init__.py +++ b/vllm_omni/transformers_utils/configs/__init__.py @@ -31,6 +31,7 @@ _CLASS_TO_MODULE: dict[str, str] = { "MingFlashOmniConfig": "vllm_omni.transformers_utils.configs.ming_flash_omni", "Qwen3VLMoeVisionConfig": "vllm_omni.transformers_utils.configs.ming_flash_omni", "WhisperEncoderConfig": "vllm_omni.transformers_utils.configs.ming_flash_omni", + "SenseNovaU1Config": "vllm_omni.transformers_utils.configs.sensenova_u1", } __all__ = [ @@ -55,6 +56,7 @@ __all__ = [ "MingFlashOmniConfig", "Qwen3VLMoeVisionConfig", "WhisperEncoderConfig", + "SenseNovaU1Config", ] @@ -82,5 +84,6 @@ from vllm_omni.transformers_utils.configs import mammoth_moda2 as _mammoth_moda2 from vllm_omni.transformers_utils.configs import ming_flash_omni as _ming_flash_omni # noqa: F401, E402 from vllm_omni.transformers_utils.configs import minimax_music3 as _minimax_music3 # noqa: F401, E402 from vllm_omni.transformers_utils.configs import omnivoice as _omnivoice # noqa: F401, E402 +from vllm_omni.transformers_utils.configs import sensenova_u1 as _sensenova_u1 # noqa: F401, E402 from vllm_omni.transformers_utils.configs import voxcpm2 as _voxcpm2 # noqa: F401, E402 from vllm_omni.transformers_utils.configs import voxtral_tts as _voxtral_tts # noqa: F401, E402 diff --git a/vllm_omni/transformers_utils/configs/sensenova_u1.py b/vllm_omni/transformers_utils/configs/sensenova_u1.py new file mode 100644 index 000000000..81e8b3406 --- /dev/null +++ b/vllm_omni/transformers_utils/configs/sensenova_u1.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""HuggingFace-style configuration classes for SenseNova-U1.""" + +from transformers import AutoConfig, PretrainedConfig, Qwen3Config + +_SENSENOVA_U1_LLM_HIDDEN_DEFAULT = 4096 + + +# Adapted from: https://github.com/OpenSenseNova/SenseNova-U1/blob/main/src/sensenova_u1/models/neo_unify/configuration_neo_chat.py +class SenseNovaU1LLMConfig(Qwen3Config): + """Qwen3-based LLM backbone config with 3D RoPE extensions.""" + + model_type = "sensenova_u1_llm" + + def __init__( + self, + rope_theta: float = 10000.0, + rope_theta_hw: float = 10000.0, + max_position_embeddings_hw: int = 10000, + **kwargs, + ): + self.rope_theta = rope_theta + self.rope_theta_hw = rope_theta_hw + self.max_position_embeddings_hw = max_position_embeddings_hw + super().__init__(**kwargs) + + +# Adapted from https://github.com/OpenSenseNova/SenseNova-U1/blob/main/src/sensenova_u1/models/neo_unify/configuration_neo_vit.py#L10 +class SenseNovaU1VisionConfig(PretrainedConfig): + """Vision embedding config (2D RoPE + conv patch embed, no transformer).""" + + model_type = "sensenova_u1_vision" + + def __init__( + self, + num_channels: int = 3, + patch_size: int = 16, + hidden_size: int = 1024, + llm_hidden_size: list[int] | None = None, + downsample_ratio: list[float] | None = None, + rope_theta_vision: float = 10000.0, + max_position_embeddings_vision: int = 10000, + **kwargs, + ): + self.num_channels = num_channels + self.patch_size = patch_size + self.hidden_size = hidden_size + self.llm_hidden_size = llm_hidden_size if llm_hidden_size is not None else [_SENSENOVA_U1_LLM_HIDDEN_DEFAULT] + self.downsample_ratio = downsample_ratio if downsample_ratio is not None else [0.5] + self.rope_theta_vision = rope_theta_vision + self.max_position_embeddings_vision = max_position_embeddings_vision + super().__init__(**kwargs) + + +# Adapted from https://github.com/OpenSenseNova/SenseNova-U1/blob/main/training/sensenovavl/model/sensenovavl_moe_chat/configuration_sensenovavl_chat.py#L17 +class SenseNovaU1Config(PretrainedConfig): + """Top-level composite config for SenseNova-U1. + + Nests ``llm_config`` and ``vision_config`` sub-configs alongside the + flow-matching / diffusion parameters. When constructed from a dict + (e.g. via ``from_pretrained``), sub-dicts are automatically promoted + to their typed config objects. + """ + + model_type = "sensenova_u1" + + def __init__( + self, + llm_config: dict | SenseNovaU1LLMConfig | None = None, + vision_config: dict | SenseNovaU1VisionConfig | None = None, + downsample_ratio: float = 0.5, + template: str = "neo1_0", + fm_head_layers: int = 2, + fm_head_dim: int = 4096, + fm_head_mlp_ratio: float = 1.0, + use_pixel_head: bool = False, + noise_scale: float = 1.0, + noise_scale_mode: str = "none", + noise_scale_base_image_seq_len: int = 256, + noise_scale_max_value: float = 10.0, + add_noise_scale_embedding: bool = False, + time_schedule: str = "standard", + time_shift_type: str = "exponential", + base_shift: float = 0.5, + max_shift: float = 1.15, + base_image_seq_len: int = 256, + max_image_seq_len: int = 4096, + concat_time_token_num: int = 0, + t_eps: float = 0.02, + **kwargs, + ): + if isinstance(llm_config, dict): + llm_config = SenseNovaU1LLMConfig(**llm_config) + self.llm_config = llm_config or SenseNovaU1LLMConfig() + + if isinstance(vision_config, dict): + vision_config = SenseNovaU1VisionConfig(**vision_config) + self.vision_config = vision_config or SenseNovaU1VisionConfig() + + self.downsample_ratio = downsample_ratio + self.template = template + self.fm_head_layers = fm_head_layers + self.fm_head_dim = fm_head_dim + self.fm_head_mlp_ratio = fm_head_mlp_ratio + self.use_pixel_head = use_pixel_head + self.noise_scale = noise_scale + self.noise_scale_mode = noise_scale_mode + self.noise_scale_base_image_seq_len = noise_scale_base_image_seq_len + self.noise_scale_max_value = noise_scale_max_value + self.add_noise_scale_embedding = add_noise_scale_embedding + self.time_schedule = time_schedule + self.time_shift_type = time_shift_type + self.base_shift = base_shift + self.max_shift = max_shift + self.base_image_seq_len = base_image_seq_len + self.max_image_seq_len = max_image_seq_len + self.concat_time_token_num = concat_time_token_num + self.t_eps = t_eps + super().__init__(**kwargs) + + +AutoConfig.register("sensenova_u1", SenseNovaU1Config) + +__all__ = [ + "SenseNovaU1Config", + "SenseNovaU1LLMConfig", + "SenseNovaU1VisionConfig", +] From 07ceb3324a52061d6fb0cd4293b573783d287896 Mon Sep 17 00:00:00 2001 From: Doug Smith Date: Tue, 18 Aug 2026 18:22:11 -0400 Subject: [PATCH 063/267] [BugFix][TTS] Move user input from INFO to DEBUG in TTS/audio log lines (#6329) --- .../openai_api/test_serving_speech.py | 4 +--- .../openai/serving_audio_generate.py | 13 ++++++---- vllm_omni/entrypoints/openai/serving_chat.py | 10 ++++++-- .../entrypoints/openai/serving_speech.py | 24 ++++++++++++------- 4 files changed, 33 insertions(+), 18 deletions(-) diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 604eb54f3..ab348f189 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -4059,9 +4059,7 @@ class TestTTSAsyncOffloading: assert adapter_model_type != legacy_tts_model_type assert any( - call.args - and call.args[0] == "TTS speech request %s: text=%r, model=%s" - and call.args[3] == adapter_model_type + call.args and call.args[0] == "TTS speech request %s: model=%s" and call.args[2] == adapter_model_type for call in log_info.call_args_list ) diff --git a/vllm_omni/entrypoints/openai/serving_audio_generate.py b/vllm_omni/entrypoints/openai/serving_audio_generate.py index 594b8c137..1975f1f16 100644 --- a/vllm_omni/entrypoints/openai/serving_audio_generate.py +++ b/vllm_omni/entrypoints/openai/serving_audio_generate.py @@ -95,11 +95,14 @@ class OmniOpenAIServingAudioGenerate(OpenAIServing, AudioMixin): "audio_end_in_s": audio_end_in_s, } - logger.info( - "Audio generation request %s: prompt=%r", - request_id, - request.input[:50] + "..." if len(request.input) > 50 else request.input, - ) + logger.info("Audio generation request %s", request_id) + _rl = getattr(self, "request_logger", None) + if _rl: + base_len = len(f"Audio generation request {request_id}: prompt=") + raw_max = getattr(_rl, "max_log_len", None) + cap = raw_max if isinstance(raw_max, int) else 200 + text = request.input[: max(cap - base_len, 0)] + logger.debug("Audio generation request %s: prompt=%r", request_id, text) generator = self.engine_client.generate( prompt=prompt, diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index 423ca6d82..d408e11a9 100755 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -3528,12 +3528,18 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): return self._create_error_response(str(exc), status_code=400) logger.info( - "Diffusion chat request %s: prompt=%r, ref_images=%d, params=%s", + "Diffusion chat request %s: ref_images=%d, params=%s", request_id, - prompt[:50] + "..." if len(prompt) > 50 else prompt, len(reference_images), {key: value for key, value in extra_body.items() if value is not None}, ) + _rl = getattr(self, "request_logger", None) + if _rl: + base_len = len(f"Diffusion chat request {request_id}: prompt=") + raw_max = getattr(_rl, "max_log_len", None) + cap = raw_max if isinstance(raw_max, int) else 200 + text = prompt[: max(cap - base_len, 0)] + logger.debug("Diffusion chat request %s: prompt=%r", request_id, text) # Decode reference images if provided pil_images: list[Image.Image] = [] diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index a3069c38c..9f58359ec 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -3080,12 +3080,14 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): model_type = tts_params.get("task_type", ["unknown"])[0] else: model_type = "generic" - logger.info( - "TTS speech request %s: text=%r, model=%s", - request_id, - request.input[:50] + "..." if len(request.input) > 50 else request.input, - model_type, - ) + logger.info("TTS speech request %s: model=%s", request_id, model_type) + _rl = getattr(self, "request_logger", None) + if _rl: + base_len = len(f"TTS speech request {request_id}: text=") + raw_max = getattr(_rl, "max_log_len", None) + cap = raw_max if isinstance(raw_max, int) else 200 + text = request.input[: max(cap - base_len, 0)] + logger.debug("TTS speech request %s: text=%r", request_id, text) # Apply model-specific extra parameters if request.extra_params is not None and sampling_params_list: @@ -3393,11 +3395,17 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): prompt["instruct"] = request.instructions logger.info( - "Diffusion TTS speech request %s: text=%r, voice_clone=%s", + "Diffusion TTS speech request %s: voice_clone=%s", request_id, - request.input[:50] + "..." if len(request.input) > 50 else request.input, "ref_audio" in prompt, ) + _rl = getattr(self, "request_logger", None) + if _rl: + base_len = len(f"Diffusion TTS speech request {request_id}: text=") + raw_max = getattr(_rl, "max_log_len", None) + cap = raw_max if isinstance(raw_max, int) else 200 + text = request.input[: max(cap - base_len, 0)] + logger.debug("Diffusion TTS speech request %s: text=%r", request_id, text) if request.extra_params is not None and not isinstance(request.extra_params, dict): raise ValueError("extra_params must be a JSON object/dict.") extra = dict(request.extra_params or {}) From 9683b7f4b44997c20c4e865ec89b9775bb60a49b Mon Sep 17 00:00:00 2001 From: Canlin Guo Date: Wed, 19 Aug 2026 11:12:10 +0800 Subject: [PATCH 064/267] [Perf] Fuse MiniMax H3 SwiGLU activation (#6283) Signed-off-by: Canlin Guo --- vllm_omni/diffusion/layers/activation.py | 37 +++++++++++++++++++ .../minimax_h3/minimax_h3_transformer.py | 5 ++- 2 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 vllm_omni/diffusion/layers/activation.py diff --git a/vllm_omni/diffusion/layers/activation.py b/vllm_omni/diffusion/layers/activation.py new file mode 100644 index 000000000..c9658cf6e --- /dev/null +++ b/vllm_omni/diffusion/layers/activation.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Custom activation functions for diffusion models.""" + +import torch +import torch.nn.functional as F + +from vllm_omni.diffusion.layers.custom_op import CustomOp +from vllm_omni.platforms import current_omni_platform + + +class SiluAndMul(CustomOp): + """SwiGLU activation dispatched by the current vLLM-Omni platform.""" + + def __init__(self) -> None: + super().__init__() + if current_omni_platform.is_cuda() or current_omni_platform.is_rocm() or current_omni_platform.is_musa(): + self.op = torch.ops._C.silu_and_mul + + @staticmethod + def forward_native(x: torch.Tensor) -> torch.Tensor: + d = x.shape[-1] // 2 + return F.silu(x[..., :d]) * x[..., d:] + + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: + if x.device.type != "cuda": + return self.forward_native(x) + d = x.shape[-1] // 2 + output_shape = x.shape[:-1] + (d,) + out = torch.empty(output_shape, dtype=x.dtype, device=x.device) + self.op(out, x) + return out + + def forward_npu(self, x: torch.Tensor) -> torch.Tensor: + return self.forward_native(x) + + def forward_xpu(self, x: torch.Tensor) -> torch.Tensor: + return self.forward_native(x) diff --git a/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py b/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py index 52f9ab589..2116182ee 100644 --- a/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py +++ b/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py @@ -33,6 +33,7 @@ from vllm_omni.diffusion.distributed.sp_plan import ( SequenceParallelInput, SequenceParallelOutput, ) +from vllm_omni.diffusion.layers.activation import SiluAndMul from vllm_omni.diffusion.layers.fused_qk_norm_rope import fused_qk_norm_rope from vllm_omni.diffusion.layers.norm import RMSNorm from vllm_omni.diffusion.layers.rope import RotaryEmbedding @@ -586,6 +587,7 @@ class MiniMaxH3MLP(nn.Module): quant_config=quant_config, prefix=f"{prefix}.fc1", ) + self.act_fn = SiluAndMul() # Chunk the fused fc1 output as [gate, up], then compute # silu(gate) * up. self.fc2 = RowParallelLinear( @@ -600,8 +602,7 @@ class MiniMaxH3MLP(nn.Module): def forward(self, x: torch.Tensor) -> torch.Tensor: hidden, _ = self.fc1(x) - gate, up = hidden.chunk(2, dim=-1) - hidden = nn.functional.silu(gate) * up + hidden = self.act_fn(hidden) out, _ = self.fc2(hidden) return out From 14f2c16fb28e1e0c8615372675f2757466260ad8 Mon Sep 17 00:00:00 2001 From: Canlin Guo Date: Wed, 19 Aug 2026 11:13:54 +0800 Subject: [PATCH 065/267] [Perf] Fuse MiniMax H3 modulation with FP32 accumulation (#6281) Signed-off-by: Canlin Guo --- .../attention/ops/minimax_h3_modulation.py | 305 ++++++++++++++++++ .../diffusion/cache/teacache/extractors.py | 10 +- .../minimax_h3/minimax_h3_transformer.py | 58 ++-- 3 files changed, 335 insertions(+), 38 deletions(-) create mode 100644 vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py diff --git a/vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py b/vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py new file mode 100644 index 000000000..53d3eb9b9 --- /dev/null +++ b/vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py @@ -0,0 +1,305 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Fused MiniMax H3 modulation with FP32 accumulation.""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _indexed_scale_shift_kernel( + output_ptr, + x_ptr, + shift_ptr, + scale_ptr, + indices_ptr, + hidden_size, + stride_x_row, + stride_shift_row, + stride_scale_row, + stride_indices, + block_n: tl.constexpr, +): + row = tl.program_id(0) + columns = tl.arange(0, block_n) + mask = columns < hidden_size + index = tl.load(indices_ptr + row * stride_indices) + + x = tl.load(x_ptr + row * stride_x_row + columns, mask=mask, other=0.0).to(tl.float32) + shift = tl.load(shift_ptr + index * stride_shift_row + columns, mask=mask, other=0.0).to(tl.float32) + scale = tl.load(scale_ptr + index * stride_scale_row + columns, mask=mask, other=0.0).to(tl.float32) + + tl.store( + output_ptr + row * stride_x_row + columns, + x * (1.0 + scale) + shift, + mask=mask, + ) + + +@triton.jit +def _indexed_gate_kernel( + output_ptr, + x_ptr, + gate_ptr, + other_ptr, + indices_ptr, + hidden_size, + stride_output_row, + stride_x_row, + stride_gate_row, + stride_other_row, + stride_indices, + block_n: tl.constexpr, +): + row = tl.program_id(0) + columns = tl.arange(0, block_n) + mask = columns < hidden_size + index = tl.load(indices_ptr + row * stride_indices) + + x = tl.load(x_ptr + row * stride_x_row + columns, mask=mask, other=0.0).to(tl.float32) + gate = tl.load(gate_ptr + index * stride_gate_row + columns, mask=mask, other=0.0).to(tl.float32) + other = tl.load(other_ptr + row * stride_other_row + columns, mask=mask, other=0.0).to(tl.float32) + + tl.store( + output_ptr + row * stride_output_row + columns, + x + gate * other, + mask=mask, + ) + + +@triton.jit +def _rms_norm_indexed_scale_shift_kernel( + output_ptr, + x_ptr, + weight_ptr, + shift_ptr, + scale_ptr, + indices_ptr, + hidden_size: tl.constexpr, + eps: tl.constexpr, + stride_x_row, + stride_shift_row, + stride_scale_row, + stride_indices, + block_n: tl.constexpr, +): + row = tl.program_id(0) + columns = tl.arange(0, block_n) + mask = columns < hidden_size + index = tl.load(indices_ptr + row * stride_indices) + + x = tl.load(x_ptr + row * stride_x_row + columns, mask=mask, other=0.0).to(tl.float32) + weight = tl.load(weight_ptr + columns, mask=mask, other=0.0).to(tl.float32) + variance = tl.sum(x * x, axis=0) / hidden_size + normalized = x * tl.rsqrt(variance + eps) * weight + + shift = tl.load(shift_ptr + index * stride_shift_row + columns, mask=mask, other=0.0).to(tl.float32) + scale = tl.load(scale_ptr + index * stride_scale_row + columns, mask=mask, other=0.0).to(tl.float32) + tl.store( + output_ptr + row * hidden_size + columns, + normalized * (1.0 + scale) + shift, + mask=mask, + ) + + +@triton.jit +def _indexed_gate_rms_norm_scale_shift_kernel( + residual_out_ptr, + modulated_out_ptr, + residual_ptr, + gate_ptr, + branch_ptr, + weight_ptr, + shift_ptr, + scale_ptr, + indices_ptr, + hidden_size: tl.constexpr, + eps: tl.constexpr, + stride_residual_row, + stride_gate_row, + stride_branch_row, + stride_shift_row, + stride_scale_row, + stride_indices, + block_n: tl.constexpr, +): + row = tl.program_id(0) + columns = tl.arange(0, block_n) + mask = columns < hidden_size + index = tl.load(indices_ptr + row * stride_indices) + + residual = tl.load(residual_ptr + row * stride_residual_row + columns, mask=mask, other=0.0).to(tl.float32) + gate = tl.load(gate_ptr + index * stride_gate_row + columns, mask=mask, other=0.0).to(tl.float32) + branch = tl.load(branch_ptr + row * stride_branch_row + columns, mask=mask, other=0.0).to(tl.float32) + updated = residual + gate * branch + tl.store(residual_out_ptr + row * hidden_size + columns, updated, mask=mask) + + weight = tl.load(weight_ptr + columns, mask=mask, other=0.0).to(tl.float32) + variance = tl.sum(updated * updated, axis=0) / hidden_size + normalized = updated * tl.rsqrt(variance + eps) * weight + shift = tl.load(shift_ptr + index * stride_shift_row + columns, mask=mask, other=0.0).to(tl.float32) + scale = tl.load(scale_ptr + index * stride_scale_row + columns, mask=mask, other=0.0).to(tl.float32) + tl.store( + modulated_out_ptr + row * hidden_size + columns, + normalized * (1.0 + scale) + shift, + mask=mask, + ) + + +def indexed_scale_shift_( + x: torch.Tensor, + shift: torch.Tensor, + scale: torch.Tensor, + indices: torch.Tensor, +) -> torch.Tensor: + """Apply indexed scale/shift in-place to a disposable contiguous input.""" + if x.is_cpu: + x.copy_((x * (1.0 + scale.index_select(0, indices)) + shift.index_select(0, indices)).to(x.dtype)) + return x + rows, hidden_size = x.shape + if rows == 0: + return x + _indexed_scale_shift_kernel[(rows,)]( + x, + x, + shift, + scale, + indices, + hidden_size, + x.stride(0), + shift.stride(0), + scale.stride(0), + indices.stride(0), + block_n=triton.next_power_of_2(hidden_size), + num_warps=8, + ) + return x + + +def indexed_gate( + x: torch.Tensor, + gate: torch.Tensor, + other: torch.Tensor, + indices: torch.Tensor, +) -> torch.Tensor: + """Return ``x + gate[indices] * other`` without indexed temporaries.""" + if x.is_cpu: + return (x + gate.index_select(0, indices) * other).to(x.dtype) + output = torch.empty_like(x) + rows, hidden_size = x.shape + if rows == 0: + return output + _indexed_gate_kernel[(rows,)]( + output, + x, + gate, + other, + indices, + hidden_size, + output.stride(0), + x.stride(0), + gate.stride(0), + other.stride(0), + indices.stride(0), + block_n=triton.next_power_of_2(hidden_size), + num_warps=8, + ) + return output + + +def rms_norm_indexed_scale_shift( + x: torch.Tensor, + weight: torch.Tensor, + shift: torch.Tensor, + scale: torch.Tensor, + indices: torch.Tensor, + eps: float, +) -> torch.Tensor: + """Fuse H3 RMSNorm with its indexed AdaLN affine transform.""" + if x.is_cpu: + input_dtype = x.dtype + normalized = x.float() + variance = normalized.pow(2).mean(-1, keepdim=True) + normalized = normalized * torch.rsqrt(variance + eps) + normalized = (weight.float() * normalized).to(input_dtype) + return (normalized * (1.0 + scale.index_select(0, indices)) + shift.index_select(0, indices)).to(input_dtype) + output = torch.empty_like(x) + rows, hidden_size = x.shape + if rows: + _rms_norm_indexed_scale_shift_kernel[(rows,)]( + output, + x, + weight, + shift, + scale, + indices, + hidden_size, + eps, + x.stride(0), + shift.stride(0), + scale.stride(0), + indices.stride(0), + block_n=triton.next_power_of_2(hidden_size), + num_warps=8, + ) + return output + + +def indexed_gate_rms_norm_scale_shift( + residual: torch.Tensor, + gate: torch.Tensor, + branch: torch.Tensor, + weight: torch.Tensor, + shift: torch.Tensor, + scale: torch.Tensor, + indices: torch.Tensor, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor]: + """Fuse gated residual, the following RMSNorm, and AdaLN affine.""" + if residual.is_cpu: + input_dtype = residual.dtype + residual_out = (residual + gate.index_select(0, indices) * branch).to(input_dtype) + normalized = residual_out.float() + variance = normalized.pow(2).mean(-1, keepdim=True) + normalized = normalized * torch.rsqrt(variance + eps) + normalized = (weight.float() * normalized).to(input_dtype) + modulated_out = (normalized * (1.0 + scale.index_select(0, indices)) + shift.index_select(0, indices)).to( + input_dtype + ) + return residual_out, modulated_out + residual_out = torch.empty_like(residual) + modulated_out = torch.empty_like(residual) + rows, hidden_size = residual.shape + if rows: + _indexed_gate_rms_norm_scale_shift_kernel[(rows,)]( + residual_out, + modulated_out, + residual, + gate, + branch, + weight, + shift, + scale, + indices, + hidden_size, + eps, + residual.stride(0), + gate.stride(0), + branch.stride(0), + shift.stride(0), + scale.stride(0), + indices.stride(0), + block_n=triton.next_power_of_2(hidden_size), + num_warps=8, + ) + return residual_out, modulated_out + + +__all__ = [ + "indexed_gate", + "indexed_gate_rms_norm_scale_shift", + "indexed_scale_shift_", + "rms_norm_indexed_scale_shift", +] diff --git a/vllm_omni/diffusion/cache/teacache/extractors.py b/vllm_omni/diffusion/cache/teacache/extractors.py index 93e306ebb..dea36fff5 100644 --- a/vllm_omni/diffusion/cache/teacache/extractors.py +++ b/vllm_omni/diffusion/cache/teacache/extractors.py @@ -1259,12 +1259,11 @@ def extract_minimax_h3_context( preprocessing via ``_embed``, cacheable ``blocks`` loop, and ``final_layer`` postprocessing with row selection and update masks. """ + from vllm_omni.diffusion.attention.ops.minimax_h3_modulation import indexed_scale_shift_ from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import ( - _BF16_DTYPE, _FORWARD_SUPPORTED_KWARGS, MINIMAX_H3_ADALN_MODALITY_NUM, _build_rope_table, - _modulate_scale_shift, _required_kwarg, ) @@ -1341,13 +1340,12 @@ def extract_minimax_h3_context( cu_seqlens = cu_seqlens.to(device) shift_msa, scale_msa, *_ = module.blocks[0].adaln_proj(t_emb) - modulated_hidden = module.blocks[0].norm1(decoder_input) - modulated_input = _modulate_scale_shift( - modulated_hidden, + modulated_input = module.blocks[0].norm1(decoder_input) + modulated_input = indexed_scale_shift_( + modulated_input, shift_msa, scale_msa, combined_indices, - dtype=_BF16_DTYPE, ) def run_transformer_blocks() -> tuple[torch.Tensor, ...]: diff --git a/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py b/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py index 2116182ee..af5f9b1dd 100644 --- a/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py +++ b/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py @@ -28,6 +28,12 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata, VideoTokenLayout from vllm_omni.diffusion.attention.layer import Attention +from vllm_omni.diffusion.attention.ops.minimax_h3_modulation import ( + indexed_gate, + indexed_gate_rms_norm_scale_shift, + indexed_scale_shift_, + rms_norm_indexed_scale_shift, +) from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig from vllm_omni.diffusion.distributed.sp_plan import ( SequenceParallelInput, @@ -190,30 +196,6 @@ def _norm(size: int, *, eps: float, dtype: torch.dtype = _BF16_DTYPE) -> RMSNorm return RMSNorm(size, eps=eps, dtype=dtype) -def _modulate_scale_shift( - x: torch.Tensor, - shift: torch.Tensor, - scale: torch.Tensor, - indices: torch.Tensor, - *, - dtype: torch.dtype, -) -> torch.Tensor: - # Apply per-index affine modulation: x * (1 + scale[idx]) + shift[idx]. - return (x * (1.0 + scale.index_select(0, indices)) + shift.index_select(0, indices)).to(dtype) - - -def _modulate_gate( - x: torch.Tensor, - gate: torch.Tensor, - other: torch.Tensor, - indices: torch.Tensor, - *, - dtype: torch.dtype, -) -> torch.Tensor: - # Apply the per-index gated residual: x + gate[idx] * other. - return (x + gate.index_select(0, indices) * other).to(dtype) - - def _sequence_parallel_local_span( seq_len: int, *, @@ -795,8 +777,14 @@ class MiniMaxH3DiTBlock(nn.Module): ) = self.adaln_proj(t_emb) residual = x - h = self.norm1(x) - h = _modulate_scale_shift(h, shift_msa, scale_msa, combined_indices, dtype=_BF16_DTYPE) + h = rms_norm_indexed_scale_shift( + x, + self.norm1.weight, + shift_msa, + scale_msa, + combined_indices, + self.norm1.variance_epsilon, + ) h = self.attn( h, rope_table=rope_table, @@ -806,13 +794,19 @@ class MiniMaxH3DiTBlock(nn.Module): sp_seq_lens=sp_seq_lens, video_layout=video_layout, ) - x = _modulate_gate(residual, gate_msa, h, combined_indices, dtype=_BF16_DTYPE) - + x, h = indexed_gate_rms_norm_scale_shift( + residual, + gate_msa, + h, + self.norm2.weight, + shift_mlp, + scale_mlp, + combined_indices, + self.norm2.variance_epsilon, + ) residual = x - h = self.norm2(x) - h = _modulate_scale_shift(h, shift_mlp, scale_mlp, combined_indices, dtype=_BF16_DTYPE) h = self.mlp(h) - return _modulate_gate(residual, gate_mlp, h, combined_indices, dtype=_BF16_DTYPE) + return indexed_gate(residual, gate_mlp, h, combined_indices) class MiniMaxH3FinalLayer(nn.Module): @@ -867,7 +861,7 @@ class MiniMaxH3FinalLayer(nn.Module): """ shift, scale = self.adaln_proj(t_emb) h = self.norm(x) - h = _modulate_scale_shift(h, shift, scale, inverse_indices, dtype=_BF16_DTYPE) + h = indexed_scale_shift_(h, shift, scale, inverse_indices) # Preserve full precision through both final output projections. h = h.to(_FP32_DTYPE) video, _ = self.video_out(h) From 14bd8f8772dfb2b6ccb2117d6b9078bbb2edbe66 Mon Sep 17 00:00:00 2001 From: bjf-frz Date: Wed, 19 Aug 2026 11:23:10 +0800 Subject: [PATCH 066/267] [REFACT]Refactor diffusion parallel state (#5531) Signed-off-by: bjf-frz Co-authored-by: Hongsheng Liu --- docs/user_guide/diffusion/parallelism/hsdp.md | 3 +- tests/config/test_omni_config.py | 7 +- .../test_distributed_vae_executor.py | 8 - .../test_expert_parallel_layout.py | 36 ++- tests/diffusion/distributed/test_hsdp.py | 156 +---------- .../test_diffusion_config_propagation.py | 24 ++ .../test_async_omni_diffusion_config.py | 23 +- vllm_omni/config/omni_config.py | 17 +- vllm_omni/diffusion/data.py | 71 +++-- vllm_omni/diffusion/distributed/__init__.py | 9 - .../autoencoders/distributed_vae_executor.py | 13 +- .../distributed/group_coordinator.py | 24 ++ vllm_omni/diffusion/distributed/hsdp.py | 45 ++-- .../diffusion/distributed/parallel_state.py | 242 ++++++++---------- .../models/minimax_h3/pipeline_minimax_h3.py | 6 +- vllm_omni/diffusion/models/minimax_h3/vae.py | 4 +- vllm_omni/diffusion/offloader/base.py | 2 +- .../diffusion/worker/diffusion_worker.py | 3 +- vllm_omni/engine/async_omni_engine.py | 9 +- 19 files changed, 315 insertions(+), 387 deletions(-) diff --git a/docs/user_guide/diffusion/parallelism/hsdp.md b/docs/user_guide/diffusion/parallelism/hsdp.md index 8a8aa5ef9..9491ed9f5 100644 --- a/docs/user_guide/diffusion/parallelism/hsdp.md +++ b/docs/user_guide/diffusion/parallelism/hsdp.md @@ -152,7 +152,8 @@ In `DiffusionParallelConfig`: **Constraints:** - `hsdp_replicate_size × hsdp_shard_size == world_size` -- HSDP cannot be used with Tensor Parallelism (`tensor_parallel_size` must be 1) +- HSDP cannot be used with Tensor Parallelism, ordinary Data Parallelism, + Pipeline Parallelism, or Expert Parallelism. --- diff --git a/tests/config/test_omni_config.py b/tests/config/test_omni_config.py index 108d997a5..3ee1d40a1 100644 --- a/tests/config/test_omni_config.py +++ b/tests/config/test_omni_config.py @@ -733,8 +733,7 @@ def test_diffusion_parallel_config_matches_diffusion_parallel_world_size_for_vae def test_diffusion_parallel_config_supports_diffusion_hsdp_auto_sharding(): cfg = OmniStageDiffusionParallelConfig( - pipeline_parallel_size=2, - ulysses_degree=2, + ulysses_degree=4, use_hsdp=True, hsdp_shard_size=-1, hsdp_replicate_size=2, @@ -745,10 +744,10 @@ def test_diffusion_parallel_config_supports_diffusion_hsdp_auto_sharding(): def test_diffusion_parallel_config_rejects_hsdp_with_tp_or_dp(): - with pytest.raises(ValueError, match="cannot be used with TP or DP"): + with pytest.raises(ValueError, match="not compatible with TP"): OmniStageDiffusionParallelConfig(tensor_parallel_size=2, use_hsdp=True, hsdp_shard_size=2) - with pytest.raises(ValueError, match="cannot be used with TP or DP"): + with pytest.raises(ValueError, match="not compatible with DP"): OmniStageDiffusionParallelConfig(data_parallel_size=2, use_hsdp=True, hsdp_shard_size=2) diff --git a/tests/diffusion/distributed/test_distributed_vae_executor.py b/tests/diffusion/distributed/test_distributed_vae_executor.py index b2ee7c10d..c122e9105 100644 --- a/tests/diffusion/distributed/test_distributed_vae_executor.py +++ b/tests/diffusion/distributed/test_distributed_vae_executor.py @@ -74,14 +74,6 @@ def mock_dist(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(dist, "broadcast", lambda *args, **kwargs: None) -@pytest.fixture(autouse=True) -def mock_dit_group(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr( - "vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor.get_dit_group", - lambda: None, - ) - - @pytest.fixture(autouse=True) def mock_dist_vae_executor(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(DistributedVaeExecutor, "gather_tensors", lambda self, x: [x]) diff --git a/tests/diffusion/distributed/test_expert_parallel_layout.py b/tests/diffusion/distributed/test_expert_parallel_layout.py index 74f30f156..d7b66be5d 100644 --- a/tests/diffusion/distributed/test_expert_parallel_layout.py +++ b/tests/diffusion/distributed/test_expert_parallel_layout.py @@ -92,9 +92,8 @@ def test_moe_ep_maps_diffusion_sp_cfg_dp_to_vllm_groups(monkeypatch): monkeypatch.setattr(parallel_state, "get_forward_context", lambda: fake_forward_context) monkeypatch.setattr(parallel_state, "init_model_parallel_group", fake_init_model_parallel_group) monkeypatch.setattr(parallel_state, "init_vllm_model_parallel_group", fake_init_vllm_model_parallel_group) - monkeypatch.setattr(parallel_state, "init_dit_group", lambda *_args, **_kwargs: None) - for name in ("_DP", "_CFG", "_SP", "_PP", "_FS", "_EXPERT_PARALLEL_GROUP_RANKS"): + for name in ("_DP", "_CFG", "_SP", "_PP", "_EXPERT_PARALLEL_GROUP_RANKS"): monkeypatch.setattr(parallel_state, name, None) for name in ("_TP", "_PCP", "_DP", "_EP", "_PP"): monkeypatch.setattr(parallel_state.vllm_parallel_state, name, None, raising=False) @@ -229,9 +228,8 @@ def test_cfg_parallel_keeps_diffusion_dp_without_ep(monkeypatch): monkeypatch.setattr(parallel_state.torch.distributed, "new_group", lambda ranks: tuple(ranks)) monkeypatch.setattr(parallel_state, "get_world_group", lambda: fake_world_group) monkeypatch.setattr(parallel_state, "init_model_parallel_group", fake_init_model_parallel_group) - monkeypatch.setattr(parallel_state, "init_dit_group", lambda *_args, **_kwargs: None) - for name in ("_DP", "_CFG", "_SP", "_PP", "_FS", "_EXPERT_PARALLEL_GROUP_RANKS"): + for name in ("_DP", "_CFG", "_SP", "_PP", "_EXPERT_PARALLEL_GROUP_RANKS"): monkeypatch.setattr(parallel_state, name, None) for name in ("_TP", "_PCP", "_DP", "_EP", "_PP"): monkeypatch.setattr(parallel_state.vllm_parallel_state, name, None, raising=False) @@ -259,6 +257,33 @@ def test_cfg_parallel_keeps_diffusion_dp_without_ep(monkeypatch): ] +@pytest.mark.cpu +@pytest.mark.core_model +def test_destroy_model_parallel_clears_vllm_pipeline_group(monkeypatch): + """A destroyed diffusion PP group must not block the next initialization.""" + + class _DestroyableGroup: + def __init__(self) -> None: + self.destroy_calls = 0 + + def destroy(self) -> None: + self.destroy_calls += 1 + + pipeline_group = _DestroyableGroup() + for name in ("_DP", "_CFG", "_SP"): + monkeypatch.setattr(parallel_state, name, None) + monkeypatch.setattr(parallel_state, "_PP", pipeline_group) + monkeypatch.setattr(parallel_state, "_EXPERT_PARALLEL_GROUP_RANKS", None) + for name in ("_DP", "_PCP", "_TP", "_EP"): + monkeypatch.setattr(parallel_state.vllm_parallel_state, name, None, raising=False) + monkeypatch.setattr(parallel_state.vllm_parallel_state, "_PP", pipeline_group, raising=False) + + parallel_state.destroy_model_parallel() + + assert pipeline_group.destroy_calls == 1 + assert parallel_state.vllm_parallel_state._PP is None + + @pytest.mark.cpu @pytest.mark.core_model def test_non_moe_ep_fails_before_vllm_ep_remap(monkeypatch): @@ -295,9 +320,8 @@ def test_non_moe_ep_fails_before_vllm_ep_remap(monkeypatch): monkeypatch.setattr(parallel_state, "get_world_group", lambda: fake_world_group) monkeypatch.setattr(parallel_state, "get_forward_context", lambda: fake_forward_context) monkeypatch.setattr(parallel_state, "init_model_parallel_group", fake_init_model_parallel_group) - monkeypatch.setattr(parallel_state, "init_dit_group", lambda *_args, **_kwargs: None) - for name in ("_DP", "_CFG", "_SP", "_PP", "_FS", "_EXPERT_PARALLEL_GROUP_RANKS"): + for name in ("_DP", "_CFG", "_SP", "_PP", "_EXPERT_PARALLEL_GROUP_RANKS"): monkeypatch.setattr(parallel_state, name, None) for name in ("_TP", "_PCP", "_DP", "_EP", "_PP"): monkeypatch.setattr(parallel_state.vllm_parallel_state, name, None, raising=False) diff --git a/tests/diffusion/distributed/test_hsdp.py b/tests/diffusion/distributed/test_hsdp.py index 91c6784fc..0703d19b8 100644 --- a/tests/diffusion/distributed/test_hsdp.py +++ b/tests/diffusion/distributed/test_hsdp.py @@ -234,7 +234,7 @@ class TestDiffusionParallelConfigHSDP: def test_hsdp_cannot_use_with_tp(self): """Test that HSDP and Tensor Parallelism cannot be used together.""" - with pytest.raises(ValueError, match="cannot be used with TP or DP"): + with pytest.raises(ValueError, match="not compatible with TP"): DiffusionParallelConfig( tensor_parallel_size=2, use_hsdp=True, @@ -243,7 +243,7 @@ class TestDiffusionParallelConfigHSDP: def test_hsdp_cannot_use_with_dp(self): """Test that HSDP and Data Parallelism cannot be used together.""" - with pytest.raises(ValueError, match="cannot be used with TP or DP"): + with pytest.raises(ValueError, match="not compatible with DP"): DiffusionParallelConfig( data_parallel_size=2, use_hsdp=True, @@ -264,158 +264,6 @@ class TestDiffusionParallelConfigHSDP: assert config.hsdp_shard_size == 2 # auto: 4 // 2 -class TestStandaloneHSDPDetection: - """Tests for standalone HSDP detection and dit_parallel_size calculation. - - These tests verify the logic used in initialize_model_parallel() to detect - standalone HSDP mode and calculate effective parallel sizes. - - Standalone HSDP is when all non-HSDP parallelism dimensions are 1. - """ - - @staticmethod - def compute_standalone_hsdp_params( - data_parallel_size: int = 1, - cfg_parallel_size: int = 1, - sequence_parallel_size: int = 1, - pipeline_parallel_size: int = 1, - tensor_parallel_size: int = 1, - fully_shard_degree: int = 1, - hsdp_replicate_size: int = 1, - ) -> dict: - """Compute standalone HSDP detection parameters. - - This mirrors the logic in initialize_model_parallel(). - """ - dit_parallel_size = ( - data_parallel_size - * cfg_parallel_size - * sequence_parallel_size - * pipeline_parallel_size - * tensor_parallel_size - ) - - # Check for standalone HSDP: all non-HSDP parallelism dimensions are 1 - is_standalone_hsdp = dit_parallel_size == 1 and fully_shard_degree > 1 - - # For standalone HSDP: use (fully_shard_degree * hsdp_replicate_size) - if is_standalone_hsdp: - effective_dit_parallel_size = fully_shard_degree * hsdp_replicate_size - else: - effective_dit_parallel_size = dit_parallel_size - - effective_dp_size = (fully_shard_degree * hsdp_replicate_size) if is_standalone_hsdp else data_parallel_size - - return { - "original_dit_parallel_size": dit_parallel_size, - "is_standalone_hsdp": is_standalone_hsdp, - "effective_dit_parallel_size": effective_dit_parallel_size, - "effective_dp_size": effective_dp_size, - } - - def test_standalone_hsdp_basic(self): - """Test basic standalone HSDP detection (shard_size=4, replicate=1).""" - result = self.compute_standalone_hsdp_params( - fully_shard_degree=4, - hsdp_replicate_size=1, - ) - assert result["original_dit_parallel_size"] == 1 - assert result["is_standalone_hsdp"] is True - assert result["effective_dit_parallel_size"] == 4 - assert result["effective_dp_size"] == 4 - - def test_standalone_hsdp_with_replicate(self): - """Test standalone HSDP with replication (shard_size=4, replicate=2).""" - result = self.compute_standalone_hsdp_params( - fully_shard_degree=4, - hsdp_replicate_size=2, - ) - assert result["original_dit_parallel_size"] == 1 - assert result["is_standalone_hsdp"] is True - assert result["effective_dit_parallel_size"] == 8 # 4 * 2 - assert result["effective_dp_size"] == 8 - - def test_combined_hsdp_sp_not_standalone(self): - """Test HSDP combined with SP is NOT detected as standalone. - - This is a regression test for the bug where the condition - `dit_parallel_size == fully_shard_degree` incorrectly matched - combined modes like SP=4 + HSDP=4. - """ - result = self.compute_standalone_hsdp_params( - sequence_parallel_size=4, - fully_shard_degree=4, - hsdp_replicate_size=1, - ) - assert result["original_dit_parallel_size"] == 4 - assert result["is_standalone_hsdp"] is False - # Should NOT override dp_size for combined mode - assert result["effective_dp_size"] == 1 # original data_parallel_size - - def test_combined_hsdp_cfg_not_standalone(self): - """Test HSDP combined with CFG is NOT detected as standalone.""" - result = self.compute_standalone_hsdp_params( - cfg_parallel_size=2, - fully_shard_degree=4, - hsdp_replicate_size=1, - ) - assert result["original_dit_parallel_size"] == 2 - assert result["is_standalone_hsdp"] is False - assert result["effective_dp_size"] == 1 - - def test_combined_hsdp_pp_not_standalone(self): - """Test HSDP combined with PP is NOT detected as standalone.""" - result = self.compute_standalone_hsdp_params( - pipeline_parallel_size=2, - fully_shard_degree=4, - hsdp_replicate_size=1, - ) - assert result["original_dit_parallel_size"] == 2 - assert result["is_standalone_hsdp"] is False - assert result["effective_dp_size"] == 1 - - def test_no_hsdp_not_standalone(self): - """Test that no HSDP (fully_shard_degree=1) is NOT standalone.""" - result = self.compute_standalone_hsdp_params( - fully_shard_degree=1, - ) - assert result["original_dit_parallel_size"] == 1 - assert result["is_standalone_hsdp"] is False - assert result["effective_dp_size"] == 1 - - def test_combined_multiple_parallelism_not_standalone(self): - """Test HSDP combined with multiple parallelism is NOT standalone.""" - result = self.compute_standalone_hsdp_params( - sequence_parallel_size=2, - cfg_parallel_size=2, - fully_shard_degree=4, - hsdp_replicate_size=1, - ) - assert result["original_dit_parallel_size"] == 4 # 2 * 2 - assert result["is_standalone_hsdp"] is False - assert result["effective_dp_size"] == 1 - - def test_standalone_hsdp_large_shard(self): - """Test standalone HSDP with large shard size.""" - result = self.compute_standalone_hsdp_params( - fully_shard_degree=8, - hsdp_replicate_size=1, - ) - assert result["is_standalone_hsdp"] is True - assert result["effective_dit_parallel_size"] == 8 - assert result["effective_dp_size"] == 8 - - def test_standalone_hsdp_large_replicate(self): - """Test standalone HSDP with large replicate size.""" - result = self.compute_standalone_hsdp_params( - fully_shard_degree=4, - hsdp_replicate_size=4, - ) - assert result["is_standalone_hsdp"] is True - assert result["effective_dit_parallel_size"] == 16 # 4 * 4 - assert result["effective_dp_size"] == 16 - - class TestHSDPShardConditions: """Tests for _hsdp_shard_conditions matching logic.""" diff --git a/tests/diffusion/test_diffusion_config_propagation.py b/tests/diffusion/test_diffusion_config_propagation.py index e456e302b..dfe4a41e8 100644 --- a/tests/diffusion/test_diffusion_config_propagation.py +++ b/tests/diffusion/test_diffusion_config_propagation.py @@ -92,6 +92,30 @@ class TestParallelConfigPropagation: od = _roundtrip_diffusion_config(model="x", parallel_config=pc) assert od.num_gpus == 2 + def test_dp_is_inferred_from_num_gpus(self): + pc = DiffusionParallelConfig(tensor_parallel_size=2, ulysses_degree=2) + od = OmniDiffusionConfig.from_kwargs(model="x", parallel_config=pc, num_gpus=8) + assert od.parallel_config.data_parallel_size == 2 + assert od.parallel_config.world_size == 8 + + def test_explicit_dp_is_validated_against_num_gpus(self): + pc = DiffusionParallelConfig(tensor_parallel_size=2, data_parallel_size=2) + od = OmniDiffusionConfig.from_kwargs(model="x", parallel_config=pc, num_gpus=4) + assert od.parallel_config.data_parallel_size == 2 + + with pytest.raises(ValueError, match="does not match WORLD-derived value"): + OmniDiffusionConfig.from_kwargs( + model="x", + parallel_config=DiffusionParallelConfig(tensor_parallel_size=2, data_parallel_size=2), + num_gpus=8, + ) + + def test_hsdp_does_not_infer_ordinary_dp(self): + pc = DiffusionParallelConfig(use_hsdp=True, hsdp_shard_size=4) + od = OmniDiffusionConfig.from_kwargs(model="x", parallel_config=pc, num_gpus=4) + assert od.parallel_config.data_parallel_size == 1 + assert od.parallel_config.world_size == 4 + class TestCreateDefaultDiffusion: """Verify engine_args structure from create_default_diffusion.""" diff --git a/tests/entrypoints/test_async_omni_diffusion_config.py b/tests/entrypoints/test_async_omni_diffusion_config.py index 1daca1e99..1d5d58011 100644 --- a/tests/entrypoints/test_async_omni_diffusion_config.py +++ b/tests/entrypoints/test_async_omni_diffusion_config.py @@ -58,6 +58,23 @@ def test_default_stage_devices_from_sequence_parallel(): assert stage_cfg["runtime"]["devices"] == "0,1,2,3" +def test_default_stage_devices_and_dp_from_num_gpus(): + """Resolve omitted DP before deriving devices from an explicit GPU count.""" + stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg( + { + "num_gpus": 8, + "tensor_parallel_size": 2, + "ulysses_degree": 2, + } + )[0] + + parallel_config = stage_cfg["engine_args"]["parallel_config"] + assert parallel_config.data_parallel_size == 2 + assert parallel_config.world_size == 8 + assert stage_cfg["engine_args"]["num_gpus"] == 8 + assert stage_cfg["runtime"]["devices"] == "0,1,2,3,4,5,6,7" + + def test_default_stage_config_uses_parallel_size_kwargs(): """Ensure default diffusion parallel_config uses CLI/API parallel sizes.""" stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg( @@ -76,8 +93,8 @@ def test_default_stage_config_uses_parallel_size_kwargs(): assert parallel_config.enable_expert_parallel is True -def test_default_stage_config_defaults_nullified_parallel_size_kwargs(): - """Ensure nullified diffusion parallel-size kwargs fall back to defaults.""" +def test_default_stage_config_preserves_omitted_dp_for_runtime_inference(): + """Keep omitted DP unresolved until the runtime WORLD size is known.""" stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg( { "pipeline_parallel_size": None, @@ -92,7 +109,7 @@ def test_default_stage_config_defaults_nullified_parallel_size_kwargs(): parallel_config = stage_cfg["engine_args"]["parallel_config"] assert parallel_config.pipeline_parallel_size == 1 - assert parallel_config.data_parallel_size == 1 + assert parallel_config.data_parallel_size is None assert parallel_config.tensor_parallel_size == 1 assert parallel_config.enable_expert_parallel is False assert stage_cfg["engine_args"]["enforce_eager"] is False diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index 9cd7ed518..8dd9c3db6 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -600,12 +600,17 @@ class OmniStageDiffusionParallelConfig(OmniStageParallelConfig): * self.cfg_parallel_size ) if self.use_hsdp: - if self.tensor_parallel_size > 1 or self.data_parallel_size > 1: - raise ValueError( - "HSDP (use_hsdp=True) cannot be used with TP or DP " - f"(tensor_parallel_size={self.tensor_parallel_size}, " - f"data_parallel_size={self.data_parallel_size})" - ) + incompatible = [] + if self.tensor_parallel_size > 1: + incompatible.append("TP") + if self.data_parallel_size > 1: + incompatible.append("DP") + if self.pipeline_parallel_size > 1: + incompatible.append("PP") + if self.enable_expert_parallel: + incompatible.append("EP") + if incompatible: + raise ValueError("HSDP (FSDP2) is not compatible with " + ", ".join(incompatible)) if self.hsdp_shard_size == -1: if other_parallel_world_size == 1: raise ValueError("Cannot auto-calculate hsdp_shard_size when other parallelism is all 1") diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index 34c8ccb96..2abd478c7 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -154,8 +154,8 @@ class DiffusionParallelConfig: pipeline_parallel_size: int = 1 """Number of pipeline parallel stages.""" - data_parallel_size: int = 1 - """Number of data parallel groups.""" + data_parallel_size: int | None = None + """Number of data parallel groups. Inferred from WORLD when omitted.""" tensor_parallel_size: int = 1 """Number of tensor parallel groups.""" @@ -243,7 +243,7 @@ class DiffusionParallelConfig: def _validate_parallel_config(self) -> Self: """Validates the config relationships among the parallel strategies.""" assert self.pipeline_parallel_size > 0, "Pipeline parallel size must be > 0" - assert self.data_parallel_size > 0, "Data parallel size must be > 0" + assert self.data_parallel_size is None or self.data_parallel_size > 0, "Data parallel size must be > 0" assert self.tensor_parallel_size > 0, "Tensor parallel size must be > 0" assert self.sequence_parallel_size > 0, "Sequence parallel size must be > 0" assert self.ulysses_degree > 0, "Ulysses degree must be > 0" @@ -283,10 +283,12 @@ class DiffusionParallelConfig: self.allgather_degree if self.allgather_degree > 1 else self.ulysses_degree * self.ring_degree ) - # Calculate world_size from other parallelism dimensions + # Until the runtime WORLD size is known, an omitted DP dimension means + # one replica. OmniDiffusionConfig resolves it against num_gpus below. + data_parallel_size = self.data_parallel_size or 1 other_parallel_world_size = ( self.pipeline_parallel_size - * self.data_parallel_size + * data_parallel_size * self.tensor_parallel_size * self.sequence_parallel_size * self.cfg_parallel_size @@ -297,13 +299,17 @@ class DiffusionParallelConfig: # 1. Standalone: when other parallelism is all 1, HSDP determines world_size # 2. Combined: HSDP overlays on top of other parallelism if self.use_hsdp: - if self.tensor_parallel_size > 1 or self.data_parallel_size > 1: - raise ValueError( - "HSDP (use_hsdp=True) cannot be used with TP or DP " - f"(tensor_parallel_size={self.tensor_parallel_size}, " - f"data_parallel_size={self.data_parallel_size}). " - "Set tensor_parallel_size=1 and data_parallel_size=1 when using HSDP." - ) + incompatible = [] + if self.tensor_parallel_size > 1: + incompatible.append("TP") + if data_parallel_size > 1: + incompatible.append("DP") + if self.pipeline_parallel_size > 1: + incompatible.append("PP") + if self.enable_expert_parallel: + incompatible.append("EP") + if incompatible: + raise ValueError("HSDP (FSDP2) is not compatible with " + ", ".join(incompatible)) if self.hsdp_shard_size == -1: # Auto-calculate: use other_parallel_world_size as shard_size if self.hsdp_replicate_size <= 0: @@ -338,6 +344,42 @@ class DiffusionParallelConfig: else: self.world_size = other_parallel_world_size + def resolve_data_parallel_size(self, world_size: int) -> int: + """Resolve or validate diffusion DP against the actual WORLD size.""" + if world_size <= 0: + raise ValueError(f"WORLD size must be > 0, but got {world_size}") + + if self.use_hsdp: + if self.data_parallel_size not in (None, 1): + raise ValueError("HSDP (FSDP2) requires data_parallel_size to be 1") + expected_world_size = self.hsdp_replicate_size * self.hsdp_shard_size + if world_size != expected_world_size: + raise ValueError( + f"WORLD size ({world_size}) must equal HSDP size " + f"({self.hsdp_replicate_size} x {self.hsdp_shard_size} = {expected_world_size})" + ) + self.data_parallel_size = 1 + self.world_size = world_size + return 1 + + non_dp_size = ( + self.pipeline_parallel_size + * self.tensor_parallel_size + * self.sequence_parallel_size + * self.cfg_parallel_size + ) + if world_size % non_dp_size != 0: + raise ValueError(f"WORLD size ({world_size}) must be divisible by non-DP parallel size ({non_dp_size})") + inferred_data_parallel_size = world_size // non_dp_size + if self.data_parallel_size is not None and self.data_parallel_size != inferred_data_parallel_size: + raise ValueError( + f"data_parallel_size ({self.data_parallel_size}) does not match WORLD-derived value " + f"({inferred_data_parallel_size})" + ) + self.data_parallel_size = inferred_data_parallel_size + self.world_size = world_size + return inferred_data_parallel_size + @classmethod def from_dict(cls, data: dict[str, Any]) -> "DiffusionParallelConfig": """ @@ -999,10 +1041,7 @@ class OmniDiffusionConfig: else: self.num_gpus = 1 - if self.num_gpus < self.parallel_config.world_size: - raise ValueError( - f"num_gpus ({self.num_gpus}) < parallel_config.world_size ({self.parallel_config.world_size})" - ) + self.parallel_config.resolve_data_parallel_size(self.num_gpus) if self.diffusion_compile_granularity == "full": incompatible_features = [] diff --git a/vllm_omni/diffusion/distributed/__init__.py b/vllm_omni/diffusion/distributed/__init__.py index 668ace06d..2a1159026 100644 --- a/vllm_omni/diffusion/distributed/__init__.py +++ b/vllm_omni/diffusion/distributed/__init__.py @@ -6,11 +6,6 @@ from vllm_omni.diffusion.distributed.hsdp import ( HSDPInferenceConfig, apply_hsdp_to_model, ) -from vllm_omni.diffusion.distributed.parallel_state import ( - get_fs_group, - get_fully_shard_rank, - get_fully_shard_world_size, -) from vllm_omni.diffusion.distributed.sp_plan import ( SequenceParallelConfig, SequenceParallelInput, @@ -46,8 +41,4 @@ __all__ = [ "get_sharding_validator", "HSDPInferenceConfig", "apply_hsdp_to_model", - # FS (Fully Shard) utilities - "get_fs_group", - "get_fully_shard_rank", - "get_fully_shard_world_size", ] diff --git a/vllm_omni/diffusion/distributed/autoencoders/distributed_vae_executor.py b/vllm_omni/diffusion/distributed/autoencoders/distributed_vae_executor.py index d804740f9..9d84cdf53 100644 --- a/vllm_omni/diffusion/distributed/autoencoders/distributed_vae_executor.py +++ b/vllm_omni/diffusion/distributed/autoencoders/distributed_vae_executor.py @@ -9,8 +9,6 @@ import torch import torch.distributed as dist from vllm.logger import init_logger -from vllm_omni.diffusion.distributed.parallel_state import get_dit_group - logger = init_logger(__name__) @@ -45,9 +43,10 @@ class DistributedVaeExecutor: """ def __init__(self): - self.group = get_dit_group() - self.world_size = dist.get_world_size(self.group) - self.rank = dist.get_rank(self.group) + # VAE tiles are scheduled across the complete worker WORLD. + self.group = None + self.world_size = dist.get_world_size() + self.rank = dist.get_rank() self.parallel_size = 1 self.parallel_mode = "tile" @@ -189,7 +188,7 @@ class DistributedVaeMixin: if self.distributed_executor.parallel_size > pp_size: logger.warning( f"vae_patch_parallel_size={self.distributed_executor.parallel_size} " - f"is greater than dit_group={world_size};" - f" using dit_group size={world_size}" + f"is greater than WORLD={world_size};" + f" using WORLD size={world_size}" ) return True diff --git a/vllm_omni/diffusion/distributed/group_coordinator.py b/vllm_omni/diffusion/distributed/group_coordinator.py index 14847007f..ac2059f56 100644 --- a/vllm_omni/diffusion/distributed/group_coordinator.py +++ b/vllm_omni/diffusion/distributed/group_coordinator.py @@ -991,6 +991,18 @@ class PipelineGroupCoordinator(GroupCoordinator): def _pipeline_isend_skip(self, tensor: torch.tensor): return torch.distributed.isend(tensor, dst=self.skip_rank, group=self.skip_device_group) + def destroy(self): + groups = [*self.device_groups, *self.cpu_groups, self.skip_device_group] + seen = {id(self.device_group), id(self.cpu_group)} + for group in groups: + if group is not None and id(group) not in seen: + torch.distributed.destroy_process_group(group) + seen.add(id(group)) + self.device_groups = [] + self.cpu_groups = [] + self.skip_device_group = None + super().destroy() + class SequenceParallelGroupCoordinator(GroupCoordinator): def __init__( @@ -1031,3 +1043,15 @@ class SequenceParallelGroupCoordinator(GroupCoordinator): self.ring_rank = torch.distributed.get_rank(self.ring_group) self.allgather_world_size = torch.distributed.get_world_size(self.allgather_group) self.allgather_rank = torch.distributed.get_rank(self.allgather_group) + + def destroy(self): + groups = [self.ulysses_group, self.ring_group, self.allgather_group] + seen = {id(self.device_group), id(self.cpu_group)} + for group in groups: + if group is not None and id(group) not in seen: + torch.distributed.destroy_process_group(group) + seen.add(id(group)) + self.ulysses_group = None + self.ring_group = None + self.allgather_group = None + super().destroy() diff --git a/vllm_omni/diffusion/distributed/hsdp.py b/vllm_omni/diffusion/distributed/hsdp.py index cae4356f4..c496804bc 100644 --- a/vllm_omni/diffusion/distributed/hsdp.py +++ b/vllm_omni/diffusion/distributed/hsdp.py @@ -15,12 +15,7 @@ from torch.distributed.fsdp import ( ) from vllm.logger import init_logger -from vllm_omni.diffusion.distributed.parallel_state import ( - get_fs_group, - get_fully_shard_rank, - get_fully_shard_world_size, - get_world_group, -) +from vllm_omni.diffusion.distributed.parallel_state import get_world_group from vllm_omni.platforms import current_omni_platform logger = init_logger(__name__) @@ -54,36 +49,36 @@ class HSDPInferenceConfig: def _create_hsdp_mesh( device_type: str, replicate_size: int, - shard_pg: torch.distributed.ProcessGroup, + shard_size: int, ) -> DeviceMesh: - """Create a 2D DeviceMesh for HSDP using an existing ProcessGroup for the shard dimension. + """Create the FSDP2 DeviceMesh; it owns the HSDP process groups. Args: device_type: The device type (e.g., "cuda", "npu") replicate_size: Number of replica groups - shard_pg: The ProcessGroup for the shard dimension (from FS GroupCoordinator) + shard_size: Number of ranks in each FSDP shard group Returns: A 2D DeviceMesh with dimensions ("replicate", "shard") """ - shard_size = torch.distributed.get_world_size(shard_pg) world_size = replicate_size * shard_size + actual_world_size = torch.distributed.get_world_size() + if world_size != actual_world_size: + raise ValueError( + f"HSDP mesh dimensions ({replicate_size} x {shard_size} = {world_size}) " + f"must equal WORLD size ({actual_world_size})" + ) # Build 2D mesh tensor: shape (replicate_size, shard_size) # Ranks are arranged so that each row is a shard group mesh_tensor = torch.arange(world_size).reshape(replicate_size, shard_size) - # Create DeviceMesh with the shard ProcessGroup - # For the shard dimension, we reuse the existing FS ProcessGroup device_mesh = init_device_mesh( device_type, mesh_shape=(replicate_size, shard_size), mesh_dim_names=("replicate", "shard"), ) - # Note: init_device_mesh creates new ProcessGroups internally. - # For consistency, we verify the mesh structure matches our FS group. - # In a future optimization, we could pass the existing ProcessGroups directly. logger.debug( "Created HSDP mesh: replicate_size=%d, shard_size=%d, mesh=%s", replicate_size, @@ -119,34 +114,24 @@ def apply_hsdp_to_model( if not hsdp_config.enabled: raise ValueError("HSDP is not enabled in config") - # Use GroupCoordinator for distributed info world_group = get_world_group() - fs_group = get_fs_group() - world_size = world_group.world_size rank = world_group.rank_in_group - fs_world_size = get_fully_shard_world_size() - fs_rank = get_fully_shard_rank() hsdp_replicate_size = hsdp_config.hsdp_replicate_size hsdp_shard_size = hsdp_config.hsdp_shard_size - # Validate that the FS group matches the HSDP shard size - if fs_world_size != hsdp_shard_size: + if hsdp_replicate_size * hsdp_shard_size != world_size: raise ValueError( - f"FS group world_size ({fs_world_size}) does not match " - f"HSDP shard_size ({hsdp_shard_size}). " - "Ensure fully_shard_degree is set correctly in initialize_model_parallel." + f"HSDP mesh dimensions ({hsdp_replicate_size} x {hsdp_shard_size}) must equal WORLD size ({world_size})" ) logger.info( - "HSDP Inference: replicate_size=%d, shard_size=%d, world_size=%d, rank=%d, fs_world_size=%d, fs_rank=%d", + "HSDP Inference (FSDP2): replicate_size=%d, shard_size=%d, world_size=%d, rank=%d", hsdp_replicate_size, hsdp_shard_size, world_size, rank, - fs_world_size, - fs_rank, ) # When the model contains FP8 parameters (online quantization), let FSDP @@ -163,14 +148,14 @@ def apply_hsdp_to_model( device_type = current_omni_platform.device_type - # Create 2D DeviceMesh for HSDP using the FS group's ProcessGroup for shard dimension + # DeviceMesh is the single source of truth for FSDP2 shard/replicate groups # The mesh shape is (replicate, shard) where: # - replicate: groups of ranks that hold the same shard (for gradient all-reduce in training) # - shard: groups of ranks that each hold different shards (for parameter all-gather) device_mesh = _create_hsdp_mesh( device_type=device_type, replicate_size=hsdp_replicate_size, - shard_pg=fs_group.device_group, + shard_size=hsdp_shard_size, ) hsdp_shard_conditions = getattr(model, "_hsdp_shard_conditions", None) diff --git a/vllm_omni/diffusion/distributed/parallel_state.py b/vllm_omni/diffusion/distributed/parallel_state.py index 55324ccf4..76d458848 100644 --- a/vllm_omni/diffusion/distributed/parallel_state.py +++ b/vllm_omni/diffusion/distributed/parallel_state.py @@ -29,11 +29,11 @@ If you only need to use the distributed environment without model parallelism, """ import inspect +from math import prod import torch import torch.distributed import vllm.distributed.parallel_state as vllm_parallel_state -from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm_omni.diffusion import envs @@ -59,8 +59,6 @@ _SP: SequenceParallelGroupCoordinator | None = None _PP: PipelineGroupCoordinator | None = None _CFG: GroupCoordinator | None = None _DP: GroupCoordinator | None = None -_FS: GroupCoordinator | None = None # Fully Sharded (HSDP shard dimension) -_DIT: GroupCoordinator | None = None # Rank-layout metadata for expert parallelism. This is not a process group; # it is reused by platform-specific runtimes that must build companion groups @@ -184,7 +182,6 @@ class RankGenerator: pp: int, cfg: int, dp: int, - fs: int = 1, order: str = "tp-sp-pp-cfg-dp", rank_offset: int = 0, ) -> None: @@ -193,9 +190,7 @@ class RankGenerator: self.pp = pp self.cfg = cfg self.dp = dp - self.fs = fs self.rank_offset = rank_offset - self.world_size = tp * sp * pp * cfg * dp self.name_to_size = { "tp": self.tp, @@ -203,15 +198,10 @@ class RankGenerator: "pp": self.pp, "cfg": self.cfg, "dp": self.dp, - "fs": self.fs, } order = order.lower() for name in self.name_to_size.keys(): - # Skip 'fs' validation - it's handled separately with independent_ranks=True - # and doesn't participate in the main orthogonal rank generation - if name == "fs": - continue if name not in order and self.name_to_size[name] != 1: raise RuntimeError( f"The size of ({name}) is ({self.name_to_size[name]}), " @@ -234,7 +224,7 @@ class RankGenerator: mask[ordered_token.index(t)] = True return mask - def get_ranks(self, token, independent_ranks: bool = False): + def get_ranks(self, token): """Get rank group by input token. Arguments: @@ -243,27 +233,11 @@ class RankGenerator: to obtain multiple parallel types, we can use a hyphen '-' to separate them. For example, if we want to obtain the TP_DP group, the token should be 'tp-dp'. - independent_ranks (bool): - If True, generate independent rank groups that divide the world - into groups of the specified size. Used for FS (fully shard) groups - which operate independently from the main parallelism hierarchy. """ - if independent_ranks and token == "fs": - # FS groups divide world into groups of size fs - # e.g., world_size=8, fs=4 -> [[0,1,2,3], [4,5,6,7]] - ranks = [] - num_groups = self.world_size // self.fs - for i in range(num_groups): - group = list(range(i * self.fs + self.rank_offset, (i + 1) * self.fs + self.rank_offset)) - ranks.append(group) - return ranks - mask = self.get_mask(self.order, token) - ranks = generate_masked_orthogonal_rank_groups(self.world_size, self.ordered_size, mask) - if self.rank_offset > 0: - for rank_group in ranks: - for i in range(len(rank_group)): - rank_group[i] += self.rank_offset + ranks = generate_masked_orthogonal_rank_groups(prod(self.ordered_size), self.ordered_size, mask) + if self.rank_offset: + return [[rank + self.rank_offset for rank in rank_group] for rank_group in ranks] return ranks @@ -376,22 +350,6 @@ def get_data_parallel_rank(): return get_dp_group().rank_in_group -# FS (Fully Shard / HSDP shard dimension) -def get_fs_group() -> GroupCoordinator: - assert _FS is not None, "fully shard group is not initialized" - return _FS - - -def get_fully_shard_world_size(): - """Return world size for the fully shard group.""" - return get_fs_group().world_size - - -def get_fully_shard_rank(): - """Return my rank for the fully shard group.""" - return get_fs_group().rank_in_group - - def is_dp_last_group(): """Return True if in the last data parallel group, False otherwise.""" return ( @@ -401,17 +359,6 @@ def is_dp_last_group(): ) -def get_dit_world_size(): - """Return world size for the DiT model.""" - return ( - get_data_parallel_world_size() - * get_classifier_free_guidance_world_size() - * get_sequence_parallel_world_size() - * get_pipeline_parallel_world_size() - * get_tensor_model_parallel_world_size() - ) - - # * SET @@ -498,7 +445,6 @@ def init_model_parallel_group( "expert", "sequence", "classifier_free_guidance", - "fully_shard", ], f"parallel_mode {parallel_mode} is not supported" if parallel_mode == "pipeline": return PipelineGroupCoordinator( @@ -546,19 +492,6 @@ def init_vllm_model_parallel_group( ) -def init_dit_group( - dit_parallel_size: int, - backend: str, -): - global _DIT - _DIT = torch.distributed.new_group(ranks=list(range(dit_parallel_size)), backend=backend) - - -def get_dit_group(): - assert _DIT is not None, "DIT group is not initialized" - return _DIT - - # adapted from https://github.com/feifeibear/long-context-attention/blob/main/yunchang/globals.py def set_seq_parallel_pg( sp_ulysses_degree: int, @@ -737,8 +670,8 @@ def set_seq_parallel_pg( return ulysses_pg, ring_pg, allgather_pg -def initialize_model_parallel( - data_parallel_size: int = 1, +def _initialize_model_parallel( + data_parallel_size: int | None = None, cfg_parallel_size: int = 1, sequence_parallel_size: int | None = None, ulysses_degree: int = 1, @@ -746,9 +679,8 @@ def initialize_model_parallel( allgather_degree: int = 1, tensor_parallel_size: int = 1, pipeline_parallel_size: int = 1, - fully_shard_degree: int = 1, - hsdp_replicate_size: int = 1, enable_expert_parallel: bool = False, + use_hsdp: bool = False, backend: str | None = None, ) -> None: if backend is None: @@ -768,7 +700,6 @@ def initialize_model_parallel( (causal=False only). Mutually exclusive with ulysses/ring in v1. tensor_parallel_size: number of GPUs used for tensor parallelism. pipeline_parallel_size: number of GPUs used for pipeline parallelism. - fully_shard_degree: number of GPUs used for fully sharded data parallelism (HSDP shard dimension). backend: distributed backend of pytorch collective comm. Let's say we have a total of 16 GPUs denoted by g0 ... g15 and we @@ -798,16 +729,20 @@ def initialize_model_parallel( ranks 8 to 15 belong to the second box. """ # Get world size and rank. Ensure some consistencies. - assert torch.distributed.is_initialized() + if not torch.distributed.is_initialized(): + raise RuntimeError("torch.distributed must be initialized before model parallel initialization") + if model_parallel_is_initialized(): + raise RuntimeError("model parallel groups are already initialized") world_size: int = torch.distributed.get_world_size() backend = backend or torch.distributed.get_backend(get_world_group().device_group) if allgather_degree > 1: - assert ulysses_degree == 1 and ring_degree == 1, ( - "AllGather-KV (allgather_degree>1) is mutually exclusive with Ulysses/Ring in v1. " - f"Got ulysses_degree={ulysses_degree}, ring_degree={ring_degree}, " - f"allgather_degree={allgather_degree}." - ) + if ulysses_degree != 1 or ring_degree != 1: + raise ValueError( + "AllGather-KV (allgather_degree>1) is mutually exclusive with Ulysses/Ring in v1. " + f"Got ulysses_degree={ulysses_degree}, ring_degree={ring_degree}, " + f"allgather_degree={allgather_degree}." + ) expected_sequence_parallel_size = allgather_degree if allgather_degree > 1 else ring_degree * ulysses_degree if sequence_parallel_size is None: @@ -820,42 +755,38 @@ def initialize_model_parallel( f"but got {sequence_parallel_size}" ) - dit_parallel_size = ( - data_parallel_size * cfg_parallel_size * sequence_parallel_size * pipeline_parallel_size * tensor_parallel_size - ) - - # Check for standalone HSDP: all non-HSDP parallelism dimensions are 1 - is_standalone_hsdp = dit_parallel_size == 1 and fully_shard_degree > 1 - - # For standalone HSDP: use (fully_shard_degree * hsdp_replicate_size) as dit_parallel_size - # This ensures orthogonal rank generation works correctly for all HSDP workers - if is_standalone_hsdp: - dit_parallel_size = fully_shard_degree * hsdp_replicate_size - - if world_size < dit_parallel_size: - raise RuntimeError( - f"world_size ({world_size}) is less than " - f"tensor_parallel_size ({tensor_parallel_size}) x " - f"pipeline_parallel_size ({pipeline_parallel_size}) x" - f"sequence_parallel_size ({sequence_parallel_size}) x" - f"cfg_parallel_size " - f"({cfg_parallel_size}) x" - f"data_parallel_size ({data_parallel_size})" - ) - - # For standalone HSDP, use (fully_shard_degree * hsdp_replicate_size) as data_parallel_size - # so that RankGenerator.world_size matches the actual number of workers - effective_dp_size = (fully_shard_degree * hsdp_replicate_size) if is_standalone_hsdp else data_parallel_size + non_dp_size = cfg_parallel_size * sequence_parallel_size * pipeline_parallel_size * tensor_parallel_size + if world_size % non_dp_size != 0: + raise ValueError(f"WORLD size ({world_size}) must be divisible by non-DP parallel size ({non_dp_size})") + if use_hsdp: + if data_parallel_size not in (None, 1): + raise ValueError("HSDP (FSDP2) requires data_parallel_size to be 1") + if non_dp_size not in (1, world_size): + raise ValueError(f"HSDP non-DP parallel size must be 1 or WORLD size ({world_size}), but got {non_dp_size}") + data_parallel_size = 1 + else: + inferred_data_parallel_size = world_size // non_dp_size + if data_parallel_size is not None and data_parallel_size != inferred_data_parallel_size: + raise ValueError( + f"data_parallel_size ({data_parallel_size}) does not match WORLD-derived value " + f"({inferred_data_parallel_size})" + ) + data_parallel_size = inferred_data_parallel_size rank_generator: RankGenerator = RankGenerator( tensor_parallel_size, sequence_parallel_size, pipeline_parallel_size, cfg_parallel_size, - effective_dp_size, - fs=fully_shard_degree, + data_parallel_size, order="tp-sp-pp-cfg-dp", ) + + def get_rank_groups(token: str) -> list[list[int]]: + if use_hsdp and non_dp_size == 1: + return [[rank] for rank in range(world_size)] + return rank_generator.get_ranks(token) + use_moe_parallel_mapping = False if enable_expert_parallel: od_config = get_forward_context().omni_diffusion_config @@ -863,11 +794,11 @@ def initialize_model_parallel( if not use_moe_parallel_mapping: raise RuntimeError("Expert parallelism enabled for a non-MoE model") - sp_group_ranks = rank_generator.get_ranks("sp") + sp_group_ranks = get_rank_groups("sp") global _DP assert _DP is None, "data parallel group is already initialized" _DP = init_model_parallel_group( - group_ranks=rank_generator.get_ranks("dp"), + group_ranks=get_rank_groups("dp"), local_rank=get_world_group().local_rank, backend=backend, parallel_mode="data", @@ -877,7 +808,7 @@ def initialize_model_parallel( global _CFG assert _CFG is None, "classifier_free_guidance group is already initialized" _CFG = init_model_parallel_group( - group_ranks=rank_generator.get_ranks("cfg"), + group_ranks=get_rank_groups("cfg"), local_rank=get_world_group().local_rank, backend=backend, parallel_mode="classifier_free_guidance", @@ -885,7 +816,7 @@ def initialize_model_parallel( global _PP assert _PP is None, "pipeline model parallel group is already initialized" _PP = init_model_parallel_group( - group_ranks=rank_generator.get_ranks("pp"), + group_ranks=get_rank_groups("pp"), local_rank=get_world_group().local_rank, backend=backend, parallel_mode="pipeline", @@ -899,7 +830,7 @@ def initialize_model_parallel( sp_ring_degree=ring_degree, sp_allgather_degree=allgather_degree, rank=get_world_group().rank_in_group, - world_size=dit_parallel_size, + world_size=world_size, sp_group_ranks=sp_group_ranks, ) _SP = init_model_parallel_group( @@ -925,7 +856,7 @@ def initialize_model_parallel( ) assert vllm_parallel_state._TP is None, "Tensor parallel group is already initialized" - tp_group_ranks = rank_generator.get_ranks("tp") + tp_group_ranks = get_rank_groups("tp") if use_moe_parallel_mapping: vllm_parallel_state._TP = init_vllm_model_parallel_group( group_ranks=tp_group_ranks, @@ -944,22 +875,14 @@ def initialize_model_parallel( # CFG is a diffusion-specific replica dimension. Fold it into vLLM DP # only when constructing the vLLM EP layout for expert-parallel paths. vllm_parallel_state._DP = init_vllm_model_parallel_group( - group_ranks=rank_generator.get_ranks("cfg-dp"), + group_ranks=get_rank_groups("cfg-dp"), local_rank=get_world_group().local_rank, backend=backend, group_name="dp", ) - global _FS, _EXPERT_PARALLEL_GROUP_RANKS - assert _FS is None, "fully shard group is already initialized" - _FS = init_model_parallel_group( - group_ranks=rank_generator.get_ranks("fs", independent_ranks=True), - local_rank=get_world_group().local_rank, - backend=backend, - parallel_mode="fully_shard", - ) - - _EXPERT_PARALLEL_GROUP_RANKS = rank_generator.get_ranks("tp-sp-cfg-dp") + global _EXPERT_PARALLEL_GROUP_RANKS + _EXPERT_PARALLEL_GROUP_RANKS = get_rank_groups("tp-sp-cfg-dp") if use_moe_parallel_mapping: vllm_parallel_state._EP = init_vllm_model_parallel_group( group_ranks=_EXPERT_PARALLEL_GROUP_RANKS, @@ -969,12 +892,63 @@ def initialize_model_parallel( use_all2all=True, ) - init_dit_group(dit_parallel_size, backend) + +def initialize_model_parallel( + data_parallel_size: int | None = None, + cfg_parallel_size: int = 1, + sequence_parallel_size: int | None = None, + ulysses_degree: int = 1, + ring_degree: int = 1, + allgather_degree: int = 1, + tensor_parallel_size: int = 1, + pipeline_parallel_size: int = 1, + enable_expert_parallel: bool = False, + use_hsdp: bool = False, + backend: str | None = None, +) -> None: + """Atomically initialize diffusion parallel groups. + + Configuration is validated before group creation. Any failure tears down + groups created by this attempt, including vLLM-bound coordinators. + """ + existing_groups = { + "dp": _DP, + "cfg": _CFG, + "sp": _SP, + "pp": _PP, + "vllm_tp": vllm_parallel_state._TP, + "vllm_dp": vllm_parallel_state._DP, + "vllm_pp": vllm_parallel_state._PP, + "vllm_ep": vllm_parallel_state._EP, + "vllm_pcp": vllm_parallel_state._PCP, + } + initialized_names = [name for name, group in existing_groups.items() if group is not None] + if initialized_names: + raise RuntimeError( + "model parallel state must be empty before initialization; found " + ", ".join(initialized_names) + ) + try: + _initialize_model_parallel( + data_parallel_size=data_parallel_size, + cfg_parallel_size=cfg_parallel_size, + sequence_parallel_size=sequence_parallel_size, + ulysses_degree=ulysses_degree, + ring_degree=ring_degree, + allgather_degree=allgather_degree, + tensor_parallel_size=tensor_parallel_size, + pipeline_parallel_size=pipeline_parallel_size, + enable_expert_parallel=enable_expert_parallel, + use_hsdp=use_hsdp, + backend=backend, + ) + except BaseException: + destroy_model_parallel() + raise def destroy_model_parallel(): """Set the groups to none and destroy them.""" - global _DP, _CFG, _SP, _PP, _FS, _EXPERT_PARALLEL_GROUP_RANKS + global _DP, _CFG, _SP, _PP, _EXPERT_PARALLEL_GROUP_RANKS if vllm_parallel_state._DP and vllm_parallel_state._DP is not _DP: vllm_parallel_state._DP.destroy() @@ -1005,14 +979,14 @@ def destroy_model_parallel(): vllm_parallel_state._EP = None _EXPERT_PARALLEL_GROUP_RANKS = None + if vllm_parallel_state._PP and vllm_parallel_state._PP is not _PP: + vllm_parallel_state._PP.destroy() + vllm_parallel_state._PP = None + if _PP: _PP.destroy() _PP = None - if _FS: - _FS.destroy() - _FS = None - def destroy_distributed_environment(): global _WORLD diff --git a/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py b/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py index f639f3ebb..57fc26177 100644 --- a/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py +++ b/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py @@ -29,7 +29,7 @@ from vllm_omni.diffusion.cache.cachedit import ( ) from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig from vllm_omni.diffusion.distributed.parallel_state import ( - get_dit_group, + get_world_group, init_world_group, ) from vllm_omni.diffusion.distributed.utils import get_local_device @@ -423,7 +423,7 @@ def _validate_reference_image(image: Image.Image) -> None: def _dit_rank_world() -> tuple[Any, int, int]: if not dist.is_initialized(): return None, 0, 1 - group = get_dit_group() + group = get_world_group().device_group return group, dist.get_rank(group), dist.get_world_size(group) @@ -1715,7 +1715,7 @@ class MiniMaxH3Pipeline( dist.broadcast( has_audio_tensor, src=0, - group=get_dit_group(), + group=get_world_group().device_group, ) has_audio = [bool(value) for value in has_audio_tensor.tolist()] diff --git a/vllm_omni/diffusion/models/minimax_h3/vae.py b/vllm_omni/diffusion/models/minimax_h3/vae.py index e28450c90..bf75ab7f6 100644 --- a/vllm_omni/diffusion/models/minimax_h3/vae.py +++ b/vllm_omni/diffusion/models/minimax_h3/vae.py @@ -18,7 +18,7 @@ from transformers.dynamic_module_utils import get_class_from_dynamic_module from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import ( DistributedVaeMixin, ) -from vllm_omni.diffusion.distributed.parallel_state import get_dit_group +from vllm_omni.diffusion.distributed.parallel_state import get_world_group from vllm_omni.diffusion.offloader.module_residency import PinnedModuleStager from .packed_tokens import minimax_h3_patchify_video_latent @@ -157,7 +157,7 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin): ) -> None: if mode != "tile": raise ValueError(f"MiniMax H3 VAE supports its native tile parallel mode only, got {mode!r}") - group = get_dit_group() + group = get_world_group().device_group world_size = dist.get_world_size(group) rank = dist.get_rank(group) parallel_size = int(parallel_size) diff --git a/vllm_omni/diffusion/offloader/base.py b/vllm_omni/diffusion/offloader/base.py index ea74fbfb7..a750d79f5 100644 --- a/vllm_omni/diffusion/offloader/base.py +++ b/vllm_omni/diffusion/offloader/base.py @@ -62,7 +62,7 @@ class OffloadConfig: dp_size = 1 if parallel_config is not None: dp_size = getattr(parallel_config, "data_parallel_size", 1) - # HSDP's fully_shard_degree also contributes to effective DP + # HSDP shard and replica sizes determine the effective group size. hsdp_shard_size = getattr(parallel_config, "hsdp_shard_size", -1) if use_hsdp else -1 hsdp_replicate_size = getattr(parallel_config, "hsdp_replicate_size", 1) if use_hsdp else 1 if use_hsdp and hsdp_shard_size > 0: diff --git a/vllm_omni/diffusion/worker/diffusion_worker.py b/vllm_omni/diffusion/worker/diffusion_worker.py index d9030e891..faa35002d 100644 --- a/vllm_omni/diffusion/worker/diffusion_worker.py +++ b/vllm_omni/diffusion/worker/diffusion_worker.py @@ -279,9 +279,8 @@ class DiffusionWorker: allgather_degree=parallel_config.allgather_degree, tensor_parallel_size=parallel_config.tensor_parallel_size, pipeline_parallel_size=parallel_config.pipeline_parallel_size, - fully_shard_degree=parallel_config.hsdp_shard_size if parallel_config.use_hsdp else 1, - hsdp_replicate_size=parallel_config.hsdp_replicate_size if parallel_config.use_hsdp else 1, enable_expert_parallel=parallel_config.enable_expert_parallel, + use_hsdp=parallel_config.use_hsdp, ) if ( getattr(self.od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY) diff --git a/vllm_omni/engine/async_omni_engine.py b/vllm_omni/engine/async_omni_engine.py index e35315f7e..d30db03cb 100644 --- a/vllm_omni/engine/async_omni_engine.py +++ b/vllm_omni/engine/async_omni_engine.py @@ -971,7 +971,7 @@ class AsyncOmniEngine: ulysses_mode = normalized_kwargs.get("ulysses_mode") or "strict" sequence_parallel_size = normalized_kwargs.get("sequence_parallel_size") pipeline_parallel_size = normalized_kwargs.get("pipeline_parallel_size") or 1 - data_parallel_size = normalized_kwargs.get("data_parallel_size") or 1 + data_parallel_size = normalized_kwargs.get("data_parallel_size") tensor_parallel_size = normalized_kwargs.get("tensor_parallel_size") or 1 cfg_parallel_size = normalized_kwargs.get("cfg_parallel_size") or 1 pipeline_parallel_size = normalized_kwargs.get("pipeline_parallel_size") or 1 @@ -1004,6 +1004,11 @@ class AsyncOmniEngine: hsdp_replicate_size=hsdp_replicate_size, ) + num_gpus = normalized_kwargs.get("num_gpus") + if num_gpus is not None: + num_gpus = int(num_gpus) + parallel_config.resolve_data_parallel_size(num_gpus) + num_devices = max(1, int(parallel_config.world_size)) devices = ",".join(str(i) for i in range(num_devices)) model_class_name = kwargs.get("model_class_name", None) @@ -1092,6 +1097,8 @@ class AsyncOmniEngine: else {} ), } + if num_gpus is not None: + stage_engine_args["num_gpus"] = num_gpus # Only set dtype if it was already explicitly passed and normalized if "dtype" in normalized_kwargs: stage_engine_args["dtype"] = normalized_kwargs["dtype"] From adf6d0c1ea33f74cf5e8ae7860cee8c9547c6bac Mon Sep 17 00:00:00 2001 From: BruceLoveDecimal <39156883+BruceLoveDecimal@users.noreply.github.com> Date: Wed, 19 Aug 2026 13:57:55 +0800 Subject: [PATCH 067/267] [Model] Add SANA-WM support(Stage-1 only) (#4061) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: BruceLoveDecimal Signed-off-by: 汪志鹏 Signed-off-by: liuqihao Co-authored-by: 汪志鹏 --- docs/models/supported_models.md | 1 + docs/user_guide/diffusion_features.md | 10 +- recipes/NVIDIA/SANA-WM.md | 193 ++ recipes/README.md | 1 + tests/diffusion/models/sana_wm/__init__.py | 0 .../models/sana_wm/test_cam_reduce_scatter.py | 129 + .../models/sana_wm/test_fused_qk_rms_norm.py | 111 + .../diffusion/models/sana_wm/test_request.py | 142 ++ tests/e2e/offline_inference/test_sana_wm.py | 159 ++ tests/e2e/online_serving/test_sana_wm.py | 106 + .../openai_api/test_video_server.py | 25 + tests/model_tests/diffusion/test_alignment.py | 1 + vllm_omni/diffusion/model_metadata.py | 4 + .../diffusion/models/sana_wm/__init__.py | 25 + .../models/sana_wm/camera_control.py | 356 +++ vllm_omni/diffusion/models/sana_wm/config.py | 93 + .../models/sana_wm/pipeline_sana_wm.py | 942 ++++++++ vllm_omni/diffusion/models/sana_wm/request.py | 269 +++ .../models/sana_wm/sana_wm_transformer.py | 2065 +++++++++++++++++ vllm_omni/diffusion/models/sana_wm/ucpe.py | 536 +++++ vllm_omni/diffusion/registry.py | 6 + 21 files changed, 5173 insertions(+), 1 deletion(-) create mode 100644 recipes/NVIDIA/SANA-WM.md create mode 100644 tests/diffusion/models/sana_wm/__init__.py create mode 100644 tests/diffusion/models/sana_wm/test_cam_reduce_scatter.py create mode 100644 tests/diffusion/models/sana_wm/test_fused_qk_rms_norm.py create mode 100644 tests/diffusion/models/sana_wm/test_request.py create mode 100644 tests/e2e/offline_inference/test_sana_wm.py create mode 100644 tests/e2e/online_serving/test_sana_wm.py create mode 100644 vllm_omni/diffusion/models/sana_wm/__init__.py create mode 100644 vllm_omni/diffusion/models/sana_wm/camera_control.py create mode 100644 vllm_omni/diffusion/models/sana_wm/config.py create mode 100644 vllm_omni/diffusion/models/sana_wm/pipeline_sana_wm.py create mode 100644 vllm_omni/diffusion/models/sana_wm/request.py create mode 100644 vllm_omni/diffusion/models/sana_wm/sana_wm_transformer.py create mode 100644 vllm_omni/diffusion/models/sana_wm/ucpe.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 31f3ac35d..58bca4809 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -39,6 +39,7 @@ th { | `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | | `LingBotWorldCausalDMDPipeline` | LingBot-World 2.0 v1 (experimental) | `robbyant/lingbot-world-v2-14b-causal-fast-diffusers` | Experimental | | | | — | | `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, V2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | +| `SanaWmPipeline` | SANA-WM | `BBBBruce/SANA-WM_bidirectional-stage1-diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-WM.md) | | `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | | `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | | — | diff --git a/docs/user_guide/diffusion_features.md b/docs/user_guide/diffusion_features.md index 5844d1114..34b642b9e 100644 --- a/docs/user_guide/diffusion_features.md +++ b/docs/user_guide/diffusion_features.md @@ -168,6 +168,14 @@ The following tables show which models support each feature: | **Cosmos3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (encode/decode) | ✅ | ❌ | | **LongCat-Video-Avatar-1.5** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | **MiniMax-H3** | ✅ (FL2VA) | ✅ | ✅ | ❌ | ✅ (DiT/TE) | ❌ | ✅ | ✅ | ✅ (tile) | ✅ (DiT) | ❌ | +| **SANA-WM** | ❌ | ❌ | ❌5 | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | + +> Notes: +> 5. SANA-WM cannot support sequence parallelism: its bidirectional gated delta +> recurrence carries state across frames, so a rank cannot denoise a slice of +> the token sequence in isolation. Doing so would need a distributed scan or +> an all-gather before every GDN block. The remaining ❌ columns are simply +> unvalidated on this model, not known-broken. > **Step execution note:** Helios supports single-request step execution only; > use `max_num_seqs=1`. @@ -175,7 +183,7 @@ The following tables show which models support each feature: **Frame Interpolation Support** - **Supported**: Wan2.2 text-to-video, image-to-video, and TI2V pipelines -- **Not supported**: Wan2.1-VACE, LTX-2, LTX-2.3, LTX-2.5, Helios, HunyuanVideo-1.5, DreamID-Omni +- **Not supported**: Wan2.1-VACE, LTX-2, LTX-2.3, LTX-2.5, Helios, HunyuanVideo-1.5, DreamID-Omni, SANA-WM ### AudioGen diff --git a/recipes/NVIDIA/SANA-WM.md b/recipes/NVIDIA/SANA-WM.md new file mode 100644 index 000000000..2926cc46f --- /dev/null +++ b/recipes/NVIDIA/SANA-WM.md @@ -0,0 +1,193 @@ +# SANA-WM + +> Camera-controllable first-frame image-to-video world model. + +## Summary + +- Vendor: Efficient-Large-Model / NVlabs SANA +- Model: `BBBBruce/SANA-WM_bidirectional-stage1-diffusers` (standard diffusers layout, converted offline from the NVlabs release; Stage-1 transformer + VAE only) +- Task: First-frame image-to-video generation with camera control +- Mode: Online serving with the OpenAI-compatible video API +- Model weights: about 13 GB for the Stage-1 transformer (10 GB) and VAE (2.3 GB) +- Local disk: reserve about 40 GB for the Hugging Face cache and runtime artifacts +- Recommended GPU: 24 GB or larger CUDA GPU +- Maintainer: Community + +## When to use this recipe + +Use this recipe when you want to serve SANA-WM through `/v1/videos` or +`/v1/videos/sync`. The model takes a text prompt, a first-frame image, and +either an action DSL string or explicit camera poses. vLLM-Omni serves the +SANA-WM Stage-1 DiT, decoded through the SANA VAE. The optional LTX-2 refiner +stage is not supported by this integration; it is a planned follow-up. + +## References + +- Upstream model card: +- Video API: [`docs/serving/videos_api.md`](../../docs/serving/videos_api.md) + +## Hardware Support + +## GPU + +### 1x NVIDIA RTX PRO 6000 Blackwell 96GB + +#### Capacity + +- Model storage: the Stage-1 transformer is about 10 GB and the VAE about + 2.3 GB. The Gemma text encoder is a separate 4.9 GB repo. The engine prefetches + the whole model repo at startup (`allow_patterns=["*"]`), which is why the + Stage-1 weights live in their own repo — the two-stage one carries an + additional 84 GB `refiner/` that this path never loads. +- Text encoder: the pipeline tries `google/gemma-2-2b-it` first, then falls back + to the ungated mirror `Efficient-Large-Model/gemma-2-2b-it`. The first repo is + gated, so without an accepted licence and `hf auth login` you will see one + failed load before the fallback succeeds — that is expected, not an error to + chase. Set `VLLM_OMNI_SANA_WM_STAGE1_TEXT_ENCODER` to pin a specific repo or a + local path and skip both. +- Disk sizing: provision about 40 GB of local disk or Hugging Face cache volume + so the model, temporary downloads, and generated artifacts fit without cache + eviction. +- GPU sizing: the default 1280x704, 161-frame, 60-step serving profile peaks at + 22.6 GB of device memory and takes about 133 s to generate on one RTX PRO + 6000 Blackwell. The peak lands in the VAE decode, which is why the pipeline + forces VAE tiling on regardless of `vae_use_tiling` — without it the same + request costs about 9 GB more, and a 321-frame one OOMs outright. On smaller + GPUs, lower `width`, `height`, or `num_frames` before serving production + requests. + +#### Environment + +- OS: Linux +- Python: 3.10+ +- Driver / runtime: NVIDIA driver with CUDA runtime supported by your PyTorch + build +- Recommended operator library: Triton, installed through the vLLM/vLLM-Omni + Python environment +- vLLM version: Match the repository requirements for your checkout +- vLLM-Omni version or commit: Use the commit you are deploying from + +#### Command + +The repo ships the standard Diffusers layout (`model_index.json` + +`transformer/`, `vae/`), and its `model_index.json` names `SanaWmPipeline`, so +the pipeline class resolves on its own. + +```bash +CUDA_VISIBLE_DEVICES=0 \ +vllm serve BBBBruce/SANA-WM_bidirectional-stage1-diffusers \ + --omni \ + --host 0.0.0.0 \ + --port 8091 +``` + +No deploy config: single-stage diffusion models are deliberately absent from +`OMNI_PIPELINES` (`vllm_omni/config/pipeline_registry.py`), so stage resolution +falls back to the default stage config and a YAML's stage settings — including +`default_sampling_params` — would not be applied. The production generation +settings therefore live in the model (`num_inference_steps=60`, +`guidance_scale=5.0`); a request that omits a field gets them. The examples +below still pass every field explicitly so the numbers are visible. + +If you point this at the older two-stage repo +(`BBBBruce/SANA-WM_bidirectional-diffusers`), startup fails with `Model class +SanaWmTwoStagesPipeline not found in diffusion model registry`, because that +repo's `model_index.json` names a class this build does not register. Add +`--model-class-name SanaWmPipeline` to override it. + +#### Verification + +Use a short smoke request first: + +```bash +curl -sS -X POST http://localhost:8091/v1/videos/sync \ + -H "Accept: video/mp4" \ + -F "prompt=A slow forward camera move through a quiet city street." \ + -F "negative_prompt=blurry, low quality, distorted, watermark" \ + -F "input_reference=@/path/to/first_frame.png;type=image/png" \ + -F "width=1280" \ + -F "height=704" \ + -F "num_frames=9" \ + -F "fps=16" \ + -F "num_inference_steps=2" \ + -F "guidance_scale=5.0" \ + -F "seed=42" \ + --form-string 'extra_params={"sana_wm":{"action":"w-8","translation_speed":0.055,"rotation_speed_deg":1.2,"intrinsics":{"fx":640,"fy":640,"cx":640,"cy":352}}}' \ + -o sana_wm_smoke.mp4 +``` + +For a production-length request, note that the action durations must sum to +`num_frames - 1` — the rollout includes the identity start pose — and a +mismatch is rejected rather than padded or truncated: + +```bash +curl -sS -X POST http://localhost:8091/v1/videos/sync \ + -H "Accept: video/mp4" \ + -F "prompt=A slow forward camera move through a quiet city street." \ + -F "negative_prompt=blurry, low quality, distorted, watermark" \ + -F "input_reference=@/path/to/first_frame.png;type=image/png" \ + -F "width=1280" \ + -F "height=704" \ + -F "num_frames=161" \ + -F "fps=16" \ + -F "num_inference_steps=60" \ + -F "guidance_scale=5.0" \ + -F "seed=42" \ + --form-string 'extra_params={"sana_wm":{"action":"w-160","translation_speed":0.055,"rotation_speed_deg":1.2,"intrinsics":{"fx":640,"fy":640,"cx":640,"cy":352}}}' \ + -o sana_wm_output.mp4 +``` + +Use `POST /v1/videos` instead when you want job storage and polling rather than +inline MP4 bytes. It accepts the same form fields as `/v1/videos/sync`. + +```bash +create_response=$(curl -sS -X POST http://localhost:8091/v1/videos \ + -H "Accept: application/json" \ + -F "prompt=A slow forward camera move through a quiet city street." \ + -F "negative_prompt=blurry, low quality, distorted, watermark" \ + -F "input_reference=@/path/to/first_frame.png;type=image/png" \ + -F "width=1280" \ + -F "height=704" \ + -F "num_frames=161" \ + -F "fps=16" \ + -F "num_inference_steps=60" \ + -F "guidance_scale=5.0" \ + -F "seed=42" \ + --form-string 'extra_params={"sana_wm":{"action":"w-160","translation_speed":0.055,"rotation_speed_deg":1.2,"intrinsics":{"fx":640,"fy":640,"cx":640,"cy":352}}}') + +video_id=$(echo "$create_response" | jq -r '.id') +curl -sS "http://localhost:8091/v1/videos/${video_id}" | jq . +curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o sana_wm_output.mp4 +``` + +#### Notes + +- Sequence parallelism is not supported. The bidirectional gated delta + recurrence carries state across frames, so a rank cannot denoise a slice + of the token sequence in isolation; supporting it needs a distributed scan + or an all-gather before the GDN blocks. + +- `input_reference` is required for the first frame. Use `image_reference` only + when you need a JSON-safe image URL or data URL instead of a multipart upload. +- `sana_wm` must provide exactly one of `action` or `camera`. +- Action strings use comma-separated `-` segments. Supported + keys are `w`, `a`, `s`, `d` for translation and `i`, `j`, `k`, `l` for + pitch/yaw rotation. The durations must sum to `num_frames - 1`. +- Explicit camera control (alternative to `action`): pass + `"camera": {"poses": [...]}` where `poses` is a list of `num_frames` + camera-to-world 4x4 matrices (row-major, OpenCV `+X right, +Y down, +Z forward` + convention), e.g. + `extra_params={"sana_wm":{"camera":{"poses":[[[1,0,0,0],[0,1,0,0],[0,0,1,0],[0,0,0,1]], ...]},"intrinsics":{...}}}`. + Most callers should prefer `action`; explicit poses exist for callers that + already have a per-frame trajectory. +- Explicit `intrinsics` are recommended and take the mapping form + `{"fx":640,"fy":640,"cx":640,"cy":352}` (for 1280x704). This `{fx,fy,cx,cy}` + mapping is the only accepted intrinsics form; omit `intrinsics` to derive them + from the output resolution. All four values must be finite, and `fx`/`fy` + must be positive — the ray map divides by them. +- The video API returns decoded MP4 bytes and has no `output_type` field, so + raw Stage-1 latents are reachable only from the offline API + (`OmniDiffusionSamplingParams(output_type="latent")`); see + [`tests/e2e/offline_inference/test_sana_wm.py`](../../tests/e2e/offline_inference/test_sana_wm.py). + Putting `output_type` in `extra_params` does not work: it lands in + `sampling_params.extra_args`, while the pipeline reads the top-level field. diff --git a/recipes/README.md b/recipes/README.md index 32c68e8c1..8b93124d0 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -70,6 +70,7 @@ recipes/ | [`OpenMOSS/MOSS-TTS.md`](./OpenMOSS/MOSS-TTS.md) | Online + offline multilingual TTS (MOSS-TTS family, 8B) | 1x H100 80GB | | [`NVIDIA/Nemotron-Labs-Audex.md`](./NVIDIA/Nemotron-Labs-Audex.md) | TTS / text-to-audio / audio understanding / cascaded S2S (Audex 2B + 30B-A3B) | 1x H100 80GB | | [`NVIDIA/NemotronLabs-VoiceChat.md`](./NVIDIA/NemotronLabs-VoiceChat.md) | Offline speech-to-speech voice chat (11B, frame-locked 12.5 Hz, 3-stage thinker/talker/code2wav) | 1x H100 80GB | +| [`NVIDIA/SANA-WM.md`](./NVIDIA/SANA-WM.md) | First-frame image-to-video serving with camera control | 1x 24GB+ CUDA GPU; ~40GB disk | | [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt mixed FP8/NVFP4 | 1x A100 80GB / 2x B200 | | [`Qwen/Qwen-Image-2512.md`](./Qwen/Qwen-Image-2512.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt FP8 / mixed FP8/NVFP4 | 1x A800 80GB / 2x B200 | | [`Qwen/Qwen-Image-Edit.md`](./Qwen/Qwen-Image-Edit.md) | Text-guided single-image editing | 1x or 2x H200 141GB | diff --git a/tests/diffusion/models/sana_wm/__init__.py b/tests/diffusion/models/sana_wm/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/diffusion/models/sana_wm/test_cam_reduce_scatter.py b/tests/diffusion/models/sana_wm/test_cam_reduce_scatter.py new file mode 100644 index 000000000..cea8aeb76 --- /dev/null +++ b/tests/diffusion/models/sana_wm/test_cam_reduce_scatter.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Equivalence tests for SANA-WM's camera-branch reduce-scatter. + +``out_proj_cam`` is row-parallel, so every rank produces a partial sum over the +full hidden width, while the GDN stream it is added to is TP-local. Instead of +all-reducing and then discarding all but one slice, the branch reduce-scatters +straight to the local shard. These tests pin the shard each rank ends up with, +including the bias slice, on both code paths: + +* the process-group path, which drives ``torch.distributed.reduce_scatter_tensor`` + directly because vLLM-Omni's diffusion ``GroupCoordinator`` does not implement + ``reduce_scatter``; +* the all-reduce-and-slice fallback taken when the coordinator exposes no + ``device_group``. + +CPU-only: the collectives and the rank/world-size lookups are mocked, so a +single process can simulate an arbitrary TP degree. +""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from vllm_omni.diffusion.models.sana_wm.sana_wm_transformer import SanaWmSelfAttention + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_MODULE = "vllm_omni.diffusion.models.sana_wm.sana_wm_transformer" + +HIDDEN = 24 + + +def _host(bias: torch.Tensor | None) -> SimpleNamespace: + """A stand-in ``self`` exposing only what the method reads.""" + return SimpleNamespace(out_proj_cam=SimpleNamespace(bias=bias)) + + +def _fake_reduce_scatter_tensor(rank: int, tp_size: int): + """Model ``reduce_scatter_tensor``: split dim 0 and keep chunk ``rank``. + + The test drives a single process, so the other ranks' partials are already + folded into the input; scattering is all this has to reproduce. + """ + + def _impl(output: torch.Tensor, input_: torch.Tensor, group=None) -> None: + output.copy_(input_.chunk(tp_size, dim=0)[rank]) + + return _impl + + +def _tp_group(with_device_group: bool): + return SimpleNamespace(device_group=object() if with_device_group else None) + + +@pytest.mark.parametrize("tp_size", [2, 4]) +@pytest.mark.parametrize("rank", [0, 1]) +def test_process_group_path_matches_all_reduce_then_slice(tp_size: int, rank: int) -> None: + """The scattered shard equals the slice the all-reduce path would take.""" + local_width = HIDDEN // tp_size + torch.manual_seed(0) + contrib = torch.randn(2, 3, HIDDEN) + bias = torch.randn(HIDDEN) + reference = torch.zeros(2, 3, local_width) + + with ( + patch(f"{_MODULE}.get_tensor_model_parallel_world_size", return_value=tp_size), + patch(f"{_MODULE}.get_tensor_model_parallel_rank", return_value=rank), + patch(f"{_MODULE}.get_tp_group", return_value=_tp_group(True)), + patch("torch.distributed.reduce_scatter_tensor", side_effect=_fake_reduce_scatter_tensor(rank, tp_size)), + ): + out = SanaWmSelfAttention._reduce_scatter_cam_contrib(_host(bias), contrib, reference) + + start = rank * local_width + expected = (contrib + bias)[..., start : start + local_width] + assert out.shape == (2, 3, local_width) + torch.testing.assert_close(out, expected) + + +@pytest.mark.parametrize("rank", [0, 1]) +def test_fallback_path_matches_the_process_group_path(rank: int) -> None: + """Without a device_group the method all-reduces and slices, same result.""" + tp_size = 2 + local_width = HIDDEN // tp_size + torch.manual_seed(1) + contrib = torch.randn(2, 3, HIDDEN) + bias = torch.randn(HIDDEN) + reference = torch.zeros(2, 3, local_width) + + with ( + patch(f"{_MODULE}.get_tensor_model_parallel_world_size", return_value=tp_size), + patch(f"{_MODULE}.get_tensor_model_parallel_rank", return_value=rank), + patch(f"{_MODULE}.get_tp_group", return_value=_tp_group(False)), + patch(f"{_MODULE}.tensor_model_parallel_all_reduce", side_effect=lambda t: t), + ): + out = SanaWmSelfAttention._reduce_scatter_cam_contrib(_host(bias), contrib, reference) + + start = rank * local_width + torch.testing.assert_close(out, (contrib + bias)[..., start : start + local_width]) + + +def test_tp1_adds_bias_without_a_collective() -> None: + contrib = torch.randn(2, 3, HIDDEN) + bias = torch.randn(HIDDEN) + + with ( + patch(f"{_MODULE}.get_tensor_model_parallel_world_size", return_value=1), + patch("torch.distributed.reduce_scatter_tensor") as collective, + ): + out = SanaWmSelfAttention._reduce_scatter_cam_contrib(_host(bias), contrib, contrib) + + assert collective.call_count == 0 + torch.testing.assert_close(out, contrib + bias) + + +def test_rejects_a_width_that_does_not_shard() -> None: + """A local width that is not 1/tp of the contribution is a wiring bug.""" + contrib = torch.randn(1, 2, HIDDEN) + reference = torch.zeros(1, 2, HIDDEN // 3) # tp=2 would need HIDDEN // 2 + + with ( + patch(f"{_MODULE}.get_tensor_model_parallel_world_size", return_value=2), + patch("torch.distributed.reduce_scatter_tensor"), + pytest.raises(ValueError, match="camera contribution width mismatch"), + ): + SanaWmSelfAttention._reduce_scatter_cam_contrib(_host(None), contrib, reference) diff --git a/tests/diffusion/models/sana_wm/test_fused_qk_rms_norm.py b/tests/diffusion/models/sana_wm/test_fused_qk_rms_norm.py new file mode 100644 index 000000000..b01ef02ce --- /dev/null +++ b/tests/diffusion/models/sana_wm/test_fused_qk_rms_norm.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Equivalence tests for SANA-WM's TensorParallelRMSNorm q/k fusion. + +`fused_qk_rms_norm` packs the q and k per-token sum-of-squares into one tensor +and issues a single tensor-parallel all-reduce instead of two. These tests lock +in that it is numerically identical to applying `norm_q`/`norm_k` separately, +and that it really issues exactly one (fused) all-reduce. + +CPU-only: the collective is mocked, so a single process simulates one rank +(all-reduce is identity). This exercises the cat/split logic a real multi-rank +run depends on. +""" + +from unittest.mock import patch + +import pytest +import torch +from torch import nn + +from vllm_omni.diffusion.layers.norm import RMSNorm +from vllm_omni.diffusion.models.sana_wm.sana_wm_transformer import ( + TensorParallelRMSNorm, + fused_qk_rms_norm, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_MODULE = "vllm_omni.diffusion.models.sana_wm.sana_wm_transformer" + + +def _make_norms(dim: int, tp_size: int) -> tuple[TensorParallelRMSNorm, TensorParallelRMSNorm]: + """Two norms with distinct random weights so a swapped q/k slice is caught.""" + torch.manual_seed(0) + norm_q = TensorParallelRMSNorm(dim, tp_size=tp_size) + norm_k = TensorParallelRMSNorm(dim, tp_size=tp_size) + norm_q.weight.data.normal_() + norm_k.weight.data.normal_() + return norm_q, norm_k + + +def _identity_all_reduce(tensor: torch.Tensor) -> torch.Tensor: + """Single-rank all-reduce: the reduction over one rank is the identity.""" + return tensor + + +def test_fused_matches_separate() -> None: + dim = 64 + norm_q, norm_k = _make_norms(dim, tp_size=2) + q = torch.randn(2, 5, dim) + k = torch.randn(2, 5, dim) + + with patch(f"{_MODULE}.tensor_model_parallel_all_reduce", side_effect=_identity_all_reduce): + ref_q, ref_k = norm_q(q), norm_k(k) + fused_q, fused_k = fused_qk_rms_norm(norm_q, norm_k, q, k) + + torch.testing.assert_close(fused_q, ref_q, rtol=0, atol=0) + torch.testing.assert_close(fused_k, ref_k, rtol=0, atol=0) + + +def test_fused_issues_one_all_reduce() -> None: + dim = 32 + norm_q, norm_k = _make_norms(dim, tp_size=4) + q = torch.randn(1, 3, dim) + k = torch.randn(1, 3, dim) + + with patch(f"{_MODULE}.tensor_model_parallel_all_reduce", side_effect=_identity_all_reduce) as collective: + fused_qk_rms_norm(norm_q, norm_k, q, k) + assert collective.call_count == 1 + # Both sums travel in one tensor: last dim is 2, not 1. + assert collective.call_args.args[0].shape[-1] == 2 + + collective.reset_mock() + norm_q(q), norm_k(k) + assert collective.call_count == 2 + + +@pytest.mark.parametrize( + "norms", + [ + (RMSNorm(16), RMSNorm(16)), + (nn.Identity(), nn.Identity()), + ], + ids=["plain_rms_norm", "identity"], +) +def test_falls_back_for_non_tp_norms(norms: tuple[nn.Module, nn.Module]) -> None: + """qk_norm=False and TP=1 both hand in norms the fused path cannot use.""" + norm_q, norm_k = norms + q = torch.randn(2, 4, 16) + k = torch.randn(2, 4, 16) + + with patch(f"{_MODULE}.tensor_model_parallel_all_reduce", side_effect=_identity_all_reduce) as collective: + fused_q, fused_k = fused_qk_rms_norm(norm_q, norm_k, q, k) + assert collective.call_count == 0 + + torch.testing.assert_close(fused_q, norm_q(q), rtol=0, atol=0) + torch.testing.assert_close(fused_k, norm_k(k), rtol=0, atol=0) + + +def test_tp1_skips_the_collective() -> None: + norm_q, norm_k = _make_norms(16, tp_size=1) + q = torch.randn(2, 4, 16) + k = torch.randn(2, 4, 16) + + with patch(f"{_MODULE}.tensor_model_parallel_all_reduce", side_effect=_identity_all_reduce) as collective: + fused_q, fused_k = fused_qk_rms_norm(norm_q, norm_k, q, k) + assert collective.call_count == 0 + + torch.testing.assert_close(fused_q, norm_q(q), rtol=0, atol=0) + torch.testing.assert_close(fused_k, norm_k(k), rtol=0, atol=0) diff --git a/tests/diffusion/models/sana_wm/test_request.py b/tests/diffusion/models/sana_wm/test_request.py new file mode 100644 index 000000000..7e40eedaf --- /dev/null +++ b/tests/diffusion/models/sana_wm/test_request.py @@ -0,0 +1,142 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for SANA-WM request normalization.""" + +import pytest + +from vllm_omni.diffusion.models.sana_wm.request import ( + SANA_WM_DEFAULT_HEIGHT, + SANA_WM_DEFAULT_NUM_FRAMES, + SANA_WM_DEFAULT_WIDTH, + normalize_sana_wm_payload, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _prompt(**sana_wm): + """A valid request, with the action rollout kept in step with num_frames.""" + block = {"num_frames": SANA_WM_DEFAULT_NUM_FRAMES, **sana_wm} + block.setdefault("action", f"w-{block['num_frames'] - 1}") + return { + "prompt": "a quiet city street", + "multi_modal_data": {"image": object()}, + "sana_wm": block, + } + + +def _identity_poses(count): + return [[[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]] for _ in range(count)] + + +def _camera_prompt(**camera): + prompt = _prompt(num_frames=9) + prompt["sana_wm"].pop("action") + prompt["sana_wm"]["camera"] = {"poses": _identity_poses(9), **camera} + return prompt + + +def test_defaults_are_latent_aligned(): + payload = normalize_sana_wm_payload(_prompt())["additional_information"]["sana_wm"] + assert payload["num_frames"] == SANA_WM_DEFAULT_NUM_FRAMES + assert payload["height"] == SANA_WM_DEFAULT_HEIGHT + assert payload["width"] == SANA_WM_DEFAULT_WIDTH + + +@pytest.mark.parametrize("num_frames", [9, 33, 161]) +def test_accepts_aligned_geometry(num_frames): + payload = normalize_sana_wm_payload(_prompt(num_frames=num_frames))["additional_information"]["sana_wm"] + assert payload["num_frames"] == num_frames + + +@pytest.mark.parametrize("num_frames", [8, 10, 160]) +def test_rejects_unaligned_num_frames(num_frames): + # The VAE compresses 8x temporally; an unaligned count would be floored. + with pytest.raises(ValueError, match="num_frames"): + normalize_sana_wm_payload(_prompt(num_frames=num_frames)) + + +@pytest.mark.parametrize(("height", "width"), [(700, 1280), (704, 1270), (703, 1279)]) +def test_rejects_unaligned_resolution(height, width): + with pytest.raises(ValueError, match="divisible by 32"): + normalize_sana_wm_payload(_prompt(height=height, width=width)) + + +def test_normalization_is_idempotent(): + once = normalize_sana_wm_payload(_prompt(num_frames=33)) + twice = normalize_sana_wm_payload(once) + assert once["additional_information"]["sana_wm"] == twice["additional_information"]["sana_wm"] + + +def test_rejects_action_shorter_than_num_frames(): + # Used to shrink the camera tensor silently, then fail deep in the + # transformer on a camera/latent frame mismatch. + with pytest.raises(ValueError, match="rolls out 33 frames but num_frames is 161"): + normalize_sana_wm_payload(_prompt(action="w-32")) + + +def test_rejects_action_longer_than_num_frames(): + with pytest.raises(ValueError, match="rolls out 161 frames but num_frames is 9"): + normalize_sana_wm_payload(_prompt(num_frames=9, action="w-160")) + + +def test_rejects_oversized_action_without_expanding_it(): + # A 12-byte request body: expanding this rollout before checking the frame + # count allocated ~16 GB. Rejection has to be arithmetic, so this test also + # fails (by hanging/OOM) if the duration is ever expanded again. + with pytest.raises(ValueError, match="rolls out 1000000001 frames"): + normalize_sana_wm_payload(_prompt(action="w-1000000000")) + + +@pytest.mark.parametrize("action", ["", " ", "w-0", "w-abc", "w", "x-4"]) +def test_rejects_malformed_action(action): + with pytest.raises(ValueError): + normalize_sana_wm_payload(_prompt(action=action)) + + +@pytest.mark.parametrize("value", [0, -1, float("nan"), float("inf")]) +def test_rejects_unusable_focal_length(value): + # compute_raymap divides by fx/fy; a zero or non-finite focal length turns + # the whole Plucker map into NaN and the response into a NaN video. + intrinsics = {"fx": value, "fy": 640, "cx": 640, "cy": 352} + with pytest.raises(ValueError, match="fx"): + normalize_sana_wm_payload(_prompt(intrinsics=intrinsics)) + + +@pytest.mark.parametrize("value", [float("nan"), float("inf")]) +def test_rejects_non_finite_principal_point(value): + intrinsics = {"fx": 640, "fy": 640, "cx": value, "cy": 352} + with pytest.raises(ValueError, match="must be finite"): + normalize_sana_wm_payload(_prompt(intrinsics=intrinsics)) + + +@pytest.mark.parametrize("value", [float("nan"), float("inf")]) +def test_rejects_non_finite_speeds(value): + # `parsed <= 0` is False for NaN, so the positivity check alone let it pass. + with pytest.raises(ValueError, match="must be finite"): + normalize_sana_wm_payload(_prompt(translation_speed=value)) + + +def test_rejects_non_finite_pose_entry(): + prompt = _camera_prompt() + prompt["sana_wm"]["camera"]["poses"][3][2][1] = float("nan") + with pytest.raises(ValueError, match="must be finite"): + normalize_sana_wm_payload(prompt) + + +def test_accepts_default_camera_metadata(): + payload = normalize_sana_wm_payload(_camera_prompt())["additional_information"]["sana_wm"] + assert payload["camera"]["poses"] + # The unread fields are not carried into the canonical payload. + assert set(payload["camera"]) == {"poses"} + + +@pytest.mark.parametrize( + ("field", "value"), + [("format", "w2c_4x4"), ("coordinate_system", "opengl")], +) +def test_rejects_unsupported_camera_convention(field, value): + # Nothing downstream reads these, so accepting one would mean interpreting + # the poses under the wrong convention. + with pytest.raises(ValueError, match=field.split("_")[0]): + normalize_sana_wm_payload(_camera_prompt(**{field: value})) diff --git a/tests/e2e/offline_inference/test_sana_wm.py b/tests/e2e/offline_inference/test_sana_wm.py new file mode 100644 index 000000000..8b1c460c2 --- /dev/null +++ b/tests/e2e/offline_inference/test_sana_wm.py @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import os +from typing import Any + +import numpy as np +import pytest + +pytestmark = [ + pytest.mark.advanced_model, + pytest.mark.diffusion, + pytest.mark.gpu, +] + + +def _make_sana_wm_e2e_image() -> Any: + from PIL import Image + + from vllm_omni.diffusion.models.sana_wm import SANA_WM_OUTPUT_HEIGHT, SANA_WM_OUTPUT_WIDTH + + return Image.new("RGB", (SANA_WM_OUTPUT_WIDTH, SANA_WM_OUTPUT_HEIGHT), (96, 128, 160)) + + +def _sana_wm_e2e_tensor_parallel_size() -> int: + return int( + os.environ.get( + "SANA_WM_E2E_TENSOR_PARALLEL_SIZE", + os.environ.get("SANA_WM_E2E_TP", "1"), + ) + ) + + +def _coerce_video_array(video: Any) -> np.ndarray: + import torch + + if isinstance(video, list): + assert video, "SANA-WM e2e produced an empty video list." + video = video[0] + if isinstance(video, torch.Tensor): + video = video.detach().cpu().float().numpy() + video_array = np.asarray(video) + if video_array.ndim == 5: + assert video_array.shape[0] == 1 + video_array = video_array[0] + assert video_array.ndim == 4 + return video_array + + +def _assert_sana_wm_e2e_shape( + video: np.ndarray, + *, + output_type: str, + num_frames: int, +) -> None: + if output_type == "latent": + # Latent output is channel-first: (C, T, H, W) after removing the + # batch dimension. Stage-1 emits LTX-2 latents with 128 channels. + assert video.shape[0] == 128 + assert 0 < video.shape[1] <= num_frames + return + assert 0 < video.shape[0] <= num_frames + + +def _run_sana_wm_e2e(*, output_type: str) -> np.ndarray: + import torch + + from vllm_omni.diffusion.models.sana_wm import ( + SANA_WM_MODEL_ID, + SANA_WM_OUTPUT_HEIGHT, + SANA_WM_OUTPUT_WIDTH, + ) + from vllm_omni.entrypoints.omni import Omni + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + from vllm_omni.outputs import OmniRequestOutput + + if not torch.cuda.is_available(): + pytest.skip("Sana-WM e2e requires CUDA.") + + model = os.environ.get("SANA_WM_E2E_MODEL", SANA_WM_MODEL_ID) + model_class_name = os.environ.get("SANA_WM_E2E_MODEL_CLASS", "SanaWmPipeline") + num_frames = int(os.environ.get("SANA_WM_E2E_NUM_FRAMES", "9")) + stage1_steps = int( + os.environ.get( + "SANA_WM_E2E_STAGE1_STEPS", + os.environ.get("SANA_WM_E2E_NUM_INFERENCE_STEPS", "1"), + ) + ) + action = os.environ.get("SANA_WM_E2E_ACTION", f"w-{max(num_frames - 1, 1)}") + tensor_parallel_size = _sana_wm_e2e_tensor_parallel_size() + image = _make_sana_wm_e2e_image() + omni = Omni( + model=model, + model_class_name=model_class_name, + enforce_eager=True, + tensor_parallel_size=tensor_parallel_size, + ) + extra_args: dict[str, Any] = {} + # The native Stage-1 path is latent-token capped; raise it for full-size + # e2e runs (704x1280 exceeds the default 4096-token cap). + native_max_tokens = os.environ.get("SANA_WM_E2E_NATIVE_MAX_TOKENS", "").strip() + if native_max_tokens: + extra_args["sana_wm_native_max_tokens"] = int(native_max_tokens) + # Return the requested output type so the shape assertion matches the + # pipeline output instead of the raw Stage-1 latent default. + + output = omni.generate( + { + "prompt": "A slow forward camera move through a quiet city street.", + "multi_modal_data": {"image": image}, + "sana_wm": { + "action": action, + "num_frames": num_frames, + "translation_speed": 0.055, + "rotation_speed_deg": 1.2, + # Avoid the optional Pi3X dependency by passing deterministic + # camera intrinsics directly. + "intrinsics": { + "fx": SANA_WM_OUTPUT_WIDTH / 2, + "fy": SANA_WM_OUTPUT_WIDTH / 2, + "cx": SANA_WM_OUTPUT_WIDTH / 2, + "cy": SANA_WM_OUTPUT_HEIGHT / 2, + }, + }, + }, + OmniDiffusionSamplingParams( + height=SANA_WM_OUTPUT_HEIGHT, + width=SANA_WM_OUTPUT_WIDTH, + num_frames=num_frames, + seed=0, + fps=16, + num_inference_steps=stage1_steps, + guidance_scale=1.0, + guidance_scale_provided=True, + output_type=output_type, + extra_args=extra_args, + ), + ) + + request_output = output[0] if isinstance(output, list) else output + assert isinstance(request_output, OmniRequestOutput) + assert request_output.error is None + assert request_output.images + + frames = request_output.images[0] + video = _coerce_video_array(frames) + _assert_sana_wm_e2e_shape(video, output_type=output_type, num_frames=num_frames) + return video + + +def test_sana_wm_native_generates_video() -> None: + output_type = os.environ.get("SANA_WM_E2E_OUTPUT_TYPE", "np") + video = _run_sana_wm_e2e(output_type=output_type) + if output_type == "latent": + assert video.ndim == 4 + else: + assert video.shape[-1] in (3, 4) diff --git a/tests/e2e/online_serving/test_sana_wm.py b/tests/e2e/online_serving/test_sana_wm.py new file mode 100644 index 000000000..883e4c367 --- /dev/null +++ b/tests/e2e/online_serving/test_sana_wm.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +Online serving smoke for SANA-WM image-to-video (first-frame I2V via ``/v1/videos``). + +Boots the server straight from the model id — the Stage-1 repo's +``model_index.json`` names ``SanaWmPipeline``, so the class resolves on its own +— submits one async ``/v1/videos`` job with a first-frame reference image, and +asserts the job completes and returns video bytes. + +From ``tests/``:: + + pytest -s -v e2e/online_serving/test_sana_wm.py -m "advanced_model and diffusion" --run-level=advanced_model +""" + +import base64 +import json +import os +from io import BytesIO + +import pytest + +from tests.helpers.mark import hardware_marks +from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler +from vllm_omni.diffusion.models.sana_wm import ( + SANA_WM_MODEL_ID, + SANA_WM_OUTPUT_HEIGHT, + SANA_WM_OUTPUT_WIDTH, +) + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +MODEL = SANA_WM_MODEL_ID +PROMPT = "A slow forward camera move through a quiet city street." +NEGATIVE_PROMPT = "blurry, low quality, distorted, watermark" + +# Smoke knobs: a handful of frames / steps so the job finishes quickly. +SMOKE_NUM_FRAMES = 9 +SMOKE_NUM_INFERENCE_STEPS = 2 +# First-frame I2V needs a camera trajectory; "w-" is the forward-move action DSL. +# Passed via the ``sana_wm`` form field (a JSON string parsed server-side); explicit +# intrinsics avoid the optional Pi3X camera-calibration dependency. +SANA_WM_PARAMS = { + "action": f"w-{SMOKE_NUM_FRAMES - 1}", + "translation_speed": 0.055, + "rotation_speed_deg": 1.2, + "intrinsics": { + "fx": SANA_WM_OUTPUT_WIDTH / 2, + "fy": SANA_WM_OUTPUT_WIDTH / 2, + "cx": SANA_WM_OUTPUT_WIDTH / 2, + "cy": SANA_WM_OUTPUT_HEIGHT / 2, + }, +} + +SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) + + +def _first_frame_data_url() -> str: + """A solid-color first frame encoded as a PNG ``data:`` URL for I2V.""" + from PIL import Image + + image = Image.new("RGB", (SANA_WM_OUTPUT_WIDTH, SANA_WM_OUTPUT_HEIGHT), (96, 128, 160)) + buffer = BytesIO() + image.save(buffer, format="PNG") + encoded = base64.b64encode(buffer.getvalue()).decode("ascii") + return f"data:image/png;base64,{encoded}" + + +def _get_diffusion_feature_cases(model: str): + """Single default server row.""" + return [ + pytest.param( + OmniServerParams(model=model), + id="default", + marks=SINGLE_CARD_FEATURE_MARKS, + ), + ] + + +@pytest.mark.advanced_model +@pytest.mark.diffusion +@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) +def test_image_to_video_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: + """Default SANA-WM I2V smoke: async ``/v1/videos`` job completes and returns video bytes.""" + request_config = { + "model": omni_server.model, + "image_reference": _first_frame_data_url(), + "form_data": { + "prompt": PROMPT, + "negative_prompt": NEGATIVE_PROMPT, + # The camera payload rides inside extra_params; there is no + # top-level sana_wm form field on the video endpoints, so sending + # one is silently dropped and the request fails in the + # preprocessor for missing camera control. + "extra_params": json.dumps({"sana_wm": SANA_WM_PARAMS}), + "height": SANA_WM_OUTPUT_HEIGHT, + "width": SANA_WM_OUTPUT_WIDTH, + "num_frames": SMOKE_NUM_FRAMES, + "fps": 16, + "num_inference_steps": SMOKE_NUM_INFERENCE_STEPS, + "guidance_scale": 5.0, + "seed": 42, + }, + } + openai_client.send_video_diffusion_request(request_config) diff --git a/tests/entrypoints/openai_api/test_video_server.py b/tests/entrypoints/openai_api/test_video_server.py index b9b7df4df..a83edfcb9 100644 --- a/tests/entrypoints/openai_api/test_video_server.py +++ b/tests/entrypoints/openai_api/test_video_server.py @@ -2168,6 +2168,31 @@ def test_sync_sampling_params_pass_through(test_client, mocker: MockerFixture): assert captured.quality == "high" +def test_sync_sana_wm_extra_params_payload_passes_to_engine_prompt(test_client, mocker: MockerFixture): + _mock_encode_video_bytes(mocker) + response = test_client.post( + "/v1/videos/sync", + data={ + "prompt": "drive forward", + "extra_params": json.dumps( + { + "sana_wm": {"action": "d-4", "rotation_speed_deg": 1.5}, + "sana_wm_native_max_tokens": 30000, + } + ), + }, + ) + + assert response.status_code == 200 + engine = test_client.app.state.openai_serving_video._engine_client + captured_params = engine.captured_sampling_params_list[0] + # The camera block reaches the model through extra_args; the Sana preprocess + # hook is what lifts it onto the prompt. + assert captured_params.extra_args["sana_wm"]["action"] == "d-4" + assert captured_params.extra_args["sana_wm"]["rotation_speed_deg"] == 1.5 + assert captured_params.extra_args["sana_wm_native_max_tokens"] == 30000 + + def test_sync_frame_interpolation_params_pass_to_sampling_params(test_client, mocker: MockerFixture): """Frame interpolation parameters should be forwarded on the sync path.""" encode_mock = _mock_encode_video_bytes(mocker) diff --git a/tests/model_tests/diffusion/test_alignment.py b/tests/model_tests/diffusion/test_alignment.py index 7faf175f8..737eba753 100644 --- a/tests/model_tests/diffusion/test_alignment.py +++ b/tests/model_tests/diffusion/test_alignment.py @@ -80,6 +80,7 @@ EXCLUDED_MODELS = [ "Pi0Pipeline", "SoulXSingerPipeline", "SoulXSingerSVCPipeline", + "SanaWmPipeline", ] diff --git a/vllm_omni/diffusion/model_metadata.py b/vllm_omni/diffusion/model_metadata.py index ecc53cdc6..e11a7541e 100644 --- a/vllm_omni/diffusion/model_metadata.py +++ b/vllm_omni/diffusion/model_metadata.py @@ -56,6 +56,10 @@ _DIFFUSION_MODEL_METADATA: dict[str, DiffusionModelMetadata] = { "WanImageToVideoPipeline": DiffusionModelMetadata(attention_mask_free=True), "WanVACEPipeline": DiffusionModelMetadata(attention_mask_free=True), "WanS2VPipeline": DiffusionModelMetadata(attention_mask_free=True), + "SanaWmPipeline": DiffusionModelMetadata( + supports_multimodal_inputs=True, + max_multimodal_image_inputs=1, + ), } diff --git a/vllm_omni/diffusion/models/sana_wm/__init__.py b/vllm_omni/diffusion/models/sana_wm/__init__.py new file mode 100644 index 000000000..f6f776194 --- /dev/null +++ b/vllm_omni/diffusion/models/sana_wm/__init__.py @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""SANA-WM diffusion model integration.""" + +from vllm_omni.diffusion.models.sana_wm.config import SanaWmConfig +from vllm_omni.diffusion.models.sana_wm.pipeline_sana_wm import ( + SANA_WM_MODEL_ID, + SANA_WM_OUTPUT_HEIGHT, + SANA_WM_OUTPUT_WIDTH, + SanaWmPipeline, + get_sana_wm_pre_process_func, +) +from vllm_omni.diffusion.models.sana_wm.request import normalize_sana_wm_payload +from vllm_omni.diffusion.models.sana_wm.sana_wm_transformer import SanaWmTransformer3DModel + +__all__ = [ + "SANA_WM_MODEL_ID", + "SANA_WM_OUTPUT_HEIGHT", + "SANA_WM_OUTPUT_WIDTH", + "SanaWmConfig", + "SanaWmPipeline", + "SanaWmTransformer3DModel", + "get_sana_wm_pre_process_func", + "normalize_sana_wm_payload", +] diff --git a/vllm_omni/diffusion/models/sana_wm/camera_control.py b/vllm_omni/diffusion/models/sana_wm/camera_control.py new file mode 100644 index 000000000..914fbc0d3 --- /dev/null +++ b/vllm_omni/diffusion/models/sana_wm/camera_control.py @@ -0,0 +1,356 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Camera-control helpers for SANA-WM. + +This module mirrors the public NVlabs/Sana SANA-WM camera preparation path: +WASD/IJKL action rollout or explicit camera-to-world poses are converted into +relative poses, per-latent-frame ray metadata, and per-VAE-chunk Plucker maps. +Model-local projection layers consume these tensors in the Stage-1 DiT. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Any + +import numpy as np +import torch + +SANA_WM_DEFAULT_TRANSLATION_SPEED = 0.05 +SANA_WM_DEFAULT_ROTATION_SPEED_DEG = 1.2 +SANA_WM_DEFAULT_PITCH_LIMIT_DEG = 85.0 +SANA_WM_ALLOWED_ACTION_KEYS = frozenset("wasdijkl") +SANA_WM_DEFAULT_VAE_STRIDE = (8, 32, 32) + + +@dataclass(frozen=True) +class SanaWmCameraCondition: + poses: Any | None = None + intrinsics: Any | None = None + action: str | None = None + num_frames: int | None = None + height: int = 704 + width: int = 1280 + translation_speed: float = SANA_WM_DEFAULT_TRANSLATION_SPEED + rotation_speed_deg: float = SANA_WM_DEFAULT_ROTATION_SPEED_DEG + pitch_limit_deg: float = SANA_WM_DEFAULT_PITCH_LIMIT_DEG + + +def _rot_x(angle_rad: float) -> np.ndarray: + c, s = np.cos(angle_rad), np.sin(angle_rad) + return np.array([[1.0, 0.0, 0.0], [0.0, c, -s], [0.0, s, c]], dtype=np.float64) + + +def _rot_y(angle_rad: float) -> np.ndarray: + c, s = np.cos(angle_rad), np.sin(angle_rad) + return np.array([[c, 0.0, s], [0.0, 1.0, 0.0], [-s, 0.0, c]], dtype=np.float64) + + +def _parse_action_segments(action: str) -> list[tuple[list[str], int]]: + """Parse an action string into ``(keys, duration)`` pairs without expanding it. + + A single segment names an unbounded rollout (``"w-1000000000"`` is valid + syntax), so expanding here would allocate one list entry per frame before + anything checks the frame count. Callers total the durations and validate + against ``num_frames`` first; only :func:`_parse_action_string` expands. + """ + cleaned = "".join(action.replace("，", ",").split()) + if not cleaned: + raise ValueError("Sana-WM action string is empty.") + + segments: list[tuple[list[str], int]] = [] + for segment in cleaned.split(","): + if not segment or "-" not in segment: + raise ValueError(f"Invalid Sana-WM action segment {segment!r}: expected '-'.") + keys_part, duration = segment.rsplit("-", 1) + if not duration.isdigit() or int(duration) <= 0: + raise ValueError(f"Sana-WM action segment {segment!r} has a non-positive duration {duration!r}.") + keys_lower = keys_part.lower() + if keys_lower == "none": + keys: list[str] = [] + else: + bad = sorted({key for key in keys_lower if key not in SANA_WM_ALLOWED_ACTION_KEYS}) + if bad: + allowed = "".join(sorted(SANA_WM_ALLOWED_ACTION_KEYS)) + raise ValueError(f"Sana-WM action segment {segment!r} contains unknown keys {bad}; allowed: {allowed}.") + keys = sorted(set(keys_lower)) + segments.append((keys, int(duration))) + return segments + + +def action_rollout_num_frames(action: str) -> int: + """Frames ``action`` rolls out: the identity start pose plus one per step. + + Totals the durations arithmetically, so an oversized request is rejected + before anything allocates per-frame state. + """ + return sum(duration for _, duration in _parse_action_segments(action)) + 1 + + +def _parse_action_string(action: str) -> list[list[str]]: + per_frame: list[list[str]] = [] + for keys, duration in _parse_action_segments(action): + per_frame.extend([keys] * duration) + return per_frame + + +def action_string_to_c2w( + action: str, + *, + translation_speed: float = SANA_WM_DEFAULT_TRANSLATION_SPEED, + rotation_speed_deg: float = SANA_WM_DEFAULT_ROTATION_SPEED_DEG, + pitch_limit_deg: float = SANA_WM_DEFAULT_PITCH_LIMIT_DEG, +) -> np.ndarray: + """Roll out an OpenCV-convention ``(N + 1, 4, 4)`` c2w trajectory.""" + + per_frame = _parse_action_string(action) + rotate_rad = math.radians(rotation_speed_deg) + pitch_limit_rad = math.radians(pitch_limit_deg) + current = np.eye(4, dtype=np.float64) + current_pitch = 0.0 + poses = [current.copy()] + + for keys in per_frame: + held = set(keys) + rotation = current[:3, :3] + translation = current[:3, 3] + + pitch_delta = (rotate_rad if "i" in held else 0.0) - (rotate_rad if "k" in held else 0.0) + new_pitch = current_pitch + pitch_delta + if -pitch_limit_rad <= new_pitch <= pitch_limit_rad: + current_pitch = new_pitch + else: + pitch_delta = 0.0 + + yaw_delta = (rotate_rad if "l" in held else 0.0) - (rotate_rad if "j" in held else 0.0) + new_rotation = _rot_y(yaw_delta) @ rotation @ _rot_x(pitch_delta) + + forward = new_rotation[:, 2].copy() + forward[1] = 0.0 + right = new_rotation[:, 0].copy() + right[1] = 0.0 + forward_norm = float(np.linalg.norm(forward)) + right_norm = float(np.linalg.norm(right)) + if forward_norm > 0: + forward /= forward_norm + 1e-6 + if right_norm > 0: + right /= right_norm + 1e-6 + + move = np.zeros(3, dtype=np.float64) + if "w" in held: + move += forward * translation_speed + if "s" in held: + move -= forward * translation_speed + if "d" in held: + move += right * translation_speed + if "a" in held: + move -= right * translation_speed + + current = np.eye(4, dtype=np.float64) + current[:3, :3] = new_rotation + current[:3, 3] = translation + move + poses.append(current.copy()) + + return np.stack(poses, axis=0).astype(np.float32) + + +def get_pose_inverse(transform: torch.Tensor) -> torch.Tensor: + """Invert homogeneous rigid transforms with shape ``(..., 4, 4)``.""" + + rotation = transform[..., :3, :3] + translation = transform[..., :3, 3] + rotation_inv = rotation.transpose(-1, -2) + translation_inv = -torch.matmul(rotation_inv, translation.unsqueeze(-1)).squeeze(-1) + output = torch.eye(4, dtype=transform.dtype, device=transform.device).repeat(transform.shape[:-2] + (1, 1)) + output[..., :3, :3] = rotation_inv + output[..., :3, 3] = translation_inv + return output + + +def compute_raymap( + intrinsics: torch.Tensor, + poses: torch.Tensor, + height: int, + width: int, + *, + use_plucker: bool = True, +) -> torch.Tensor: + """Compute SANA-WM geometry ray maps. + + Args: + intrinsics: ``(T, 4)`` tensor in ``[fx, fy, cx, cy]`` order. + poses: ``(T, 4, 4)`` camera-to-world matrices. + height: raymap height in latent pixels. + width: raymap width in latent pixels. + use_plucker: return ``[direction, moment]`` if true, otherwise + ``[origin, direction]``. + """ + + if intrinsics.ndim != 2 or intrinsics.shape[-1] != 4: + raise ValueError(f"Sana-WM intrinsics must have shape (T, 4), got {tuple(intrinsics.shape)}.") + if poses.ndim != 3 or poses.shape[-2:] != (4, 4): + raise ValueError(f"Sana-WM poses must have shape (T, 4, 4), got {tuple(poses.shape)}.") + if intrinsics.shape[0] != poses.shape[0]: + raise ValueError("Sana-WM intrinsics and poses must have the same temporal length.") + + frames = intrinsics.shape[0] + device = intrinsics.device + dtype = intrinsics.dtype + y_grid, x_grid = torch.meshgrid( + torch.arange(height, device=device, dtype=dtype), + torch.arange(width, device=device, dtype=dtype), + indexing="ij", + ) + x_grid = x_grid[None].expand(frames, -1, -1) + y_grid = y_grid[None].expand(frames, -1, -1) + + fx = intrinsics[:, 0].view(frames, 1, 1) + fy = intrinsics[:, 1].view(frames, 1, 1) + cx = intrinsics[:, 2].view(frames, 1, 1) + cy = intrinsics[:, 3].view(frames, 1, 1) + dirs_cam = torch.stack([(x_grid - cx) / fx, (y_grid - cy) / fy, torch.ones_like(x_grid)], dim=-1) + + rotation = poses[:, :3, :3] + translation = poses[:, :3, 3] + dirs_world = torch.einsum("tij,thwj->thwi", rotation, dirs_cam) + dirs_world = dirs_world / torch.norm(dirs_world, dim=-1, keepdim=True).clamp_min(1e-12) + origins = translation.view(frames, 1, 1, 3).expand_as(dirs_world) + + if use_plucker: + moments = torch.cross(origins, dirs_world, dim=-1) + return torch.cat([dirs_world, moments], dim=-1) + return torch.cat([origins, dirs_world], dim=-1) + + +def intrinsics_to_vec4_array(intrinsics: Any, *, num_frames: int, height: int, width: int) -> np.ndarray: + """Normalize the intrinsics payload to an ``(F, 4)`` ``[fx, fy, cx, cy]`` array. + + Accepts either ``None`` (derive from the output resolution) or the + ``{fx, fy, cx, cy}`` mapping — the only form the request contract exposes. + """ + + if intrinsics is None: + focal = float(max(height, width)) + vec = np.array([focal, focal, width / 2.0, height / 2.0], dtype=np.float32) + return np.broadcast_to(vec, (num_frames, 4)).copy() + + if isinstance(intrinsics, dict): + vec = np.array( + [intrinsics["fx"], intrinsics["fy"], intrinsics["cx"], intrinsics["cy"]], + dtype=np.float32, + ) + return np.broadcast_to(vec, (num_frames, 4)).copy() + + raise ValueError("Sana-WM intrinsics must be a {fx, fy, cx, cy} mapping.") + + +def _pack_camera_conditions( + poses: torch.Tensor, + intrinsics_latent: torch.Tensor, + *, + num_frames: int, + latent_height: int, + latent_width: int, + vae_time_stride: int, +) -> dict[str, torch.Tensor]: + # This yields exactly the latent frame count the VAE produces: + # ``len(arange(0, n, s)) == ceil(n / s) == (n - 1) // s + 1``. + time_indices = torch.arange(0, num_frames, vae_time_stride) + + raymap = torch.cat([poses[time_indices].reshape(len(time_indices), -1), intrinsics_latent[time_indices]], dim=-1) + + chunks: list[torch.Tensor] = [] + spatial_raymap_frames: list[torch.Tensor] = [] + chunk_starts = time_indices - (vae_time_stride - 1) + for start, t_idx in zip(chunk_starts, time_indices): + start_idx = max(0, int(start)) + end_idx = start_idx + vae_time_stride + chunk_poses = poses[start_idx:end_idx] + chunk_intrinsics = intrinsics_latent[start_idx:end_idx] + if chunk_poses.shape[0] < vae_time_stride: + pad = vae_time_stride - chunk_poses.shape[0] + chunk_poses = torch.cat([chunk_poses, chunk_poses[-1:].repeat(pad, 1, 1)], dim=0) + chunk_intrinsics = torch.cat([chunk_intrinsics, chunk_intrinsics[-1:].repeat(pad, 1)], dim=0) + plucker = compute_raymap(chunk_intrinsics, chunk_poses, latent_height, latent_width, use_plucker=True) + chunks.append(plucker.permute(0, 3, 1, 2).reshape(-1, latent_height, latent_width)) + + # Per-frame spatial ray-direction map for raymap_embedder: take the + # representative frame in each chunk (the anchor at t_idx) and use only + # the direction channels (last 3 of 6 in the origin+direction layout). + t = int(t_idx) + od = compute_raymap( + intrinsics_latent[t : t + 1], + poses[t : t + 1], + latent_height, + latent_width, + use_plucker=False, + ) # [1, H, W, 6] + spatial_raymap_frames.append(od[0, :, :, 3:]) # [H, W, 3] — direction only + + chunk_plucker = torch.stack(chunks).permute(1, 0, 2, 3) + # spatial_raymap: [3, F_latent, H, W] — consumed by raymap_embedder + spatial_raymap = torch.stack(spatial_raymap_frames).permute(3, 0, 1, 2) + return {"raymap": raymap, "chunk_plucker": chunk_plucker, "spatial_raymap": spatial_raymap} + + +def build_plucker_condition( + condition: SanaWmCameraCondition, + *, + vae_stride: tuple[int, int, int] | list[int] = SANA_WM_DEFAULT_VAE_STRIDE, +) -> dict[str, torch.Tensor]: + """Build native Stage-1 camera tensors from normalized request metadata.""" + + if bool(condition.action) == (condition.poses is not None): + raise ValueError("Sana-WM camera condition requires exactly one of action or poses.") + + if condition.action: + poses_c2w = action_string_to_c2w( + condition.action, + translation_speed=condition.translation_speed, + rotation_speed_deg=condition.rotation_speed_deg, + pitch_limit_deg=condition.pitch_limit_deg, + ) + else: + poses_c2w = np.asarray(condition.poses, dtype=np.float32) + + if poses_c2w.ndim != 3 or poses_c2w.shape[-2:] != (4, 4): + raise ValueError(f"Sana-WM c2w poses must have shape (F, 4, 4), got {poses_c2w.shape}.") + + num_frames = condition.num_frames or int(poses_c2w.shape[0]) + if poses_c2w.shape[0] < num_frames: + if condition.action: + num_frames = int(poses_c2w.shape[0]) + else: + raise ValueError(f"Sana-WM c2w poses length {poses_c2w.shape[0]} is shorter than num_frames {num_frames}.") + poses_c2w = poses_c2w[:num_frames] + + intrinsics = intrinsics_to_vec4_array( + condition.intrinsics, + num_frames=num_frames, + height=condition.height, + width=condition.width, + ) + + vae_time_stride = int(vae_stride[0]) + vae_spatial_stride = int(vae_stride[-1]) + latent_height = condition.height // vae_spatial_stride + latent_width = condition.width // vae_spatial_stride + + poses = torch.from_numpy(poses_c2w).float() + first_inv = get_pose_inverse(poses[0:1]).squeeze(0) + poses_rel = torch.matmul(first_inv, poses[1:]) + poses = torch.cat([torch.eye(4, dtype=poses.dtype).unsqueeze(0), poses_rel], dim=0) + + intrinsics_latent = torch.from_numpy(intrinsics).float() + intrinsics_latent[:, [0, 2]] *= latent_width / float(condition.width) + intrinsics_latent[:, [1, 3]] *= latent_height / float(condition.height) + + return _pack_camera_conditions( + poses, + intrinsics_latent, + num_frames=num_frames, + latent_height=latent_height, + latent_width=latent_width, + vae_time_stride=vae_time_stride, + ) diff --git a/vllm_omni/diffusion/models/sana_wm/config.py b/vllm_omni/diffusion/models/sana_wm/config.py new file mode 100644 index 000000000..bf4eb428c --- /dev/null +++ b/vllm_omni/diffusion/models/sana_wm/config.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Config for the SANA-WM Stage-1 transformer. + +Loaded from the standard diffusers ``transformer/config.json`` (a flat dict +keyed by the dataclass field names). The bespoke ``config.yaml`` release format +is handled once, offline, at checkpoint-conversion time — the runtime only ever +sees the converted, diffusers-standard config. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass, field, fields +from pathlib import Path +from typing import Any + + +def _as_tuple3(value: Any, default: tuple[int, int, int]) -> tuple[int, int, int]: + if value is None: + return default + if isinstance(value, int): + return (value, value, value) + if isinstance(value, (list, tuple)): + if len(value) == 2: + return (1, int(value[0]), int(value[1])) + if len(value) == 3: + return (int(value[0]), int(value[1]), int(value[2])) + return default + + +# LTX-2 VAE compression ratios (SANA-WM ships ``AutoencoderKLLTX2Video``). +# They live here rather than in the pipeline so the request-normalisation module +# can validate latent geometry without importing the pipeline. +SANA_WM_VAE_SPATIAL_COMPRESSION = 32 +SANA_WM_VAE_TEMPORAL_COMPRESSION = 8 + + +@dataclass(frozen=True) +class SanaWmConfig: + """SANA-WM Stage-1 architecture and runtime config. + + Defaults mirror the first public HF release. ``from_json`` (reading + ``transformer/config.json``) is the normal construction path. + """ + + architecture_name: str | None = "SanaMSVideoCamCtrl_1600M_P1_D20" + num_blocks: int = 20 + hidden_size: int = 2240 + mlp_ratio: float = 3.0 + attn_type: str = "BidirectionalGDNTriton" + softmax_every_n: int = 4 + linear_head_dim: int = 112 + conv_kernel_size: int = 4 + t_kernel_size: int = 3 + k_conv_only: bool = True + ffn_type: str = "GLUMBConvTemp" + pos_embed_type: str = "wan_rope" + patch_size: tuple[int, int, int] = (1, 1, 1) + qk_norm: bool = True + cross_norm: bool = True + mixed_precision: str = "bf16" + fp32_attention: bool = True + image_size: int = 720 + cam_attn_compress: int = 1 + use_chunk_plucker_post_attn: bool = True + chunk_plucker_channels: int = 48 + chunk_plucker_post_attn_blocks: int = 20 + inference_flow_shift: float = 9.8 + scheduler_type: str = "flow_dpm-solver" + chi_prompt: list[str] = field(default_factory=list) + model_max_length: int = 300 + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> SanaWmConfig: + """Build from a flat mapping of field name -> value. + + Unknown keys (e.g. the diffusers ``_class_name`` marker or the + ``latent_channels`` / ``prompt_channels`` constructor kwargs) are + ignored. ``patch_size`` is coerced from a JSON list back to a tuple. + """ + field_names = {f.name for f in fields(cls)} + known = {k: v for k, v in data.items() if k in field_names} + if "patch_size" in known: + known["patch_size"] = _as_tuple3(known["patch_size"], cls.patch_size) + return cls(**known) + + @classmethod + def from_json(cls, path: str | Path) -> SanaWmConfig: + with open(path, encoding="utf-8") as f: + data = json.load(f) + return cls.from_dict(data if isinstance(data, Mapping) else {}) diff --git a/vllm_omni/diffusion/models/sana_wm/pipeline_sana_wm.py b/vllm_omni/diffusion/models/sana_wm/pipeline_sana_wm.py new file mode 100644 index 000000000..9c2843040 --- /dev/null +++ b/vllm_omni/diffusion/models/sana_wm/pipeline_sana_wm.py @@ -0,0 +1,942 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Sana-WM pipeline integration. + +This module wires the registry-visible surface, release-layout validation, and +the Stage-1 sampling loop. The DiT, Gated-DeltaNet and camera stack are +vLLM-Omni-native layers (``sana_wm_transformer.py`` / ``ucpe.py``); nothing here +calls into NVlabs code at runtime. +""" + +from __future__ import annotations + +import os +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any, ClassVar + +import torch +import torch.nn.functional as F +from torch import nn +from vllm.logger import init_logger +from vllm.model_executor.models.utils import AutoWeightsLoader + +from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin +from vllm_omni.diffusion.distributed.utils import get_local_device +from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader +from vllm_omni.diffusion.models.interface import SupportImageInput, SupportsComponentDiscovery +from vllm_omni.diffusion.models.ltx2.ltx2_latents import ( + denormalize_latents, + normalize_latents, + pack_latents, + resolve_video_latent_shape, + unpack_latents, +) +from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin +from vllm_omni.diffusion.models.sana_wm.camera_control import ( + SanaWmCameraCondition, + build_plucker_condition, +) +from vllm_omni.diffusion.models.sana_wm.config import ( + SANA_WM_VAE_SPATIAL_COMPRESSION, + SANA_WM_VAE_TEMPORAL_COMPRESSION, + SanaWmConfig, +) +from vllm_omni.diffusion.models.sana_wm.request import normalize_sana_wm_payload +from vllm_omni.diffusion.models.sana_wm.sana_wm_transformer import ( + SANA_WM_STAGE1_PROMPT_CHANNELS, + SanaWmTransformer3DModel, +) +from vllm_omni.diffusion.models.schedulers import FlowMatchEulerDiscreteScheduler +from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery +from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import ( + DiffusionPipelineProfilerMixin, +) +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific + +logger = init_logger(__name__) + +# The loader expects the standard diffusers layout (transformer/config.json + +# transformer/diffusion_pytorch_model.safetensors + model_index.json), so this +# repo is an offline conversion of the NVlabs release into that layout. It holds +# Stage-1 only — transformer/ + vae/, with model_index.json naming +# SanaWmPipeline — because startup prefetches the whole model repo +# (allow_patterns=["*"]); a repo that also carried the refiner would pull tens of +# GB this path never loads. +SANA_WM_MODEL_ID = "BBBBruce/SANA-WM_bidirectional-stage1-diffusers" + +SANA_WM_STAGE1_DIT_FILE = "transformer/diffusion_pytorch_model.safetensors" +SANA_WM_CONFIG_FILE = "transformer/config.json" +SANA_WM_VAE_CONFIG_FILE = "vae/config.json" +SANA_WM_VAE_WEIGHT_FILE = "vae/diffusion_pytorch_model.safetensors" +SANA_WM_STAGE1_TEXT_ENCODER_ID = "google/gemma-2-2b-it" +SANA_WM_STAGE1_TEXT_ENCODER_FALLBACK_ID = "Efficient-Large-Model/gemma-2-2b-it" +SANA_WM_STAGE1_TEXT_ENCODER_ENV = "VLLM_OMNI_SANA_WM_STAGE1_TEXT_ENCODER" +SANA_WM_OUTPUT_HEIGHT = 704 +SANA_WM_OUTPUT_WIDTH = 1280 +SANA_WM_NUM_TRAIN_TIMESTEPS = 1000 +# Maximum supported clip length (20s @16fps) — the native envelope, not a +# request default. Request-side defaults live in +# ``models/sana_wm/request.py``. +SANA_WM_NATIVE_NUM_FRAMES = 321 +# Fail-fast latent-token cap for the native Stage-1 path, sized to the native +# 704x1280 output at the maximum clip length so it only rejects genuinely +# oversized requests. Overridable per-request via ``sana_wm_native_max_tokens``. +SANA_WM_NATIVE_MAX_TOKENS = ( + ((SANA_WM_NATIVE_NUM_FRAMES - 1) // SANA_WM_VAE_TEMPORAL_COMPRESSION + 1) + * (SANA_WM_OUTPUT_HEIGHT // SANA_WM_VAE_SPATIAL_COMPRESSION) + * (SANA_WM_OUTPUT_WIDTH // SANA_WM_VAE_SPATIAL_COMPRESSION) +) +# The NVlabs production inference settings for this release. They belong in the +# model rather than in a deploy YAML: single-stage diffusion models resolve +# through ``create_default_diffusion``, which never reads a YAML's +# ``default_sampling_params``, so a caller that omits either field lands here. +SANA_WM_DEFAULT_NUM_INFERENCE_STEPS = 60 +SANA_WM_DEFAULT_GUIDANCE_SCALE = 5.0 + +SANA_WM_STAGE1_PATTERNS = ( + SANA_WM_CONFIG_FILE, + SANA_WM_STAGE1_DIT_FILE, + SANA_WM_VAE_CONFIG_FILE, + SANA_WM_VAE_WEIGHT_FILE, +) +SANA_WM_STAGE1_DIT_SUBFOLDER = "transformer" +SANA_WM_STAGE1_DIT_BASENAME = Path(SANA_WM_STAGE1_DIT_FILE).name + + +@dataclass(frozen=True) +class SanaWmLocalPaths: + """Resolved local file paths for a SANA-WM snapshot.""" + + root: Path + config: Path + stage1_dit: Path + vae_config: Path + vae_weights: Path + + +@dataclass(frozen=True) +class SanaWmNativeParams: + """Resolved generation settings for one Stage-1 request. + + Built by ``_native_params`` from the request payload and sampling params, + and consumed by ``_run_native_backend`` — this is the production path. + """ + + height: int + width: int + num_frames: int + num_inference_steps: int + seed: int + # Defaults to the NVlabs production scale, so a caller that never mentions + # guidance still gets the two-branch CFG forward the model was tuned for. + # An explicit ``guidance_scale <= 1.0`` still selects the single-branch path. + cfg_scale: float = SANA_WM_DEFAULT_GUIDANCE_SCALE + + +def build_sana_wm_download_patterns() -> tuple[str, ...]: + """Return the minimal HF allow-patterns needed for SANA-WM.""" + + return tuple(SANA_WM_STAGE1_PATTERNS) + + +def resolve_sana_wm_local_paths(snapshot_dir: str | Path) -> SanaWmLocalPaths: + root = Path(snapshot_dir) + return SanaWmLocalPaths( + root=root, + config=root / SANA_WM_CONFIG_FILE, + stage1_dit=root / SANA_WM_STAGE1_DIT_FILE, + vae_config=root / SANA_WM_VAE_CONFIG_FILE, + vae_weights=root / SANA_WM_VAE_WEIGHT_FILE, + ) + + +def validate_sana_wm_local_paths(paths: SanaWmLocalPaths) -> None: + required = [ + paths.config, + paths.stage1_dit, + paths.vae_config, + paths.vae_weights, + ] + missing = [path for path in required if not path.exists()] + if missing: + joined = ", ".join(str(path) for path in missing) + raise FileNotFoundError(f"Missing SANA-WM checkpoint files: {joined}") + + +def resolve_or_download_sana_wm_checkpoint( + model: str = SANA_WM_MODEL_ID, + *, + revision: str | None = None, + cache_dir: str | None = None, +) -> SanaWmLocalPaths: + """Resolve a local SANA-WM tree or download the required HF files.""" + + root = Path(model) + if root.is_dir(): + snapshot_dir = root + else: + snapshot_dir = Path( + download_weights_from_hf_specific( + model, + cache_dir, + list(build_sana_wm_download_patterns()), + revision=revision, + require_all=True, + ) + ) + paths = resolve_sana_wm_local_paths(snapshot_dir) + validate_sana_wm_local_paths(paths) + return paths + + +def build_sana_wm_output_envelope( + *, + output: Any, + output_type: str, + metadata: dict[str, Any], +) -> dict[str, Any]: + """Wrap a Stage-1 result in the canonical output envelope. + + ``normalize_diffusion_postprocess_output`` splits ``{"payload": ..., + "metadata": ...}`` into the API-facing payload and the metadata groups, so + the model-specific diagnostics ride along under a ``sana_wm`` group instead + of the removed ``DiffusionOutput.custom_output`` field. + """ + payload_key = "latents" if output_type == "latent" else "video" + return {"payload": {payload_key: output}, "metadata": {"sana_wm": metadata}} + + +def get_sana_wm_pre_process_func(od_config: OmniDiffusionConfig): + del od_config + + def pre_process_func(request: Any) -> Any: + from vllm_omni.inputs.data import OmniTextPrompt + + sampling_params = getattr(request, "sampling_params", None) + prompt = request.prompt + prompt_mapping = OmniTextPrompt(prompt=prompt) if isinstance(prompt, str) else prompt + prompt_mapping = dict(prompt_mapping) + + # The camera block travels as extra_params["sana_wm"], which serving + # merges into extra_args. Offline callers may also put it straight on + # the prompt. + if sampling_params is not None: + extra_args = getattr(sampling_params, "extra_args", None) + if isinstance(extra_args, Mapping): + block = extra_args.get("sana_wm") + if isinstance(block, Mapping): + merged = dict(block) + merged.update(prompt_mapping.get("sana_wm") or {}) + prompt_mapping["sana_wm"] = merged + # Geometry always comes from the standard sampling fields. + for field in ("num_frames", "height", "width"): + value = getattr(sampling_params, field, None) + if value is not None and not (field == "num_frames" and value == 1): + prompt_mapping[field] = value + + request.prompt = normalize_sana_wm_payload(prompt_mapping) + + # Preprocess is the only place the request is normalised, so publish the + # canonical geometry back onto sampling_params for the rest of the stack. + if sampling_params is not None: + payload = request.prompt["additional_information"]["sana_wm"] + sampling_params.num_frames = payload["num_frames"] + sampling_params.height = payload["height"] + sampling_params.width = payload["width"] + return request + + return pre_process_func + + +class SanaWmPipeline( + nn.Module, + CFGParallelMixin, + SupportImageInput, + SupportsComponentDiscovery, + ProgressBarMixin, + DiffusionPipelineProfilerMixin, +): + """Stage-1 SANA-WM image-to-video pipeline.""" + + support_image_input: ClassVar[bool] = True + color_format: ClassVar[str] = "RGB" + # Skip the startup warmup: the generic dummy request carries a blank image + # with no camera poses and no action string, which SANA-WM's request + # validation always rejects. The first real request does the warming. + dummy_run_num_frames: ClassVar[int] = 0 + _dit_modules: ClassVar[list[str]] = ["transformer"] + _encoder_modules: ClassVar[list[str]] = ["text_encoder"] + _vae_modules: ClassVar[list[str]] = ["vae"] + _resident_modules: ClassVar[list[str]] = [] + + def __init__(self, *, od_config: OmniDiffusionConfig | None = None, prefix: str = "") -> None: + super().__init__() + self.od_config = od_config + self.prefix = prefix + + self.sana_wm_config = SanaWmConfig() + self.quant_config = getattr(od_config, "quantization_config", None) if od_config is not None else None + + self.tokenizer: Any | None = None + self.text_encoder: nn.Module | None = None + self.vae: nn.Module | None = None + self.release_paths: SanaWmLocalPaths | None = None + self.weights_sources = [] + if od_config is not None and od_config.model is not None: + # Resolve the checkpoint once at construction (the standard + # startup-time flow, like every other diffusion pipeline) so the + # transformer below is built with the transformer/config.json shape + # rather than defaults. + self.resolve_checkpoint() + self.weights_sources = [ + DiffusersPipelineLoader.ComponentSource( + model_or_path=od_config.model, + subfolder=SANA_WM_STAGE1_DIT_SUBFOLDER, + revision=od_config.revision, + # The subfolder file carries module-local keys (standard + # diffusers layout); prefix them into the pipeline + # namespace for the loader's strict-coverage check. + prefix="transformer.", + fall_back_to_pt=False, + allow_patterns_overrides=[SANA_WM_STAGE1_DIT_BASENAME], + ) + ] + # Built eagerly: the loader constructs the pipeline under the target + # device / default-dtype context, and the weight loader then streams + # checkpoint tensors directly into these modules. + self.transformer = SanaWmTransformer3DModel( + config=self.sana_wm_config, + quant_config=self.quant_config, + prefix=f"{prefix}.transformer" if prefix else "transformer", + ) + self.device = get_local_device() + if od_config is not None and od_config.model is not None: + self._build_aux_components() + + def resolve_checkpoint(self) -> SanaWmLocalPaths: + if self.release_paths is not None: + return self.release_paths + if self.od_config is None or self.od_config.model is None: + raise ValueError("Sana-WM checkpoint resolution requires od_config.model.") + self.release_paths = resolve_or_download_sana_wm_checkpoint( + self.od_config.model, + revision=self.od_config.revision, + ) + self.sana_wm_config = SanaWmConfig.from_json(self.release_paths.config) + return self.release_paths + + @staticmethod + def _extra_args(sampling_params: Any | None) -> dict[str, Any]: + extra_args = getattr(sampling_params, "extra_args", None) if sampling_params is not None else None + return dict(extra_args or {}) + + def _native_params(self, payload: dict[str, Any], sampling_params: Any | None) -> SanaWmNativeParams: + extra_args = self._extra_args(sampling_params) + height = int(getattr(sampling_params, "height", None) or payload["height"]) + width = int(getattr(sampling_params, "width", None) or payload["width"]) + num_frames = int(getattr(sampling_params, "num_frames", None) or payload["num_frames"]) + steps = int( + getattr(sampling_params, "num_inference_steps", None) + or extra_args.get("num_inference_steps") + or SANA_WM_DEFAULT_NUM_INFERENCE_STEPS + ) + seed = int(getattr(sampling_params, "seed", None) or extra_args.get("seed", 0)) + if min(height, width, num_frames, steps) <= 0: + raise ValueError("Sana-WM native height, width, num_frames, and steps must be positive.") + cfg_scale = SANA_WM_DEFAULT_GUIDANCE_SCALE + if sampling_params is not None and getattr(sampling_params, "guidance_scale_provided", False): + cfg_scale = float(getattr(sampling_params, "guidance_scale", 1.0) or 1.0) + return SanaWmNativeParams( + height=height, + width=width, + num_frames=num_frames, + num_inference_steps=steps, + seed=seed, + cfg_scale=cfg_scale, + ) + + def _native_dtype(self, device: torch.device) -> torch.dtype: + dtype = getattr(self.od_config, "dtype", None) if self.od_config is not None else None + if isinstance(dtype, torch.dtype): + return dtype + return torch.float32 if device.type == "cpu" else torch.bfloat16 + + def _runtime_device_dtype(self) -> tuple[torch.device, torch.dtype]: + return self.device, self._native_dtype(self.device) + + def _build_aux_components(self) -> None: + """Construct the text encoder and VAE at startup, on CPU. + + ``ModuleDiscovery`` skips attributes that are ``None``, so components + created lazily on the first request are invisible to sequential + offload, HSDP auxiliary placement, and the shared device/memory + lifecycle. Building them here and deferring placement to the common + logic mirrors LTX-2's ``initialize_pipeline_components``. + """ + dtype = self._native_dtype(self.device) + self._load_stage1_text_encoder(dtype=dtype) + self._load_vae(dtype=dtype) + self._place_aux_components() + + def _place_aux_components(self) -> None: + """Move the discovered auxiliary components onto the local device. + + Skipped when offload or HSDP owns placement — those paths expect the + components to still be on CPU when they take over. + """ + parallel_config = getattr(self.od_config, "parallel_config", None) + if ( + getattr(self.od_config, "enable_cpu_offload", False) + or getattr(self.od_config, "enable_layerwise_offload", False) + or getattr(parallel_config, "use_hsdp", False) + ): + return + modules = ModuleDiscovery.discover(self) + for module in (*modules.encoders, *modules.vaes, *modules.resident_modules): + module.to(self.device) + + @staticmethod + def _component_local_files_only(model_id: Any) -> bool: + """Decide download policy per component, from its own model id. + + The Stage-1 release directory is local once ``resolve_checkpoint`` has + run, but the Gemma text encoder lives in a separate Hub repo. Deriving + locality from the release paths forced ``local_files_only=True`` onto + Gemma as well, which forbids the download on a box that has not cached + it yet. + """ + return Path(str(model_id)).expanduser().exists() + + def _load_stage1_text_encoder(self, *, dtype: torch.dtype) -> None: + from transformers import AutoModel, AutoTokenizer + + model_ids = [os.environ.get(SANA_WM_STAGE1_TEXT_ENCODER_ENV, "").strip()] + model_ids.extend([SANA_WM_STAGE1_TEXT_ENCODER_ID, SANA_WM_STAGE1_TEXT_ENCODER_FALLBACK_ID]) + errors: list[str] = [] + for model_id in dict.fromkeys(model_id for model_id in model_ids if model_id): + try: + self.tokenizer = AutoTokenizer.from_pretrained( + model_id, + local_files_only=self._component_local_files_only(model_id), + ) + text_encoder_model_id = model_id + break + except OSError as exc: + errors.append(f"{model_id}: {exc}") + else: + raise OSError("Could not load SANA-WM Stage-1 text encoder. Tried: " + "; ".join(errors)) + if getattr(self.tokenizer, "pad_token", None) is None: + self.tokenizer.pad_token = self.tokenizer.eos_token + self.tokenizer.padding_side = "right" + with torch.device("cpu"): + # AutoModel, not AutoModelForCausalLM: only the final hidden state is + # used, so the LM head would compute a [seq, 256k] logits tensor + # that is thrown away on every prompt encode. + self.text_encoder = AutoModel.from_pretrained( + text_encoder_model_id, + torch_dtype=dtype, + local_files_only=self._component_local_files_only(text_encoder_model_id), + ) + + def _ensure_stage1_text_encoder(self, *, device: torch.device, dtype: torch.dtype) -> None: + if self.text_encoder is None: + self._load_stage1_text_encoder(dtype=dtype) + self.text_encoder.to(device=device, dtype=dtype) + + def _load_vae(self, *, dtype: torch.dtype) -> None: + if self.release_paths is None and self.od_config is not None and self.od_config.model is not None: + self.resolve_checkpoint() + if self.release_paths is None: + raise ValueError("Sana-WM VAE loading requires a resolved checkpoint.") + + from diffusers import AutoencoderKLLTX2Video + + with torch.device("cpu"): + self.vae = AutoencoderKLLTX2Video.from_pretrained( + str(self.release_paths.root), + subfolder="vae", + torch_dtype=dtype, + local_files_only=self._component_local_files_only(self.release_paths.root), + ) + self.vae.use_framewise_encoding = True + self.vae.use_framewise_decoding = True + # NVlabs reads these from its YAML VAE config with 64/96 defaults; the + # diffusers VAE config.json only ships scaling_factor, so the defaults + # are load-bearing (the keys are genuinely absent here). + self.vae.tile_sample_stride_num_frames = int(getattr(self.vae.config, "tile_sample_stride_num_frames", 64)) + self.vae.tile_sample_min_num_frames = int(getattr(self.vae.config, "tile_sample_min_num_frames", 96)) + + def _ensure_vae(self, *, device: torch.device, dtype: torch.dtype) -> None: + if self.vae is None: + self._load_vae(dtype=dtype) + # Tiling is re-asserted here, not in _load_vae: registry.py assigns + # ``model.vae.use_tiling = od_config.vae_use_tiling`` after the pipeline + # is constructed, and that flag defaults to False. Matching NVlabs' + # inference_video_scripts/inference_sana_wm.py, the SANA VAE has to + # tile — a 161-frame decode costs about 9 GiB more without it, and a + # 321-frame request (41 latent frames) decodes as one 3D volume and + # OOMs. Overriding is deliberate, but say so rather than dropping the + # caller's setting silently. + if not self.vae.use_tiling: + logger.warning("Sana-WM forces VAE tiling on; ignoring vae_use_tiling=False (the VAE OOMs without it).") + self.vae.use_tiling = True + self.vae.to(device=device, dtype=dtype) + + @staticmethod + def _preprocess_first_frame( + image: Any, + *, + height: int, + width: int, + device: torch.device, + dtype: torch.dtype, + ) -> torch.Tensor: + """Return the first-frame image as ``[1, 3, 1, H, W]`` in ``[-1, 1]``. + + ``VideoProcessor.preprocess`` covers the PIL / ndarray / tensor cases. + ``resample="bicubic"`` keeps the filter PIL's ``Image.resize`` defaulted + to; the processor's own default is lanczos, which would move the + conditioning latent. + """ + import numpy as np + from diffusers.video_processor import VideoProcessor + + # The processor normalises to [-1, 1] assuming float inputs in [0, 1]. + if isinstance(image, np.ndarray) and image.dtype == np.uint8: + image = image.astype(np.float32) / 255.0 + + processor = VideoProcessor(vae_scale_factor=SANA_WM_VAE_SPATIAL_COMPRESSION, resample="bicubic") + frame = processor.preprocess(image, height=height, width=width) + return frame.unsqueeze(2).to(device=device, dtype=dtype) + + def _vae_normalize_latent(self, latent: torch.Tensor) -> torch.Tensor: + """LTX-2 VAE per-channel latent normalisation matching NVlabs. + + ``z_norm = (z_raw - latents_mean) * scaling_factor / latents_std`` + + The LTX-2 VAE (``AutoencoderKLLTX2Video`` in diffusers) ships with + per-channel ``latents_mean`` and ``latents_std`` tensors that must be + applied alongside ``scaling_factor`` for the round-trip to be + identity. Applying `* scaling_factor` alone gives decoded videos a + systematic G/B colour shift (pred RGB means + ~(110, 220, 208) vs ref ~(94, 114, 138)). + + Returns the normalised latent in the same dtype as the input. + """ + if self.vae is None: + raise RuntimeError("Sana-WM VAE did not initialize.") + return normalize_latents( + latent, + self.vae.latents_mean, + self.vae.latents_std, + float(self.vae.config.scaling_factor), + ) + + def _vae_denormalize_latent(self, latent: torch.Tensor) -> torch.Tensor: + """Inverse of :meth:`_vae_normalize_latent` — match NVlabs decode.""" + if self.vae is None: + raise RuntimeError("Sana-WM VAE did not initialize.") + return denormalize_latents( + latent, + self.vae.latents_mean, + self.vae.latents_std, + float(self.vae.config.scaling_factor), + ) + + def _vae_encode_first_frame( + self, + image: Any, + *, + height: int, + width: int, + latent_height: int, + latent_width: int, + device: torch.device, + dtype: torch.dtype, + ) -> torch.Tensor: + """Encode the first-frame image to latent space via the LTX-2 VAE. + + Returns shape ``[1, 128, 1, latent_height, latent_width]``. + """ + self._ensure_vae(device=device, dtype=dtype) + if self.vae is None: + raise RuntimeError("Sana-WM VAE did not initialize.") + frame = self._preprocess_first_frame(image, height=height, width=width, device=device, dtype=dtype) + with torch.inference_mode(): + encoded = self.vae.encode(frame) + first_latent = encoded.latent_dist.mean + first_latent = self._vae_normalize_latent(first_latent.to(dtype=dtype)) + # The encoder's own rounding can land a pixel off the latent grid the + # noise tensor was built on, and the two are concatenated below. + if first_latent.shape[-2:] != (latent_height, latent_width): + first_latent = F.interpolate( + first_latent.squeeze(2), + size=(latent_height, latent_width), + mode="bilinear", + align_corners=False, + ).unsqueeze(2) + return first_latent # [1, 128, 1, lh, lw] + + def _stage1_prompt_text(self, prompt: dict[str, Any]) -> str: + user_prompt = str(prompt.get("prompt") or "") + chi_prompt = "\n".join(part for part in self.sana_wm_config.chi_prompt if part) + if chi_prompt: + return chi_prompt + user_prompt + return user_prompt + + def _native_prompt_embeds( + self, + prompt: dict[str, Any], + *, + device: torch.device, + dtype: torch.dtype, + ) -> torch.Tensor: + prompt_text = self._stage1_prompt_text(prompt) + self._ensure_stage1_text_encoder(device=device, dtype=dtype) + tokenizer = self.tokenizer + if tokenizer is None or self.text_encoder is None: + raise RuntimeError("Sana-WM Stage-1 text encoder did not initialize.") + chi_prompt = "\n".join(part for part in self.sana_wm_config.chi_prompt if part) + model_max_length = int(self.sana_wm_config.model_max_length) + max_length_all = model_max_length + if chi_prompt: + # NVlabs encodes the chi-prompt-prefixed text with extra room for + # the chi tokens, then keeps BOS plus the final model window. + max_length_all = len(tokenizer.encode(chi_prompt)) + model_max_length - 2 + encoded = tokenizer( + [prompt_text], + padding="max_length", + max_length=max_length_all, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ).to(device) + # last_hidden_state is the same tensor the CausalLM wrapper exposed as + # hidden_states[-1] (both are post-final-norm), without retaining every + # intermediate layer. + hidden_states = self.text_encoder(**encoded).last_hidden_state + attention_mask = encoded.attention_mask + if chi_prompt: + select_index = [0] + list(range(-model_max_length + 1, 0)) + hidden_states = hidden_states[:, select_index] + attention_mask = attention_mask[:, select_index] + if hidden_states.shape[-1] != SANA_WM_STAGE1_PROMPT_CHANNELS: + raise ValueError( + "Sana-WM Stage-1 Gemma hidden size mismatch: expected " + f"{SANA_WM_STAGE1_PROMPT_CHANNELS}, got {hidden_states.shape[-1]}." + ) + # Return RAW Gemma hidden states: the transformer's ``attention_y_norm`` + # RMSNorm does the normalisation, so normalising here double-normalises. + prompt_attention_mask = attention_mask.to(device=device, dtype=torch.float32) + # An all-ones mask carries no information. Keep it ``None`` so + # cross-attention takes the mask-free attention path instead of handing + # every block, on every step, a mask the backend must reshape to no + # effect. Evaluated once per prompt encode, so the device sync is paid + # twice per request (cond + uncond), not once per denoising step. + if bool(prompt_attention_mask.all()): + prompt_attention_mask = None + self._last_prompt_attention_mask = prompt_attention_mask + return hidden_states.to(device=device, dtype=dtype) + + def predict_noise(self, **kwargs: Any) -> torch.Tensor: + """Single transformer forward. + + Overrides the mixin default, which assumes a tuple-returning transformer + and takes ``result[0]``; SANA-WM's returns the noise prediction directly. + """ + return self.transformer(**kwargs) + + def _decode_native_latents( + self, + latents: torch.Tensor, + *, + output_type: str, + device: torch.device, + dtype: torch.dtype, + ) -> Any: + if output_type == "latent": + return latents + self._ensure_vae(device=device, dtype=dtype) + if self.vae is None: + raise RuntimeError("Sana-WM VAE did not initialize.") + # NVlabs always passes ``temb=None`` to ``vae.decode`` for this VAE + # (``builder.py::vae_decode -> LTX2VAE_diffusers``); match that. + denorm = self._vae_denormalize_latent(latents.to(getattr(self.vae, "dtype", dtype))) + video = self.vae.decode(denorm, temb=None, return_dict=False)[0] + # Post-process to the requested output_type (np/pil/pt). Let failures + # surface: silently returning the raw decoder tensor would hand the + # caller a wrong-format/range video with no error. + from diffusers.video_processor import VideoProcessor + + processor = VideoProcessor(vae_scale_factor=getattr(self.vae, "spatial_compression_ratio", 32)) + return processor.postprocess_video(video, output_type=output_type) + + def _run_native_backend( + self, + *, + prompt: dict[str, Any], + payload: dict[str, Any], + sampling_params: Any | None, + ) -> DiffusionOutput: + params = self._native_params(payload, sampling_params) + latent_frames, latent_height, latent_width = resolve_video_latent_shape( + params.height, + params.width, + params.num_frames, + vae_spatial_compression_ratio=SANA_WM_VAE_SPATIAL_COMPRESSION, + vae_temporal_compression_ratio=SANA_WM_VAE_TEMPORAL_COMPRESSION, + ) + token_count = latent_frames * latent_height * latent_width + extra_args = self._extra_args(sampling_params) + max_tokens = int(extra_args.get("sana_wm_native_max_tokens", SANA_WM_NATIVE_MAX_TOKENS)) + if token_count > max_tokens: + raise ValueError( + "Sana-WM native latent token count exceeds the configured cap. " + f"Requested latent tokens={token_count}, max={max_tokens}. Request a smaller " + "`height`/`width`/`num_frames`, or raise the cap via " + "`sana_wm_native_max_tokens`." + ) + + device, dtype = self._runtime_device_dtype() + generator = torch.Generator(device=device) + generator.manual_seed(params.seed) + noise = torch.randn( + (1, 128, latent_frames, latent_height, latent_width), + device=device, + dtype=dtype, + generator=generator, + ) + + # Shared flow-matching Euler scheduler. NVlabs ``LTXFlowEuler`` drives + # diffusers' ``FlowMatchEulerDiscreteScheduler``; this is vLLM-Omni's + # vendored copy of it, including the ``per_token_timesteps`` step branch. + # The schedule applies the shift twice (once via the shifted ``sigma_min`` + # persisted by ``__init__``, once in ``set_timesteps``), which is what + # reproduces the NVlabs timestep table (e.g. shift=9.8, N=3 -> + # [1000, 909.0, 87.7]). + scheduler = FlowMatchEulerDiscreteScheduler( + num_train_timesteps=SANA_WM_NUM_TRAIN_TIMESTEPS, + shift=self.sana_wm_config.inference_flow_shift, + ) + scheduler.set_timesteps(params.num_inference_steps, device=device) + timesteps = scheduler.timesteps + + # ``normalize_sana_wm_payload`` guarantees the key is present but says + # nothing about its type. Falling back to pure noise on an unusable + # image would silently drop the conditioning and produce a + # plausible-looking but wrong video, so this raises instead. + import numpy as _np + + first_frame_image = (prompt.get("multi_modal_data") or {}).get("image") + if not (hasattr(first_frame_image, "convert") or isinstance(first_frame_image, (_np.ndarray, torch.Tensor))): + raise TypeError( + "Sana-WM first-frame image must be a PIL Image, numpy ndarray, or " + f"torch.Tensor; got {type(first_frame_image).__name__}." + ) + # Per-frame timestep contract matching NVlabs ``LTXFlowEuler.sample``: + # frame 0 is the clean VAE-encoded conditioning latent and its timestep + # is held at 0 while the sampling loop drives the other frames from + # noise. + first_latent = self._vae_encode_first_frame( + first_frame_image, + height=params.height, + width=params.width, + latent_height=latent_height, + latent_width=latent_width, + device=device, + dtype=dtype, + ) + # Place the CLEAN VAE-encoded first frame at frame 0 and rely on + # per-frame timesteps + the `condition_mask` torch.where restore (below) + # to keep it invariant across denoising steps. + latents = torch.cat([first_latent, noise[:, :, 1:]], dim=2) + + prompt_embeds = self._native_prompt_embeds( + prompt, + device=device, + dtype=dtype, + ) + prompt_attention_mask = self._last_prompt_attention_mask + + # Under CFG, encode the negative prompt up front; each denoise step + # then runs a two-branch transformer forward. The negative prompt comes + # from the request (``prompt["negative_prompt"]``, set by + # ``serving_video``); absent, it defaults to the empty string matching + # NVlabs ``GenerationParams.negative_prompt = ""``. + do_cfg = params.cfg_scale > 1.0 + if do_cfg: + negative_prompt_obj = {"prompt": str(prompt.get("negative_prompt") or "")} + negative_prompt_embeds = self._native_prompt_embeds( + negative_prompt_obj, + device=device, + dtype=dtype, + ) + negative_prompt_attention_mask = self._last_prompt_attention_mask + # Restore the cond mask so later readers of + # `_last_prompt_attention_mask` see the positive branch. + self._last_prompt_attention_mask = prompt_attention_mask + else: + negative_prompt_embeds = None + negative_prompt_attention_mask = None + + camera = payload.get("camera") or {} + condition = SanaWmCameraCondition( + poses=camera.get("poses") if isinstance(camera, dict) else None, + intrinsics=payload.get("intrinsics"), + action=payload.get("action"), + num_frames=params.num_frames, + height=params.height, + width=params.width, + translation_speed=float(payload.get("translation_speed", 0.05)), + rotation_speed_deg=float(payload.get("rotation_speed_deg", 1.2)), + ) + camera_tensors = build_plucker_condition(condition) + plucker = camera_tensors["chunk_plucker"].to(device=device, dtype=dtype) + raymap = camera_tensors["raymap"].to(device=device, dtype=dtype) + spatial_raymap = camera_tensors.get("spatial_raymap") + if spatial_raymap is not None: + spatial_raymap = spatial_raymap.to(device=device, dtype=dtype) + + # Build a per-frame condition mask matching NVlabs ``LTXFlowEuler``: + # frame 0 is the conditioning frame at sigma=0 (preserved after + # each ``scheduler.step``), all other frames carry the current + # sampling sigma. We omit the legacy `add_noise_to_image_conditioning_latents` + # motion-continuity term because the public SANA-WM config sets + # ``condition_frame_info={0: 0.0}`` (image_cond_noise_scale=0). + cam_batch = latents.shape[0] + cam_frames = latents.shape[2] + condition_mask = torch.zeros(cam_batch, 1, cam_frames, 1, 1, device=latents.device, dtype=latents.dtype) + condition_mask[:, :, 0] = 1.0 + + for timestep in timesteps: + # (B, 1, F) per-frame timestep, frame 0 forced to 0. Stays fp32: + # casting through the bf16 latent dtype would quantise it before + # ``SanaWmTimestepEmbedder``'s sinusoidal embed. + model_timestep = timestep.float().expand(cam_batch, 1, cam_frames).clone() + model_timestep[:, :, 0] = 0.0 + positive_kwargs = { + "hidden_states": latents, + "timestep": model_timestep, + "encoder_hidden_states": prompt_embeds, + "encoder_attention_mask": prompt_attention_mask, + "plucker": plucker, + "raymap": raymap, + "spatial_raymap": spatial_raymap, + } + negative_kwargs = ( + { + **positive_kwargs, + "encoder_hidden_states": negative_prompt_embeds, + "encoder_attention_mask": negative_prompt_attention_mask, + } + if do_cfg + else None + ) + # ``cfg_normalize=False`` keeps the plain NVlabs combine, + # ``uncond + scale * (cond - uncond)``. Under cfg_parallel_size > 1 + # the mixin runs one branch per rank and all-gathers instead of + # evaluating both branches serially. + noise_pred = self.predict_noise_maybe_with_cfg( + do_cfg, + params.cfg_scale, + positive_kwargs, + negative_kwargs, + cfg_normalize=False, + ) + + # The per-frame contract pairs with a per-token flow-matching Euler + # step. Both step branches end at ``prev = sample + dt * + # model_output``, but the scheduler defines dt as + # ``sigma - sigma_next`` per-token against ``sigma_next - sigma`` + # standard, so the per-token call takes the sign-flipped prediction + # (matching NVlabs ``LTXFlowEuler.sample``); passing the standard + # sign here would integrate towards more noise. + # + # Broadcast the (B, 1, F) timestep to (B, 1, F, H, W) and flatten + # F*H*W; frame-0 conditioning tokens are already 0. The flatten + # order (F, H, W) matches ``pack_latents``. + pt_t = ( + model_timestep.unsqueeze(-1) + .unsqueeze(-1) + .expand(cam_batch, 1, cam_frames, latents.shape[3], latents.shape[4]) + .reshape(cam_batch, -1) + ) + # The per-token branch consumes/returns the packed (B, N, C) layout + # and keeps its result in fp32; cast back so the latent dtype stays + # stable across steps. + stepped_packed = scheduler.step( + -pack_latents(noise_pred), + timestep, + pack_latents(latents), + per_token_timesteps=pt_t, + return_dict=False, + )[0] + stepped = unpack_latents( + stepped_packed.to(latents.dtype), + cam_frames, + latents.shape[3], + latents.shape[4], + ) + + # Match NVlabs LTXFlowEuler exactly. This is stricter than a plain + # condition-frame restore: with bf16 latents, the `t=1000` + # comparison rounds `1 - 1e-6` to `1`, so the first full-noise step + # is discarded for generated tokens as well. + tokens_to_denoise_mask = timestep / float(SANA_WM_NUM_TRAIN_TIMESTEPS) - 1e-6 < (1.0 - condition_mask) + latents = torch.where(tokens_to_denoise_mask, stepped, latents) + + output_type = getattr(sampling_params, "output_type", None) or "np" + output = self._decode_native_latents(latents, output_type=output_type, device=device, dtype=dtype) + return DiffusionOutput( + output=build_sana_wm_output_envelope( + output=output, + output_type=output_type, + metadata={ + "backend": "native_gdn", + "output_space": output_type, + "chi_prompt_applied": bool(self.sana_wm_config.chi_prompt), + "num_frames": params.num_frames, + "height": params.height, + "width": params.width, + "latent_tokens": token_count, + "sampling_steps": params.num_inference_steps, + }, + ), + ) + + def forward(self, req: DiffusionRequestBatch, *args: Any, **kwargs: Any) -> DiffusionOutput: + del args, kwargs + if len(req.prompts) != 1: + raise ValueError("Sana-WM native backend currently supports exactly one prompt per request.") + + prompt = req.prompts[0] + if isinstance(prompt, str): + raise ValueError("Sana-WM requires a mapping prompt with first-frame image and camera/action metadata.") + # get_sana_wm_pre_process_func is the single canonical entry point: it + # normalises the request and republishes the geometry onto + # sampling_params. normalize_sana_wm_payload is idempotent, so this only + # covers offline callers that bypass the preprocess hook. + prompt = normalize_sana_wm_payload(prompt) + payload = prompt["additional_information"]["sana_wm"] + + # The checkpoint is resolved once in __init__; this is a cached no-op + # kept as a guard for pipelines constructed without od_config.model. + if self.od_config is not None and self.od_config.model is not None: + self.resolve_checkpoint() + + return self._run_native_backend( + prompt=prompt, + payload=payload, + sampling_params=req.sampling_params, + ) + + def load_weights(self, weights: Iterable[tuple[str, Any]]) -> set[str]: + # Keys arrive pipeline-prefixed (``transformer.*``, per the component + # source). AutoWeightsLoader streams them to the submodule and returns + # the prefixed names the loader's strict-coverage check compares. + loader = AutoWeightsLoader(self) + return loader.load_weights(weights) diff --git a/vllm_omni/diffusion/models/sana_wm/request.py b/vllm_omni/diffusion/models/sana_wm/request.py new file mode 100644 index 000000000..010675045 --- /dev/null +++ b/vllm_omni/diffusion/models/sana_wm/request.py @@ -0,0 +1,269 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Request normalization for Sana-WM image-to-video. + +The released SANA-WM checkpoint is first-frame image-to-video. This module keeps +request-side normalization lightweight: it validates the image/camera/action +contract and stores a canonical payload under +``additional_information["sana_wm"]``. Model-internal raymap / Plucker +projection lives with the transformer. + +This lives beside the model rather than under ``stage_input_processors`` +because nothing loads it as one: the pipeline's ``pre_process_func`` is +registered from ``pipeline_sana_wm``, the same as every other diffusion model, +and no deploy config wires a ``custom_process_input_func`` here. Keeping it in +the stage package forced the pipeline to import *upwards* — the only such +import in ``diffusion/models/`` — which in turn made the VAE compression +constants impossible to share and so duplicated. +""" + +from __future__ import annotations + +import math +from collections.abc import Mapping, Sequence +from typing import Any + +from vllm_omni.diffusion.models.sana_wm.camera_control import action_rollout_num_frames +from vllm_omni.diffusion.models.sana_wm.config import ( + SANA_WM_VAE_SPATIAL_COMPRESSION, + SANA_WM_VAE_TEMPORAL_COMPRESSION, +) + +SANA_WM_CANONICAL_KEY = "sana_wm" +# Request fallbacks used whenever the field is absent, mirroring the deploy YAML +# (161 frames) and the model's native resolution. Both are genuinely reachable: +# a served request that omits num_frames never gets one injected, and callers +# may ask for a different resolution. +SANA_WM_DEFAULT_NUM_FRAMES = 161 +SANA_WM_DEFAULT_HEIGHT = 704 +SANA_WM_DEFAULT_WIDTH = 1280 +SANA_WM_DEFAULT_CAMERA_FORMAT = "c2w_4x4" +SANA_WM_DEFAULT_COORDINATE_SYSTEM = "official" + + +def _unwrap_single(value: Any) -> Any: + if isinstance(value, list): + return value[0] if value else None + return value + + +def _as_dict(value: Any, *, name: str) -> dict[str, Any]: + if value is None: + return {} + if not isinstance(value, Mapping): + raise ValueError(f"Sana-WM {name} must be a mapping, got {type(value).__name__}.") + return dict(value) + + +def _is_sequence(value: Any) -> bool: + return isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)) + + +def _as_finite_float(value: Any, *, name: str) -> float: + """Parse a float and reject NaN/inf. + + Non-finite geometry does not fail loudly downstream: it propagates through + the raymap into the Plucker maps and comes back as an all-NaN video with a + 200 response. + """ + try: + parsed = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"Sana-WM {name} must be a number, got {value!r}.") from exc + if not math.isfinite(parsed): + raise ValueError(f"Sana-WM {name} must be finite, got {parsed!r}.") + return parsed + + +def _validate_numeric_matrix4x4(matrix: Any, *, name: str) -> None: + if not _is_sequence(matrix) or len(matrix) != 4: + raise ValueError(f"Sana-WM {name} must be a 4x4 matrix.") + for row_idx, row in enumerate(matrix): + if not _is_sequence(row) or len(row) != 4: + raise ValueError(f"Sana-WM {name}[{row_idx}] must contain 4 values.") + for col_idx, value in enumerate(row): + _as_finite_float(value, name=f"{name}[{row_idx}][{col_idx}]") + + +def _validate_camera_poses(poses: Any, *, num_frames: int | None = None) -> None: + if not _is_sequence(poses) or len(poses) == 0: + raise ValueError("Sana-WM camera poses must be a non-empty sequence of 4x4 matrices.") + if num_frames is not None and len(poses) != num_frames: + raise ValueError(f"Sana-WM camera poses length {len(poses)} must equal num_frames {num_frames}.") + for idx, pose in enumerate(poses): + _validate_numeric_matrix4x4(pose, name=f"camera.poses[{idx}]") + + +def _validate_intrinsics(intrinsics: Any) -> None: + # Only the ``{fx, fy, cx, cy}`` mapping is accepted (the one form the recipe + # and examples use). Camera intrinsics are optional; when omitted the model + # derives them from the output resolution. + if intrinsics is None: + return + if not isinstance(intrinsics, Mapping): + raise ValueError("Sana-WM intrinsics must be a {fx, fy, cx, cy} mapping.") + required = {"fx", "fy", "cx", "cy"} + missing = sorted(required - set(intrinsics)) + if missing: + raise ValueError(f"Sana-WM intrinsics mapping is missing keys: {missing}.") + for key in required: + value = _as_finite_float(intrinsics[key], name=f"intrinsics[{key!r}]") + # ``compute_raymap`` divides by fx/fy. cx/cy stay unconstrained beyond + # finiteness — a principal point outside the frame is unusual, not wrong. + if key in ("fx", "fy") and value <= 0: + raise ValueError(f"Sana-WM intrinsics[{key!r}] must be positive, got {value}.") + + +def _as_positive_int(value: Any, *, name: str) -> int: + try: + parsed = int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"Sana-WM {name} must be an integer, got {value!r}.") from exc + if parsed <= 0: + raise ValueError(f"Sana-WM {name} must be positive, got {parsed}.") + return parsed + + +def _as_positive_float(value: Any, *, name: str) -> float: + # Finiteness first: every comparison against NaN is False, so a bare + # ``parsed <= 0`` would accept it. + parsed = _as_finite_float(value, name=name) + if parsed <= 0: + raise ValueError(f"Sana-WM {name} must be positive, got {parsed}.") + return parsed + + +def _extract_image(mm_data: Mapping[str, Any]) -> Any: + return _unwrap_single(mm_data.get("image")) + + +def _first_present(*values: Any) -> Any: + for value in values: + if value is not None: + return value + return None + + +def _validate_latent_geometry(*, num_frames: int, height: int, width: int) -> None: + """Reject geometry the VAE would silently floor. + + The SANA-WM VAE compresses 32x spatially and 8x temporally, so a request + that is not aligned loses pixels/frames instead of failing. + """ + if height % SANA_WM_VAE_SPATIAL_COMPRESSION or width % SANA_WM_VAE_SPATIAL_COMPRESSION: + raise ValueError( + f"Sana-WM height and width must be divisible by {SANA_WM_VAE_SPATIAL_COMPRESSION}, " + f"got {height} and {width}." + ) + if (num_frames - 1) % SANA_WM_VAE_TEMPORAL_COMPRESSION: + raise ValueError( + f"Sana-WM num_frames must satisfy (num_frames - 1) % {SANA_WM_VAE_TEMPORAL_COMPRESSION} == 0, " + f"got {num_frames}." + ) + + +def normalize_sana_wm_payload(prompt: Mapping[str, Any]) -> dict[str, Any]: + """Return a prompt copy with canonical Sana-WM request metadata. + + The ``sana_wm`` block is read from the top-level ``sana_wm`` key, falling + back to ``additional_information["sana_wm"]`` so the function is idempotent + when called again on its own output. The first-frame image is read from + ``multi_modal_data["image"]``. + """ + + if not isinstance(prompt, Mapping): + raise ValueError(f"Sana-WM prompt must be a mapping, got {type(prompt).__name__}.") + + result = dict(prompt) + mm_data = _as_dict(result.get("multi_modal_data"), name="multi_modal_data") + image = _extract_image(mm_data) + if image is None: + raise ValueError("Sana-WM requires a first-frame image in `multi_modal_data['image']`.") + + additional = _as_dict(result.get("additional_information"), name="additional_information") + raw = _first_present( + result.get(SANA_WM_CANONICAL_KEY), + additional.get(SANA_WM_CANONICAL_KEY), + {}, + ) + raw = _as_dict(raw, name=SANA_WM_CANONICAL_KEY) + + num_frames = _as_positive_int( + _first_present(raw.get("num_frames"), result.get("num_frames"), SANA_WM_DEFAULT_NUM_FRAMES), + name="num_frames", + ) + height = _as_positive_int( + _first_present(raw.get("height"), result.get("height"), SANA_WM_DEFAULT_HEIGHT), + name="height", + ) + width = _as_positive_int(_first_present(raw.get("width"), result.get("width"), SANA_WM_DEFAULT_WIDTH), name="width") + _validate_latent_geometry(num_frames=num_frames, height=height, width=width) + + camera = raw.get("camera") + camera = _as_dict(camera, name="camera") if camera is not None else None + action = raw.get("action") + if action is not None: + if not isinstance(action, str) or not action.strip(): + raise ValueError("Sana-WM action must be a non-empty string when provided.") + # Rejected here rather than in the model: a mismatch used to shorten the + # camera tensor silently and then fail deep in the transformer on a + # camera/latent frame mismatch, and an unbounded duration used to be + # expanded one entry per frame before any of that. + rollout_frames = action_rollout_num_frames(action) + if rollout_frames != num_frames: + raise ValueError( + f"Sana-WM action rolls out {rollout_frames} frames but num_frames is {num_frames}. " + "Action segment durations must sum to num_frames - 1." + ) + if camera is not None and action is not None: + raise ValueError("Sana-WM accepts exactly one of `camera` or `action`, not both.") + if camera is None and action is None: + raise ValueError("Sana-WM requires either explicit camera poses or an action DSL string.") + + canonical_camera = None + if camera is not None: + poses = camera.get("poses") + if poses is None: + raise ValueError("Sana-WM camera payload requires `poses`.") + _validate_camera_poses(poses, num_frames=num_frames) + # Only the defaults are accepted: nothing downstream reads these two + # fields, so honouring another value would mean interpreting w2c poses + # as c2w and silently generating the wrong trajectory. + camera_format = camera.get("format", SANA_WM_DEFAULT_CAMERA_FORMAT) + if camera_format != SANA_WM_DEFAULT_CAMERA_FORMAT: + raise ValueError( + f"Sana-WM only accepts camera format {SANA_WM_DEFAULT_CAMERA_FORMAT!r}; " + f"convert the poses before sending, got {camera_format!r}." + ) + coordinate_system = camera.get("coordinate_system", SANA_WM_DEFAULT_COORDINATE_SYSTEM) + if coordinate_system != SANA_WM_DEFAULT_COORDINATE_SYSTEM: + raise ValueError( + f"Sana-WM only accepts coordinate system {SANA_WM_DEFAULT_COORDINATE_SYSTEM!r}, " + f"got {coordinate_system!r}." + ) + canonical_camera = {"poses": poses} + + intrinsics = raw.get("intrinsics") + _validate_intrinsics(intrinsics) + translation_speed = raw.get("translation_speed") + rotation_speed_deg = raw.get("rotation_speed_deg") + + canonical = { + "num_frames": num_frames, + "height": height, + "width": width, + "camera": canonical_camera, + "action": action, + "intrinsics": intrinsics, + } + if translation_speed is not None: + canonical["translation_speed"] = _as_positive_float(translation_speed, name="translation_speed") + if rotation_speed_deg is not None: + canonical["rotation_speed_deg"] = _as_positive_float(rotation_speed_deg, name="rotation_speed_deg") + + mm_data["image"] = image + result["multi_modal_data"] = mm_data + additional[SANA_WM_CANONICAL_KEY] = canonical + result["additional_information"] = additional + result.pop(SANA_WM_CANONICAL_KEY, None) + return result diff --git a/vllm_omni/diffusion/models/sana_wm/sana_wm_transformer.py b/vllm_omni/diffusion/models/sana_wm/sana_wm_transformer.py new file mode 100644 index 000000000..07537ec09 --- /dev/null +++ b/vllm_omni/diffusion/models/sana_wm/sana_wm_transformer.py @@ -0,0 +1,2065 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""SANA-WM Stage-1 transformer. + +Native vLLM-Omni port of the NVlabs SANA-WM DiT. Modules are built eagerly at +construction (under the loader's target-device context) and checkpoint tensors +are streamed in via ``load_weights``. The Bidirectional Gated DeltaNet +recurrence is implemented in pure PyTorch in this module. +""" + +from __future__ import annotations + +import math +from collections.abc import Iterable +from typing import Any, ClassVar + +import torch +import torch.nn.functional as F +from diffusers.models.embeddings import Timesteps +from torch import nn +from vllm.distributed import ( + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce, +) +from vllm.distributed.parallel_state import get_tp_group +from vllm.model_executor.layers.linear import ( + ColumnParallelLinear, + MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear, +) +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig +from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader + +from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata +from vllm_omni.diffusion.attention.layer import Attention +from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNorm +from vllm_omni.diffusion.layers.norm import RMSNorm +from vllm_omni.diffusion.models.sana_wm.config import SanaWmConfig +from vllm_omni.diffusion.models.sana_wm.ucpe import ( + SanaWmCamGeometry, + cam_prep_func, + prepare_cam_geometry, +) + +SANA_WM_STAGE1_LATENT_CHANNELS = 128 +SANA_WM_STAGE1_PROMPT_CHANNELS = 2304 +SANA_WM_STAGE1_TIMESTEP_CHANNELS = 256 + + +def _shard_param_across_tp(param: torch.Tensor | None, dim: int = 0) -> None: + """Attach vLLM's TP shard loader to a plain (non-parallel-layer) parameter. + + vLLM parallel layers narrow full checkpoint tensors to the local shard via + their own ``weight_loader``. SANA-WM also has plain parameters that become + TP-local - the GDN vectors (``A_log``, ``dt_bias``), the depthwise temporal + convs and the q/k norms - so they need the same behavior at load time. + Attached only under TP > 1: at TP = 1 the checkpoint tensor already matches + the parameter and ``sharded_weight_loader`` would require an initialized TP + group that standalone/unit-test builds do not have. + """ + if param is None or get_tensor_model_parallel_world_size() <= 1: + return + param.weight_loader = sharded_weight_loader(dim) + + +def _is_sana_wm_transformer_block(name: str, module: Any) -> bool: + del module + parts = name.split(".") + return len(parts) == 2 and parts[0] == "blocks" and parts[1].isdigit() + + +def _to_3tuple(value: int | tuple[int, int] | tuple[int, int, int]) -> tuple[int, int, int]: + if isinstance(value, int): + return (value, value, value) + if len(value) == 2: + return (1, int(value[0]), int(value[1])) + return (int(value[0]), int(value[1]), int(value[2])) + + +# --------------------------------------------------------------------------- +# Bidirectional Gated DeltaNet recurrence (pure PyTorch) +# +# Ported from the Apache-2.0 NVlabs/Sana reference. SANA-WM's bidirectional +# video-latent recurrence is a different contract than vLLM's autoregressive +# Qwen3-Next GDN cache path, so it stays model-local. +# --------------------------------------------------------------------------- + + +def _validate_gdn_inputs( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + beta: torch.Tensor, + decay: torch.Tensor, + spatial_tokens: int, +) -> tuple[int, int, int, int, int]: + if query.ndim != 4 or key.ndim != 4 or value.ndim != 4: + raise ValueError("Sana-WM GDN query/key/value must be shaped [B, H, D, N].") + if query.shape != key.shape or query.shape != value.shape: + raise ValueError( + "Sana-WM GDN query/key/value shapes must match, got " + f"{tuple(query.shape)}, {tuple(key.shape)}, {tuple(value.shape)}." + ) + if beta.ndim != 4: + raise ValueError("Sana-WM GDN beta must be shaped [B, H, T, S].") + if decay.ndim != 3: + raise ValueError("Sana-WM GDN decay must be shaped [B, H, T].") + if spatial_tokens <= 0: + raise ValueError("Sana-WM GDN spatial_tokens must be positive.") + + batch_size, num_heads, head_dim, token_count = query.shape + if token_count % spatial_tokens != 0: + raise ValueError(f"Sana-WM GDN token count {token_count} is not divisible by spatial_tokens={spatial_tokens}.") + frames = token_count // spatial_tokens + if beta.shape != (batch_size, num_heads, frames, spatial_tokens): + raise ValueError( + "Sana-WM GDN beta shape mismatch: expected " + f"{(batch_size, num_heads, frames, spatial_tokens)}, got {tuple(beta.shape)}." + ) + if decay.shape != (batch_size, num_heads, frames): + raise ValueError( + f"Sana-WM GDN decay shape mismatch: expected {(batch_size, num_heads, frames)}, got {tuple(decay.shape)}." + ) + return batch_size, num_heads, head_dim, token_count, frames + + +def _delta_scan( + query_rot: torch.Tensor, + key_rot: torch.Tensor, + value: torch.Tensor, + beta: torch.Tensor, + decay: torch.Tensor, + *, + spatial_tokens: int, + query: torch.Tensor | None = None, + key: torch.Tensor | None = None, + skip_z: bool = False, + flip_output: bool = False, +) -> tuple[torch.Tensor, torch.Tensor | None]: + """One-directional gated delta-rule recurrence over frames. + + The KV stream consumes only the rotary-embedded Q/K; the Z (denominator) + stream consumes only the non-rotary Q/K. ``skip_z=True`` drops the Z stream + entirely and returns ``(numerator, None)`` — the numerator-only variant the + camera branch needs (NVlabs ``torch_recurrent_cam_single_path_delta_rule``), + so ``query``/``key`` may be omitted there. + + ``flip_output=True`` emits the frames in reverse of the order they were + computed, which is what a caller running on flipped inputs wants; it costs a + list reversal instead of a flip over the assembled result. + """ + if not skip_z and (query is None or key is None): + raise ValueError("Sana-WM delta scan needs non-rotary query/key unless skip_z is set.") + + batch_size, num_heads, head_dim, token_count = query_rot.shape + frames = beta.shape[2] + if token_count != frames * spatial_tokens: + raise ValueError(f"Sana-WM delta scan token_count={token_count} != frames*S={frames * spatial_tokens}.") + + def to_frames(tensor: torch.Tensor) -> torch.Tensor: + return tensor.view(batch_size, num_heads, head_dim, frames, spatial_tokens).permute(0, 1, 3, 2, 4) + + query_rot_f = to_frames(query_rot) + key_rot_f = to_frames(key_rot) + value_f = to_frames(value) + state_kv = torch.zeros(batch_size, num_heads, head_dim, head_dim, device=query_rot.device, dtype=query_rot.dtype) + numerators: list[torch.Tensor] = [] + + if skip_z: + query_f = key_f = None + state_z = None + denominators = None + else: + query_f = to_frames(query) + key_f = to_frames(key) + state_z = torch.zeros(batch_size, num_heads, head_dim, 1, device=query_rot.device, dtype=query_rot.dtype) + denominators = [] + + for frame_idx in range(frames): + query_rot_t = query_rot_f[:, :, frame_idx] + key_rot_t = key_rot_f[:, :, frame_idx] + value_t = value_f[:, :, frame_idx] + beta_t = beta[:, :, frame_idx].unsqueeze(2) + decay_t = decay[:, :, frame_idx].view(batch_size, num_heads, 1, 1) + + state_kv = state_kv * decay_t + value_pred = torch.matmul(state_kv, key_rot_t) + delta_value = (value_t - value_pred) * beta_t + state_kv = state_kv + torch.matmul(delta_value, key_rot_t.transpose(-1, -2)) + numerators.append(torch.matmul(state_kv, query_rot_t)) + + if skip_z: + continue + query_t = query_f[:, :, frame_idx] + key_t = key_f[:, :, frame_idx] + state_z = state_z * decay_t + z_pred = torch.matmul(state_z.transpose(-1, -2), key_t) + delta_z = (1.0 - z_pred) * beta_t + state_z = state_z + torch.matmul(key_t, delta_z.transpose(-1, -2)) + denominators.append(torch.matmul(state_z.transpose(-1, -2), query_t)) + + def restore(tensors: list[torch.Tensor], dim: int) -> torch.Tensor: + stacked = torch.stack(tensors[::-1] if flip_output else tensors, dim=2) + return stacked.permute(0, 1, 3, 2, 4).reshape(batch_size, num_heads, dim, token_count) + + return restore(numerators, head_dim), (None if skip_z else restore(denominators, 1)) + + +def _bidirectional_delta_scan( + query_rot: torch.Tensor, + key_rot: torch.Tensor, + value: torch.Tensor, + beta: torch.Tensor, + decay: torch.Tensor, + *, + spatial_tokens: int, + query: torch.Tensor | None = None, + key: torch.Tensor | None = None, + skip_z: bool = False, +) -> tuple[torch.Tensor, torch.Tensor | None]: + """Forward + backward delta scan, summed. + + The backward direction shifts K/V/beta by one frame (zero pad) and the decay + by one frame (neutral 1.0 pad), matching the NVlabs ``flip_and_shift`` + convention. The backward scan also emits its frames in forward order, so the + two directions add directly instead of flipping the assembled result back. + """ + frames = beta.shape[2] + + def reverse(tensor: torch.Tensor, *, shift_value: float | None = None) -> torch.Tensor: + return _reverse_frames(tensor, frames=frames, spatial_tokens=spatial_tokens, shift_value=shift_value) + + num_fwd, den_fwd = _delta_scan( + query_rot, + key_rot, + value, + beta, + decay, + spatial_tokens=spatial_tokens, + query=query, + key=key, + skip_z=skip_z, + ) + + bwd_kwargs: dict[str, torch.Tensor] = {} + if not skip_z: + bwd_kwargs["query"] = reverse(query) + bwd_kwargs["key"] = reverse(key, shift_value=0.0) + + num_bwd, den_bwd = _delta_scan( + reverse(query_rot), + reverse(key_rot, shift_value=0.0), + reverse(value, shift_value=0.0), + _flip_and_shift(beta, dim=2, shift_value=0.0), + _flip_and_shift(decay, dim=2, shift_value=1.0), + spatial_tokens=spatial_tokens, + skip_z=skip_z, + flip_output=True, + **bwd_kwargs, + ) + + return num_fwd + num_bwd, (None if skip_z else den_fwd + den_bwd) + + +def _reverse_frames( + tensor: torch.Tensor, + *, + frames: int, + spatial_tokens: int, + shift_value: float | None = None, +) -> torch.Tensor: + """Reverse frame order on a ``[..., T*S]`` tensor, optionally shifting by one frame. + + The frame axis is the outer half of the flat token axis, so unfolding it in + place keeps the tensor's own layout: the flip writes a contiguous result and + the reshape back is a view. Routing this through a ``[..., T, D, S]`` + permutation instead would cost a second full materialization. + """ + lead = tensor.shape[:-1] + reversed_ = torch.flip(tensor.reshape(*lead, frames, spatial_tokens), dims=[-2]) + if shift_value is not None: + padding = torch.full((*lead, 1, spatial_tokens), shift_value, device=tensor.device, dtype=tensor.dtype) + reversed_ = torch.cat([padding, reversed_.narrow(-2, 0, frames - 1)], dim=-2) + return reversed_.reshape(*lead, frames * spatial_tokens) + + +def _flip_and_shift(tensor: torch.Tensor, *, dim: int, shift_value: float) -> torch.Tensor: + flipped = torch.flip(tensor, dims=[dim]) + shifted = flipped.narrow(dim, 0, tensor.shape[dim] - 1) + pad_shape = list(tensor.shape) + pad_shape[dim] = 1 + padding = torch.full(pad_shape, shift_value, device=tensor.device, dtype=tensor.dtype) + return torch.cat([padding, shifted], dim=dim) + + +def reference_bidirectional_gated_delta_net( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + *, + beta: torch.Tensor, + decay: torch.Tensor, + spatial_tokens: int, + query_rot: torch.Tensor | None = None, + key_rot: torch.Tensor | None = None, + eps: float = 1e-8, +) -> torch.Tensor: + """Run the SANA-WM bidirectional gated delta recurrence in PyTorch. + + Inputs follow the layout used by the official Stage-1 operator: + ``query/key/value/query_rot/key_rot`` are ``[B, H, D, T*S]``, ``beta`` is + ``[B, H, T, S]``, and ``decay`` is ``[B, H, T]``. + """ + + _validate_gdn_inputs(query, key, value, beta, decay, spatial_tokens) + if query_rot is None: + query_rot = query + if key_rot is None: + key_rot = key + if query_rot.shape != query.shape or key_rot.shape != key.shape: + raise ValueError("Sana-WM GDN rotary query/key shapes must match query/key.") + + dtype_orig = query.dtype + numerator, denominator = _bidirectional_delta_scan( + query_rot.float(), + key_rot.float(), + value.float(), + beta.float(), + decay.float(), + spatial_tokens=spatial_tokens, + query=query.float(), + key=key.float(), + ) + return (numerator / (denominator + eps)).to(dtype_orig) + + +class TensorParallelRMSNorm(nn.Module): + """RMSNorm whose statistics span a tensor-parallel sharded last dimension. + + Models that normalize q/k across *all* heads (``qk_norm="rms_norm_across_heads"``) + cannot use a plain RMSNorm once the q/k projections are column-parallel: each + rank would then compute the RMS over its local head shard only. This + all-reduces the squared sum so the denominator matches the global width, + while the affine weight stays local. + + Model-local on purpose: LTX-2 and DreamZero carry their own near-identical + copies, and converging all three belongs in its own change rather than in + the SANA-WM port. + """ + + def __init__( + self, + hidden_size: int, + eps: float = 1e-6, + elementwise_affine: bool = True, + tp_size: int = 1, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.tp_size = max(int(tp_size), 1) + self.global_hidden_size = hidden_size * self.tp_size + self.eps = eps + if elementwise_affine: + self.weight = nn.Parameter(torch.ones(hidden_size)) + else: + self.register_parameter("weight", None) + + def _local_sum_sq(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + x_float = x.float() + return x_float, x_float.pow(2).sum(dim=-1, keepdim=True) + + def _scale(self, x_float: torch.Tensor, global_sum_sq: torch.Tensor, input_dtype: torch.dtype) -> torch.Tensor: + out = x_float * torch.rsqrt(global_sum_sq / self.global_hidden_size + self.eps) + if self.weight is not None: + out = out * self.weight.float() + return out.to(dtype=input_dtype) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x_float, sum_sq = self._local_sum_sq(x) + if self.tp_size > 1: + sum_sq = tensor_model_parallel_all_reduce(sum_sq) + return self._scale(x_float, sum_sq, x.dtype) + + +def fused_qk_rms_norm( + norm_q: nn.Module, + norm_k: nn.Module, + q: torch.Tensor, + k: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Apply q/k :class:`TensorParallelRMSNorm` with a SINGLE fused all-reduce. + + Self-attention normalizes q and k every step, and under TP each norm issues + its own tiny all-reduce of the per-token sum-of-squares. These collectives + are latency-bound, so packing both sums into one tensor halves the count + (2 collectives -> 1) for free. + + Numerically identical to ``norm_q(q), norm_k(k)``: all-reduce is + elementwise, so packing along the last dim reduces each slice independently + with the same fp32 accumulation. Falls back to independent application when + either norm is not a :class:`TensorParallelRMSNorm` (e.g. ``nn.Identity`` + when qk_norm is off, or plain ``RMSNorm`` at TP=1). + """ + if not (isinstance(norm_q, TensorParallelRMSNorm) and isinstance(norm_k, TensorParallelRMSNorm)): + return norm_q(q), norm_k(k) + if norm_q.tp_size <= 1: + return norm_q(q), norm_k(k) + + q_float, q_sum_sq = norm_q._local_sum_sq(q) + k_float, k_sum_sq = norm_k._local_sum_sq(k) + packed = tensor_model_parallel_all_reduce(torch.cat([q_sum_sq, k_sum_sq], dim=-1)) + q_sum_sq, k_sum_sq = packed[..., 0:1], packed[..., 1:2] + return norm_q._scale(q_float, q_sum_sq, q.dtype), norm_k._scale(k_float, k_sum_sq, k.dtype) + + +def _make_sharded_qk_rms_norm(hidden_size: int, eps: float = 1e-6) -> nn.Module: + """Build the q/k norm, TP-aware. + + SANA-WM normalizes q/k across all heads, so once the projections are + column-parallel the RMS denominator has to span the TP ranks. + """ + tp_size = get_tensor_model_parallel_world_size() + if tp_size > 1: + return TensorParallelRMSNorm(hidden_size, eps=eps, tp_size=tp_size) + return RMSNorm(hidden_size, eps=eps) + + +class SanaWmTextProjection(nn.Module): + def __init__( + self, + prompt_channels: int, + hidden_size: int, + *, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.fc1 = ColumnParallelLinear( + prompt_channels, + hidden_size, + bias=True, + gather_output=False, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.fc1" if prefix else "fc1", + ) + self.fc2 = RowParallelLinear( + hidden_size, + hidden_size, + bias=True, + input_is_parallel=True, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.fc2" if prefix else "fc2", + ) + self.act = nn.SiLU() + + def forward(self, prompt_embeds: torch.Tensor) -> torch.Tensor: + return self.fc2(self.act(self.fc1(prompt_embeds))) + + +class SanaWmTextEmbedder(nn.Module): + def __init__( + self, + prompt_channels: int, + hidden_size: int, + max_length: int, + *, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.y_embedding = nn.Parameter(torch.zeros(max_length, prompt_channels)) + self.y_proj = SanaWmTextProjection( + prompt_channels, + hidden_size, + quant_config=quant_config, + prefix=f"{prefix}.y_proj" if prefix else "y_proj", + ) + + def forward( + self, + prompt_embeds: torch.Tensor | None, + *, + batch_size: int, + dtype: torch.dtype, + ) -> torch.Tensor: + if prompt_embeds is None: + prompt_embeds = self.y_embedding.unsqueeze(0).expand(batch_size, -1, -1) + return self.y_proj(prompt_embeds.to(dtype=dtype)) + + +class SanaWmTimestepEmbedder(nn.Module): + def __init__( + self, + in_features: int, + hidden_size: int, + *, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + linear_1 = ColumnParallelLinear( + in_features, + hidden_size, + bias=True, + gather_output=False, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.mlp.0" if prefix else "mlp.0", + ) + linear_2 = RowParallelLinear( + hidden_size, + hidden_size, + bias=True, + input_is_parallel=True, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.mlp.2" if prefix else "mlp.2", + ) + self.act = nn.SiLU() + self.mlp = nn.ModuleList([linear_1, self.act, linear_2]) + # diffusers' Timesteps reproduces this model's sinusoid exactly: + # downscale_freq_shift=0 makes the exponent denominator half_dim, and + # flip_sin_to_cos=True emits [cos, sin] in that order. The MLP stays + # model-local because its checkpoint keys are mlp.0/mlp.2 and it is + # tensor-parallel. + self.timesteps_proj = Timesteps(num_channels=in_features, flip_sin_to_cos=True, downscale_freq_shift=0) + + def forward(self, timestep: torch.Tensor) -> torch.Tensor: + emb = self.timesteps_proj(timestep.flatten()) + hidden_states = self.mlp[0](emb.to(dtype=self.mlp[0].weight.dtype)) + hidden_states = self.act(hidden_states) + return self.mlp[2](hidden_states) + + +class SanaWmPatchEmbedMS3D(nn.Module): + """Official-style 3D patch embedder used by SANA-WM Stage-1.""" + + def __init__( + self, + patch_size: tuple[int, int, int], + in_channels: int, + hidden_size: int, + *, + kernel_size: tuple[int, int, int] | None = None, + bias: bool = True, + ) -> None: + super().__init__() + self.patch_size = _to_3tuple(patch_size) + self.kernel_size = _to_3tuple(kernel_size or patch_size) + self.proj = nn.Conv3d( + in_channels, + hidden_size, + kernel_size=self.kernel_size, + stride=self.patch_size, + bias=bias, + ) + self.norm = nn.Identity() + + def project_with_shape(self, latents: torch.Tensor) -> tuple[torch.Tensor, tuple[int, int, int]]: + hidden_states = self.norm(self.proj(latents)) + _, _, frames, height, width = hidden_states.shape + return hidden_states.flatten(2).transpose(1, 2), (frames, height, width) + + def forward(self, latents: torch.Tensor) -> torch.Tensor: + return self.project_with_shape(latents)[0] + + +class SanaWmWanRotaryPosEmbed(nn.Module): + """Wan-style 3D RoPE table used by the official SANA-WM GDN blocks.""" + + def __init__( + self, + attention_head_dim: int, + *, + max_seq_len: int = 1024, + theta: float = 10000.0, + ) -> None: + super().__init__() + self.theta = theta + h_dim = w_dim = 2 * (attention_head_dim // 6) + t_dim = attention_head_dim - h_dim - w_dim + self._split_sizes = ( + t_dim // 2, + h_dim // 2, + w_dim // 2, + ) + self.freqs = self._build_freqs(max_seq_len) + + def _build_1d_freq(self, dim: int, positions: torch.Tensor) -> torch.Tensor: + if dim <= 0: + return torch.empty(positions.shape[0], 0, dtype=torch.complex128) + freqs = 1.0 / ( + self.theta ** (torch.arange(0, dim, 2, dtype=torch.float64, device=positions.device)[: dim // 2] / dim) + ) + phase = torch.outer(positions.to(torch.float64), freqs) + return torch.polar(torch.ones_like(phase), phase) + + def _build_freqs(self, max_seq_len: int) -> torch.Tensor: + positions = torch.arange(max_seq_len, dtype=torch.float64) + dims = ( + self._split_sizes[0] * 2, + self._split_sizes[1] * 2, + self._split_sizes[2] * 2, + ) + return torch.cat([self._build_1d_freq(dim, positions) for dim in dims], dim=1) + + def forward(self, spatial_shape: tuple[int, int, int], device: torch.device) -> torch.Tensor: + frames, height, width = spatial_shape + if max(spatial_shape) > self.freqs.shape[0]: + self.freqs = self._build_freqs(max(spatial_shape)).to(self.freqs.device) + freqs = self.freqs.to(device=device) + freqs_f, freqs_h, freqs_w = freqs.split(self._split_sizes, dim=1) + f_dim, h_dim, w_dim = self._split_sizes + parts = [ + freqs_f[:frames].view(frames, 1, 1, f_dim).expand(frames, height, width, f_dim), + freqs_h[:height].view(1, height, 1, h_dim).expand(frames, height, width, h_dim), + freqs_w[:width].view(1, 1, width, w_dim).expand(frames, height, width, w_dim), + ] + return torch.cat(parts, dim=-1).reshape(1, 1, frames * height * width, -1) + + +class SanaWmSelfAttention(nn.Module): + def __init__( + self, + config: SanaWmConfig, + *, + use_gdn: bool = True, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + hidden_size = config.hidden_size + self.total_num_heads = max(hidden_size // max(config.linear_head_dim, 1), 1) + self.head_dim = hidden_size // self.total_num_heads + self.num_heads = self.total_num_heads + self.num_kv_heads = self.total_num_heads + self.eps = 1e-8 + self.use_gdn = use_gdn and "GDN" in config.attn_type + self.patch_size = _to_3tuple(config.patch_size) + # Total camera branch width from the checkpoint. TP ColumnParallel + # layers expose a per-rank local slice at runtime; keep both sizes so + # layer construction uses the global contract while local conv/norm + # tensors match q/k/v outputs. + self.total_cam_dim = hidden_size // max(config.cam_attn_compress, 1) + self.cam_dim = self.total_cam_dim + self.qkv = QKVParallelLinear( + hidden_size=hidden_size, + head_size=self.head_dim, + total_num_heads=self.total_num_heads, + bias=False, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.qkv" if prefix else "qkv", + ) + self.num_heads = self.qkv.num_heads + self.num_kv_heads = self.qkv.num_kv_heads + self.proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=True, + input_is_parallel=True, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.proj" if prefix else "proj", + ) + self.beta_proj = ColumnParallelLinear( + hidden_size, + self.total_num_heads, + bias=True, + gather_output=False, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.beta_proj" if prefix else "beta_proj", + ) + self.gate_proj = ColumnParallelLinear( + hidden_size, + self.total_num_heads, + bias=True, + gather_output=False, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.gate_proj" if prefix else "gate_proj", + ) + self.output_gate = ColumnParallelLinear( + hidden_size, + self.total_num_heads * self.head_dim, + bias=True, + gather_output=False, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.output_gate" if prefix else "output_gate", + ) + self.q_proj_cam = ColumnParallelLinear( + hidden_size, + self.total_cam_dim, + bias=True, + gather_output=False, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.q_proj_cam" if prefix else "q_proj_cam", + ) + self.k_proj_cam = ColumnParallelLinear( + hidden_size, + self.total_cam_dim, + bias=True, + gather_output=False, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.k_proj_cam" if prefix else "k_proj_cam", + ) + self.v_proj_cam = ColumnParallelLinear( + hidden_size, + self.total_cam_dim, + bias=True, + gather_output=False, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.v_proj_cam" if prefix else "v_proj_cam", + ) + self.out_proj_cam = RowParallelLinear( + self.total_cam_dim, + hidden_size, + bias=True, + input_is_parallel=True, + # The camera contribution is summed into the TP-local GDN stream, so + # the cross-rank sum and the slice down to the local width collapse + # into a single reduce-scatter (see _reduce_scatter_cam_contrib). + # RowParallelLinear must therefore not all-reduce, and its bias is + # applied after the scatter rather than inside the matmul. + reduce_results=False, + skip_bias_add=True, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.out_proj_cam" if prefix else "out_proj_cam", + ) + local_inner_dim = self.num_heads * self.head_dim + norm_cls = _make_sharded_qk_rms_norm if config.qk_norm else (lambda *_args, **_kwargs: nn.Identity()) + self.q_norm = norm_cls(local_inner_dim) + self.k_norm = norm_cls(local_inner_dim) + + # These names mirror the official GDN checkpoint. + self.A_log = nn.Parameter(torch.zeros(self.num_heads)) + self.dt_bias = nn.Parameter(torch.zeros(self.num_heads)) + self.register_buffer("recall_gate", torch.zeros(1)) + self.conv_k = ( + nn.Conv1d( + local_inner_dim, + local_inner_dim, + kernel_size=config.conv_kernel_size, + groups=local_inner_dim, + bias=False, + ) + if self.use_gdn and config.conv_kernel_size > 0 + else None + ) + self.conv_q = None + self.conv_v = None + + self.cam_dim = int(getattr(self.q_proj_cam, "output_size_per_partition", self.total_cam_dim)) + + cam_compress = max(config.cam_attn_compress, 1) + self.cam_heads = max(self.num_heads // cam_compress, 1) + if self.cam_dim % self.cam_heads != 0: + raise ValueError(f"Sana-WM local cam_dim={self.cam_dim} must be divisible by cam_heads={self.cam_heads}.") + self.cam_head_dim = self.cam_dim // self.cam_heads + # Under TP, cam_head_dim must equal main head_dim so the WAN RoPE + # (built for main head_dim) can be correctly sliced for the cam branch. + # A mismatch indicates the cam ColumnParallelLinear layers were not + # partitioned consistently with the main QKVParallelLinear. + if self.cam_head_dim != self.head_dim: + raise ValueError( + f"Sana-WM cam_head_dim={self.cam_head_dim} != head_dim={self.head_dim}. " + f"Under TP the cam layers must be partitioned by the same TP degree as the " + f"main QKV (cam_dim={self.cam_dim}, cam_heads={self.cam_heads}, " + f"total_cam_dim={self.total_cam_dim})." + ) + self.q_norm_cam = norm_cls(self.cam_dim) + self.k_norm_cam = norm_cls(self.cam_dim) + self.conv_k_cam = ( + nn.Conv1d( + self.cam_dim, + self.cam_dim, + kernel_size=config.conv_kernel_size, + groups=self.cam_dim, + bias=False, + ) + if self.use_gdn and config.conv_kernel_size > 0 + else None + ) + self.conv_q_cam = None + self.conv_v_cam = None + # Softmax hybrid blocks (every ``softmax_every_n``-th) and the camera + # branch both run plain non-causal self-attention, so both go through + # the shared Attention for platform backend selection and the fp32 + # fallback. The camera branch gets its own module because + # ``cam_attn_compress`` may give it fewer heads. + self.softmax_attn = Attention( + num_heads=self.num_heads, + head_size=self.head_dim, + num_kv_heads=self.num_kv_heads, + softmax_scale=self.head_dim**-0.5, + causal=False, + role="self", + qkv_layout="BSND", + skip_sequence_parallel=True, + prefix=prefix, + ) + self.softmax_attn_cam = Attention( + num_heads=self.cam_heads, + head_size=self.cam_head_dim, + num_kv_heads=self.cam_heads, + softmax_scale=self.cam_head_dim**-0.5, + causal=False, + role="self", + qkv_layout="BSND", + skip_sequence_parallel=True, + prefix=prefix, + ) + self._init_short_convs() + self._mark_tp_sharded_params() + + def _mark_tp_sharded_params(self) -> None: + """Declare which plain parameters are TP-local (see ``_shard_param_across_tp``).""" + # Per-head GDN vectors: one entry per local head. + _shard_param_across_tp(self.A_log) + _shard_param_across_tp(self.dt_bias) + # Depthwise temporal convs and q/k norms: one entry per local channel. + for module in (self.conv_q, self.conv_k, self.conv_v, self.conv_q_cam, self.conv_k_cam, self.conv_v_cam): + if module is not None: + _shard_param_across_tp(module.weight) + for norm in (self.q_norm, self.k_norm, self.q_norm_cam, self.k_norm_cam): + _shard_param_across_tp(getattr(norm, "weight", None)) + + @staticmethod + def _init_short_conv(conv: nn.Conv1d | None) -> None: + if conv is None: + return + with torch.no_grad(): + conv.weight.zero_() + conv.weight[:, 0, -1] = 1.0 + + def _init_short_convs(self) -> None: + for conv in (self.conv_q, self.conv_k, self.conv_v, self.conv_q_cam, self.conv_k_cam, self.conv_v_cam): + self._init_short_conv(conv) + + @staticmethod + def _apply_rotary_emb(hidden_states: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor: + rotated = torch.view_as_complex(hidden_states.permute(0, 1, 3, 2).to(torch.float64).unflatten(3, (-1, 2))) + output = torch.view_as_real(rotated * freqs).flatten(3, 4).permute(0, 1, 3, 2) + return output.type_as(hidden_states) + + @staticmethod + def _reshape_to_temporal( + hidden_states: torch.Tensor, + spatial_shape: tuple[int, int, int], + ) -> tuple[torch.Tensor, int, int, int]: + batch_size, token_count, hidden_size = hidden_states.shape + frames, height, width = spatial_shape + spatial_tokens = height * width + if token_count != frames * spatial_tokens: + raise ValueError(f"Sana-WM temporal conv expects N=T*H*W, got N={token_count}, THW={spatial_shape}.") + hidden_states = hidden_states.reshape(batch_size, frames, spatial_tokens, hidden_size) + hidden_states = hidden_states.permute(0, 2, 1, 3).reshape(batch_size * spatial_tokens, frames, hidden_size) + return hidden_states, batch_size, spatial_tokens, frames + + @staticmethod + def _reshape_from_temporal( + hidden_states: torch.Tensor, + batch_size: int, + spatial_tokens: int, + frames: int, + ) -> torch.Tensor: + hidden_size = hidden_states.shape[-1] + return ( + hidden_states.reshape(batch_size, spatial_tokens, frames, hidden_size) + .permute(0, 2, 1, 3) + .reshape(batch_size, frames * spatial_tokens, hidden_size) + ) + + @staticmethod + def _causal_conv_1d(hidden_states: torch.Tensor, conv: nn.Conv1d) -> torch.Tensor: + dtype = hidden_states.dtype + conv_input = hidden_states.transpose(1, 2).to(conv.weight.dtype) + conv_input = F.pad(conv_input, (conv.kernel_size[0] - 1, 0)) + output = conv(conv_input).transpose(1, 2) + return output.to(dtype) + + def _bidirectional_temporal_short_conv( + self, + hidden_states: torch.Tensor, + conv: nn.Conv1d, + spatial_shape: tuple[int, int, int], + ) -> torch.Tensor: + hidden_states, batch_size, spatial_tokens, frames = self._reshape_to_temporal(hidden_states, spatial_shape) + forward_states = self._causal_conv_1d(hidden_states, conv) + backward_states = self._causal_conv_1d(hidden_states.flip(1), conv).flip(1) + center_weight = conv.weight[:, 0, -1].to(hidden_states.dtype) + center_states = hidden_states * center_weight.view(1, 1, -1) + output = forward_states + backward_states - center_states + return self._reshape_from_temporal(output, batch_size, spatial_tokens, frames) + + def _compute_frame_gates( + self, + hidden_states: torch.Tensor, + spatial_shape: tuple[int, int, int], + ) -> tuple[torch.Tensor, torch.Tensor]: + batch_size, token_count, hidden_size = hidden_states.shape + frames, height, width = spatial_shape + spatial_tokens = height * width + if token_count != frames * spatial_tokens: + raise ValueError(f"Sana-WM GDN token layout mismatch: N={token_count}, expected {frames * spatial_tokens}.") + beta = torch.sigmoid(self.beta_proj(hidden_states)) + beta = beta.reshape(batch_size, frames, spatial_tokens, self.num_heads).permute(0, 3, 1, 2) + frame_states = hidden_states.reshape(batch_size, frames, spatial_tokens, hidden_size).mean(dim=2) + gate = self.gate_proj(frame_states).float() + decay = torch.exp( + -self.A_log.float().exp().view(1, 1, -1) * F.softplus(gate + self.dt_bias.float().view(1, 1, -1)) + ) + return beta, decay.transpose(1, 2) + + def _forward_gdn_raw( + self, + hidden_states: torch.Tensor, + spatial_shape: tuple[int, int, int], + rotary_emb: torch.Tensor | None, + *, + precomputed_gates: tuple[torch.Tensor, torch.Tensor] | None = None, + ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]: + """Run main-branch bidirectional GDN and return the (B, N, q_size) + raw output BEFORE ``output_gate`` and ``proj`` are applied. + + Returns ``(raw_output, (beta, decay))`` so the caller can share the + beta/decay gates with the camera branch (matches NVlabs' shared + ``precomputed_gates`` plumbing). + """ + batch_size, token_count, _hidden_size = hidden_states.shape + frames, height, width = spatial_shape + spatial_tokens = height * width + if token_count != frames * spatial_tokens: + raise ValueError(f"Sana-WM GDN expects N=T*H*W, got N={token_count}, THW={spatial_shape}.") + + qkv = self.qkv(hidden_states) + q_size = self.num_heads * self.head_dim + kv_size = self.num_kv_heads * self.head_dim + query, key, value = qkv.split([q_size, kv_size, kv_size], dim=-1) + if self.conv_k is not None: + key = self._bidirectional_temporal_short_conv(key, self.conv_k, spatial_shape) + if precomputed_gates is None: + beta, decay = self._compute_frame_gates(hidden_states, spatial_shape) + else: + beta, decay = precomputed_gates + + query, key = fused_qk_rms_norm(self.q_norm, self.k_norm, query, key) + query = query.reshape(batch_size, token_count, self.num_heads, self.head_dim) + key = key.reshape(batch_size, token_count, self.num_heads, self.head_dim) + value = value.reshape(batch_size, token_count, self.num_heads, self.head_dim) + + query = F.relu(query).permute(0, 2, 3, 1) + key = F.relu(key).permute(0, 2, 3, 1) + value = value.permute(0, 2, 3, 1) + key = key * ((self.head_dim**-0.5) * (spatial_tokens**-0.5)) + if rotary_emb is not None: + query_rot = self._apply_rotary_emb(query, rotary_emb) + key_rot = self._apply_rotary_emb(key, rotary_emb) + else: + query_rot = query + key_rot = key + + output = reference_bidirectional_gated_delta_net( + query, + key, + value, + beta=beta, + decay=decay, + spatial_tokens=spatial_tokens, + query_rot=query_rot, + key_rot=key_rot, + eps=self.eps, + ) + output = output.permute(0, 3, 1, 2).reshape(batch_size, token_count, q_size) + return output, (beta, decay) + + def _apply_output_gate_and_proj( + self, + combined: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + """Apply SiLU output gate (driven by hidden_states) + final projection. + + The gate reads ``hidden_states``, not ``combined``, and NVlabs applies + it once to the main+camera sum rather than per branch. + """ + gate = F.silu(self.output_gate(hidden_states).float()) + gated = combined * gate + return self.proj(gated.to(self.proj.weight.dtype)) + + def _reduce_scatter_cam_contrib(self, contrib: torch.Tensor, reference: torch.Tensor) -> torch.Tensor: + """Reduce the partial camera projection straight down to the local shard. + + ``out_proj_cam`` is row-parallel over the camera features, so each rank + produces a partial sum spanning the full hidden width. The GDN stream it + is added to is TP-local (``output_gate`` and ``proj`` both expect the + local width), so an all-reduce would hand every rank the full result + only for all but one slice of it to be discarded. Reduce-scatter does + both steps in one collective and moves 1/tp of the bytes. + + ``skip_bias_add`` keeps the bias out of the matmul so it can be applied + to the scattered shard; that slice is the only remaining use of the TP + rank here, and it is a 1-D view rather than a copy of the hidden tensor. + + ``tensor_model_parallel_reduce_scatter`` is not usable here: it calls + ``get_tp_group().reduce_scatter()``, and vLLM-Omni's diffusion + ``GroupCoordinator`` deliberately does not implement that method (see + the note in ``diffusion/distributed/parallel_state.py``). Drive + ``torch.distributed`` on the group's process group instead, and fall + back to all-reduce plus a slice if the group is not exposed. + """ + bias = self.out_proj_cam.bias + tp_size = get_tensor_model_parallel_world_size() + if tp_size == 1: + return contrib if bias is None else contrib + bias + + local_width = reference.shape[-1] + if local_width <= 0 or contrib.shape[-1] != local_width * tp_size: + raise ValueError( + "Sana-WM TP camera contribution width mismatch: " + f"contrib={contrib.shape[-1]} expected={local_width * tp_size} (local={local_width}, tp={tp_size})." + ) + + device_group = getattr(get_tp_group(), "device_group", None) + if device_group is None: + out = tensor_model_parallel_all_reduce(contrib.contiguous()) + start = get_tensor_model_parallel_rank() * local_width + out = out[..., start : start + local_width].contiguous() + else: + # reduce_scatter_tensor splits along dim 0, so move the rank axis to + # the front first. Rank r then receives the summed columns + # [r * local_width, (r + 1) * local_width), matching what the + # all-reduce-and-slice fallback produces. + batch, tokens, _ = contrib.shape + source = ( + contrib.reshape(batch, tokens, tp_size, local_width) + .permute(2, 0, 1, 3) + .contiguous() + .reshape(tp_size * batch, tokens, local_width) + ) + out = torch.empty((batch, tokens, local_width), device=contrib.device, dtype=contrib.dtype) + torch.distributed.reduce_scatter_tensor(out, source, group=device_group) + + if bias is not None: + start = get_tensor_model_parallel_rank() * local_width + out = out + bias[start : start + local_width] + return out + + def _forward_gdn( + self, + hidden_states: torch.Tensor, + spatial_shape: tuple[int, int, int], + rotary_emb: torch.Tensor | None, + ) -> torch.Tensor: + """Main-branch only GDN forward with output_gate + proj applied.""" + raw, _ = self._forward_gdn_raw(hidden_states, spatial_shape, rotary_emb) + return self._apply_output_gate_and_proj(raw, hidden_states) + + @staticmethod + def _downscale_to_reference_rms( + ref: torch.Tensor, + transformed: torch.Tensor, + eps: float = 1e-6, + ) -> torch.Tensor: + """Downscale ``transformed`` so its per-token channel RMS does not + exceed ``ref``'s. Mirrors NVlabs ``_downscale_to_reference_rms``. + + Inputs are ``(B, H, N, D)`` (channel dim last). The RMS-scale is + clamped at 1.0 — the function only ever shrinks, never amplifies. + """ + ref_rms = ref.square().mean(dim=-1, keepdim=True).add(eps).sqrt() + tr_rms = transformed.square().mean(dim=-1, keepdim=True).add(eps).sqrt() + scale = (ref_rms / tr_rms.clamp_min(eps)).clamp(max=1.0) + return transformed * scale + + @staticmethod + def _ucpe_rotary_freqs(rotary_emb: torch.Tensor | None) -> torch.Tensor | None: + # SanaWmRope emits (1, 1, N, D//2) complex; the cam path wants (N, D//2). + if rotary_emb is None: + return None + return rotary_emb.squeeze(0).squeeze(0) + + def _forward_softmax_raw( + self, + hidden_states: torch.Tensor, + spatial_shape: tuple[int, int, int], + rotary_emb: torch.Tensor | None, + ) -> torch.Tensor: + """Softmax self-attention raw output before output_gate/proj. + + Mirrors NVlabs ``_forward_softmax_attn(..., apply_output_gate=False)`` + for every-N-th hybrid blocks. GDN-specific gates are intentionally not + used here. + """ + batch_size, token_count, hidden_size = hidden_states.shape + frames, height, width = spatial_shape + if token_count != frames * height * width: + raise ValueError(f"Sana-WM softmax attention expects N=T*H*W, got N={token_count}, THW={spatial_shape}.") + + qkv = self.qkv(hidden_states) + q_size = self.num_heads * self.head_dim + kv_size = self.num_kv_heads * self.head_dim + query, key, value = qkv.split([q_size, kv_size, kv_size], dim=-1) + + query, key = fused_qk_rms_norm(self.q_norm, self.k_norm, query, key) + query = query.reshape(batch_size, token_count, self.num_heads, self.head_dim) + key = key.reshape(batch_size, token_count, self.num_kv_heads, self.head_dim) + value = value.reshape(batch_size, token_count, self.num_kv_heads, self.head_dim) + + if rotary_emb is not None: + query = self._apply_rotary_emb(query.permute(0, 2, 3, 1), rotary_emb).permute(0, 3, 1, 2) + key = self._apply_rotary_emb(key.permute(0, 2, 3, 1), rotary_emb).permute(0, 3, 1, 2) + + # NVlabs runs the attention in bf16 even under ``fp32_attention=True``, + # since FlashAttention only supports bf16/fp16 and fp32 falls back to + # the math backend. Keep Q/K/V at the module dtype so an upstream step + # that promoted to fp32 does not silently change the kernel. See + # ``sana_gdn_camctrl_blocks.py::_forward_softmax_attn_sdpa``. + dtype_orig = hidden_states.dtype + attn = self.softmax_attn(query.to(dtype_orig), key.to(dtype_orig), value.to(dtype_orig)) + return attn.reshape(batch_size, token_count, q_size) + + def _forward_softmax_cam_branch( + self, + hidden_states: torch.Tensor, + spatial_shape: tuple[int, int, int], + cam_geometry: SanaWmCamGeometry, + rotary_emb: torch.Tensor | None, + ) -> torch.Tensor: + """UCPE camera branch for softmax hybrid blocks. + + Matches NVlabs ``_forward_cam_branch_softmax``: camera Q/K/V are + projected from the same hidden stream, transformed by UCPE, run through + SDPA, inverse-transformed, then returned as raw ``cam_dim`` features. + """ + batch_size, token_count, _ = hidden_states.shape + frames, height, width = spatial_shape + if token_count != frames * height * width: + raise ValueError(f"Sana-WM softmax cam branch expects N=T*H*W, got N={token_count}, THW={spatial_shape}.") + + q_cam = self.q_proj_cam(hidden_states) + k_cam = self.k_proj_cam(hidden_states) + v_cam = self.v_proj_cam(hidden_states) + + if self.conv_q_cam is not None: + q_cam = self._bidirectional_temporal_short_conv(q_cam, self.conv_q_cam, spatial_shape) + if self.conv_k_cam is not None: + k_cam = self._bidirectional_temporal_short_conv(k_cam, self.conv_k_cam, spatial_shape) + if self.conv_v_cam is not None: + v_cam = self._bidirectional_temporal_short_conv(v_cam, self.conv_v_cam, spatial_shape) + + q_cam, k_cam = fused_qk_rms_norm(self.q_norm_cam, self.k_norm_cam, q_cam, k_cam) + q_cam = q_cam.reshape(batch_size, token_count, self.cam_heads, self.cam_head_dim) + k_cam = k_cam.reshape(batch_size, token_count, self.cam_heads, self.cam_head_dim) + v_cam = v_cam.reshape(batch_size, token_count, self.cam_heads, self.cam_head_dim) + + q_cam = q_cam.transpose(1, 2).contiguous() # (B, H_cam, N, D) + k_cam = k_cam.transpose(1, 2).contiguous() + v_cam = v_cam.transpose(1, 2).contiguous() + + q_cam_trans = cam_geometry.apply_q(q_cam) + kv_cam_trans = cam_geometry.apply_kv(torch.cat([k_cam, v_cam], dim=1)) + k_cam_trans, v_cam_trans = torch.chunk(kv_cam_trans, chunks=2, dim=1) + + q_cam_trans = self._downscale_to_reference_rms(q_cam, q_cam_trans) + k_cam_trans = self._downscale_to_reference_rms(k_cam, k_cam_trans) + v_cam_trans = self._downscale_to_reference_rms(v_cam, v_cam_trans) + + # The shared Attention takes BSND and selects a backend that supports + # ``cam_head_dim`` directly, so odd head dims need no padding up to an + # SDPA-friendly size. Q/K/V stay at the module dtype for the same + # reason as the main softmax path above. + dtype_orig = hidden_states.dtype + out = self.softmax_attn_cam( + q_cam_trans.transpose(1, 2).to(dtype_orig), + k_cam_trans.transpose(1, 2).to(dtype_orig), + v_cam_trans.transpose(1, 2).to(dtype_orig), + ) + out = cam_geometry.apply_output(out.transpose(1, 2)) + return out.transpose(1, 2).reshape(batch_size, token_count, self.cam_dim) + + def _forward_cam_branch( + self, + hidden_states: torch.Tensor, + spatial_shape: tuple[int, int, int], + cam_geometry: SanaWmCamGeometry, + rotary_emb: torch.Tensor | None, + *, + precomputed_gates: tuple[torch.Tensor, torch.Tensor], + ) -> torch.Tensor: + """UCPE camera branch — matches the NVlabs production + ``BidirectionalGDNUCPESinglePathLiteLABothTriton`` variant declared + in the SANA-WM 1600M release config. + + Two things differ from the main GDN branch and are easy to get wrong: + Q/K/V all come from ``hidden_states`` (UCPE is not cross-attention — + camera information enters through the per-pixel projection matrices), + and the recurrence is single-path, i.e. numerator only, with no Z + denominator and no final divide. + + Returns ``(B, N, cam_dim)`` raw — caller applies ``out_proj_cam`` and + the shared ``output_gate`` + ``proj``. + """ + batch_size, token_count, _ = hidden_states.shape + frames, height, width = spatial_shape + spatial_tokens = height * width + if token_count != frames * spatial_tokens: + raise ValueError(f"Sana-WM cam branch expects N=T*H*W, got N={token_count}, THW={spatial_shape}.") + + q_cam = self.q_proj_cam(hidden_states) + k_cam = self.k_proj_cam(hidden_states) + v_cam = self.v_proj_cam(hidden_states) + + if self.conv_k_cam is not None: + k_cam = self._bidirectional_temporal_short_conv(k_cam, self.conv_k_cam, spatial_shape) + if self.conv_q_cam is not None: + q_cam = self._bidirectional_temporal_short_conv(q_cam, self.conv_q_cam, spatial_shape) + if self.conv_v_cam is not None: + v_cam = self._bidirectional_temporal_short_conv(v_cam, self.conv_v_cam, spatial_shape) + + # Normalise through the model's own norm modules: under TP these are + # TensorParallelRMSNorm, which all-reduces the squared sum so the + # denominator spans the global channel width. Doing the RMS inside + # cam_prep_func would normalise over this rank's head shard only. q and + # k share a shape here, so the two reductions fuse into one. + q_cam, k_cam = fused_qk_rms_norm(self.q_norm_cam, self.k_norm_cam, q_cam, k_cam) + q_normed = q_cam.reshape(batch_size, token_count, self.cam_heads, self.cam_head_dim) + k_normed = k_cam.reshape(batch_size, token_count, self.cam_heads, self.cam_head_dim) + + # cam_prep_func returns NVlabs' ``(B, H, D, N)`` scan layout directly, + # so nothing needs permuting before the recurrence below. + v_raw = v_cam.reshape(batch_size, token_count, self.cam_heads, self.cam_head_dim).contiguous() + k_scale = (self.cam_head_dim**-0.5) * (spatial_tokens**-0.5) + q_rot_bhdn, k_rot_bhdn, v_bhdn, inflation_sq = cam_prep_func( + q_normed.contiguous(), + k_normed.contiguous(), + v_raw, + proj_q=cam_geometry.proj_q, + proj_kv=cam_geometry.proj_kv, + rope_cos=cam_geometry.rope_cos, + rope_sin=cam_geometry.rope_sin, + k_scale=k_scale, + ) + + # Discount beta by the raw K inflation instead of shrinking Q/K/V first: + # the released NVlabs ``BothTriton`` cam path skips the Python + # ``PostUCPERenorm`` step and feeds the scan the raw UCPE-transformed + # tensors. + beta, decay = precomputed_gates + inflation_per_token = inflation_sq.reshape(batch_size, self.cam_heads, frames, spatial_tokens) + frame_inflation_sq = inflation_per_token.mean(dim=-1) # (B, H_cam, T) + # β/decay come from the main branch, so the cam branch can only reuse + # them head-for-head. cam_attn_compress > 1 would give the cam branch + # fewer heads and needs a grouping rule this port does not implement; + # __init__ already rejects it via the cam_head_dim check. + if beta.shape[1] != self.cam_heads: + raise ValueError( + "Sana-WM cam branch requires cam_heads == main heads (cam_attn_compress=1)," + f" got cam_heads={self.cam_heads} main_heads={beta.shape[1]}." + ) + beta_cam = beta / frame_inflation_sq.unsqueeze(-1).clamp_min(1.0) + decay_cam = decay + + cam_out_bhdn, _ = _bidirectional_delta_scan( + q_rot_bhdn.float(), + k_rot_bhdn.float(), + v_bhdn.float(), + beta_cam.float(), + decay_cam.float(), + spatial_tokens=spatial_tokens, + skip_z=True, + ) + cam_out_bhdn = cam_out_bhdn.to(q_rot_bhdn.dtype) + + # ``apply_output`` expects (B, H, N, D); the scan emits (B, H, D, N). + cam_out_bhnd = cam_out_bhdn.transpose(-1, -2).contiguous() + cam_out_bhnd = cam_geometry.apply_output(cam_out_bhnd) + cam_out_bhdn = cam_out_bhnd.transpose(-1, -2).contiguous() + + # (B, H_cam, D_cam, N) → (B, N, cam_dim) + return cam_out_bhdn.permute(0, 3, 1, 2).reshape(batch_size, token_count, self.cam_dim) + + # The GDN/UCPE branch runs a sequential recurrence whose trip count is the + # latent frame count, i.e. data-dependent at trace time. Run the whole + # attention eagerly (graph break) rather than letting torch.compile unroll + # it per shape; the rest of the block (MLP / norms / cross-attention) still + # compiles. + @torch.compiler.disable + def forward( + self, + hidden_states: torch.Tensor, + spatial_shape: tuple[int, int, int], + rotary_emb: torch.Tensor | None = None, + cam_geometry: SanaWmCamGeometry | None = None, + ) -> torch.Tensor: + """Forward with dual-branch GDN+UCPE when ``cam_geometry`` is + provided, otherwise main-branch-only GDN or softmax fallback. + + ``cam_geometry`` holds the per-pixel 4x4 transforms built once per + transformer forward from the raw ``(B, T_latent, 20)`` raymap. The cam + branch consumes ``hidden_states`` as its sole Q/K/V source and applies + those transforms to the Q/K/V channels via UCPE. + """ + if self.use_gdn: + if cam_geometry is None: + # No camera conditioning on this request — main branch only. + return self._forward_gdn(hidden_states, spatial_shape, rotary_emb) + # β/decay are computed here rather than inside each branch: NVlabs + # drives both recurrences from one set of gates, and the camera + # branch then discounts beta by its own K inflation. + beta, decay = self._compute_frame_gates(hidden_states, spatial_shape) + main_raw, _ = self._forward_gdn_raw( + hidden_states, + spatial_shape, + rotary_emb, + precomputed_gates=(beta, decay), + ) + cam_raw = self._forward_cam_branch( + hidden_states, + spatial_shape, + cam_geometry, + rotary_emb, + precomputed_gates=(beta, decay), + ) + cam_contrib = self.out_proj_cam(cam_raw) + cam_contrib = self._reduce_scatter_cam_contrib(cam_contrib, main_raw) + combined = main_raw + cam_contrib.to(main_raw.dtype) + attn_out = self._apply_output_gate_and_proj(combined, hidden_states) + return attn_out + + main_raw = self._forward_softmax_raw(hidden_states, spatial_shape, rotary_emb) + if cam_geometry is not None: + cam_raw = self._forward_softmax_cam_branch( + hidden_states, + spatial_shape, + cam_geometry, + rotary_emb, + ) + cam_contrib = self.out_proj_cam(cam_raw) + cam_contrib = self._reduce_scatter_cam_contrib(cam_contrib, main_raw) + main_raw = main_raw + cam_contrib.to(main_raw.dtype) + return self._apply_output_gate_and_proj(main_raw, hidden_states) + + +class SanaWmCrossAttention(nn.Module): + def __init__( + self, + config: SanaWmConfig, + *, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + hidden_size = config.hidden_size + self.total_num_heads = max(hidden_size // max(config.linear_head_dim, 1), 1) + self.head_dim = hidden_size // self.total_num_heads + self.num_heads = self.total_num_heads + inner = self.total_num_heads * self.head_dim + self.q_linear = ColumnParallelLinear( + hidden_size, + inner, + bias=True, + gather_output=False, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.q_linear" if prefix else "q_linear", + ) + # The checkpoint stores K and V fused as one (2 * inner, hidden) + # tensor. A plain ColumnParallelLinear would shard that by the + # concatenated output dim, handing all of K to one rank and all of V + # to another; MergedColumnParallelLinear knows the block structure + # and shards K and V by heads independently, so `chunk(2)` below + # yields this rank's K and V directly - no gather, no manual slice. + self.kv_linear = MergedColumnParallelLinear( + hidden_size, + [inner, inner], + bias=True, + gather_output=False, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.kv_linear" if prefix else "kv_linear", + ) + self.num_heads = self.q_linear.output_size_per_partition // self.head_dim + self.proj = RowParallelLinear( + inner, + hidden_size, + bias=True, + input_is_parallel=True, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.proj" if prefix else "proj", + ) + local_inner = self.num_heads * self.head_dim + norm_cls = _make_sharded_qk_rms_norm if config.cross_norm else (lambda *_args, **_kwargs: nn.Identity()) + self.q_norm = norm_cls(local_inner) + self.k_norm = norm_cls(local_inner) + # q is column-parallel and k is sliced to the local width before k_norm, + # so both norms hold one weight entry per local channel. + _shard_param_across_tp(getattr(self.q_norm, "weight", None)) + _shard_param_across_tp(getattr(self.k_norm, "weight", None)) + self.softmax_attn = Attention( + num_heads=self.num_heads, + head_size=self.head_dim, + num_kv_heads=self.num_heads, + softmax_scale=1.0 / (self.head_dim**0.5), + causal=False, + role="cross", + qkv_layout="BSND", + skip_sequence_parallel=True, + disable_kv_quant=True, + prefix=prefix, + ) + + def _reshape_to_seq_heads(self, tensor: torch.Tensor) -> torch.Tensor: + batch, seq_len, hidden_size = tensor.shape + return tensor.reshape(batch, seq_len, self.num_heads, hidden_size // self.num_heads) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + query = self.q_norm(self.q_linear(hidden_states)) + # MergedColumnParallelLinear emits [K_local; V_local], so the chunk is + # already this rank's head shard. + key, value = self.kv_linear(encoder_hidden_states).chunk(2, dim=-1) + key = self.k_norm(key) + if attention_mask is not None and attention_mask.ndim != 2: + raise ValueError( + f"Sana-WM cross-attention mask must be shaped [B, text_len], got {tuple(attention_mask.shape)}." + ) + query = self._reshape_to_seq_heads(query) + key = self._reshape_to_seq_heads(key) + value = self._reshape_to_seq_heads(value) + # The prompt padding mask goes to the shared Attention through + # ``AttentionMetadata``: backends take the [B, text_len] form directly + # and convert/reshape it themselves (``_maybe_reshape_attn_mask``), so + # cross-attention keeps backend selection. ``None`` keeps the mask-free + # fast path - callers normalise an all-ones mask to ``None`` rather than + # paying for a no-op mask. + attn_metadata = AttentionMetadata(attn_mask=attention_mask) if attention_mask is not None else None + attn = self.softmax_attn(query, key, value, attn_metadata) + return self.proj(attn.flatten(2, 3)) + + +class _ConvWrapper(nn.Module): + def __init__(self, conv: nn.Module) -> None: + super().__init__() + self.conv = conv + + +class SanaWmMbConvFfn(nn.Module): + def __init__(self, config: SanaWmConfig) -> None: + super().__init__() + hidden_size = config.hidden_size + expanded = int(hidden_size * config.mlp_ratio) * 2 + t_padding = config.t_kernel_size // 2 + self.glu_act = nn.SiLU() + self.inverted_conv = _ConvWrapper(nn.Conv2d(hidden_size, expanded, kernel_size=1)) + self.depth_conv = _ConvWrapper(nn.Conv2d(expanded, expanded, kernel_size=3, padding=1, groups=expanded)) + self.point_conv = _ConvWrapper(nn.Conv2d(expanded // 2, hidden_size, kernel_size=1, bias=False)) + self.t_conv = nn.Conv2d( + hidden_size, + hidden_size, + kernel_size=(config.t_kernel_size, 1), + padding=(t_padding, 0), + bias=False, + ) + + def forward(self, hidden_states: torch.Tensor, spatial_shape: tuple[int, int, int]) -> torch.Tensor: + """Match NVlabs ``GLUMBConvTemp.forward`` exactly: + + 1. Reshape ``(B, N=F*H*W, C) → (B*F, H, W, C) → (B*F, C, H, W)`` so the + spatial MBConv runs PER FRAME with proper 2D (H, W) geometry — the + 3×3 ``depth_conv`` must not span an (F, H*W) plane. + 2. Apply expand → depthwise 3×3 → GLU → contract on per-frame + (B*F, C, H, W). + 3. Temporal aggregation: reshape back to ``(B, C, F, H*W)`` + and add ``t_conv`` (kernel ``(3, 1)``) along the F axis. + 4. Reshape to ``(B, N, C)``. + """ + batch, _, hidden_size = hidden_states.shape + frames, height, width = spatial_shape + spatial_tokens = height * width + # (B, F*H*W, C) → (B*F, H, W, C) → (B*F, C, H, W) + x = ( + hidden_states.reshape(batch * frames, spatial_tokens, hidden_size) + .reshape(batch * frames, height, width, hidden_size) + .permute(0, 3, 1, 2) + ) + # Keep this NHWC-derived NCHW view non-contiguous like NVlabs: on bf16 + # CUDA convs the stride selects a different kernel, and a contiguous + # copy breaks late-step parity. + # NVlabs ``ConvLayer`` is `Conv → norm → act` with + # `act=("silu", "silu", None)`, so inverted_conv activates and the other + # two do not. ``_ConvWrapper`` stores only the raw Conv2d (to match the + # `inverted_conv.conv.weight` checkpoint key), hence the explicit SiLU. + x_inv = self.glu_act(self.inverted_conv.conv(x)) + x_dep = self.depth_conv.conv(x_inv) + value, gate = x_dep.chunk(2, dim=1) + x_glu = value * self.glu_act(gate) + x_pt = self.point_conv.conv(x_glu) # (B*F, hidden, H, W) + x = x_pt + # Temporal aggregation: (B*F, C, H, W) → (B, F, C, H*W) → (B, C, F, H*W) + x_temporal = x.reshape(batch, frames, hidden_size, spatial_tokens).permute(0, 2, 1, 3) + t_conv_out = self.t_conv(x_temporal) + x_temporal = x_temporal + t_conv_out + # → (B, F, H*W, C) → (B, N, C) + final = x_temporal.permute(0, 2, 3, 1).reshape(batch, frames * spatial_tokens, hidden_size).contiguous() + return final + + +class SanaWmBlock(nn.Module): + def __init__( + self, + config: SanaWmConfig, + *, + block_idx: int = 0, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + hidden_size = config.hidden_size + use_gdn = config.softmax_every_n <= 0 or (block_idx + 1) % config.softmax_every_n != 0 + use_plucker_proj = config.use_chunk_plucker_post_attn and ( + config.chunk_plucker_post_attn_blocks < 0 or block_idx < config.chunk_plucker_post_attn_blocks + ) + self.norm1 = AdaLayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.attn = SanaWmSelfAttention( + config, + use_gdn=use_gdn, + quant_config=quant_config, + prefix=f"{prefix}.attn" if prefix else "attn", + ) + self.cross_attn = SanaWmCrossAttention( + config, + quant_config=quant_config, + prefix=f"{prefix}.cross_attn" if prefix else "cross_attn", + ) + self.norm2 = AdaLayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.mlp = SanaWmMbConvFfn(config) + if use_plucker_proj: + self.plucker_proj: nn.Module | None = ColumnParallelLinear( + hidden_size, + hidden_size, + bias=True, + gather_output=True, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.plucker_proj" if prefix else "plucker_proj", + ) + else: + self.plucker_proj = None + self.scale_shift_table = nn.Parameter(torch.zeros(6, hidden_size)) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + timestep_modulation: torch.Tensor, + spatial_shape: tuple[int, int, int], + rotary_emb: torch.Tensor | None = None, + camera_hidden_states: torch.Tensor | None = None, + cam_geometry: SanaWmCamGeometry | None = None, + encoder_attention_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + # Per-frame timestep contract (see SanaWmTransformer3DModel.forward). + if timestep_modulation.ndim > 2: + return self._forward_frame_aware( + hidden_states, + encoder_hidden_states, + timestep_modulation, + spatial_shape, + rotary_emb, + camera_hidden_states, + cam_geometry, + encoder_attention_mask, + ) + batch_size = hidden_states.shape[0] + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + self.scale_shift_table[None] + timestep_modulation.reshape(batch_size, 6, -1) + ).chunk(6, dim=1) + attn_input = self.norm1(hidden_states, scale_msa, shift_msa) + attn_output = self.attn(attn_input, spatial_shape, rotary_emb, cam_geometry) + if camera_hidden_states is not None and self.plucker_proj is not None: + attn_output = attn_output + self.plucker_proj(camera_hidden_states) + hidden_states = hidden_states + gate_msa * attn_output + hidden_states = hidden_states + self.cross_attn( + hidden_states, + encoder_hidden_states, + encoder_attention_mask, + ) + mlp_input = self.norm2(hidden_states, scale_mlp, shift_mlp) + hidden_states = hidden_states + gate_mlp * self.mlp(mlp_input, spatial_shape) + return hidden_states + + def _forward_frame_aware( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + timestep_modulation: torch.Tensor, + spatial_shape: tuple[int, int, int], + rotary_emb: torch.Tensor | None, + camera_hidden_states: torch.Tensor | None, + cam_geometry: SanaWmCamGeometry | None, + encoder_attention_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + """Per-frame timestep modulation path matching NVlabs + ``SanaVideoMSCamCtrlBlock.forward_frame_aware``. + + ``timestep_modulation`` is ``(B, 1, F, 6*D)``. We split into the + six per-frame ``(B, F, 1, D)`` chunks via ``scale_shift_table`` + and apply ``shift/scale/gate`` per frame, broadcasting over the + spatial tokens within each frame. + """ + batch_size, token_count, hidden_size = hidden_states.shape + frames, height, width = spatial_shape + spatial_tokens = height * width + if token_count != frames * spatial_tokens: + raise ValueError(f"Sana-WM frame-aware block expects N=T*H*W, got N={token_count}, THW={spatial_shape}.") + if timestep_modulation.shape[2] != frames: + raise ValueError( + "Sana-WM frame-aware block: timestep frame axis must match spatial frames, " + f"got modulation frames={timestep_modulation.shape[2]} vs spatial frames={frames}." + ) + + # (B, 1, F, 6*D) -> (B, F, 6, D), then add the (1, 1, 6, D) table. + t_per_frame = timestep_modulation.reshape(batch_size, frames, 6, -1) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + self.scale_shift_table[None, None, :, :] + t_per_frame + ).chunk(6, dim=-2) # each: (B, F, 1, D) + + # Apply per-frame modulation by reshaping x to (B, F, S, D), broadcasting + # scale/shift over the spatial axis, then flattening back. The reshape + # comes first because the norm only reduces over the last axis, so it is + # unaffected, and the modulation then broadcasts as written. + x_4d = hidden_states.reshape(batch_size, frames, spatial_tokens, hidden_size) + x_msa_in = self.norm1(x_4d, scale_msa, shift_msa).reshape(batch_size, token_count, hidden_size) + attn_output = self.attn(x_msa_in, spatial_shape, rotary_emb, cam_geometry) + if camera_hidden_states is not None and self.plucker_proj is not None: + attn_output = attn_output + self.plucker_proj(camera_hidden_states) + attn_output_4d = attn_output.reshape(batch_size, frames, spatial_tokens, hidden_size) + hidden_states = hidden_states + (gate_msa * attn_output_4d).reshape(batch_size, token_count, hidden_size) + + hidden_states = hidden_states + self.cross_attn( + hidden_states, + encoder_hidden_states, + encoder_attention_mask, + ) + + x_4d = hidden_states.reshape(batch_size, frames, spatial_tokens, hidden_size) + x_mlp_in = self.norm2(x_4d, scale_mlp, shift_mlp).reshape(batch_size, token_count, hidden_size) + mlp_out = self.mlp(x_mlp_in, spatial_shape) + mlp_out_4d = mlp_out.reshape(batch_size, frames, spatial_tokens, hidden_size) + hidden_states = hidden_states + (gate_mlp * mlp_out_4d).reshape(batch_size, token_count, hidden_size) + return hidden_states + + +class SanaWmFinalLayer(nn.Module): + def __init__( + self, + hidden_size: int, + patch_size: tuple[int, int, int], + out_channels: int, + *, + quant_config: Any = None, + prefix: str = "", + ) -> None: + super().__init__() + self.norm_final = AdaLayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.scale_shift_table = nn.Parameter(torch.zeros(2, hidden_size)) + self.out_channels = out_channels + out_features = math.prod(patch_size) * out_channels + self.linear: nn.Module = ColumnParallelLinear( + hidden_size, + out_features, + bias=True, + gather_output=True, + return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.linear" if prefix else "linear", + ) + + def forward( + self, + hidden_states: torch.Tensor, + timestep_embed: torch.Tensor, + spatial_shape: tuple[int, int, int] | None = None, + ) -> torch.Tensor: + # Per-frame timestep contract (see SanaWmTransformer3DModel.forward). + if timestep_embed.ndim > 2: + return self._forward_frame_aware(hidden_states, timestep_embed, spatial_shape) + shift, scale = (self.scale_shift_table[None] + timestep_embed[:, None]).chunk(2, dim=1) + return self.linear(self.norm_final(hidden_states, scale, shift)) + + def _forward_frame_aware( + self, + hidden_states: torch.Tensor, + timestep_embed: torch.Tensor, + spatial_shape: tuple[int, int, int] | None, + ) -> torch.Tensor: + """Per-frame final-layer modulation matching NVlabs + ``T2IFinalLayer.forward_frame_aware``. + + ``timestep_embed`` is ``(B, 1, F, D)``. We transpose to + ``(B, F, 1, D)`` so it adds correctly to the ``(1, 1, 2, D)`` + ``scale_shift_table`` and produces per-frame ``(B, F, 1, D)`` + ``shift`` / ``scale`` that broadcast over the spatial tokens + within each frame. + """ + batch_size, token_count, hidden_size = hidden_states.shape + frames = timestep_embed.shape[2] + if spatial_shape is not None: + spatial_tokens = spatial_shape[1] * spatial_shape[2] + else: + spatial_tokens = token_count // frames + if frames * spatial_tokens != token_count: + raise ValueError( + "Sana-WM frame-aware final layer: token count mismatch " + f"(N={token_count}, F={frames}, S={spatial_tokens})." + ) + # (B, 1, F, D) -> (B, F, 1, D); add (1, 1, 2, D); chunk into shift/scale: each (B, F, 1, D). + t_per_frame = timestep_embed.transpose(1, 2) + shift, scale = (self.scale_shift_table[None, None, :, :] + t_per_frame).chunk(2, dim=-2) + x_4d = hidden_states.reshape(batch_size, frames, spatial_tokens, hidden_size) + x_mod = self.norm_final(x_4d, scale, shift).reshape(batch_size, token_count, hidden_size) + return self.linear(x_mod) + + +class SanaWmTransformer3DModel(nn.Module): + """SANA-WM Stage-1 DiT: bidirectional Gated DeltaNet blocks with a softmax + attention block every ``softmax_every_n``.""" + + _repeated_blocks: ClassVar[list[str]] = ["SanaWmBlock"] + _layerwise_offload_blocks_attrs: ClassVar[list[str]] = ["blocks"] + _hsdp_shard_conditions: ClassVar[list[Any]] = [_is_sana_wm_transformer_block] + + def __init__( + self, + config: SanaWmConfig | None = None, + *, + device: torch.device | str | None = None, + dtype: torch.dtype | None = None, + latent_channels: int = SANA_WM_STAGE1_LATENT_CHANNELS, + prompt_channels: int = SANA_WM_STAGE1_PROMPT_CHANNELS, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config or SanaWmConfig() + self.quant_config = quant_config + self.prefix = prefix + self._latent_channels = latent_channels + self._prompt_channels = prompt_channels + self.patch_size = _to_3tuple(self.config.patch_size) + self.x_embedder = SanaWmPatchEmbedMS3D(self.patch_size, self._latent_channels, self.config.hidden_size) + self.y_embedder = SanaWmTextEmbedder( + self._prompt_channels, + self.config.hidden_size, + self.config.model_max_length, + quant_config=self.quant_config, + prefix=f"{self.prefix}.y_embedder" if self.prefix else "y_embedder", + ) + self.t_embedder = SanaWmTimestepEmbedder( + SANA_WM_STAGE1_TIMESTEP_CHANNELS, + self.config.hidden_size, + quant_config=self.quant_config, + prefix=f"{self.prefix}.t_embedder" if self.prefix else "t_embedder", + ) + # t_block: SiLU → Linear(hidden, 6*hidden). Weight key is t_block.1.{weight,bias} + # to match the Stage-1 checkpoint layout (nn.Sequential index 1). + _t_block_linear: nn.Module = ColumnParallelLinear( + self.config.hidden_size, + 6 * self.config.hidden_size, + bias=True, + gather_output=True, + return_bias=False, + quant_config=self.quant_config, + prefix=f"{self.prefix}.t_block.1" if self.prefix else "t_block.1", + ) + self.t_block = nn.Sequential(nn.SiLU(), _t_block_linear) + self.plucker_embedder = SanaWmPatchEmbedMS3D( + self.patch_size, + self.config.chunk_plucker_channels, + self.config.hidden_size, + ) + self.raymap_embedder = SanaWmPatchEmbedMS3D(self.patch_size, 3, self.config.hidden_size) + self.blocks = nn.ModuleList( + [ + SanaWmBlock( + self.config, + block_idx=i, + quant_config=self.quant_config, + prefix=f"{self.prefix}.blocks.{i}" if self.prefix else f"blocks.{i}", + ) + for i in range(self.config.num_blocks) + ] + ) + self.final_layer = SanaWmFinalLayer( + self.config.hidden_size, + self.patch_size, + self._latent_channels, + quant_config=self.quant_config, + prefix=f"{self.prefix}.final_layer" if self.prefix else "final_layer", + ) + self.pos_embed = nn.Parameter(torch.zeros(1, 484, self.config.hidden_size)) + self.rope = SanaWmWanRotaryPosEmbed(self.config.linear_head_dim) + self.attention_y_norm = RMSNorm(self.config.hidden_size) + if device is not None or dtype is not None: + self.to(device=device, dtype=dtype) + + def _positional_embedding(self, token_count: int, dtype: torch.dtype, device: torch.device) -> torch.Tensor: + pos_embed = self.pos_embed.to(device=device, dtype=dtype) + if pos_embed.shape[1] == token_count: + return pos_embed + pos_embed = pos_embed.transpose(1, 2) + pos_embed = F.interpolate(pos_embed, size=token_count, mode="linear", align_corners=False) + return pos_embed.transpose(1, 2) + + @staticmethod + def _match_tokens(hidden_states: torch.Tensor, expected_tokens: int) -> torch.Tensor: + if hidden_states.shape[1] == expected_tokens: + return hidden_states + if hidden_states.shape[1] > expected_tokens: + return hidden_states[:, :expected_tokens] + pad = hidden_states[:, -1:].expand(-1, expected_tokens - hidden_states.shape[1], -1) + return torch.cat([hidden_states, pad], dim=1) + + def _camera_hidden_states_from_conditions( + self, + *, + plucker: torch.Tensor | None, + spatial_raymap: torch.Tensor | None, + spatial_shape: tuple[int, int, int], + batch_size: int, + dtype: torch.dtype, + device: torch.device, + ) -> torch.Tensor | None: + if plucker is None: + return None + if plucker.ndim == 4: + plucker = plucker.unsqueeze(0) + plucker = plucker.to(device=device, dtype=dtype) + if plucker.shape[0] == 1 and batch_size > 1: + plucker = plucker.expand(batch_size, -1, -1, -1, -1) + camera_hidden_states = self.plucker_embedder(plucker) + expected_tokens = spatial_shape[0] * spatial_shape[1] * spatial_shape[2] + camera_hidden_states = self._match_tokens(camera_hidden_states, expected_tokens) + + # Fuse per-pixel ray-direction map via raymap_embedder only on the + # non-plucker path. NVlabs skips the absmap/raymap embedder whenever + # chunk-plucker post-attention injection is enabled. + if spatial_raymap is not None and not self.config.use_chunk_plucker_post_attn: + if spatial_raymap.ndim == 4: # [C, F, H, W] → [1, C, F, H, W] + spatial_raymap = spatial_raymap.unsqueeze(0) + spatial_raymap = spatial_raymap.to(device=device, dtype=dtype) + if spatial_raymap.shape[0] == 1 and batch_size > 1: + spatial_raymap = spatial_raymap.expand(batch_size, -1, -1, -1, -1) + ray_hidden = self._match_tokens(self.raymap_embedder(spatial_raymap), expected_tokens) + camera_hidden_states = camera_hidden_states + ray_hidden + + return camera_hidden_states + + def _unpatchify(self, hidden_states: torch.Tensor, spatial_shape: tuple[int, int, int]) -> torch.Tensor: + batch_size = hidden_states.shape[0] + frames, height, width = spatial_shape + patch_frames, patch_height, patch_width = self.patch_size + hidden_states = hidden_states.reshape( + batch_size, + frames, + height, + width, + patch_frames, + patch_height, + patch_width, + self._latent_channels, + ) + hidden_states = torch.einsum("nfhwopqc->ncfohpwq", hidden_states) + return hidden_states.reshape( + batch_size, + self._latent_channels, + frames * patch_frames, + height * patch_height, + width * patch_width, + ) + + def forward( + self, + hidden_states: torch.Tensor, + timestep: torch.Tensor | float | int, + *, + encoder_hidden_states: torch.Tensor | None = None, + encoder_attention_mask: torch.Tensor | None = None, + camera_hidden_states: torch.Tensor | None = None, + plucker: torch.Tensor | None = None, + raymap: torch.Tensor | None = None, + spatial_raymap: torch.Tensor | None = None, + ) -> torch.Tensor: + if hidden_states.ndim != 5: + raise ValueError("Sana-WM transformer expects latent input shaped [B, C, F, H, W].") + batch_size = hidden_states.shape[0] + latent_shape = hidden_states.shape[2:] + hidden_states, spatial_shape = self.x_embedder.project_with_shape(hidden_states) + rotary_emb = None + if self.config.pos_embed_type == "wan_rope": + rotary_emb = self.rope(spatial_shape, hidden_states.device) + else: + hidden_states = hidden_states + self._positional_embedding( + hidden_states.shape[1], hidden_states.dtype, hidden_states.device + ) + + encoder_hidden_states = self.y_embedder( + encoder_hidden_states.to(device=hidden_states.device) if encoder_hidden_states is not None else None, + batch_size=batch_size, + dtype=hidden_states.dtype, + ) + encoder_hidden_states = self.attention_y_norm(encoder_hidden_states) + # No mask means "attend to every prompt token"; keep it ``None`` so + # cross-attention takes the mask-free attention path rather than + # building an all-ones mask the backend would have to reshape per block. + if encoder_attention_mask is not None: + encoder_attention_mask = encoder_attention_mask.to(device=hidden_states.device) + if encoder_attention_mask.shape != encoder_hidden_states.shape[:2]: + raise ValueError( + "Sana-WM encoder_attention_mask must match text tokens " + f"{tuple(encoder_hidden_states.shape[:2])}, got {tuple(encoder_attention_mask.shape)}." + ) + + if not torch.is_tensor(timestep): + timestep = torch.tensor([timestep], device=hidden_states.device, dtype=torch.float32) + timestep = timestep.to(device=hidden_states.device) + if timestep.ndim == 0: + timestep = timestep.expand(batch_size) + elif timestep.ndim == 1 and timestep.shape[0] == 1 and batch_size > 1: + timestep = timestep.expand(batch_size) + # Per-frame timestep contract: a rank > 1 ``timestep`` (``(B, 1, F)`` + # from the LTX flow-matching sampler) embeds on the flattened token axis + # and unflattens back, giving ``(B, 1, F, D)`` / ``(B, 1, F, 6*D)``. + # SanaWmBlock and SanaWmFinalLayer dispatch on ``ndim > 2`` to the + # frame-aware paths that broadcast modulation over spatial tokens. + timestep_shape = tuple(timestep.shape) + time_embed = self.t_embedder(timestep) # (numel, D) + _t_silu = self.t_block[0](time_embed) + timestep_modulation = self.t_block[1](_t_silu).to(hidden_states.dtype) + if len(timestep_shape) > 1: + time_embed = time_embed.unflatten(0, timestep_shape) + timestep_modulation = timestep_modulation.unflatten(0, timestep_shape) + + if camera_hidden_states is None: + camera_hidden_states = self._camera_hidden_states_from_conditions( + plucker=plucker, + spatial_raymap=spatial_raymap, + spatial_shape=spatial_shape, + batch_size=batch_size, + dtype=hidden_states.dtype, + device=hidden_states.device, + ) + + # Build (B, T_latent, 20) camera_conditions from the raw raymap, then + # derive the UCPE geometry ONCE for the whole forward. The ray grid, + # its SE(3) inverse and the camera RoPE tables are identical in every + # block, so building them per block cost ~2400 rebuilds per request + # (20 blocks x 60 steps x 2 CFG branches). When the caller passes + # pre-embedded camera_hidden_states without raymap, the cam branch + # becomes a no-op (main-branch only). + cam_geometry: SanaWmCamGeometry | None = None + if raymap is not None: + # ``_pack_camera_conditions`` emits an unbatched (T_latent, 20). + camera_conditions = raymap.unsqueeze(0) + if batch_size > 1: + camera_conditions = camera_conditions.expand(batch_size, -1, -1) + camera_conditions = camera_conditions.to(device=hidden_states.device, dtype=hidden_states.dtype) + # cam_head_dim comes from the attention module rather than being + # re-derived here: __init__ already asserts it equals the main + # head_dim and is consistent across blocks. + attn = self.blocks[0].attn + cam_geometry = prepare_cam_geometry( + camera_conditions=camera_conditions, + spatial_shape=spatial_shape, + patch_size=self.patch_size, + head_dim=attn.cam_head_dim, + rotary_emb=attn._ucpe_rotary_freqs(rotary_emb), + ) + + for block in self.blocks: + hidden_states = block( + hidden_states, + encoder_hidden_states, + timestep_modulation, + spatial_shape, + rotary_emb, + camera_hidden_states, + cam_geometry, + encoder_attention_mask, + ) + + hidden_states = self.final_layer(hidden_states, time_embed.to(hidden_states.dtype), spatial_shape) + hidden_states = self._unpatchify(hidden_states, spatial_shape) + if hidden_states.shape[2:] != latent_shape: + hidden_states = F.interpolate(hidden_states, size=latent_shape, mode="trilinear", align_corners=False) + return hidden_states + + def load_weights(self, weights: Iterable[tuple[str, Any]]) -> set[str]: + """Stream checkpoint tensors into the eagerly-built modules. + + Follows the wan2_2 idiom: one ``named_parameters`` lookup table plus the + per-tensor ``weight_loader``, which narrows full checkpoint tensors to + the TP-local shard at copy time for both vLLM parallel layers and the + plain parameters marked by ``_shard_param_across_tp``. The source tensor + is dropped each iteration — no copy of the checkpoint is retained. + """ + params_dict = dict(self.named_parameters()) + buffers_dict = dict(self.named_buffers()) + loaded: set[str] = set() + unmapped: list[str] = [] + duplicates: list[str] = [] + + for source_name, tensor in weights: + # Weights ship in the standard diffusers ``transformer/`` layout, so + # the checkpoint key is already the module-local parameter name. + if source_name in loaded: + duplicates.append(source_name) + continue + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Sana-WM weight {source_name!r} must be a torch.Tensor, got {type(tensor).__name__}.") + target = params_dict.get(source_name) + if target is None: + target = buffers_dict.get(source_name) + if target is None: + unmapped.append(source_name) + continue + weight_loader = getattr(target, "weight_loader", None) + if callable(weight_loader): + # vLLM parallel layers and the plain params marked by + # `_shard_param_across_tp` narrow to the local shard here. + weight_loader(target, tensor) + else: + if tuple(target.shape) != tuple(tensor.shape): + raise ValueError( + f"Sana-WM weight shape mismatch for {source_name}: " + f"expected {tuple(target.shape)}, got {tuple(tensor.shape)}." + ) + with torch.no_grad(): + target.copy_(tensor.to(device=target.device, dtype=target.dtype)) + loaded.add(source_name) + + if unmapped or duplicates: + details = [] + if unmapped: + details.append(f"unmapped={unmapped[:10]}") + if duplicates: + details.append(f"duplicates={duplicates[:10]}") + raise ValueError("Invalid SANA-WM Stage-1 checkpoint keys: " + "; ".join(details)) + return loaded diff --git a/vllm_omni/diffusion/models/sana_wm/ucpe.py b/vllm_omni/diffusion/models/sana_wm/ucpe.py new file mode 100644 index 000000000..88a3a115d --- /dev/null +++ b/vllm_omni/diffusion/models/sana_wm/ucpe.py @@ -0,0 +1,536 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""SANA-WM UCPE (Unified Camera Pose Embedding) per-block attention transforms. + +Ported from NVlabs/Sana ``sana_camctrl_blocks.py`` for the inference-only +SANA-WM bidirectional 1600M release. Scope is intentionally narrow: + +* Pinhole camera only (xi=0); the UCM ``xi`` parameter is fixed. +* Inference path only — no training-time camera-branch dropout. +* Online computation only — the precomputed ``cam_pos_embeds`` shortcut + used by NVlabs' fused kernels is omitted because vLLM-Omni builds the + matrices once per transformer forward and shares them across all blocks. +* ``apply_vo=True`` only — the SANA-WM checkpoint always uses the + inverse-output transform. + +Public surface: + ``prepare_cam_geometry(...)`` -> :class:`SanaWmCamGeometry`, built once per + transformer forward and threaded through every block. It carries both the + ``(apply_q, apply_kv, apply_output)`` closures used by the softmax camera + branch and the projection/RoPE tensors used by ``cam_prep_func(...)``, the + NVlabs fused camera-prep contract on the GDN camera branch. + +Each closure transforms a tensor of shape ``(B, num_heads, N, D)`` where +``N = T * H * W`` (matching the GDN token layout) and ``D == head_dim``, +returning a tensor of the same shape. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from functools import partial + +import torch +import torch.nn.functional as F + +# --------------------------------------------------------------------------- +# Pinhole ray-grid construction +# --------------------------------------------------------------------------- + + +def _make_pixel_grid( + height: int, + width: int, + device: torch.device, + dtype: torch.dtype, +) -> torch.Tensor: + """Return ``(H, W, 3)`` grid ``(x, y, 1)`` of pixel coordinates.""" + xs = torch.arange(width, device=device, dtype=dtype) + ys = torch.arange(height, device=device, dtype=dtype) + y_grid, x_grid = torch.meshgrid(ys, xs, indexing="ij") + return torch.stack((x_grid, y_grid, torch.ones_like(x_grid)), dim=-1) + + +def _unproject_pinhole( + fx: torch.Tensor, + fy: torch.Tensor, + cx: torch.Tensor, + cy: torch.Tensor, + height: int, + width: int, +) -> torch.Tensor: + """Unproject a pixel grid into camera-frame direction vectors. + + Args: + fx, fy, cx, cy: shape ``(B, T)`` intrinsics in token-grid units — the + caller divides all four by patch size before calling. + height, width: token-grid size. + + Returns: + ``(B, T, H, W, 3)`` direction tensor (NOT normalised). + """ + B, T = fx.shape + device = fx.device + dtype = fx.dtype + grid = _make_pixel_grid(height, width, device, dtype) # (H, W, 3) + u = grid[..., 0] # (H, W) + v = grid[..., 1] + fx_e = fx.view(B, T, 1, 1) + fy_e = fy.view(B, T, 1, 1) + cx_e = cx.view(B, T, 1, 1) + cy_e = cy.view(B, T, 1, 1) + x = (u - cx_e) / fx_e + y = (v - cy_e) / fy_e + z = torch.ones_like(x) + # Match NVlabs UCM normalisation at xi=0: alpha = 1, gamma = 1/(1+r^2), + # X = gamma*x, Y = gamma*y, Z = gamma. The resulting vector has unit + # norm. Computed compactly via direct stack + normalise. + d_cam = torch.stack((x, y, z), dim=-1) # (B, T, H, W, 3) + d_cam = d_cam / d_cam.norm(dim=-1, keepdim=True).clamp_min(1e-12) + return d_cam + + +def _world_to_ray_mats( + d_cam: torch.Tensor, # (B, T, H, W, 3) unit rays in camera frame + c2w: torch.Tensor, # (B, T, 4, 4) camera-to-world SE(3) +) -> torch.Tensor: + """Build per-pixel ``ray <- world`` SE(3) transforms ``(B, T, H, W, 4, 4)``. + + The ray frame is anchored at the camera origin with the +Z axis along + the per-pixel viewing direction and a +Y axis defined by the camera-Y + cross product. This matches NVlabs ``world_to_ray_mats``. + """ + B, T, H, W, _ = d_cam.shape + device = d_cam.device + dtype = d_cam.dtype + rot_cam = c2w[..., :3, :3] + trans_cam = c2w[..., :3, 3] + d_world = torch.einsum("btij,bthwj->bthwi", rot_cam, d_cam) + cam_y = rot_cam[..., :, 1].unsqueeze(2).unsqueeze(3).expand(B, T, H, W, 3) + z_ray = F.normalize(d_world, dim=-1, eps=1e-6) + x_ray = F.normalize(torch.cross(cam_y, z_ray, dim=-1), dim=-1, eps=1e-6) + y_ray = F.normalize(torch.cross(z_ray, x_ray, dim=-1), dim=-1, eps=1e-6) + rot_local_to_world = torch.stack((x_ray, y_ray, z_ray), dim=-1) # (B,T,H,W,3,3) + rot_world_to_local = rot_local_to_world.transpose(-1, -2) + t_world = trans_cam.unsqueeze(2).unsqueeze(3).expand(B, T, H, W, 3) + t_w2l = -torch.einsum("bthwij,bthwj->bthwi", rot_world_to_local, t_world) + raymats = torch.zeros(B, T, H, W, 4, 4, device=device, dtype=dtype) + raymats[..., :3, :3] = rot_world_to_local + raymats[..., :3, 3] = t_w2l + raymats[..., 3, 3] = 1.0 + # Guard NaNs from degenerate pixels (e.g. nan direction at infinity). + nan_mask = torch.isnan(d_world).any(-1) + if nan_mask.any(): + eye = torch.eye(4, device=device, dtype=dtype) + raymats[nan_mask] = eye + return raymats + + +def _process_camera_conditions( + camera_conditions: torch.Tensor, # (B, T, 20) + hw: tuple[int, int, int], + patch_size: tuple[int, int, int] = (1, 1, 1), +) -> torch.Tensor: + """Convert ``(B, T, 20)`` SANA-WM camera conditions to ``(B, T, H, W, 4, 4)`` + ``ray <- world`` transforms. + + Layout of the last axis is ``[c2w_flat (16), fx, fy, cx, cy]``. + Intrinsics follow the NVlabs UCPE contract: the packed camera payload is + expressed on the pre-patch latent image grid, while ``HW`` is the + transformer token grid. ``patch_size`` maps those units to the token grid + before unprojection. The released SANA-WM P1 config uses ``(1, 1, 1)``. + + Args: + camera_conditions: ``(B, T, 20)`` from ``_pack_camera_conditions``. + hw: ``(T, H, W)`` token-grid shape. Equal to the latent grid only + because the released config patches by ``(1, 1, 1)``. + """ + B, T_cond, last = camera_conditions.shape + if last != 20: + raise ValueError(f"SANA-WM camera_conditions last dim must be 20, got {last}.") + T_latent, H_latent, W_latent = hw + if T_cond != T_latent: + raise ValueError(f"SANA-WM camera_conditions frames {T_cond} != latent frames {T_latent}.") + c2w = camera_conditions[..., :16].reshape(B, T_latent, 4, 4) + patch_t, patch_h, patch_w = patch_size + del patch_t + fx = camera_conditions[..., 16] / float(patch_w) + fy = camera_conditions[..., 17] / float(patch_h) + cx = camera_conditions[..., 18] / float(patch_w) + cy = camera_conditions[..., 19] / float(patch_h) + d_cam = _unproject_pinhole(fx, fy, cx, cy, H_latent, W_latent) + raymats = _world_to_ray_mats(d_cam, c2w) + return raymats + + +def _invert_se3(transforms: torch.Tensor) -> torch.Tensor: + """Closed-form inverse of a stack of 4x4 SE(3) matrices.""" + if transforms.shape[-2:] != (4, 4): + raise ValueError(f"SE3 inverse expects (..., 4, 4), got {tuple(transforms.shape)}.") + rot_inv = transforms[..., :3, :3].transpose(-1, -2) + out = torch.zeros_like(transforms) + out[..., :3, :3] = rot_inv + out[..., :3, 3] = -torch.einsum("...ij,...j->...i", rot_inv, transforms[..., :3, 3]) + out[..., 3, 3] = 1.0 + return out + + +# --------------------------------------------------------------------------- +# Per-token transform primitives +# --------------------------------------------------------------------------- + + +def _apply_ray_projmat( + feats: torch.Tensor, # (B, num_heads, N, D) + matrix: torch.Tensor, # (B, N, 4, 4) +) -> torch.Tensor: + """Apply a per-token 4x4 projection to features grouped by 4 channels. + + ``D`` must be a multiple of 4. Out-of-place; preserves ``feats.shape``. + """ + B, num_heads, N, D = feats.shape + if D % 4 != 0: + raise ValueError(f"UCPE projmat expects D divisible by 4, got D={D}.") + feats_grouped = feats.reshape(B, num_heads, N, D // 4, 4) + # matrix: (B, N, 4, 4); feats_grouped: (B, num_heads, N, K, 4) + out = torch.einsum("bnij,bhnkj->bhnki", matrix, feats_grouped) + return out.reshape(B, num_heads, N, D) + + +def _apply_complex_rope( + hidden_states: torch.Tensor, + freqs: torch.Tensor, + *, + inverse: bool = False, +) -> torch.Tensor: + """Apply complex RoPE: treat last-dim pairs as ``(real, imag)`` and multiply + by ``freqs``. Computation runs in fp64 to match the NVlabs reference. + + Args: + hidden_states: ``(..., D)`` with ``D % 2 == 0``. + freqs: complex tensor broadcastable to ``(..., D // 2)``. + inverse: when ``True`` apply ``freqs.conj()`` (used for ``apply_o``). + """ + x_real = hidden_states.to(torch.float64).contiguous() + x_complex = torch.view_as_complex(x_real.unflatten(-1, (-1, 2))) + if inverse: + freqs = freqs.conj() + x_out = torch.view_as_real(x_complex * freqs).flatten(-2, -1) + return x_out.to(hidden_states.dtype) + + +def _apply_block_diagonal( + feats: torch.Tensor, + func_size_pairs: list[tuple[Callable[[torch.Tensor], torch.Tensor], int]], +) -> torch.Tensor: + """Split ``feats`` along the last axis by ``block_sizes`` and apply one + callable per block, then concatenate. Preserves ``feats.shape``. + """ + funcs, block_sizes = zip(*func_size_pairs) + if feats.shape[-1] != sum(block_sizes): + raise ValueError(f"UCPE block-diagonal block sum {sum(block_sizes)} != feats D {feats.shape[-1]}.") + blocks = torch.split(feats, list(block_sizes), dim=-1) + return torch.cat([func(block) for func, block in zip(funcs, blocks)], dim=-1) + + +def _slice_rope_for_cam( + rotary_emb: torch.Tensor | None, + rope_dim: int, +) -> torch.Tensor | None: + """Re-slice WAN-style RoPE frequencies for a smaller ``rope_dim`` using the + same ``(T, H, W)`` split as the main branch. Mirrors NVlabs slicing logic. + + The T/H/W band layout is read off the input shape + (``rotary_emb.shape[-1] * 2``) rather than a separately supplied head_dim, + so the split stays correct without the caller passing it. ``cam_head_dim`` + still has to equal the main ``head_dim`` — ``SanaWmSelfAttention.__init__`` + rejects anything else. + """ + if rotary_emb is None: + return None + # Derive WAN RoPE T/H/W band offsets from the actual input tensor size, + # not from a head_dim argument. This is correct regardless of whether + # cam_head_dim == main_head_dim. + in_head_dim = rotary_emb.shape[-1] * 2 + orig_t = in_head_dim // 2 - 2 * (in_head_dim // 6) + orig_h = in_head_dim // 6 + new_t = rope_dim // 2 - 2 * (rope_dim // 6) + new_h = rope_dim // 6 + new_w = rope_dim // 6 + t_part = rotary_emb[..., :new_t] + h_part = rotary_emb[..., orig_t : orig_t + new_h] + w_part = rotary_emb[..., orig_t + orig_h : orig_t + orig_h + new_w] + return torch.cat([t_part, h_part, w_part], dim=-1) + + +# --------------------------------------------------------------------------- +# Apply-fn preparation +# --------------------------------------------------------------------------- + + +def _identity(x: torch.Tensor) -> torch.Tensor: + return x + + +def _prepare_ray_apply_fns( + head_dim: int, + proj: torch.Tensor, # (B, N, 4, 4) ray <- world + proj_t: torch.Tensor, # (B, N, 4, 4) ray <- world transpose + proj_inv: torch.Tensor, # (B, N, 4, 4) world <- ray + rotary_emb: torch.Tensor | None = None, +) -> tuple[Callable, Callable, Callable]: + """Build ``(apply_q, apply_kv, apply_o)`` block-diagonal callables. + + Each callable transforms a ``(B, num_heads, N, head_dim)`` tensor: + the first ``head_dim // 2`` channels go through the per-token 4x4 + projection, the remaining ``head_dim // 2`` channels go through complex + RoPE (or identity when ``rotary_emb`` is None). + """ + if rotary_emb is not None: + rope_fwd = partial(_apply_complex_rope, freqs=rotary_emb, inverse=False) + rope_inv = partial(_apply_complex_rope, freqs=rotary_emb, inverse=True) + else: + rope_fwd = _identity + rope_inv = _identity + + half = head_dim // 2 + + transforms_q = [ + (partial(_apply_ray_projmat, matrix=proj_t), half), + (rope_fwd, half), + ] + transforms_kv = [ + (partial(_apply_ray_projmat, matrix=proj_inv), half), + (rope_fwd, half), + ] + transforms_o = [ + (partial(_apply_ray_projmat, matrix=proj), half), + (rope_inv, half), + ] + + apply_q = partial(_apply_block_diagonal, func_size_pairs=transforms_q) + apply_kv = partial(_apply_block_diagonal, func_size_pairs=transforms_kv) + apply_o = partial(_apply_block_diagonal, func_size_pairs=transforms_o) + return apply_q, apply_kv, apply_o + + +@dataclass(frozen=True) +class SanaWmCamGeometry: + """UCPE geometry shared by every block of one transformer forward. + + The ray grid, its SE(3) inverse and the camera RoPE tables depend only on + the camera payload, the latent grid, the patch size and the head dim — all + fixed for the whole forward — yet each block's camera branch used to + rebuild them. At 20 blocks x 60 steps x 2 CFG branches that is ~2400 + rebuilds of an 18480-token ray grid and its 4x4 inverses per request. + + ``apply_q``/``apply_kv``/``apply_output`` serve the softmax camera branch; + ``proj_q``/``proj_kv``/``rope_cos``/``rope_sin`` serve the fused + ``cam_prep_func`` contract on the GDN branch. Both are derived from the + same projection matrices, so they are built together. + """ + + proj_q: torch.Tensor + proj_kv: torch.Tensor + rope_cos: torch.Tensor + rope_sin: torch.Tensor + apply_q: Callable[[torch.Tensor], torch.Tensor] + apply_kv: Callable[[torch.Tensor], torch.Tensor] + apply_output: Callable[[torch.Tensor], torch.Tensor] + + +def _prepare_ucpe_rope_tables( + rotary_emb_cam: torch.Tensor, + token_count: int, + half_dim: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """Convert complex RoPE freqs to NVlabs' interleaved cos/sin tables. + + The output convention is ``out[d] = x[d] * cos[d] + x[d ^ 1] * sin[d]``. + For every complex pair, ``sin[2i] = -imag(freq[i])`` and + ``sin[2i + 1] = imag(freq[i])``. + """ + freqs = rotary_emb_cam + while freqs.ndim > 2 and freqs.shape[0] == 1: + freqs = freqs.squeeze(0) + if freqs.ndim != 2: + raise ValueError(f"Sana-WM cam RoPE freqs must be 2D after squeeze, got {tuple(freqs.shape)}.") + if freqs.shape[0] != token_count: + raise ValueError(f"Sana-WM cam RoPE token count {freqs.shape[0]} != {token_count}.") + if freqs.shape[1] * 2 != half_dim: + raise ValueError(f"Sana-WM cam RoPE dim {freqs.shape[1] * 2} != half_dim {half_dim}.") + cos_half = freqs.real.float() + sin_half = freqs.imag.float() + rope_cos = cos_half.repeat_interleave(2, dim=-1).contiguous() + rope_sin = torch.stack((-sin_half, sin_half), dim=-1).reshape(token_count, half_dim).contiguous() + return rope_cos, rope_sin + + +def prepare_cam_geometry( + *, + camera_conditions: torch.Tensor, + spatial_shape: tuple[int, int, int], + patch_size: tuple[int, int, int], + head_dim: int, + rotary_emb: torch.Tensor | None, +) -> SanaWmCamGeometry: + """Build the per-forward UCPE geometry: ray projections, RoPE tables, apply fns. + + Call once per transformer forward and share the result across every block — + see :class:`SanaWmCamGeometry` for why. + """ + if head_dim % 4 != 0: + raise ValueError(f"UCPE head_dim must be divisible by 4, got {head_dim}.") + if head_dim % 2 != 0 or (head_dim // 2) % 4 != 0: + raise ValueError(f"Sana-WM cam prep head_dim={head_dim} must satisfy D % 2 == 0 and (D/2) % 4 == 0.") + + batch = camera_conditions.shape[0] + frames, height, width = spatial_shape + token_count = frames * height * width + half_dim = head_dim // 2 + + raymats = _process_camera_conditions(camera_conditions, spatial_shape, patch_size=patch_size) + proj = raymats.reshape(batch, token_count, 4, 4).contiguous() + proj_q = proj.transpose(-1, -2).contiguous() + proj_kv = _invert_se3(proj).contiguous() + + rotary_emb_cam = _slice_rope_for_cam(rotary_emb, half_dim) + if rotary_emb_cam is None: + rope_cos = torch.ones(token_count, half_dim, device=camera_conditions.device, dtype=torch.float32) + rope_sin = torch.zeros(token_count, half_dim, device=camera_conditions.device, dtype=torch.float32) + else: + rope_cos, rope_sin = _prepare_ucpe_rope_tables(rotary_emb_cam, token_count, half_dim) + + apply_q, apply_kv, apply_output = _prepare_ray_apply_fns(head_dim, proj, proj_q, proj_kv, rotary_emb=rotary_emb_cam) + return SanaWmCamGeometry( + proj_q=proj_q, + proj_kv=proj_kv, + rope_cos=rope_cos, + rope_sin=rope_sin, + apply_q=apply_q, + apply_kv=apply_kv, + apply_output=apply_output, + ) + + +def _validate_cam_prep_inputs( + q_raw: torch.Tensor, + k_raw: torch.Tensor, + v_raw: torch.Tensor, + proj_q: torch.Tensor, + proj_kv: torch.Tensor, + rope_cos: torch.Tensor, + rope_sin: torch.Tensor, +) -> int: + """Validate the fused cam-prep contract and return ``head_dim // 2``.""" + if q_raw.shape != k_raw.shape or q_raw.shape != v_raw.shape: + raise ValueError(f"Sana-WM cam prep q/k/v shapes must match, got {q_raw.shape}, {k_raw.shape}, {v_raw.shape}.") + if q_raw.ndim != 4: + raise ValueError(f"Sana-WM cam prep q/k/v must be (B,N,H,D), got {tuple(q_raw.shape)}.") + batch, token_count, _num_heads, head_dim = q_raw.shape + if head_dim % 2 != 0 or (head_dim // 2) % 4 != 0: + raise ValueError(f"Sana-WM cam prep head_dim={head_dim} must satisfy D % 2 == 0 and (D/2) % 4 == 0.") + half_dim = head_dim // 2 + if proj_q.shape != (batch, token_count, 4, 4): + raise ValueError(f"Sana-WM cam prep proj_q shape {tuple(proj_q.shape)} != {(batch, token_count, 4, 4)}.") + if proj_kv.shape != (batch, token_count, 4, 4): + raise ValueError(f"Sana-WM cam prep proj_kv shape {tuple(proj_kv.shape)} != {(batch, token_count, 4, 4)}.") + if rope_cos.shape != (token_count, half_dim) or rope_sin.shape != (token_count, half_dim): + raise ValueError( + "Sana-WM cam prep rope table shapes must be " + f"{(token_count, half_dim)}, got {tuple(rope_cos.shape)} and {tuple(rope_sin.shape)}." + ) + return half_dim + + +def _apply_ray_projection(feats: torch.Tensor, matrix: torch.Tensor) -> torch.Tensor: + batch, token_count, num_heads, half_dim = feats.shape + grouped = feats.reshape(batch, token_count, num_heads, half_dim // 4, 4) + out = torch.einsum("bnij,bnhgj->bnhgi", matrix.float(), grouped) + return out.reshape(batch, token_count, num_heads, half_dim) + + +def _apply_interleaved_rope(feats: torch.Tensor, rope_cos: torch.Tensor, rope_sin: torch.Tensor) -> torch.Tensor: + half_dim = feats.shape[-1] + pair_index = torch.arange(half_dim, device=feats.device) ^ 1 + paired = feats.index_select(-1, pair_index) + return feats * rope_cos.reshape(1, rope_cos.shape[0], 1, half_dim) + paired * rope_sin.reshape( + 1, rope_sin.shape[0], 1, half_dim + ) + + +def cam_prep_func( + q_normed: torch.Tensor, + k_normed: torch.Tensor, + v_raw: torch.Tensor, + *, + proj_q: torch.Tensor, + proj_kv: torch.Tensor, + rope_cos: torch.Tensor, + rope_sin: torch.Tensor, + k_scale: float, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Native equivalent of NVlabs ``cam_prep_func``. + + The caller is responsible for the q/k RMSNorm, which must run through the + model's norm modules so a tensor-parallel shard still normalises over the + global channel width. This function only applies ReLU, the K scale, the + UCPE ray projection and RoPE. + + Args: + q_normed: ``(B, N, H, D)`` camera Q, already RMSNorm'd. + k_normed: ``(B, N, H, D)`` camera K, already RMSNorm'd. K must also + include the temporal short convolution. + v_raw: ``(B, N, H, D)`` raw camera V. + proj_q: ``(B, N, 4, 4)`` Q ray projection matrices. + proj_kv: ``(B, N, 4, 4)`` K/V ray projection matrices. + rope_cos: ``(N, D//2)`` interleaved-pair RoPE cosine table. + rope_sin: ``(N, D//2)`` interleaved-pair RoPE sine table. + k_scale: ``D^-0.5 * spatial_tokens^-0.5``. + + Returns: + ``q_trans``, ``k_trans``, ``v_trans`` in ``(B, H, D, N)`` layout, and + ``inflation_sq`` in ``(B, H, N)``. + """ + half_dim = _validate_cam_prep_inputs( + q_normed, + k_normed, + v_raw, + proj_q, + proj_kv, + rope_cos, + rope_sin, + ) + + q_norm = torch.relu(q_normed.float()) + k_norm = torch.relu(k_normed.float()) * float(k_scale) + value = v_raw.float() + + q_half = _apply_ray_projection(q_norm[..., :half_dim], proj_q) + k_half = _apply_ray_projection(k_norm[..., :half_dim], proj_kv) + v_half = _apply_ray_projection(value[..., :half_dim], proj_kv) + + q_rope = _apply_interleaved_rope(q_norm[..., half_dim:], rope_cos.float(), rope_sin.float()) + k_rope = _apply_interleaved_rope(k_norm[..., half_dim:], rope_cos.float(), rope_sin.float()) + v_rope = _apply_interleaved_rope(value[..., half_dim:], rope_cos.float(), rope_sin.float()) + + k_pre_sq = k_norm.square().sum(dim=-1).permute(0, 2, 1).contiguous() + k_post = torch.cat((k_half, k_rope), dim=-1) + k_post_sq = k_post.square().sum(dim=-1).permute(0, 2, 1).contiguous() + inflation_sq = k_post_sq.clamp_min(1e-12) / k_pre_sq.clamp_min(1e-12) + + def to_bhdn(tensor: torch.Tensor) -> torch.Tensor: + return tensor.permute(0, 2, 3, 1).contiguous().to(v_raw.dtype) + + q_out = to_bhdn(torch.cat((q_half, q_rope), dim=-1)) + k_out = to_bhdn(k_post) + v_out = to_bhdn(torch.cat((v_half, v_rope), dim=-1)) + return q_out, k_out, v_out, inflation_sq + + +__all__ = [ + "SanaWmCamGeometry", + "cam_prep_func", + "prepare_cam_geometry", +] diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index 47063bc4a..b248e6690 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -171,6 +171,11 @@ _DIFFUSION_MODELS = { "pipeline_ming_imagegen", "MingImagePipeline", ), + "SanaWmPipeline": ( + "sana_wm", + "pipeline_sana_wm", + "SanaWmPipeline", + ), "InternVLAA1Pipeline": ( "internvla_a1", "pipeline_internvla_a1", @@ -633,6 +638,7 @@ _DIFFUSION_PRE_PROCESS_FUNCS = { "LingBotVideoPipeline": "get_lingbot_video_pre_process_func", "HunyuanImage3ForCausalMM": "get_hunyuan_image_3_pre_process_func", "MagiHumanPipeline": "get_magi_human_pre_process_func", + "SanaWmPipeline": "get_sana_wm_pre_process_func", "Cosmos3OmniDiffusersPipeline": "get_cosmos3_pre_process_func", "Cosmos3OmniPipeline": "get_cosmos3_pre_process_func", "SoulXSingerPipeline": "get_soulxsinger_pre_process_func", From a630b0c2ec5e8ffd3be4ff490e4a85fe06f46536 Mon Sep 17 00:00:00 2001 From: Yuanheng Zhao <54058983+yuanheng-zhao@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:13:40 +0800 Subject: [PATCH 068/267] [Ming] Cleanup Ming-family shared modules (#6119) Signed-off-by: Yuanheng Signed-off-by: Yuanheng Zhao --- .../text_to_speech/ming_tts/end2end.py | 5 +- recipes/inclusionAI/Ming-omni-tts.md | 19 ++-- .../model_loader/test_resolve_model_path.py | 92 +++++++++++++++++++ .../ming_tts/test_ming_shared_modules.py | 91 +++++++++++++++++- .../model_loader/weight_utils.py | 44 +++++++++ .../models/common/ming/__init__.py | 11 +++ .../{ming_tts => common/ming}/aggregator.py | 2 +- .../fm => common/ming}/cfm_cudagraph.py | 2 +- .../fm/dit.py => common/ming/cfm_dit.py} | 2 +- .../ming/cfm_head.py} | 5 +- .../common/ming/{fm.py => cfm_solver.py} | 16 +++- .../common/ming/{dit.py => dit_blocks.py} | 16 ---- .../models/common/ming/speaker_extractor.py | 56 +++++++++++ .../ming}/spk_embedding.py | 0 .../ming_flash_omni/ming_flash_omni_talker.py | 36 ++++---- .../ming_flash_omni_thinker.py | 7 +- .../models/ming_flash_omni/talker_module.py | 61 ++---------- .../models/ming_flash_omni/voice_presets.py | 34 +++---- .../models/ming_tts/fm/__init__.py | 2 - .../models/ming_tts/ming_tts_llm.py | 13 +-- .../models/ming_tts/prompt_encoder.py | 4 +- .../models/ming_tts/speaker_extractor.py | 54 +---------- 22 files changed, 380 insertions(+), 192 deletions(-) create mode 100644 tests/model_executor/model_loader/test_resolve_model_path.py rename vllm_omni/model_executor/models/{ming_tts => common/ming}/aggregator.py (96%) rename vllm_omni/model_executor/models/{ming_tts/fm => common/ming}/cfm_cudagraph.py (98%) rename vllm_omni/model_executor/models/{ming_tts/fm/dit.py => common/ming/cfm_dit.py} (97%) rename vllm_omni/model_executor/models/{ming_tts/flowloss_head.py => common/ming/cfm_head.py} (94%) rename vllm_omni/model_executor/models/common/ming/{fm.py => cfm_solver.py} (77%) rename vllm_omni/model_executor/models/common/ming/{dit.py => dit_blocks.py} (93%) create mode 100644 vllm_omni/model_executor/models/common/ming/speaker_extractor.py rename vllm_omni/model_executor/models/{ming_flash_omni => common/ming}/spk_embedding.py (100%) delete mode 100644 vllm_omni/model_executor/models/ming_tts/fm/__init__.py diff --git a/examples/offline_inference/text_to_speech/ming_tts/end2end.py b/examples/offline_inference/text_to_speech/ming_tts/end2end.py index 1a6991e40..89d5fda73 100644 --- a/examples/offline_inference/text_to_speech/ming_tts/end2end.py +++ b/examples/offline_inference/text_to_speech/ming_tts/end2end.py @@ -13,6 +13,7 @@ import yaml from transformers import AutoTokenizer from vllm.utils.argparse_utils import FlexibleArgumentParser +from vllm_omni.model_executor.models.common.ming.speaker_extractor import SpeakerEmbeddingExtractor from vllm_omni.model_executor.models.ming_tts.config_ming_tts import ( KEY_CFG, KEY_MAX_DECODE_STEPS, @@ -21,7 +22,6 @@ from vllm_omni.model_executor.models.ming_tts.config_ming_tts import ( SAMPLE_RATE, ) from vllm_omni.model_executor.models.ming_tts.prompt_assembly import build_ming_dense_prompt -from vllm_omni.model_executor.models.ming_tts.speaker_extractor import MingSpeakerEmbeddingExtractor try: from .runner import ( @@ -142,7 +142,8 @@ def _resolve_speaker_embedding(args, case: dict, paths: list[str]): return _load_speaker_embedding(args.speaker_embedding) if not (case.get("auto_extract_speaker_embeddings") or args.extract_speaker_embeddings) or not paths: return None - embs = MingSpeakerEmbeddingExtractor(args.model).extract_many(paths) + # Runs before the engine is built, so nothing has pre-fetched campplus.onnx. + embs = SpeakerEmbeddingExtractor(args.model, allow_download=True).extract_many(paths) if not embs: raise RuntimeError("Speaker extraction produced no embeddings") return embs[0] if len(embs) == 1 else torch.stack(embs, dim=0) diff --git a/recipes/inclusionAI/Ming-omni-tts.md b/recipes/inclusionAI/Ming-omni-tts.md index 0b3584691..3666dce0f 100644 --- a/recipes/inclusionAI/Ming-omni-tts.md +++ b/recipes/inclusionAI/Ming-omni-tts.md @@ -5,20 +5,21 @@ ## Summary - Vendor: inclusionAI -- Model: `inclusionAI/Ming-omni-tts-0.5B` +- Model: `inclusionAI/Ming-omni-tts-16.8B-A3B`, `inclusionAI/Ming-omni-tts-0.5B` - Task: Text-to-speech with style, dialect, cloning, and multi-speaker controls - Mode: Online serving via the OpenAI-compatible `/v1/audio/speech` API - Maintainer: Community ## References -- [Huggingface Model card](https://huggingface.co/inclusionAI/Ming-omni-tts-0.5B) +- [Huggingface: inclusionAI/Ming-omni-tts-16.8B-A3B](https://huggingface.co/inclusionAI/Ming-omni-tts-16.8B-A3B) +- [Huggingface: inclusionAI/Ming-omni-tts-0.5B](https://huggingface.co/inclusionAI/Ming-omni-tts-0.5B) - Upstream repository [inclusionAI/Ming-omni-tts](https://github.com/inclusionAI/Ming-omni-tts) ## Hardware Support -This recipe documents a validated ROCm configuration and a CUDA configuration for the dense 0.5B two-stage TTS pipeline deployment. +This recipe documents a validated ROCm configuration and a CUDA configuration for the dense 0.5B and the MoE 16.8B two-stage TTS pipeline deployment. Other hardware is welcome as community validation lands. ## CUDA @@ -39,11 +40,7 @@ Other hardware is welcome as community validation lands. Launch the two-stage talker: ```bash -vllm serve inclusionAI/Ming-omni-tts-0.5B \ - --deploy-config vllm_omni/deploy/ming_tts.yaml \ - --omni \ - --port 8091 \ - --enforce-eager +vllm serve inclusionAI/Ming-omni-tts-16.8B-A3B --omni --port 8091 ``` #### Verification @@ -54,7 +51,7 @@ Basic synthesis (save the WAV bytes): curl -X POST http://localhost:8091/v1/audio/speech \ -H "Content-Type: application/json" \ -d '{ - "model": "inclusionAI/Ming-omni-tts-0.5B", + "model": "inclusionAI/Ming-omni-tts-16.8B-A3B", "input": "你好，这是 Ming 在线语音合成测试。", "response_format": "wav" }' --output ming_tts_basic.wav @@ -74,7 +71,7 @@ Reference-audio zero-shot cloning: REF_AUDIO="$BASE/10002287-00000094.wav" jq -n --arg ref_audio "$REF_AUDIO" \ '{ - model: "inclusionAI/Ming-omni-tts-0.5B", + model: "inclusionAI/Ming-omni-tts-16.8B-A3B", input: "我们的愿景是构建未来服务业的数字化基础设施，为世界带来更多微小而美好的改变。", ref_audio: $ref_audio, ref_text: "在此奉劝大家别乱打美白针。", @@ -107,7 +104,7 @@ jq -n \ --arg a "$REF_A" \ --arg b "$REF_B" \ '{ - model: "inclusionAI/Ming-omni-tts-0.5B", + model: "inclusionAI/Ming-omni-tts-16.8B-A3B", input: $input, ref_audio: [$a, $b], ref_text: $ref_text, diff --git a/tests/model_executor/model_loader/test_resolve_model_path.py b/tests/model_executor/model_loader/test_resolve_model_path.py new file mode 100644 index 000000000..996313b06 --- /dev/null +++ b/tests/model_executor/model_loader/test_resolve_model_path.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for resolve_model_to_local_path.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import huggingface_hub +import pytest +from huggingface_hub.errors import LocalEntryNotFoundError + +from vllm_omni.model_executor.model_loader import weight_utils +from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +# Both resolution paths are patched in every test, so nothing is ever fetched +REPO_ID = "test-org/test-model" +RESOLVED = "/cache/snapshots/deadbeef" + + +@pytest.fixture +def paths(mocker): + """The resolver's two ways out, both patched.""" + return SimpleNamespace( + cache_only=mocker.patch.object(huggingface_hub, "snapshot_download", return_value=RESOLVED), + download=mocker.patch.object(weight_utils, "download_weights_from_hf_specific", return_value=RESOLVED), + ) + + +@pytest.mark.parametrize("allow_download", [False, True]) +def test_local_directory_short_circuits(tmp_path, paths, allow_download): + """A local dir resolves to itself without either path running.""" + assert resolve_model_to_local_path(str(tmp_path), allow_download=allow_download) == str(tmp_path) + + paths.cache_only.assert_not_called() + paths.download.assert_not_called() + + +@pytest.mark.parametrize( + ("kwargs", "allow_patterns", "cache_dir"), + [ + ({}, None, None), + ({"allow_patterns": ["subdir/*"], "cache_dir": "/cache"}, ["subdir/*"], "/cache"), + ], +) +def test_repo_id_is_cache_only_by_default(paths, kwargs, allow_patterns, cache_dir): + """Without `allow_download` nothing is fetched, and narrowing is forwarded.""" + assert resolve_model_to_local_path(REPO_ID, **kwargs) == RESOLVED + + paths.cache_only.assert_called_once_with( + REPO_ID, + cache_dir=cache_dir, + allow_patterns=allow_patterns, + local_files_only=True, + ) + paths.download.assert_not_called() + + +def test_cache_miss_propagates(paths): + """A failed resolution raises instead of degrading into the raw repo id.""" + paths.cache_only.side_effect = LocalEntryNotFoundError("cold cache") + + with pytest.raises(LocalEntryNotFoundError): + resolve_model_to_local_path(REPO_ID) + + paths.download.assert_not_called() + + +@pytest.mark.parametrize( + ("kwargs", "allow_patterns"), + [ + ({}, ["*"]), + ({"allow_patterns": ["subdir/model*.safetensors"]}, ["subdir/model*.safetensors"]), + ], +) +def test_download_routes_through_the_locked_helper(paths, kwargs, allow_patterns): + """Downloads go through the vLLM helper holding the node-wide lock. + + That helper is also what honors `VLLM_USE_MODELSCOPE`; + calling `snapshot_download` directly would drop both. + """ + assert resolve_model_to_local_path(REPO_ID, allow_download=True, cache_dir="/cache", **kwargs) == RESOLVED + + paths.download.assert_called_once_with( + model_name_or_path=REPO_ID, + cache_dir="/cache", + allow_patterns=allow_patterns, + require_all=True, + ) + paths.cache_only.assert_not_called() diff --git a/tests/model_executor/models/ming_tts/test_ming_shared_modules.py b/tests/model_executor/models/ming_tts/test_ming_shared_modules.py index 7a0ad68de..21c8dfd14 100644 --- a/tests/model_executor/models/ming_tts/test_ming_shared_modules.py +++ b/tests/model_executor/models/ming_tts/test_ming_shared_modules.py @@ -7,8 +7,14 @@ import pytest import torch from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech +from vllm_omni.model_executor.models.common.ming.aggregator import Aggregator from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAEConfig -from vllm_omni.model_executor.models.common.ming.fm import Solver +from vllm_omni.model_executor.models.common.ming.cfm_cudagraph import CFMGraphExecutor, CFMSampler +from vllm_omni.model_executor.models.common.ming.cfm_dit import DiT +from vllm_omni.model_executor.models.common.ming.cfm_head import FlowLoss +from vllm_omni.model_executor.models.common.ming.cfm_solver import Solver, get_epss_timesteps +from vllm_omni.model_executor.models.common.ming.speaker_extractor import SpeakerEmbeddingExtractor +from vllm_omni.model_executor.models.common.ming.spk_embedding import SpkembExtractor from vllm_omni.model_executor.models.ming_tts.constants import ( AGGREGATOR_HIDDEN_SIZE, HISTORY_PATCH_SIZE, @@ -24,7 +30,6 @@ from vllm_omni.model_executor.models.ming_tts.constants import ( SPEAKER_EMBEDDING_DIM, VAE_PATCH_SIZE, ) -from vllm_omni.model_executor.models.ming_tts.flowloss_head import FlowLoss from vllm_omni.model_executor.models.ming_tts.ming_tts_llm import MingLLMModel from vllm_omni.model_executor.models.ming_tts.speaker_extractor import _resolve_speaker_embeddings from vllm_omni.model_executor.models.ming_tts.validation import validate_ming_tts_config @@ -44,7 +49,87 @@ def test_ming_tts_audio_vae_uses_common_config(): def test_ming_tts_cfm_solver_uses_common_implementation(): """Ming dense imports the shared solver implementation directly.""" - assert Solver.__module__ == "vllm_omni.model_executor.models.common.ming.fm" + assert Solver.__module__ == "vllm_omni.model_executor.models.common.ming.cfm_solver" + + +@pytest.mark.parametrize( + ("symbol", "expected_module"), + [ + (Aggregator, "vllm_omni.model_executor.models.common.ming.aggregator"), + (DiT, "vllm_omni.model_executor.models.common.ming.cfm_dit"), + (FlowLoss, "vllm_omni.model_executor.models.common.ming.cfm_head"), + (CFMGraphExecutor, "vllm_omni.model_executor.models.common.ming.cfm_cudagraph"), + (CFMSampler, "vllm_omni.model_executor.models.common.ming.cfm_cudagraph"), + (SpeakerEmbeddingExtractor, "vllm_omni.model_executor.models.common.ming.speaker_extractor"), + (SpkembExtractor, "vllm_omni.model_executor.models.common.ming.spk_embedding"), + ], +) +def test_ming_tts_reusable_modules_live_in_common(symbol, expected_module): + """The pieces a second Ming-family TTS model reuses live under common/ming. + + Guards the promotion: if an implementation drifts back into `ming_tts` + (or into another model package), a second model would have to reach across + packages to get it. + """ + assert symbol.__module__ == expected_module + + +def test_ming_flash_talker_uses_the_shared_aggregator(): + """Ming flash no longer carries its own copy of the aggregator. + + Both Ming packages ran near-identical aggregators one directory apart, + which is the duplication this module layout exists to remove. + """ + from vllm_omni.model_executor.models.ming_flash_omni import talker_module + + assert talker_module.Aggregator is Aggregator + assert not any( + isinstance(obj, type) and obj.__module__ == talker_module.__name__ and name == "Aggregator" + for name, obj in vars(talker_module).items() + ) + + +def test_shared_aggregator_matches_the_ming_flash_construction(): + """The shared aggregator reproduces Ming flash's former default geometry. + + `talker_module.Aggregator` defaulted to `in_channels=64` while the + shared class defaults to 4, so the call site now passes `latent_dim` + explicitly. This pins the resulting shapes rather than the default. + """ + latent_dim, llm_hidden = 64, 32 + aggregator = Aggregator( + in_channels=latent_dim, + llm_input_dim=llm_hidden, + hidden_size=16, + depth=1, + num_heads=2, + ) + + assert aggregator.x_embedder.in_features == latent_dim + assert aggregator.final_layer.linear.out_features == llm_hidden + + out = aggregator(torch.randn(1, 3, latent_dim)) + # Keeps only the CLS projection: [B, T, D] -> [B, 1, llm_hidden]. + assert out.shape == (1, 1, llm_hidden) + + +def test_shared_aggregator_rejects_mismatched_latent_width(): + """Wrong-width input fails loudly instead of silently projecting incorrect.""" + aggregator = Aggregator(in_channels=64, llm_input_dim=32, hidden_size=16, depth=1, num_heads=2) + + with pytest.raises(ValueError, match="feature dim mismatch"): + aggregator(torch.randn(1, 3, 4)) + + +def test_cfm_solver_owns_the_timestep_schedule(): + """get_epss_timesteps sits with the solver, not with the DiT blocks.""" + from vllm_omni.model_executor.models.common.ming import dit_blocks + + assert not hasattr(dit_blocks, "get_epss_timesteps") + + t = get_epss_timesteps(10, device="cpu", dtype=torch.float32) + assert t.shape == (11,) + assert float(t[0]) == 0.0 and float(t[-1]) == 1.0 def test_ming_tts_flowloss_preserves_checkpoint_prefix(): diff --git a/vllm_omni/model_executor/model_loader/weight_utils.py b/vllm_omni/model_executor/model_loader/weight_utils.py index c5225ec02..59c2cd860 100644 --- a/vllm_omni/model_executor/model_loader/weight_utils.py +++ b/vllm_omni/model_executor/model_loader/weight_utils.py @@ -1,3 +1,4 @@ +import os import time from pathlib import Path @@ -85,3 +86,46 @@ def download_weights_from_hf_specific( time_taken, ) return hf_folder + + +def resolve_model_to_local_path( + model: str, + *, + allow_download: bool = False, + allow_patterns: list[str] | None = None, + cache_dir: str | None = None, +) -> str: + """Resolve a model reference to a local directory. + Failure is always raised: whether a failure is fatal is the caller's decision. + + Args: + model: A local directory or an HF Hub repo ID. + allow_download: Fetch missing files instead of requiring a warm cache. + Downloads go through :func:`download_weights_from_hf_specific` + allow_patterns: Restrict the fetch to these glob patterns. + Defaults to the whole repository. Ignored when model is already a directory. + cache_dir: Download cache location; None uses the HF default. + + Returns: + Path to a local directory containing the model files. + """ + if os.path.isdir(model): + return model + + if allow_download: + return download_weights_from_hf_specific( + model_name_or_path=model, + cache_dir=cache_dir, + allow_patterns=allow_patterns or ["*"], + require_all=True, + ) + + # Cache-only lookups stay on huggingface_hub even under VLLM_USE_MODELSCOPE: + # ModelScope's snapshot_download has no local_files_only equivalent, + # so a ModelScope deployment that needs resolution here must pass allow_download. + return huggingface_hub.snapshot_download( + model, + cache_dir=cache_dir, + allow_patterns=allow_patterns, + local_files_only=True, + ) diff --git a/vllm_omni/model_executor/models/common/ming/__init__.py b/vllm_omni/model_executor/models/common/ming/__init__.py index 208f01a7c..5134c7cf9 100644 --- a/vllm_omni/model_executor/models/common/ming/__init__.py +++ b/vllm_omni/model_executor/models/common/ming/__init__.py @@ -1,2 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Components shared by the Ming-family model packages. + +Naming: The flow-matching pieces all use the cfm_ prefix, +because "flow matching", "fm" and "cfm" were three names for one concept here: + +* dit_blocks - DiT building blocks (DiTBlock, FinalLayer, CondEmbedder) +* cfm_dit - the DiT used as the CFM velocity model +* cfm_solver - timestep schedules and the ODE/SDE integrator +* cfm_head - inference-only CFM wrapper preserving cfm.model.* names +* cfm_cudagraph - batch-1 CUDA-graph executor for the sampling loop +""" diff --git a/vllm_omni/model_executor/models/ming_tts/aggregator.py b/vllm_omni/model_executor/models/common/ming/aggregator.py similarity index 96% rename from vllm_omni/model_executor/models/ming_tts/aggregator.py rename to vllm_omni/model_executor/models/common/ming/aggregator.py index 3233c9659..e32e8b47b 100644 --- a/vllm_omni/model_executor/models/ming_tts/aggregator.py +++ b/vllm_omni/model_executor/models/common/ming/aggregator.py @@ -7,7 +7,7 @@ import torch import torch.nn as nn from x_transformers.x_transformers import RotaryEmbedding -from vllm_omni.model_executor.models.common.ming.dit import DiTBlock, FinalLayer +from vllm_omni.model_executor.models.common.ming.dit_blocks import DiTBlock, FinalLayer class Aggregator(nn.Module): diff --git a/vllm_omni/model_executor/models/ming_tts/fm/cfm_cudagraph.py b/vllm_omni/model_executor/models/common/ming/cfm_cudagraph.py similarity index 98% rename from vllm_omni/model_executor/models/ming_tts/fm/cfm_cudagraph.py rename to vllm_omni/model_executor/models/common/ming/cfm_cudagraph.py index 7a7280034..1eb3a9db0 100644 --- a/vllm_omni/model_executor/models/ming_tts/fm/cfm_cudagraph.py +++ b/vllm_omni/model_executor/models/common/ming/cfm_cudagraph.py @@ -17,7 +17,7 @@ from __future__ import annotations import torch import torch.nn as nn -from vllm_omni.model_executor.models.common.ming.fm import build_timesteps +from vllm_omni.model_executor.models.common.ming.cfm_solver import build_timesteps class CFMSampler(nn.Module): diff --git a/vllm_omni/model_executor/models/ming_tts/fm/dit.py b/vllm_omni/model_executor/models/common/ming/cfm_dit.py similarity index 97% rename from vllm_omni/model_executor/models/ming_tts/fm/dit.py rename to vllm_omni/model_executor/models/common/ming/cfm_dit.py index 3eb13e987..c29ddf48d 100644 --- a/vllm_omni/model_executor/models/ming_tts/fm/dit.py +++ b/vllm_omni/model_executor/models/common/ming/cfm_dit.py @@ -7,7 +7,7 @@ import torch.nn as nn from x_transformers.x_transformers import RotaryEmbedding from vllm_omni.model_executor.layers.timestep_embedding import DiTTimestepEmbedding -from vllm_omni.model_executor.models.common.ming.dit import CondEmbedder, DiTBlock, FinalLayer +from vllm_omni.model_executor.models.common.ming.dit_blocks import CondEmbedder, DiTBlock, FinalLayer class DiT(nn.Module): diff --git a/vllm_omni/model_executor/models/ming_tts/flowloss_head.py b/vllm_omni/model_executor/models/common/ming/cfm_head.py similarity index 94% rename from vllm_omni/model_executor/models/ming_tts/flowloss_head.py rename to vllm_omni/model_executor/models/common/ming/cfm_head.py index 3f300cec0..14e63557e 100644 --- a/vllm_omni/model_executor/models/ming_tts/flowloss_head.py +++ b/vllm_omni/model_executor/models/common/ming/cfm_head.py @@ -5,9 +5,8 @@ import torch import torch.nn as nn -from vllm_omni.model_executor.models.common.ming.fm import Solver, build_timesteps - -from .fm.dit import DiT +from vllm_omni.model_executor.models.common.ming.cfm_dit import DiT +from vllm_omni.model_executor.models.common.ming.cfm_solver import Solver, build_timesteps class FlowLoss(nn.Module): diff --git a/vllm_omni/model_executor/models/common/ming/fm.py b/vllm_omni/model_executor/models/common/ming/cfm_solver.py similarity index 77% rename from vllm_omni/model_executor/models/common/ming/fm.py rename to vllm_omni/model_executor/models/common/ming/cfm_solver.py index d2cbc6211..5d9db4634 100644 --- a/vllm_omni/model_executor/models/common/ming/fm.py +++ b/vllm_omni/model_executor/models/common/ming/cfm_solver.py @@ -3,7 +3,21 @@ import torch -from .dit import get_epss_timesteps + +def get_epss_timesteps(n, device, dtype): + dt = 1 / 32 + predefined_timesteps = { + 5: [0, 2, 4, 8, 16, 32], + 6: [0, 2, 4, 6, 8, 16, 32], + 7: [0, 2, 4, 6, 8, 16, 24, 32], + 10: [0, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32], + 12: [0, 2, 4, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32], + 16: [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32], + } + t = predefined_timesteps.get(n, []) + if not t: + return torch.linspace(0, 1, n + 1, device=device, dtype=dtype) + return dt * torch.tensor(t, device=device, dtype=dtype) def apply_sway_sampling(t, sway_sampling_coef): diff --git a/vllm_omni/model_executor/models/common/ming/dit.py b/vllm_omni/model_executor/models/common/ming/dit_blocks.py similarity index 93% rename from vllm_omni/model_executor/models/common/ming/dit.py rename to vllm_omni/model_executor/models/common/ming/dit_blocks.py index 56e65f47f..8a093811e 100644 --- a/vllm_omni/model_executor/models/common/ming/dit.py +++ b/vllm_omni/model_executor/models/common/ming/dit_blocks.py @@ -216,19 +216,3 @@ class CondEmbedder(nn.Module): def forward(self, llm_cond): return self.cond_embedder(llm_cond) - - -def get_epss_timesteps(n, device, dtype): - dt = 1 / 32 - predefined_timesteps = { - 5: [0, 2, 4, 8, 16, 32], - 6: [0, 2, 4, 6, 8, 16, 32], - 7: [0, 2, 4, 6, 8, 16, 24, 32], - 10: [0, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32], - 12: [0, 2, 4, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32], - 16: [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32], - } - t = predefined_timesteps.get(n, []) - if not t: - return torch.linspace(0, 1, n + 1, device=device, dtype=dtype) - return dt * torch.tensor(t, device=device, dtype=dtype) diff --git a/vllm_omni/model_executor/models/common/ming/speaker_extractor.py b/vllm_omni/model_executor/models/common/ming/speaker_extractor.py new file mode 100644 index 000000000..bb06a3b58 --- /dev/null +++ b/vllm_omni/model_executor/models/common/ming/speaker_extractor.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adopted from https://github.com/inclusionAI/Ming-omni-tts/blob/main/spkemb_extractor.py +"""CAM++ speaker-embedding extraction, shared by the Ming model packages.""" + +from __future__ import annotations + +import os + +import torch +import torchaudio +from vllm.multimodal.media.audio import load_audio + +from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path +from vllm_omni.model_executor.models.common.ming.spk_embedding import SpkembExtractor + + +class SpeakerEmbeddingExtractor: + """CAM++ ONNX speaker embeddings, resampled to the extractor's rate.""" + + def __init__(self, model, target_sr=16000, *, allow_download: bool = False): + local_model_path = resolve_model_to_local_path( + model, + allow_download=allow_download, + allow_patterns=["campplus.onnx"], + ) + campplus_path = os.path.join(local_model_path, "campplus.onnx") + if not os.path.exists(campplus_path): + raise RuntimeError(f"Missing Ming speaker extractor model: {campplus_path}") + + self.target_sr = int(target_sr) + self._core = SpkembExtractor(campplus_path, target_sr=self.target_sr) + + def extract_from_waveform(self, waveform, sample_rate): + if not isinstance(waveform, torch.Tensor): + waveform = torch.as_tensor(waveform) + + tensor = waveform.detach().to(torch.float32) + if tensor.ndim == 1: + tensor = tensor.unsqueeze(0) + if int(sample_rate) != self.target_sr: + tensor = torchaudio.transforms.Resample(orig_freq=int(sample_rate), new_freq=self.target_sr)(tensor) + + embedding = self._core._extract_spk_embedding(tensor) + return embedding.squeeze(0).to(dtype=torch.float32) + + def extract_from_file(self, audio_path): + audio, sample_rate = load_audio(audio_path, sr=None, mono=False) + waveform = torch.from_numpy(audio).contiguous() + return self.extract_from_waveform(waveform, sample_rate) + + def extract_many(self, audio_paths): + return [self.extract_from_file(path) for path in audio_paths] + + +__all__ = ["SpeakerEmbeddingExtractor"] diff --git a/vllm_omni/model_executor/models/ming_flash_omni/spk_embedding.py b/vllm_omni/model_executor/models/common/ming/spk_embedding.py similarity index 100% rename from vllm_omni/model_executor/models/ming_flash_omni/spk_embedding.py rename to vllm_omni/model_executor/models/common/ming/spk_embedding.py diff --git a/vllm_omni/model_executor/models/ming_flash_omni/ming_flash_omni_talker.py b/vllm_omni/model_executor/models/ming_flash_omni/ming_flash_omni_talker.py index 8868e9d2a..d7e31beb3 100644 --- a/vllm_omni/model_executor/models/ming_flash_omni/ming_flash_omni_talker.py +++ b/vllm_omni/model_executor/models/ming_flash_omni/ming_flash_omni_talker.py @@ -25,14 +25,15 @@ from vllm.model_executor.models.utils import AutoWeightsLoader from vllm.sequence import IntermediateTensors from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin -from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific +from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path +from vllm_omni.model_executor.models.common.ming.aggregator import Aggregator from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE, AudioVAEConfig from vllm_omni.model_executor.models.ming_tts.constants import SPEAKER_EMBEDDING_DIM from vllm_omni.model_executor.models.output_templates import OmniOutput from vllm_omni.transformers_utils.configs.ming_flash_omni import MingFlashOmniTalkerConfig from .prompt_utils import DEFAULT_PROMPT as MING_DEFAULT_PROMPT -from .talker_module import CFM, Aggregator, DiT, MingAudioGenerator, build_tts_input +from .talker_module import CFM, DiT, MingAudioGenerator, build_tts_input from .text_processing import segment_and_normalize from .voice_presets import VoicePresetRegistry @@ -119,7 +120,11 @@ class MingFlashOmniTalkerForConditionalGeneration(nn.Module, CustomProcessMixin) DiT(llm_input_dim=self.hidden_size, **config.flowmodel), steps=config.steps, ) - self.aggregator = Aggregator(llm_input_dim=self.hidden_size, **config.aggregator) + # config.aggregator still wins `in_channels` if the checkpoint states it. + self.aggregator = Aggregator( + llm_input_dim=self.hidden_size, + **{"in_channels": self.latent_dim, **config.aggregator}, + ) self.stop_head = nn.Linear(self.hidden_size, 2, bias=True) self.spk_head = nn.Linear(SPEAKER_EMBEDDING_DIM, self.hidden_size, bias=True) @@ -525,27 +530,22 @@ class MingFlashOmniTalkerForConditionalGeneration(nn.Module, CustomProcessMixin) def _iter_talker_safetensors(self) -> Iterable[tuple[str, torch.Tensor]]: """Yield (name, tensor) pairs from talker/model*.safetensors.""" - model_path = self._model_path - # Try local path first - for candidate in (os.path.join(model_path, "talker"), model_path): + model_root = resolve_model_to_local_path( + self._model_path, + allow_download=True, + allow_patterns=["talker/model*.safetensors"], + cache_dir=self.vllm_config.load_config.download_dir, + ) + + # Nested talker/ layout first, then a flat checkpoint at the root. + for candidate in (os.path.join(model_root, "talker"), model_root): sf_files = sorted(glob_module.glob(os.path.join(candidate, "model*.safetensors"))) if sf_files: for sf_path in sf_files: yield from load_file(sf_path, device="cpu").items() return - # HF hub fallback: download only the talker checkpoint files - model_root = download_weights_from_hf_specific( - model_path, - self.vllm_config.load_config.download_dir, - allow_patterns=["talker/model*.safetensors"], - ) - talker_dir = os.path.join(model_root, "talker") - sf_files = sorted(glob_module.glob(os.path.join(talker_dir, "model*.safetensors"))) - if not sf_files: - raise RuntimeError(f"No talker safetensors found under {model_root}. Expected talker/model*.safetensors.") - for sf_path in sf_files: - yield from load_file(sf_path, device="cpu").items() + raise RuntimeError(f"No talker safetensors found under {model_root}. Expected talker/model*.safetensors.") def _load_vae_weights(self) -> set[str]: """Load AudioVAE weights from talker/vae/model.safetensors.""" diff --git a/vllm_omni/model_executor/models/ming_flash_omni/ming_flash_omni_thinker.py b/vllm_omni/model_executor/models/ming_flash_omni/ming_flash_omni_thinker.py index 8c527b8d9..03cf87abb 100644 --- a/vllm_omni/model_executor/models/ming_flash_omni/ming_flash_omni_thinker.py +++ b/vllm_omni/model_executor/models/ming_flash_omni/ming_flash_omni_thinker.py @@ -64,6 +64,7 @@ from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin +from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path from vllm_omni.model_executor.models.output_templates import OmniOutput from vllm_omni.transformers_utils.configs.ming_flash_omni import BailingMM2Config from vllm_omni.transformers_utils.processors.ming import ( @@ -715,12 +716,8 @@ class MingFlashOmniThinkerForConditionalGeneration( p = Path(model_path) / "mlp" / "model.safetensors" if not p.exists() and not os.path.exists(model_path): - from vllm_omni.model_executor.model_loader.weight_utils import ( - download_weights_from_hf_specific, - ) - try: - local_path = download_weights_from_hf_specific(model_path, None, ["mlp/*"]) + local_path = resolve_model_to_local_path(model_path, allow_download=True, allow_patterns=["mlp/*"]) p = Path(local_path) / "mlp" / "model.safetensors" except Exception: logger.exception("[MingFlashOmniThinker] failed to download mlp/ from %s", model_path) diff --git a/vllm_omni/model_executor/models/ming_flash_omni/talker_module.py b/vllm_omni/model_executor/models/ming_flash_omni/talker_module.py index cfb1090c2..8e3e55611 100644 --- a/vllm_omni/model_executor/models/ming_flash_omni/talker_module.py +++ b/vllm_omni/model_executor/models/ming_flash_omni/talker_module.py @@ -33,9 +33,14 @@ from vllm.platforms import current_platform from x_transformers.x_transformers import RotaryEmbedding from vllm_omni.model_executor.layers.timestep_embedding import DiTTimestepEmbedding +from vllm_omni.model_executor.models.common.ming.aggregator import Aggregator from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE -from vllm_omni.model_executor.models.common.ming.dit import CondEmbedder, DiTBlock, FinalLayer, get_epss_timesteps -from vllm_omni.model_executor.models.common.ming.fm import apply_sway_sampling, integrate_cfm_steps +from vllm_omni.model_executor.models.common.ming.cfm_solver import ( + apply_sway_sampling, + get_epss_timesteps, + integrate_cfm_steps, +) +from vllm_omni.model_executor.models.common.ming.dit_blocks import CondEmbedder, DiTBlock, FinalLayer logger = init_logger(__name__) @@ -458,58 +463,6 @@ def silence_holder( return speech_out, sil_cache -######################################################################## -# Audio Postprocess -# Ported from: -# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/talker_module/aggregator.py -######################################################################## - - -class Aggregator(nn.Module): - """Maps generated audio latent patches back to LLM embedding space.""" - - def __init__( - self, - in_channels: int = 64, - hidden_size: int = 1152, - depth: int = 28, - num_heads: int = 16, - mlp_ratio: float = 4.0, - llm_input_dim: int = 896, - **kwargs, - ): - super().__init__() - self.in_channels = in_channels - self.out_channels = in_channels - self.num_heads = num_heads - - self.word_embedder = nn.Embedding(1, hidden_size) - self.x_embedder = nn.Linear(in_channels, hidden_size) - self.hidden_size = hidden_size - - self.rotary_embed = RotaryEmbedding(hidden_size // num_heads) - - self.blocks = nn.ModuleList( - [DiTBlock(hidden_size, num_heads, mlp_ratio=mlp_ratio, **kwargs) for _ in range(depth)] - ) - self.final_layer = FinalLayer(hidden_size, llm_input_dim) - - def forward(self, x: torch.Tensor, mask: torch.Tensor | None = None) -> torch.Tensor: - x = self.x_embedder(x) - cls_embed = self.word_embedder(torch.zeros((x.shape[0], 1), dtype=torch.long, device=x.device)) - x = torch.cat([cls_embed, x], dim=1) - - rope = self.rotary_embed.forward_from_seq_len(x.shape[1]) - if mask is not None: - mask_pad = mask.clone().detach()[:, :1] - mask = torch.cat([mask_pad, mask], dim=-1) - for block in self.blocks: - x = block(x, mask, rope) - x = self.final_layer(x) - x = x[:, :1, :] - return x - - ######################################################################## # Prompt Builder # Adapted from: diff --git a/vllm_omni/model_executor/models/ming_flash_omni/voice_presets.py b/vllm_omni/model_executor/models/ming_flash_omni/voice_presets.py index 0a096df86..8f6db1afe 100644 --- a/vllm_omni/model_executor/models/ming_flash_omni/voice_presets.py +++ b/vllm_omni/model_executor/models/ming_flash_omni/voice_presets.py @@ -16,16 +16,15 @@ import torch from transformers.utils.hub import cached_file from vllm.logger import init_logger -from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific +from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path +from vllm_omni.model_executor.models.common.ming.spk_embedding import SpkembExtractor -from .spk_embedding import SpkembExtractor from .talker_module import resample if TYPE_CHECKING: + from vllm_omni.model_executor.models.common.ming.aggregator import Aggregator from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE - from .talker_module import Aggregator - logger = init_logger(__name__) @@ -273,18 +272,19 @@ class VoicePresetRegistry: if os.path.isfile(path): return path, candidate - if not os.path.isdir(self._model_path): - try: - hf_root = download_weights_from_hf_specific( - self._model_path, - self._download_dir, - allow_patterns=["talker/data/**"], - require_all=True, - ) - candidate = os.path.join(hf_root, "talker", "data", "voice_name.json") - if os.path.isfile(candidate): - return candidate, os.path.join(hf_root, "talker") - except Exception as e: # pragma: no cover - logger.info("Could not download voice presets from HF: %s", e) + try: + hf_root = resolve_model_to_local_path( + self._model_path, + allow_download=True, + allow_patterns=["talker/data/**"], + cache_dir=self._download_dir, + ) + except Exception as e: + logger.info("Could not download voice presets from HF: %s", e) + return None, None + + candidate = os.path.join(hf_root, "talker", "data", "voice_name.json") + if os.path.isfile(candidate): + return candidate, os.path.join(hf_root, "talker") return None, None diff --git a/vllm_omni/model_executor/models/ming_tts/fm/__init__.py b/vllm_omni/model_executor/models/ming_tts/fm/__init__.py deleted file mode 100644 index 208f01a7c..000000000 --- a/vllm_omni/model_executor/models/ming_tts/fm/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm_omni/model_executor/models/ming_tts/ming_tts_llm.py b/vllm_omni/model_executor/models/ming_tts/ming_tts_llm.py index c0ec5a513..b42c4e9cb 100644 --- a/vllm_omni/model_executor/models/ming_tts/ming_tts_llm.py +++ b/vllm_omni/model_executor/models/ming_tts/ming_tts_llm.py @@ -11,15 +11,21 @@ import torch import torch.nn as nn from vllm.config import VllmConfig from vllm.logger import init_logger +from vllm.model_executor.models.bailing_moe import BailingMoeModel from vllm.model_executor.models.qwen2 import Qwen2Model from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper, maybe_prefix from vllm.sequence import IntermediateTensors from vllm.v1.outputs import SamplerOutput from vllm.v1.sample.metadata import SamplingMetadata +from vllm_omni.model_executor.models.common.ming.aggregator import Aggregator +from vllm_omni.model_executor.models.common.ming.cfm_cudagraph import ( + CFMGraphExecutor, + CFMSampler, +) +from vllm_omni.model_executor.models.common.ming.cfm_head import FlowLoss from vllm_omni.model_executor.models.output_templates import OmniOutput -from .aggregator import Aggregator from .config_ming_tts import ( KEY_CFG, KEY_DECODE_STEP, @@ -34,7 +40,6 @@ from .config_ming_tts import ( MingTTSConfig, ) from .constants import SPEAKER_EMBEDDING_DIM -from .flowloss_head import FlowLoss from .patch_emission import ( MING_STOP_REASON_CODES, MING_STOP_REASON_CONTINUE, @@ -82,8 +87,6 @@ class MingLLMModel(nn.Module): self.prefix = prefix self.fm_dtype = _resolve_ming_runtime_dtype(vllm_config) if self.ming_config.model_variant == "moe": - from vllm.model_executor.models.bailing_moe import BailingMoeModel - # BailingMoeModel reads ``vllm_config.model_config.hf_config`` directly # (no get_text_config()), so re-wrap with the nested bailing_moe config. llm_config = vllm_config.model_config.hf_config.get_text_config() @@ -335,8 +338,6 @@ class MingLLMModel(nn.Module): if self._cfm_graph is not None: return self._cfm_graph try: - from .fm.cfm_cudagraph import CFMGraphExecutor, CFMSampler - # TODO(perf): # (1) compile per-block (dit_model.blocks) instead of the # whole forward for tighter fusion / fewer graph breaks; diff --git a/vllm_omni/model_executor/models/ming_tts/prompt_encoder.py b/vllm_omni/model_executor/models/ming_tts/prompt_encoder.py index 251b2e0f9..355bd8873 100644 --- a/vllm_omni/model_executor/models/ming_tts/prompt_encoder.py +++ b/vllm_omni/model_executor/models/ming_tts/prompt_encoder.py @@ -12,7 +12,7 @@ import torch from safetensors import safe_open from vllm.logger import init_logger -from vllm_omni.engine.stage_init_utils import _resolve_model_to_local_path +from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE from .audio_prep import ( @@ -78,7 +78,7 @@ def _load_prompt_encoder(wrapper: Any) -> AudioVAE: loaded_encoder_params = set() with torch.no_grad(): for shard_path in _iter_model_safetensors( - _resolve_model_to_local_path(str(wrapper.vllm_config.model_config.model)) + resolve_model_to_local_path(str(wrapper.vllm_config.model_config.model)) ): with safe_open(str(shard_path), framework="pt", device="cpu") as handle: for key in handle.keys(): diff --git a/vllm_omni/model_executor/models/ming_tts/speaker_extractor.py b/vllm_omni/model_executor/models/ming_tts/speaker_extractor.py index 8e9f69068..6b6da71fe 100644 --- a/vllm_omni/model_executor/models/ming_tts/speaker_extractor.py +++ b/vllm_omni/model_executor/models/ming_tts/speaker_extractor.py @@ -3,16 +3,13 @@ # Adopted from https://github.com/inclusionAI/Ming-omni-tts/blob/main/spkemb_extractor.py from __future__ import annotations -import os import threading from typing import TYPE_CHECKING, Any import torch -import torchaudio from vllm.logger import init_logger -from vllm.multimodal.media.audio import load_audio -from vllm_omni.model_executor.models.ming_flash_omni.spk_embedding import SpkembExtractor +from vllm_omni.model_executor.models.common.ming.speaker_extractor import SpeakerEmbeddingExtractor from .audio_prep import coerce_prompt_waveform, coerce_speaker_embeddings from .constants import ( @@ -29,48 +26,9 @@ logger = init_logger(__name__) _EXTRACTOR_LOAD_LOCK = threading.Lock() -def resolve_model_to_local_path(model): - if os.path.isdir(model): - return model - - from huggingface_hub import snapshot_download - - return snapshot_download(model) - - -class MingSpeakerEmbeddingExtractor: - def __init__(self, model, target_sr=16000): - local_model_path = resolve_model_to_local_path(model) - campplus_path = os.path.join(local_model_path, "campplus.onnx") - if not os.path.exists(campplus_path): - raise RuntimeError(f"Missing Ming speaker extractor model: {campplus_path}") - - self.target_sr = int(target_sr) - self._core = SpkembExtractor(campplus_path, target_sr=self.target_sr) - - def extract_from_waveform(self, waveform, sample_rate): - if not isinstance(waveform, torch.Tensor): - waveform = torch.as_tensor(waveform) - - tensor = waveform.detach().to(torch.float32) - if tensor.ndim == 1: - tensor = tensor.unsqueeze(0) - if int(sample_rate) != self.target_sr: - tensor = torchaudio.transforms.Resample(orig_freq=int(sample_rate), new_freq=self.target_sr)(tensor) - - embedding = self._core._extract_spk_embedding(tensor) - return embedding.squeeze(0).to(dtype=torch.float32) - - def extract_from_file(self, audio_path): - audio, sample_rate = load_audio(audio_path, sr=None, mono=False) - waveform = torch.from_numpy(audio).contiguous() - return self.extract_from_waveform(waveform, sample_rate) - - def extract_many(self, audio_paths): - return [self.extract_from_file(path) for path in audio_paths] - - -def _load_speaker_extractor(wrapper: MingTTSForConditionalGeneration) -> MingSpeakerEmbeddingExtractor: +# Ming's bundle ships the CAM++ artifact as `campplus.onnx` in the model +# directory, which is the shared extractor's default. +def _load_speaker_extractor(wrapper: MingTTSForConditionalGeneration) -> SpeakerEmbeddingExtractor: extractor = getattr(wrapper, "_speaker_extractor", None) if extractor is not None: return extractor @@ -79,7 +37,7 @@ def _load_speaker_extractor(wrapper: MingTTSForConditionalGeneration) -> MingSpe extractor = getattr(wrapper, "_speaker_extractor", None) if extractor is None: model_path = wrapper.vllm_config.model_config.model - extractor = MingSpeakerEmbeddingExtractor(model_path, target_sr=16000) + extractor = SpeakerEmbeddingExtractor(model_path, target_sr=16000) wrapper._speaker_extractor = extractor return extractor @@ -174,8 +132,6 @@ def _resolve_speaker_embeddings( __all__ = [ - "MingSpeakerEmbeddingExtractor", "_load_speaker_extractor", "_resolve_speaker_embeddings", - "resolve_model_to_local_path", ] From 26e003715c83cabebdc05ee3466b8359133f1074 Mon Sep 17 00:00:00 2001 From: Sy03 <1370724210@qq.com> Date: Tue, 18 Aug 2026 23:36:19 -0700 Subject: [PATCH 069/267] [Perf] Enable async schedule for MOSS-TTS (#6241) Signed-off-by: Sy03 <1370724210@qq.com> --- .../openai_api/test_tts_adapter.py | 18 +++++ .../models/moss_tts/test_compact_topk.py | 41 ++++++++++ .../moss_tts/test_local_depth_rope_cache.py | 44 +++++++++++ .../moss_tts/test_talker_logits_processor.py | 76 +++++++++++++++++++ .../openai/tts_adapters/moss_tts.py | 16 +++- .../moss_tts/modeling_moss_tts_local.py | 31 +++++--- .../moss_tts/modeling_moss_tts_local_depth.py | 53 +++++++++++-- .../moss_tts/modeling_moss_tts_talker.py | 22 ++++-- 8 files changed, 278 insertions(+), 23 deletions(-) create mode 100644 tests/model_executor/models/moss_tts/test_compact_topk.py create mode 100644 tests/model_executor/models/moss_tts/test_local_depth_rope_cache.py diff --git a/tests/entrypoints/openai_api/test_tts_adapter.py b/tests/entrypoints/openai_api/test_tts_adapter.py index 9ab744c83..e87cab05f 100644 --- a/tests/entrypoints/openai_api/test_tts_adapter.py +++ b/tests/entrypoints/openai_api/test_tts_adapter.py @@ -22,6 +22,10 @@ from vllm_omni.entrypoints.openai.tts_adapters.indextts2 import ( IndexTTS25Adapter, indextts2_conditioning_cache_salt, ) +from vllm_omni.entrypoints.openai.tts_adapters.moss_tts import ( + MossTTSAdapter, + MossTTSNanoAdapter, +) from vllm_omni.entrypoints.openai.tts_adapters.qwen3_tts import Qwen3TTSAdapter from vllm_omni.model_executor.models.indextts2 import prompt_utils from vllm_omni.model_executor.models.indextts2.tokenizer_v2_5 import ( @@ -94,6 +98,20 @@ def test_all_adapters_are_ar_or_diffusion(): assert cls.backend in ("ar", "diffusion") +@pytest.mark.parametrize("adapter_cls", [MossTTSAdapter, MossTTSNanoAdapter]) +def test_moss_tts_applies_request_max_new_tokens(adapter_cls): + adapter = adapter_cls(SimpleNamespace(server=object())) + stage_defaults = [SimpleNamespace(max_tokens=4096)] + + overridden = adapter.apply_sampling_overrides( + stage_defaults, + SimpleNamespace(max_new_tokens=512), + ) + + assert overridden[0].max_tokens == 512 + assert stage_defaults[0].max_tokens == 4096 + + def test_qwen3_tts_metadata(): assert Qwen3TTSAdapter.backend == "ar" assert issubclass(Qwen3TTSAdapter, ARTTSAdapter) diff --git a/tests/model_executor/models/moss_tts/test_compact_topk.py b/tests/model_executor/models/moss_tts/test_compact_topk.py new file mode 100644 index 000000000..e2b06c824 --- /dev/null +++ b/tests/model_executor/models/moss_tts/test_compact_topk.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CPU regression tests for MOSS-TTS compact top-k sampling.""" + +import pytest +import torch + +from vllm_omni.model_executor.models.moss_tts import modeling_moss_tts_local + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.tts] + + +def test_compact_topk_only_samples_retained_candidates() -> None: + logits = torch.randn(32, 1024, generator=torch.Generator().manual_seed(7)) + + sampled = modeling_moss_tts_local._sample_token( + logits, + temperature=1.7, + top_k=25, + top_p=0.8, + do_sample=True, + generator=torch.Generator().manual_seed(42), + ) + + retained = logits.topk(25, dim=-1).indices + assert torch.all((retained == sampled.unsqueeze(-1)).any(dim=-1)) + + +def test_compact_topk_tiny_nucleus_selects_argmax() -> None: + logits = torch.randn(32, 1024, generator=torch.Generator().manual_seed(17)) + + sampled = modeling_moss_tts_local._sample_token( + logits, + temperature=1.7, + top_k=25, + top_p=1e-9, + do_sample=True, + generator=torch.Generator().manual_seed(42), + ) + + torch.testing.assert_close(sampled, logits.argmax(dim=-1)) diff --git a/tests/model_executor/models/moss_tts/test_local_depth_rope_cache.py b/tests/model_executor/models/moss_tts/test_local_depth_rope_cache.py new file mode 100644 index 000000000..90c386ee0 --- /dev/null +++ b/tests/model_executor/models/moss_tts/test_local_depth_rope_cache.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CPU parity tests for MOSS-TTS Local Depth RoPE caching.""" + +import pytest +import torch +from transformers import GPT2Config + +from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_local_depth import ( + MossTTSLocalDepthTransformer, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.tts] + + +def _test_config() -> GPT2Config: + return GPT2Config( + n_embd=32, + n_head=4, + n_inner=64, + layer_norm_epsilon=1e-6, + rope_base=10000.0, + ) + + +def test_rope_cache_matches_reference_and_reuses_storage() -> None: + model = MossTTSLocalDepthTransformer(_test_config()).eval() + attn = model.h[0].attn + device = torch.device("cpu") + + attn.prepare_rope_cache(12, device, torch.float32) + cache_ptr = attn._rope_cos_cache.data_ptr() + cos, sin = attn._rope_cos_sin(4, device, torch.float32) + + positions = torch.arange(4, dtype=torch.float32) + freqs = torch.einsum("s,d->sd", positions, attn.inv_freq.float()) + expected_cos = freqs.cos().repeat_interleave(2, dim=-1).view(1, 4, 1, attn.head_dim) + expected_sin = freqs.sin().repeat_interleave(2, dim=-1).view(1, 4, 1, attn.head_dim) + torch.testing.assert_close(cos, expected_cos) + torch.testing.assert_close(sin, expected_sin) + + # Smaller slices retain the fixed allocation used by CUDA Graph. + attn.prepare_rope_cache(6, device, torch.float32) + assert attn._rope_cos_cache.data_ptr() == cache_ptr diff --git a/tests/model_executor/models/moss_tts/test_talker_logits_processor.py b/tests/model_executor/models/moss_tts/test_talker_logits_processor.py index 35c309979..b5866781c 100644 --- a/tests/model_executor/models/moss_tts/test_talker_logits_processor.py +++ b/tests/model_executor/models/moss_tts/test_talker_logits_processor.py @@ -23,6 +23,11 @@ class _RecordingLogitsProcessor: return torch.zeros((1, 4)) +class _EmbeddingBackbone(nn.Module): + def embed_tokens(self, input_ids: torch.Tensor) -> torch.Tensor: + return input_ids.to(dtype=torch.float32).unsqueeze(-1).repeat(1, 4) + + def test_moss_tts_delay_compute_logits_does_not_forward_sampling_metadata() -> None: from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_talker import ( MossTTSDelayTalkerForGeneration, @@ -41,3 +46,74 @@ def test_moss_tts_delay_compute_logits_does_not_forward_sampling_metadata() -> N assert logits.shape == (1, 4) assert model.logits_processor.args == (model.text_lm_head, hidden_states) assert model.logits_processor.kwargs == {} + + +def test_moss_tts_local_keeps_fixed_shape_rows_and_routes_stop_logits() -> None: + from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_talker import ( + MossTTSLocalTalkerForGeneration, + ) + + model = MossTTSLocalTalkerForGeneration.__new__(MossTTSLocalTalkerForGeneration) + nn.Module.__init__(model) + model.audio_pad_token_id = 1024 + model.audio_assistant_slot_token_id = 2 + model.im_end_token_id = 3 + model.text_vocab_size = 8 + model.n_vq = 4 + model._batch_state = None + model._batch_state_spans = None + model._batch_should_continue = None + + valid = torch.tensor([[1, 2, 3, 4]], dtype=torch.long) + stopped = torch.full((1, 4), model.audio_pad_token_id, dtype=torch.long) + infos = [ + {"audio_state": {}, "audio_codes": {"current": valid}}, + {"audio_state": {}, "audio_codes": {"current": stopped}}, + ] + spans = [(0, 3), (3, 4)] + + output = model.make_omni_output( + torch.randn(4, 8), + model_intermediate_buffer=infos, + request_token_spans=spans, + ) + + audio = output.multimodal_outputs["codes"]["audio"] + assert len(audio) == 2 + torch.testing.assert_close(audio[0], valid) + # Keeping this row avoids CUDA boolean-index shape discovery; the + # talker2codec raw async-chunk processor drops all-pad rows on CPU. + torch.testing.assert_close(audio[1], stopped) + assert model._batch_should_continue.tolist() == [True, False] + + logits = model.compute_logits(torch.randn(4, 8)) + + assert logits is not None + assert logits.argmax(dim=-1).tolist() == [2, 2, 2, 3] + + +def test_moss_tts_local_single_token_prefill_does_not_advance_audio() -> None: + from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_talker import ( + MossTTSLocalTalkerForGeneration, + ) + + model = MossTTSLocalTalkerForGeneration.__new__(MossTTSLocalTalkerForGeneration) + nn.Module.__init__(model) + model.model = _EmbeddingBackbone() + model.n_vq = 4 + + input_ids = torch.tensor([7], dtype=torch.long) + _, embeds, update = model.preprocess( + input_ids, + None, + _omni_is_prefill=True, + audio_state={"is_stopping": True}, + ref_offset=5, + ) + + torch.testing.assert_close(embeds, torch.full((1, 4), 7.0)) + assert update == { + "audio_state": {"is_stopping": False}, + "ref_offset": 6, + } + assert "mtp_inputs" not in update diff --git a/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py index ad43a1c01..81bda2078 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py @@ -10,7 +10,12 @@ from typing import TYPE_CHECKING from vllm.inputs import tokens_input from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, conditioning_cache_salt +from vllm_omni.entrypoints.openai.tts_adapters.base import ( + ARTTSAdapter, + PreparedRequest, + apply_max_new_tokens, + conditioning_cache_salt, +) if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -104,6 +109,15 @@ class _MossTTSAdapterBase(ARTTSAdapter): prompt["cache_salt"] = conditioning_cache_salt(request, tts_params) return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type=self.name) + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) + @register_tts_adapter class MossTTSNanoAdapter(_MossTTSAdapterBase): diff --git a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py index 2fd0beefd..31a9fced1 100644 --- a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py +++ b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py @@ -128,32 +128,45 @@ def _sample_token( do_sample: bool, generator: torch.Generator | None = None, ) -> torch.Tensor: - """Top-k + top-p sampling (matches upstream's ``sample_token`` for the - inference branch). + """Top-k + top-p sampling for the upstream inference branch. + + Nucleus filtering and multinomial sampling operate only on the retained + top-k candidates before the result is mapped back to the original + vocabulary. This preserves the categorical distribution when the top-k + boundary has no ties, but it is not seed/bit equivalent to multinomial over + a full-vocabulary tensor because random-number mapping depends on width. """ if not do_sample or temperature <= 0: return logits.argmax(dim=-1) logits = logits / max(temperature, 1e-6) + compact_indices = None if top_k and top_k > 0 and top_k < logits.shape[-1]: - top_vals, _ = torch.topk(logits, top_k, dim=-1) - thresh = top_vals[..., -1:].expand_as(logits) - logits = torch.where(logits < thresh, torch.full_like(logits, float("-inf")), logits) + top_vals, top_indices = torch.topk(logits, top_k, dim=-1, sorted=True) + # topk is already descending: keep the following nucleus filter, + # softmax, and multinomial at width k instead of the full vocab. + logits = top_vals + compact_indices = top_indices if 0.0 < top_p < 1.0: - sorted_logits, sorted_idx = torch.sort(logits, descending=True, dim=-1) - probs = F.softmax(sorted_logits, dim=-1) + sorted_indices = None + if compact_indices is None: + logits, sorted_indices = torch.sort(logits, descending=True, dim=-1) + probs = F.softmax(logits, dim=-1) cum = probs.cumsum(dim=-1) # Drop tail beyond top_p (keep at least one token). drop = cum > top_p drop[..., 1:] = drop[..., :-1].clone() drop[..., 0] = False - sorted_logits = sorted_logits.masked_fill(drop, float("-inf")) - logits = torch.full_like(logits, float("-inf")).scatter_(-1, sorted_idx, sorted_logits) + logits = logits.masked_fill(drop, float("-inf")) + if sorted_indices is not None: + logits = torch.full_like(logits, float("-inf")).scatter_(-1, sorted_indices, logits) probs = F.softmax(logits, dim=-1) flat = probs.reshape(-1, probs.shape[-1]) sampled = torch.multinomial(flat, num_samples=1, generator=generator).reshape(probs.shape[:-1]) + if compact_indices is not None: + return compact_indices.gather(-1, sampled.unsqueeze(-1)).squeeze(-1) return sampled diff --git a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local_depth.py b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local_depth.py index 7ee551f29..e2d46ce2c 100644 --- a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local_depth.py +++ b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local_depth.py @@ -52,15 +52,48 @@ class _MossTTSLocalAttention(nn.Module): self.c_proj = nn.Linear(hidden_size, hidden_size, bias=True) inv_freq = 1.0 / (rope_base ** (torch.arange(0, self.head_dim, 2, dtype=torch.float32) / self.head_dim)) self.register_buffer("inv_freq", inv_freq, persistent=False) + self.register_buffer("_rope_cos_cache", torch.empty(0), persistent=False) + self.register_buffer("_rope_sin_cache", torch.empty(0), persistent=False) - def _rope_cos_sin( - self, seq_len: int, device: torch.device, dtype: torch.dtype - ) -> tuple[torch.Tensor, torch.Tensor]: - position_ids = torch.arange(seq_len, device=device, dtype=torch.float32) - freqs = torch.einsum("s,d->sd", position_ids, self.inv_freq.to(device=device)) + def prepare_rope_cache( + self, + max_seq_len: int, + device: torch.device, + dtype: torch.dtype, + ) -> None: + """Materialize frame-local RoPE values once per device/dtype. + + Local Depth positions always restart at zero and never exceed the + number of RVQ codebooks (12 for MOSS-TTS Local v1.5). Building + arange/einsum/cos/sin inside every one of the 12 depth steps creates + many tiny kernels and also bloats the enclosing talker CUDA graph. + Keep one non-persistent cache and slice it for prefix execution. + """ + if max_seq_len <= 0: + return + cache = self._rope_cos_cache + if cache.numel() > 0 and cache.device == device and cache.dtype == dtype and int(cache.shape[1]) >= max_seq_len: + return + + position_ids = torch.arange(max_seq_len, device=device, dtype=torch.float32) + inv_freq = self.inv_freq.to(device=device, dtype=torch.float32) + freqs = torch.einsum("s,d->sd", position_ids, inv_freq) cos = freqs.cos().repeat_interleave(2, dim=-1).to(dtype) sin = freqs.sin().repeat_interleave(2, dim=-1).to(dtype) - return cos.view(1, seq_len, 1, self.head_dim), sin.view(1, seq_len, 1, self.head_dim) + self._rope_cos_cache = cos.view(1, max_seq_len, 1, self.head_dim) + self._rope_sin_cache = sin.view(1, max_seq_len, 1, self.head_dim) + + def _rope_cos_sin( + self, + seq_len: int, + device: torch.device, + dtype: torch.dtype, + ) -> tuple[torch.Tensor, torch.Tensor]: + self.prepare_rope_cache(seq_len, device, dtype) + return ( + self._rope_cos_cache[:, :seq_len], + self._rope_sin_cache[:, :seq_len], + ) @staticmethod def _rotate_half(x: torch.Tensor) -> torch.Tensor: @@ -200,9 +233,15 @@ class MossTTSLocalDepthTransformer(nn.Module): batch_size = backbone_last_hidden.shape[0] dtype = self.ln_f.weight.dtype + # Populate the complete [0, n_vq) table before torch.compile traces + # _forward_prefix or the runner captures talker_mtp. The compiled / + # captured body then contains only fixed-address cache slices rather + # than cache construction or buffer replacement. + for block in self.h: + block.attn.prepare_rope_cache(n_vq, backbone_last_hidden.device, dtype) + embeds = backbone_last_hidden.new_zeros((batch_size, n_vq, self.hidden_size), dtype=dtype) embeds[:, 0, :] = backbone_last_hidden.to(dtype) - hidden = self._run_prefix(embeds[:, :1, :]) local_hidden = hidden[:, 0, :] diff --git a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py index 171e57328..3f9ebf42f 100644 --- a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py +++ b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py @@ -1323,7 +1323,11 @@ class MossTTSLocalTalkerForGeneration(nn.Module): self.mtp_hidden_size = hidden_size self.talker_mtp_graph_safe = True self.talker_mtp_output_key = ("audio_codes", "current") - self.use_async_omni_output = False + # ``make_omni_output`` keeps code rows fixed-shape and performs all + # state updates eagerly, so the runner can safely pack/snapshot them + # before CUDA-graph buffers are reused. This moves the pinned D2H and + # output materialization off the Stage-0 execution thread. + self.use_async_omni_output = True self.eager_omni_postprocess_before_async_output = True self.omni_pooler_payload_include_hidden = False self.postprocess_uses_multimodal_outputs = False @@ -1466,7 +1470,7 @@ class MossTTSLocalTalkerForGeneration(nn.Module): ) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]: """Build per-step input embeddings (text + audio additive fusion). - Prefill: initialise the per-request ``{is_stopping, step}`` state. + Prefill: initialise the per-request ``{is_stopping}`` state. Decode: combine the forced text token (from ``compute_logits``) with the audio codes the local transformer produced last step. """ @@ -1474,8 +1478,9 @@ class MossTTSLocalTalkerForGeneration(nn.Module): span_len = int(input_ids.shape[0]) audio_state = info_dict.get("audio_state") is_first_call = not isinstance(audio_state, dict) + is_prefill = bool(info_dict.get("_omni_is_prefill", False)) - if span_len > 1 or is_first_call: + if span_len > 1 or is_first_call or is_prefill: embeds = self.model.embed_tokens(input_ids) ref_codes = (info_dict.get("codes", {}) or {}).get("ref") @@ -1668,9 +1673,14 @@ class MossTTSLocalTalkerForGeneration(nn.Module): should_continue = codes.ne(self.audio_pad_token_id).any(dim=-1) should_values.append(should_continue.reshape(-1)[:1]) have_mtp_output = True - emitted_codes = codes[should_continue] - per_req_codes.append(emitted_codes) - have_codes = have_codes or emitted_codes.numel() > 0 + # Do not boolean-index a CUDA tensor here. Even for the + # common single-row case, ``codes[should_continue]`` must + # discover a dynamic output size on the host and therefore + # synchronizes the CUDA stream once per request. Keep the + # fixed-shape row (including an all-pad stop row); the async + # chunk processor already filters all-pad rows on CPU. + per_req_codes.append(codes) + have_codes = True else: should_values.append(torch.ones((1,), device=hidden.device, dtype=torch.bool)) per_req_codes.append(hidden.new_empty((0, self.n_vq), dtype=torch.long)) From ac0067078bd01200776b74e15ee7119153ab1af8 Mon Sep 17 00:00:00 2001 From: herotai214 <68222888+herotai214@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:20:18 +0800 Subject: [PATCH 070/267] [Tests][Refactor] P0.1: Add API server surface guardrails (#6202) Signed-off-by: herotai214 --- .../openai_api/test_api_server_guards.py | 810 ++++++++++++++++++ 1 file changed, 810 insertions(+) create mode 100644 tests/entrypoints/openai_api/test_api_server_guards.py diff --git a/tests/entrypoints/openai_api/test_api_server_guards.py b/tests/entrypoints/openai_api/test_api_server_guards.py new file mode 100644 index 000000000..ed1c65e13 --- /dev/null +++ b/tests/entrypoints/openai_api/test_api_server_guards.py @@ -0,0 +1,810 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""API-server surface guards for the OpenAI Omni entrypoint. + +Purpose +------- +Freeze the *observable* server contract so helper moves (P0.2), route +extraction (P0.3), and later endpoint-family PRs cannot silently change: + +* which routes exist, +* how upstream routes are overridden at assembly time, +* unavailable / missing-handler envelopes, +* shared engine-error JSON fields, +* app-state keys wired by ``omni_init_app_state``. + +CPU-only (fakes / TestClient). No checked-in OpenAPI golden JSON. + +Common wrong changes these tests are meant to catch +---------------------------------------------------- +* Route accidentally removed, renamed, method-changed, or double-registered + after a move (census / assembled-app checks). +* OpenAPI drops an Omni HTTP path while the router still has it (or the reverse). +* Assembly forgets to strip upstream ``/v1/chat/completions`` (batch) / + ``/v1/models``, or also deletes unrelated upstream routes. +* Profiler / Omni router not mounted; storage not started; Omni engine + exception handlers not registered. +* Timestamp middleware stops stamping HTTP, or starts mutating WebSocket scopes. +* Missing-handler WS/HTTP responses change status, JSON shape, or wording + (clients treat these as capability probes). +* Engine error payloads lose ``request_id`` / ``error_stage_id``. +* Pure-diffusion vs multi-stage init drops an ``app.state`` key, or leaves a + handler as ``None`` while the route is still registered. + +Not covered here (add elsewhere if needed) +------------------------------------------ +* Deep per-endpoint success behavior (family UTs). +* Byte-identical helper bodies after a move (P0.2 AST integrity). +* Live GPU / speech byte-identity checks. +""" + +from __future__ import annotations + +import asyncio +import json +from contextlib import asynccontextmanager +from types import SimpleNamespace + +import pytest +from fastapi import FastAPI, HTTPException +from fastapi.routing import APIRoute, APIWebSocketRoute +from fastapi.testclient import TestClient +from starlette.datastructures import State +from starlette.requests import Request +from starlette.websockets import WebSocketDisconnect +from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError + +from vllm_omni.entrypoints.openai import api_server + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +# FastAPI/Starlette auto-adds HEAD on every GET route. Including HEAD here +# would invent ("HEAD", path) entries for every GET and fail the current census. +# TODO: extend this set and the expected census if @router.head / @router.options appear. +_HTTP_METHODS = {"GET", "POST", "DELETE", "PUT", "PATCH"} + +# Exact Omni router census on main. Update intentionally when routes change. +_EXPECTED_ROUTER_ROUTES = { + ("POST", "/v1/chat/completions"), + ("POST", "/v1/chat/completions/batch"), + ("POST", "/v1/audio/speech"), + ("POST", "/v1/audio/speech/batch"), + ("POST", "/v1/audio/generate"), + ("GET", "/v1/audio/voices"), + ("POST", "/v1/audio/voices"), + ("DELETE", "/v1/audio/voices/{name}"), + ("WEBSOCKET", "/v1/audio/speech/stream"), + ("WEBSOCKET", "/v1/video/chat/stream"), + ("WEBSOCKET", "/v1/realtime/video"), + ("WEBSOCKET", "/v1/realtime"), + ("WEBSOCKET", "/v1/realtime/robot/openpi"), + ("WEBSOCKET", "/v1/duplex"), + ("GET", "/health"), + ("GET", "/v1/models"), + ("POST", "/v1/images/generations"), + ("POST", "/v1/images/edits"), + ("POST", "/v1/videos"), + ("POST", "/v1/videos/sync"), + ("GET", "/v1/videos"), + ("GET", "/v1/videos/{video_id}"), + ("DELETE", "/v1/videos/{video_id}"), + ("GET", "/v1/videos/{video_id}/content"), + ("POST", "/v1/omni/sleep"), + ("POST", "/v1/omni/wakeup"), +} +_EXPECTED_PROFILER_ROUTES = { + ("POST", "/start_profile"), + ("POST", "/stop_profile"), +} + +# App-state keys that must remain available across refactor phases. +# Presence is not enough: "not wired" is written as ``state.X = None``. +_DIFFUSION_APP_STATE_KEYS = { + "engine_client", + "log_stats", + "args", + "sleeping_stages", + "stage_configs", + "vllm_config", + "diffusion_engine", + "openai_serving_models", + "serving_tokenization", + "openai_serving_chat", + "openai_serving_chat_batch", + "openai_serving_speech", + "openai_serving_audio_generate", + "openai_serving_video", + "openai_streaming_video_output", + "openai_serving_duplex", + "openai_streaming_speech", + "openai_streaming_video", + "openai_serving_realtime_robot", + "enable_server_load_tracking", + "server_load_metrics", +} +_DIFFUSION_MUST_BE_NONE = { + "vllm_config", + "serving_tokenization", + "openai_serving_duplex", + "openai_streaming_speech", + "openai_streaming_video", +} +_DIFFUSION_MUST_BE_WIRED = _DIFFUSION_APP_STATE_KEYS - _DIFFUSION_MUST_BE_NONE +_MULTISTAGE_APP_STATE_KEYS = { + "engine_client", + "log_stats", + "args", + "sleeping_stages", + "stage_configs", + "vllm_config", + "openai_serving_models", + "serving_tokenization", + "online_renderer", + "openai_serving_chat", + "openai_serving_chat_batch", + "openai_serving_speech", + "openai_serving_audio_generate", + "openai_streaming_speech", + "openai_streaming_video", + "openai_serving_duplex", + "openai_serving_realtime", + "openai_serving_video", + "openai_serving_realtime_robot", + "enable_server_load_tracking", + "server_load_metrics", +} +_MULTISTAGE_MUST_BE_NONE = { + "openai_serving_duplex", + "openai_serving_realtime_robot", +} +_MULTISTAGE_MUST_BE_WIRED = _MULTISTAGE_APP_STATE_KEYS - _MULTISTAGE_MUST_BE_NONE + + +def _route_entries(routes) -> list[tuple[str, str]]: + entries: list[tuple[str, str]] = [] + for route in routes: + if isinstance(route, APIRoute): + for method in sorted((route.methods or set()) & _HTTP_METHODS): + entries.append((method, route.path)) + elif isinstance(route, APIWebSocketRoute): + entries.append(("WEBSOCKET", route.path)) + return entries + + +def _route_manifest(routes) -> set[tuple[str, str]]: + return set(_route_entries(routes)) + + +def _assert_unique_route_entries(routes) -> None: + entries = _route_entries(routes) + duplicates = sorted({item for item in entries if entries.count(item) > 1}) + assert not duplicates, f"duplicate route registrations: {duplicates}" + + +def _matching_http_routes(routes, method: str, path: str) -> list[APIRoute]: + return [ + route + for route in routes + if isinstance(route, APIRoute) and route.path == path and method in (route.methods or set()) + ] + + +def _single_http_route(routes, method: str, path: str) -> APIRoute: + matches = _matching_http_routes(routes, method, path) + assert len(matches) == 1, f"expected exactly one {method} {path}, got {len(matches)}" + return matches[0] + + +def _assert_app_state_snapshot( + state, + *, + expected_keys: set[str], + must_be_wired: set[str], + must_be_none: set[str], +) -> None: + assert must_be_wired.isdisjoint(must_be_none) + assert must_be_wired | must_be_none <= expected_keys + present = {key for key in expected_keys if hasattr(state, key)} + assert present == expected_keys + not_wired = sorted(key for key in must_be_wired if getattr(state, key) is None) + assert not not_wired, f"app.state keys registered but not wired: {not_wired}" + unexpectedly_set = sorted(key for key in must_be_none if getattr(state, key) is not None) + assert not unexpectedly_set, f"app.state keys expected to stay None: {unexpectedly_set}" + + +def _minimal_args(**overrides) -> SimpleNamespace: + args = SimpleNamespace( + tool_parser_plugin=None, + reasoning_parser_plugin=None, + reasoning_parser=None, + structured_outputs_config=SimpleNamespace(reasoning_parser=None), + enable_ssl_refresh=False, + host="127.0.0.1", + port=8000, + uvicorn_log_level="info", + disable_uvicorn_access_log=True, + ssl_keyfile=None, + ssl_certfile=None, + ssl_ca_certs=None, + ssl_cert_reqs=0, + ssl_ciphers=None, + h11_max_incomplete_event_size=None, + h11_max_header_count=None, + log_config_file=None, + disable_access_log_for_endpoints=None, + disable_log_stats=True, + enable_log_requests=False, + max_log_len=None, + served_model_name=None, + model="demo-model", + enable_server_load_tracking=False, + log_error_stack=False, + chat_template=None, + chat_template_content_format="auto", + trust_request_chat_template=False, + tool_server=None, + lora_modules=None, + response_role="assistant", + default_chat_template_kwargs=None, + return_tokens_as_token_ids=False, + enable_auto_tool_choice=False, + exclude_tools_when_tool_choice_none=False, + tool_call_parser=None, + enable_prompt_tokens_details=False, + enable_force_include_usage=False, + enable_log_outputs=False, + enable_log_deltas=False, + tokens_only=False, + shutdown_timeout=0, + stage_configs_path=None, + deploy_config=None, + ) + for key, value in overrides.items(): + setattr(args, key, value) + return args + + +def _request_for(app: FastAPI, *, method: str = "GET", path: str = "/") -> Request: + return Request( + { + "type": "http", + "method": method, + "path": path, + "headers": [], + "app": app, + "state": {}, + } + ) + + +class _FakeSocket: + def __init__(self) -> None: + self.closed = False + + def close(self) -> None: + self.closed = True + + +class _FakeEngineClient: + def __init__(self, *, stage_configs=None, endpoint_restrictions=None, vllm_config=None) -> None: + self.stage_configs = stage_configs if stage_configs is not None else [] + self.endpoint_restrictions = endpoint_restrictions if endpoint_restrictions is not None else {} + self.model_config = SimpleNamespace() + self.input_processor = object() + self.renderer = object() + self.errored = False + self.vllm_config = vllm_config + + async def get_vllm_config(self): + return self.vllm_config + + async def get_supported_tasks(self) -> tuple[str, ...]: + return ("generate",) + + async def get_tokenizer(self): + return SimpleNamespace(chat_template="dummy") + + async def check_health(self) -> None: + return None + + +def _marker(name: str): + return SimpleNamespace(name=name) + + +def test_router_route_manifest_is_explicit_and_complete() -> None: + """Lock the Omni + profiler router census. + + Fails if a route is added/removed/renamed, a method changes, or a + WebSocket path disappears during helper/router moves. Also fails if the + same public route is registered more than once, which a set manifest alone + would collapse. + """ + # TODO(P0.3): retarget this object-level census when routes move onto family + # routers. The assembled-app census is the client-facing lock that must survive. + _assert_unique_route_entries(api_server.router.routes) + _assert_unique_route_entries(api_server.profiler_router.routes) + assert _route_manifest(api_server.router.routes) == _EXPECTED_ROUTER_ROUTES + assert _route_manifest(api_server.profiler_router.routes) == _EXPECTED_PROFILER_ROUTES + + +def test_router_openapi_paths_cover_http_manifest() -> None: + """Lock that OpenAPI advertises every HTTP census route/method. + + Fails if a decorator still registers a route but OpenAPI loses the path + (or method), e.g. after moving route bodies / response models. + """ + app = FastAPI() + app.include_router(api_server.router) + app.include_router(api_server.profiler_router) + paths = app.openapi().get("paths", {}) + + http_routes = { + (method, path) for method, path in _EXPECTED_ROUTER_ROUTES | _EXPECTED_PROFILER_ROUTES if method != "WEBSOCKET" + } + for method, path in sorted(http_routes): + assert path in paths, f"missing OpenAPI path {path}" + assert method.lower() in paths[path], f"missing OpenAPI method {method} {path}" + + +@pytest.mark.asyncio +async def test_api_server_assembly_replaces_upstream_routes_and_mounts_omni_router(monkeypatch) -> None: + """Lock worker assembly: override, mount, storage, handlers, full census. + + Fails if assembly stops replacing upstream chat/batch/models, deletes + unrelated upstream routes, forgets Omni/profiler mounts, skips storage + start, or drops Omni ``EngineDeadError`` / ``EngineGenerateError`` handlers. + """ + captured: dict[str, object] = {} + + def fake_build_openai_app(args, supported_tasks): + app = FastAPI() + + @app.post("/v1/chat/completions") + async def upstream_chat(): + return {"owner": "upstream"} + + @app.post("/v1/chat/completions/batch") + async def upstream_batch(): + return {"owner": "upstream"} + + @app.get("/v1/models") + async def upstream_models(): + return {"owner": "upstream"} + + @app.get("/v1/upstream-only") + async def upstream_only(): + return {"owner": "upstream"} + + captured["supported_tasks"] = supported_tasks + return app + + @asynccontextmanager + async def fake_build_async_omni(*_args, **_kwargs): + yield _FakeEngineClient( + stage_configs=[{"engine_args": {"profiler_config": {"profiler": "torch"}}}], + vllm_config=SimpleNamespace(parallel_config=SimpleNamespace(_api_process_rank=0)), + ) + + async def fake_omni_init_app_state(engine_client, state, args): + state.engine_client = engine_client + state.initialized_by_omni = True + + async def fake_storage_start(): + captured["storage_started"] = True + + async def fake_serve_http(app, **_kwargs): + captured["served_app"] = app + task = asyncio.create_task(asyncio.sleep(0)) + await asyncio.sleep(0) + return task + + monkeypatch.setattr(api_server, "build_openai_app", fake_build_openai_app) + monkeypatch.setattr(api_server, "build_async_omni", fake_build_async_omni) + monkeypatch.setattr(api_server, "omni_init_app_state", fake_omni_init_app_state) + monkeypatch.setattr( + api_server, + "shutdown_unsupported_routes", + lambda app, restrictions: captured.setdefault("restrictions", restrictions), + ) + monkeypatch.setattr(api_server, "STORAGE_MANAGER", SimpleNamespace(start=fake_storage_start)) + monkeypatch.setattr(api_server, "serve_http", fake_serve_http) + + sock = _FakeSocket() + await api_server.omni_run_server_worker("127.0.0.1:8000", sock, _minimal_args()) + + served_wrapper = captured["served_app"] + # Timestamp middleware wraps the FastAPI app. + assert hasattr(served_wrapper, "_inner") + served_app = served_wrapper._inner + routes = served_app.routes + _assert_unique_route_entries(routes) + manifest = _route_manifest(routes) + + assert captured["supported_tasks"] == ("generate",) + assert captured["storage_started"] is True + assert captured["restrictions"] == {} + assert sock.closed is True + assert served_app.state.initialized_by_omni is True + + # Upstream override: unrelated route kept; Omni owns overridden pairs once. + # Endpoint identity ensures the surviving route is the Omni handler, not + # the fake upstream handler with the same method/path. + assert ("GET", "/v1/upstream-only") in manifest + assert _single_http_route(routes, "POST", "/v1/chat/completions").endpoint is api_server.create_chat_completion + assert ( + _single_http_route(routes, "POST", "/v1/chat/completions/batch").endpoint + is api_server.create_batch_chat_completion + ) + assert _single_http_route(routes, "GET", "/v1/models").endpoint is api_server.show_available_models + + # Assembled app includes Omni router HTTP/WS surface + profiler. + for item in _EXPECTED_ROUTER_ROUTES: + assert item in manifest, f"assembled app missing {item}" + assert ("POST", "/start_profile") in manifest + assert ("POST", "/stop_profile") in manifest + + assert served_app.exception_handlers[EngineGenerateError] + assert served_app.exception_handlers[EngineDeadError] + + # Light OpenAPI check on the assembled FastAPI app (not the ASGI wrapper). + openapi_paths = served_app.openapi().get("paths", {}) + assert "/v1/chat/completions" in openapi_paths + assert "/v1/models" in openapi_paths + assert "/health" in openapi_paths + + +@pytest.mark.asyncio +async def test_timestamp_middleware_stamps_http_and_passes_websocket(monkeypatch) -> None: + """Lock outermost timestamp middleware behavior. + + Fails if HTTP requests lose ``request_timestamp``, or WebSocket scopes + start getting stamped / mutated incorrectly. + """ + captured: dict[str, object] = {} + + def fake_build_openai_app(args, supported_tasks): + return FastAPI() + + @asynccontextmanager + async def fake_build_async_omni(*_args, **_kwargs): + yield _FakeEngineClient() + + async def fake_omni_init_app_state(engine_client, state, args): + state.engine_client = engine_client + + async def fake_serve_http(app, **_kwargs): + captured["served_app"] = app + task = asyncio.create_task(asyncio.sleep(0)) + await asyncio.sleep(0) + return task + + monkeypatch.setattr(api_server, "build_openai_app", fake_build_openai_app) + monkeypatch.setattr(api_server, "build_async_omni", fake_build_async_omni) + monkeypatch.setattr(api_server, "omni_init_app_state", fake_omni_init_app_state) + monkeypatch.setattr(api_server, "shutdown_unsupported_routes", lambda *_a, **_k: None) + monkeypatch.setattr(api_server, "STORAGE_MANAGER", SimpleNamespace(start=lambda: asyncio.sleep(0))) + monkeypatch.setattr(api_server, "serve_http", fake_serve_http) + + await api_server.omni_run_server_worker("127.0.0.1:8000", _FakeSocket(), _minimal_args()) + middleware = captured["served_app"] + + http_scope = {"type": "http", "state": {}} + ws_scope = {"type": "websocket"} + + async def _receive(): + return {"type": "http.disconnect"} + + async def _send(_message): + return None + + called = {"http": False, "websocket": False} + + async def inner(scope, receive, send): + called[scope["type"]] = True + + middleware._inner = inner + await middleware(http_scope, _receive, _send) + await middleware(ws_scope, _receive, _send) + + assert called["http"] is True + assert called["websocket"] is True + assert "request_timestamp" in http_scope["state"] + assert isinstance(http_scope["state"]["request_timestamp"], float) + assert "request_timestamp" not in ws_scope.get("state", {}) + + +@pytest.mark.parametrize( + ("path", "payload"), + [ + ("/v1/audio/speech/stream", {"type": "error", "message": "Streaming speech is not available"}), + ("/v1/video/chat/stream", {"type": "error", "message": "Streaming video chat is not available"}), + ("/v1/realtime/video", {"type": "error", "message": "Streaming video generation is not available"}), + ("/v1/realtime", {"type": "error", "error": "Realtime API is not available", "code": "unsupported"}), + ( + "/v1/realtime/robot/openpi", + {"type": "error", "error": "Robot policy not available", "code": "unsupported"}, + ), + ("/v1/duplex", {"type": "error", "error": "Duplex API is not available", "code": "unsupported"}), + ], +) +def test_websocket_routes_emit_stable_unavailable_frames_and_close(path: str, payload: dict[str, str]) -> None: + """Lock missing-handler WebSocket capability probes. + + Fails if unavailable JSON changes (keys/wording/code) or the socket stays + open after the error frame — common when WS wrappers move between modules. + """ + app = FastAPI() + app.include_router(api_server.router) + + with TestClient(app) as client: + with client.websocket_connect(path) as websocket: + assert websocket.receive_json() == payload + with pytest.raises(WebSocketDisconnect): + websocket.receive_text() + + +def test_health_without_engine_returns_stable_unhealthy_response() -> None: + """Lock ``/health`` with no engine initialized. + + Fails if status drifts from 503 or the unhealthy payload shape/reason + string changes during health-route extraction. + """ + app = FastAPI() + response = asyncio.run(api_server.health(_request_for(app, path="/health"))) + + assert response.status_code == 503 + assert json.loads(response.body) == { + "status": "unhealthy", + "reason": "No engine initialized", + } + + +def test_models_without_handler_returns_empty_openai_list() -> None: + """Lock ``/v1/models`` when ``openai_serving_models`` is unset. + + Fails if the empty listing becomes an error, or the OpenAI list envelope + changes (``object`` / ``data``). + """ + app = FastAPI() + response = asyncio.run(api_server.show_available_models(_request_for(app, path="/v1/models"))) + + assert response.status_code == 200 + assert json.loads(response.body) == {"object": "list", "data": []} + + +def test_images_generation_without_engine_preserves_service_unavailable_error() -> None: + """Lock images generation when the multi-stage engine is missing. + + Fails if the 503 detail string changes — clients/tests use it as a + capability signal when bootstrap failed. + """ + app = FastAPI() + raw_request = _request_for(app, method="POST", path="/v1/images/generations") + request = api_server.ImageGenerationRequest(prompt="a cat", model="demo-model") + + with pytest.raises(HTTPException) as exc_info: + asyncio.run(api_server.generate_images(request, raw_request)) + + assert exc_info.value.status_code == 503 + assert exc_info.value.detail == "Multi-stage engine not initialized. Start server with a multi-stage omni model." + + +def test_images_generation_without_multistage_chat_handler_preserves_unavailable_error(monkeypatch) -> None: + """Lock images generation when chat serving was not wired. + + Fails if multi-stage images stop requiring ``openai_serving_chat``, or the + 503 detail changes after images/chat ownership splits. + """ + app = FastAPI() + app.state.engine_client = SimpleNamespace(stage_configs=[object(), object()]) + app.state.stage_configs = app.state.engine_client.stage_configs + app.state.openai_serving_models = SimpleNamespace(base_model_paths=[SimpleNamespace(name="demo-model")]) + app.state.openai_serving_chat = None + app.state.args = SimpleNamespace(max_generated_image_size=None) + + raw_request = _request_for(app, method="POST", path="/v1/images/generations") + request = api_server.ImageGenerationRequest(prompt="a cat", model="demo-model") + monkeypatch.setattr(api_server, "get_stage_type", lambda _stage_cfg: "diffusion") + + with pytest.raises(HTTPException) as exc_info: + asyncio.run(api_server.generate_images(request, raw_request)) + + assert exc_info.value.status_code == 503 + assert exc_info.value.detail == "openai_serving_chat is not initialized for multi-stage image generation." + + +def test_speech_without_handler_preserves_not_found_http_error() -> None: + """Lock speech HTTP when ``openai_serving_speech`` is unset. + + Fails if missing speech becomes a different status/detail after TTS route + wrappers move out of ``api_server``. + """ + app = FastAPI() + app.state.openai_serving_speech = None + raw_request = _request_for(app, method="POST", path="/v1/audio/speech") + request = api_server.OpenAICreateSpeechRequest(input="hello", model="demo-model", voice="alloy") + + with pytest.raises(HTTPException) as exc_info: + asyncio.run(api_server.create_speech(request, raw_request)) + + assert exc_info.value.status_code == 404 + assert exc_info.value.detail == "The model does not support Speech API" + + +def test_engine_error_json_response_includes_request_and_stage_fields(monkeypatch) -> None: + """Lock shared ``EngineGenerateError`` JSON fields used by Omni handlers. + + Fails if ``request_id`` / ``error_stage_id`` are dropped when error helpers + move (P0.2) or exception-handler wiring changes. + + Exercises the envelope through the registered handler, not the private + ``_create_engine_error_json_response`` helper, so relocating that helper + in P0.2 is not a false red. + """ + app = FastAPI() + # TODO(P0.2): retarget this call if ``_register_omni_exception_handlers`` + # leaves api_server (red here is a rename, not a surface regression). + api_server._register_omni_exception_handlers(app) + app.state.engine_client = SimpleNamespace(errored=True, engine=SimpleNamespace(is_alive=lambda: False)) + app.state.server = object() + app.state.args = SimpleNamespace(log_error_stack=False) + + req = _request_for(app, method="POST", path="/v1/chat/completions") + req.state.request_metadata = SimpleNamespace(request_id="req-123") + + monkeypatch.setattr(api_server, "terminate_if_errored", lambda **_kwargs: None) + + exc = EngineGenerateError("boom") + exc.error_stage_id = 2 # type: ignore[attr-defined] + handler = app.exception_handlers[EngineGenerateError] + response = asyncio.run(handler(req, exc)) + payload = json.loads(response.body) + + assert "error" in payload + assert payload["error"]["request_id"] == "req-123" + assert payload["error"]["error_stage_id"] == 2 + assert response.status_code >= 400 + + +def test_engine_dead_error_handler_registered_returns_json(monkeypatch) -> None: + """Lock Omni ``EngineDeadError`` handler registration + JSON envelope. + + Fails if registration is skipped or the handler stops returning OpenAI-style + JSON with ``request_id``. + """ + app = FastAPI() + # TODO(P0.2): retarget this call if ``_register_omni_exception_handlers`` + # leaves api_server (red here is a rename, not a surface regression). + api_server._register_omni_exception_handlers(app) + app.state.engine_client = SimpleNamespace(errored=True, engine=SimpleNamespace(is_alive=lambda: False)) + app.state.server = object() + app.state.args = SimpleNamespace(log_error_stack=False) + + monkeypatch.setattr(api_server, "terminate_if_errored", lambda **_kwargs: None) + + handler = app.exception_handlers[EngineDeadError] + req = _request_for(app) + req.state.request_metadata = SimpleNamespace(request_id="dead-1") + response = asyncio.run(handler(req, EngineDeadError())) + payload = json.loads(response.body) + assert payload["error"]["request_id"] == "dead-1" + assert response.status_code >= 400 + + +@pytest.mark.asyncio +async def test_pure_diffusion_app_state_key_snapshot(monkeypatch) -> None: + """Lock pure-diffusion ``app.state`` keys after init, including live vs None. + + Fails if bootstrap drops keys route owners still read (e.g. video/speech/ + models), wires a handler that must stay None, or leaves a required + handler as None. + """ + stage = SimpleNamespace(engine_args={}) + engine = _FakeEngineClient(stage_configs=[stage]) + + def _for_diffusion_factory(label: str): + @classmethod + def _factory(cls, *args, **kwargs): + return _marker(label) + + return _factory + + monkeypatch.setattr(api_server, "get_stage_type", lambda _cfg: "diffusion") + monkeypatch.setattr(api_server.OmniOpenAIServingChat, "for_diffusion", _for_diffusion_factory("chat")) + monkeypatch.setattr(api_server.OmniOpenAIServingChatBatch, "for_diffusion", _for_diffusion_factory("chat_batch")) + monkeypatch.setattr( + api_server.OmniOpenAIServingAudioGenerate, + "for_diffusion", + _for_diffusion_factory("audio_generate"), + ) + monkeypatch.setattr(api_server.OmniOpenAIServingVideo, "for_diffusion", _for_diffusion_factory("video")) + monkeypatch.setattr(api_server.OmniStreamingVideoOutputHandler, "__init__", lambda self, *a, **k: None) + monkeypatch.setattr(api_server.OmniOpenAIServingSpeech, "for_diffusion", _for_diffusion_factory("speech")) + monkeypatch.setattr( + api_server.ServingRealtimeRobotOpenPI, + "create_policy_server", + classmethod(lambda cls, *a, **k: _marker("openpi")), + ) + + state = State() + await api_server.omni_init_app_state(engine, state, _minimal_args()) + + _assert_app_state_snapshot( + state, + expected_keys=_DIFFUSION_APP_STATE_KEYS, + must_be_wired=_DIFFUSION_MUST_BE_WIRED, + must_be_none=_DIFFUSION_MUST_BE_NONE, + ) + assert state.diffusion_engine is engine + + +@pytest.mark.asyncio +async def test_multistage_app_state_key_snapshot(monkeypatch) -> None: + """Lock multi-stage ``app.state`` keys after init, including live vs None. + + Fails if chat/speech/video/realtime/tokenization keys disappear or are + left as None after init refactors — classic “route still registered, + handler never wired” failure mode. + """ + engine = _FakeEngineClient( + stage_configs=[object(), object()], + vllm_config=SimpleNamespace( + lora_config=None, + model_config=SimpleNamespace(), + parallel_config=SimpleNamespace(_api_process_rank=0), + ), + ) + + class _FakeModels: + def __init__(self, *args, **kwargs): + self.base_model_paths = kwargs.get("base_model_paths") or [] + + async def init_static_loras(self): + return None + + class _FakeCtor: + def __init__(self, *args, **kwargs): + pass + + def warmup(self): + return None + + class _FakeSpeech(_FakeCtor): + async def warmup(self): + return None + + monkeypatch.setattr(api_server, "load_chat_template", lambda *_a, **_k: None) + monkeypatch.setattr(api_server, "process_lora_modules", lambda modules, _defaults: modules or []) + monkeypatch.setattr(api_server, "OpenAIServingModels", _FakeModels) + monkeypatch.setattr(api_server, "OnlineRenderer", _FakeCtor) + monkeypatch.setattr(api_server, "OpenAIServingResponses", _FakeCtor) + monkeypatch.setattr(api_server, "OmniOpenAIServingChat", _FakeCtor) + monkeypatch.setattr(api_server, "OmniOpenAIServingChatBatch", _FakeCtor) + monkeypatch.setattr(api_server, "OpenAIServingCompletion", _FakeCtor) + monkeypatch.setattr(api_server, "ServingPooling", _FakeCtor) + monkeypatch.setattr(api_server, "OpenAIServingEmbedding", _FakeCtor) + monkeypatch.setattr(api_server, "ServingClassification", _FakeCtor) + monkeypatch.setattr(api_server, "ServingScores", _FakeCtor) + monkeypatch.setattr(api_server, "ServingTokenization", _FakeCtor) + monkeypatch.setattr(api_server, "OpenAIServingTranscription", _FakeCtor) + monkeypatch.setattr(api_server, "OpenAIServingTranslation", _FakeCtor) + monkeypatch.setattr(api_server, "AnthropicServingMessages", _FakeCtor) + monkeypatch.setattr(api_server, "ServingTokens", _FakeCtor) + monkeypatch.setattr(api_server, "OmniOpenAIServingSpeech", _FakeSpeech) + monkeypatch.setattr(api_server, "OmniOpenAIServingAudioGenerate", _FakeCtor) + monkeypatch.setattr(api_server, "OmniStreamingSpeechHandler", _FakeCtor) + monkeypatch.setattr(api_server, "create_streaming_video_handler", lambda **_k: _marker("streaming_video")) + monkeypatch.setattr(api_server, "OpenAIServingRealtime", _FakeCtor) + monkeypatch.setattr(api_server, "OmniOpenAIServingVideo", _FakeCtor) + monkeypatch.setattr(api_server, "should_enable_duplex_endpoint", lambda *_a, **_k: False) + + state = State() + await api_server.omni_init_app_state(engine, state, _minimal_args()) + + _assert_app_state_snapshot( + state, + expected_keys=_MULTISTAGE_APP_STATE_KEYS, + must_be_wired=_MULTISTAGE_MUST_BE_WIRED, + must_be_none=_MULTISTAGE_MUST_BE_NONE, + ) From 70995834040f488d63ed5c916fcee4e94268a902 Mon Sep 17 00:00:00 2001 From: BruceLoveDecimal <39156883+BruceLoveDecimal@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:26:03 +0800 Subject: [PATCH 071/267] [Bugfix][MiniCPM-o] Restore streaming audio cache and unstall the first duplex response (#6274) Signed-off-by: BruceLoveDecimal --- .../test_streaming_audio_cache.py | 156 ++++++++++++++++++ .../models/minicpmo_4_5/batched_token2wav.py | 20 +++ .../minicpmo_4_5/minicpmo_4_5_omni_llm.py | 40 +++-- 3 files changed, 205 insertions(+), 11 deletions(-) create mode 100644 tests/model_executor/models/minicpmo_4_5/test_streaming_audio_cache.py diff --git a/tests/model_executor/models/minicpmo_4_5/test_streaming_audio_cache.py b/tests/model_executor/models/minicpmo_4_5/test_streaming_audio_cache.py new file mode 100644 index 000000000..e867e0cab --- /dev/null +++ b/tests/model_executor/models/minicpmo_4_5/test_streaming_audio_cache.py @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression tests for MiniCPM-o 4.5 streaming audio cache compatibility.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch +from torch import nn +from transformers.models.whisper.modeling_whisper import WhisperConfig + +from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import ( + WHISPER_ATTENTION_CLASSES, + MiniCPMWhisperEncoder, + MiniCPMWhisperEncoderLayer, + _get_audio_cache_length, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class _LegacyWhisperAttention(nn.Module): + def __init__(self, **kwargs) -> None: + super().__init__() + self.seen_cache = None + + def forward(self, hidden_states, past_key_value=None, **kwargs): + self.seen_cache = past_key_value + return hidden_states, None, past_key_value + + +class _CurrentWhisperAttention(nn.Module): + def __init__(self, **kwargs) -> None: + super().__init__() + self.seen_cache = None + + def forward(self, hidden_states, past_key_values=None, **kwargs): + self.seen_cache = past_key_values + return hidden_states, None + + +@pytest.mark.parametrize( + ("implementation", "attention_cls"), + [ + ("test_legacy_cache", _LegacyWhisperAttention), + ("test_current_cache", _CurrentWhisperAttention), + ], +) +def test_whisper_attention_preserves_streaming_cache( + monkeypatch: pytest.MonkeyPatch, + implementation: str, + attention_cls: type[nn.Module], +) -> None: + monkeypatch.setitem(WHISPER_ATTENTION_CLASSES, implementation, attention_cls) + config = WhisperConfig( + d_model=4, + encoder_attention_heads=1, + encoder_ffn_dim=8, + encoder_layers=1, + ) + config._attn_implementation = implementation + layer = MiniCPMWhisperEncoderLayer(config, layer_idx=0) + cache = object() + + outputs = layer( + torch.randn(1, 2, config.d_model), + attention_mask=None, + layer_head_mask=None, + past_key_values=cache, + use_cache=True, + ) + + assert layer.self_attn.seen_cache is cache + assert outputs[-1] is cache + + +def test_audio_cache_length_uses_current_cache_api() -> None: + dynamic_cache = SimpleNamespace(get_seq_length=lambda: 7) + encoder_decoder_cache = SimpleNamespace(self_attention_cache=dynamic_cache) + + assert _get_audio_cache_length(encoder_decoder_cache) == 7 + + +def test_audio_cache_length_supports_legacy_cache() -> None: + key = torch.zeros(1, 1, 5, 2) + value = torch.zeros_like(key) + + assert _get_audio_cache_length(((key, value),)) == 5 + + +def _tiny_encoder_config(attn_implementation: str) -> WhisperConfig: + config = WhisperConfig( + num_mel_bins=16, + d_model=32, + encoder_layers=2, + encoder_attention_heads=2, + encoder_ffn_dim=64, + max_source_positions=100, + dropout=0.0, + attention_dropout=0.0, + ) + config._attn_implementation = attn_implementation + return config + + +@pytest.mark.parametrize("attn_implementation", ["eager", "sdpa"]) +def test_encoder_reuses_cache_across_streaming_chunks(attn_implementation: str) -> None: + """Encode two chunks against the installed Whisper attention and cache classes. + + Unlike the stub-based tests above, this drives the real transformers cache + API that ``get_audio_embedding_streaming`` relies on, so it fails whenever a + removed cache method or renamed attention kwarg breaks streaming reuse. + """ + torch.manual_seed(0) + config = _tiny_encoder_config(attn_implementation) + encoder = MiniCPMWhisperEncoder(config).eval() + + first_frames, second_frames = 20, 14 + first_len = (first_frames - 1) // 2 + 1 + second_len = (second_frames - 1) // 2 + 1 + first_mel = torch.randn(1, config.num_mel_bins, first_frames) + second_mel = torch.randn(1, config.num_mel_bins, second_frames) + + with torch.no_grad(): + first = encoder( + first_mel, + attention_mask=torch.zeros(1, 1, first_len, first_len), + past_key_values=None, + use_cache=True, + output_hidden_states=True, + ) + cache = first.past_key_values + assert cache is not None + assert _get_audio_cache_length(cache) == first_len + + second = encoder( + second_mel, + attention_mask=torch.zeros(1, 1, second_len, first_len + second_len), + past_key_values=cache, + use_cache=True, + output_hidden_states=True, + ) + assert _get_audio_cache_length(second.past_key_values) == first_len + second_len + + stateless = encoder( + second_mel, + attention_mask=torch.zeros(1, 1, second_len, second_len), + past_key_values=None, + use_cache=True, + output_hidden_states=True, + ) + + assert torch.isfinite(second.last_hidden_state).all() + assert not torch.allclose(second.last_hidden_state, stateless.last_hidden_state) diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py b/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py index 41223e864..fe5c8a57e 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py @@ -58,6 +58,25 @@ class BatchedToken2WavState: hift_cache: dict[str, torch.Tensor] +def _undecorate_dynamo(module: nn.Module, method: str) -> None: + """Restore ``method`` on ``module`` if TorchDynamo wrapped it. + + ``cosyvoice2`` decorates ``UpsampleConformerEncoderV2.forward_chunk`` with + ``torch.compile(backend="eager")``. That backend performs no Inductor + optimisation, so the wrapper only adds tracing and guard construction, and + duplex pays it again on every unseen chunk shape -- seconds inside a live + response. Dropping the wrapper leaves the original implementation, which is + what the eager backend was executing anyway. + """ + bound = getattr(module, method, None) + original = getattr(bound, "_torchdynamo_orig_callable", None) or getattr(bound, "__wrapped__", None) + if original is None: + return + function = getattr(original, "__func__", original) + module.__dict__[method] = function.__get__(module, type(module)) + logger.info("Bypassed TorchDynamo wrapper on %s.%s", type(module).__name__, method) + + class BatchedToken2Wav(nn.Module): """Drive Token2wav's modules with dynamically-sized, request-owned caches. @@ -82,6 +101,7 @@ class BatchedToken2Wav(nn.Module): self._trt_stepper = trt_stepper self.flow = token2wav.flow self.hift = token2wav.hift + _undecorate_dynamo(self.flow.encoder, "forward_chunk") hift_parameter = next(self.hift.parameters(), None) if hift_parameter is not None and hift_parameter.device.type == "cuda": # Prime the CUDA state used by HiFT during backend construction. diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py index a182848cf..490d8e0cf 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py @@ -17,6 +17,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import inspect import math import os import warnings @@ -2505,6 +2506,17 @@ def _in_projection( return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v) +def _get_audio_cache_length(past_key_values: Any) -> int: + cache = getattr(past_key_values, "self_attention_cache", past_key_values) + get_seq_length = getattr(cache, "get_seq_length", None) + if callable(get_seq_length): + try: + return int(get_seq_length()) + except TypeError: + return int(get_seq_length(0)) + return int(cache[0][0].shape[2]) + + # Copied from transformers.models.whisper.modeling_whisper.WhisperEncoderLayer and add use_cache for streaming inference class MiniCPMWhisperEncoderLayer(nn.Module): def __init__(self, config: WhisperConfig, layer_idx: int = None): @@ -2517,6 +2529,10 @@ class MiniCPMWhisperEncoderLayer(nn.Module): config=config, layer_idx=layer_idx, ) + attention_parameters = inspect.signature(self.self_attn.forward).parameters + self._past_key_values_kwarg = ( + "past_key_values" if "past_key_values" in attention_parameters else "past_key_value" + ) self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) self.dropout = config.dropout self.activation_fn = ACT2FN[config.activation_function] @@ -2554,16 +2570,18 @@ class MiniCPMWhisperEncoderLayer(nn.Module): """ residual = hidden_states hidden_states = self.self_attn_layer_norm(hidden_states) - attn_out = self.self_attn( - hidden_states=hidden_states, - attention_mask=attention_mask, - layer_head_mask=layer_head_mask, - output_attentions=output_attentions, - past_key_value=past_key_values, - ) + attention_kwargs = { + "hidden_states": hidden_states, + "attention_mask": attention_mask, + "layer_head_mask": layer_head_mask, + "output_attentions": output_attentions, + self._past_key_values_kwarg: past_key_values, + } + attn_out = self.self_attn(**attention_kwargs) hidden_states = attn_out[0] attn_weights = attn_out[1] if len(attn_out) > 1 else None - past_key_values = attn_out[2] if len(attn_out) > 2 else None + if len(attn_out) > 2: + past_key_values = attn_out[2] hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) hidden_states = residual + hidden_states @@ -2769,7 +2787,7 @@ class MiniCPMWhisperEncoder(WhisperEncoder): past_key_values = EncoderDecoderCache(past_key_values, DynamicCache()) else: pass - past_key_values_length = past_key_values.self_attention_cache.get_usable_length(inputs_embeds.shape[1]) + past_key_values_length = _get_audio_cache_length(past_key_values) if inputs_embeds.shape[1] + past_key_values_length > embed_pos.shape[0]: logger.warning("seems the audio is longer than 30s. repeating the last part of the audio") embed_pos_front = embed_pos[past_key_values_length:, :] @@ -4468,7 +4486,7 @@ class MiniCPMO45OmniLLMForConditionalGeneration(nn.Module, SupportsMultiModal, S return [] if self.audio_past_key_values is not None: - cache_length = self.audio_past_key_values[0][0].shape[2] + cache_length = _get_audio_cache_length(self.audio_past_key_values) apm_max_len = self.apm.embed_positions.weight.shape[0] if cache_length + current_seq_len >= apm_max_len: logger.warning( @@ -4480,7 +4498,7 @@ class MiniCPMO45OmniLLMForConditionalGeneration(nn.Module, SupportsMultiModal, S past_len = 0 if self.audio_past_key_values is not None: - past_len = self.audio_past_key_values[0][0].shape[2] + past_len = _get_audio_cache_length(self.audio_past_key_values) total_seq_len = past_len + current_seq_len audio_attention_mask = torch.zeros( (batch_size, 1, current_seq_len, total_seq_len), From 8dc3d504d8bae73bd6a627b4b759a4dbe6cd35fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zeyu=20Huang=20=7C=20=E9=BB=83=E6=BE=A4=E5=AE=87?= <11222265+fhfuih@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:27:58 +0800 Subject: [PATCH 072/267] [skip ci][Doc] Remove obsolete DiffusionParallelConfig in many example scripts (#6347) Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com> Co-authored-by: Hongsheng Liu --- .../image_to_image/image_edit.py | 12 ++++------ .../diffusion/lingbot_world_v2.py | 8 ++----- .../diffusion/lingbot_world_v2_realtime.py | 3 +-- .../image_to_image/image_edit.py | 15 +++++-------- .../image_to_video/image_to_video.py | 22 ++++++++----------- .../longcat_video/end2end.py | 9 -------- .../speech_to_video/speech_to_video.py | 21 +++++++----------- .../text_to_audio/text_to_audio.py | 11 +++------- .../text_to_video/text_to_video.py | 20 ++++++----------- .../x_to_video_audio/x_to_video_audio.py | 15 +++++-------- 10 files changed, 44 insertions(+), 92 deletions(-) diff --git a/examples/offline_inference/custom_pipeline/image_to_image/image_edit.py b/examples/offline_inference/custom_pipeline/image_to_image/image_edit.py index 5f4311050..851d18e18 100644 --- a/examples/offline_inference/custom_pipeline/image_to_image/image_edit.py +++ b/examples/offline_inference/custom_pipeline/image_to_image/image_edit.py @@ -53,7 +53,6 @@ from typing import Any import torch from PIL import Image -from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput @@ -139,12 +138,6 @@ async def main(): # ---- Torch setup ---- generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed) - parallel_config = DiffusionParallelConfig( - ulysses_degree=args.ulysses_degree, - ring_degree=args.ring_degree, - cfg_parallel_size=args.cfg_parallel_size, - tensor_parallel_size=args.tensor_parallel_size, - ) # ---- Cache Config ---- if args.cache_backend == "cache_dit": @@ -171,7 +164,10 @@ async def main(): vae_use_tiling=args.vae_use_tiling, cache_backend=args.cache_backend, cache_config=cache_config, - parallel_config=parallel_config, + ulysses_degree=args.ulysses_degree, + ring_degree=args.ring_degree, + cfg_parallel_size=args.cfg_parallel_size, + tensor_parallel_size=args.tensor_parallel_size, enforce_eager=args.enforce_eager, enable_cpu_offload=args.enable_cpu_offload, diffusion_load_format="dummy", diff --git a/examples/offline_inference/diffusion/lingbot_world_v2.py b/examples/offline_inference/diffusion/lingbot_world_v2.py index 145f5f641..b3a4ae0d1 100644 --- a/examples/offline_inference/diffusion/lingbot_world_v2.py +++ b/examples/offline_inference/diffusion/lingbot_world_v2.py @@ -154,8 +154,6 @@ def _positive_finite(value: float, flag: str) -> float: def build_omni_kwargs( args: argparse.Namespace, paths: LingBotPaths, - *, - parallel_config: Any, ) -> dict[str, Any]: """Build Omni configuration without overriding checkpoint class discovery.""" @@ -167,7 +165,7 @@ def build_omni_kwargs( return { "model": model, "flow_shift": flow_shift, - "parallel_config": parallel_config, + "tensor_parallel_size": args.tensor_parallel_size, "enforce_eager": args.enforce_eager, "model_config": {"lingbot_action_root": str(paths.action_root)}, } @@ -262,12 +260,10 @@ def main(argv: Sequence[str] | None = None) -> Path: # helper tests work on machines without the native vLLM runtime. from diffusers.utils import export_to_video - from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams - parallel_config = DiffusionParallelConfig(tensor_parallel_size=args.tensor_parallel_size) - omni_kwargs = build_omni_kwargs(args, paths, parallel_config=parallel_config) + omni_kwargs = build_omni_kwargs(args, paths) omni = Omni(**omni_kwargs) try: outputs = omni.generate( diff --git a/examples/offline_inference/diffusion/lingbot_world_v2_realtime.py b/examples/offline_inference/diffusion/lingbot_world_v2_realtime.py index 3d6d7a7e1..86461f9f8 100644 --- a/examples/offline_inference/diffusion/lingbot_world_v2_realtime.py +++ b/examples/offline_inference/diffusion/lingbot_world_v2_realtime.py @@ -125,7 +125,6 @@ async def run(argv: Sequence[str] | None = None) -> Path: # require a CUDA-enabled vLLM installation. import torch - from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.diffusion.models.lingbot_world.actions import LingBotCameraControlReducer from vllm_omni.entrypoints.async_omni import AsyncOmni from vllm_omni.experimental.ar_diffusion.consumer import ARDiffusionOmniTickConsumer @@ -141,7 +140,7 @@ async def run(argv: Sequence[str] | None = None) -> Path: model=args.model, engine_backend="vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine", enforce_eager=args.enforce_eager, - parallel_config=DiffusionParallelConfig(tensor_parallel_size=args.tensor_parallel_size), + tensor_parallel_size=args.tensor_parallel_size, max_num_seqs=1, model_config={ "ar_diffusion_height": args.height, diff --git a/examples/offline_inference/image_to_image/image_edit.py b/examples/offline_inference/image_to_image/image_edit.py index ae84353f7..136d99857 100644 --- a/examples/offline_inference/image_to_image/image_edit.py +++ b/examples/offline_inference/image_to_image/image_edit.py @@ -96,7 +96,6 @@ from typing import Any import torch from PIL import Image -from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.diffusion.utils.image_output import extract_images_from_outputs from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args from vllm_omni.entrypoints.omni import Omni @@ -452,14 +451,6 @@ def main(): generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed) - parallel_config = DiffusionParallelConfig( - ulysses_degree=args.ulysses_degree, - ring_degree=args.ring_degree, - cfg_parallel_size=args.cfg_parallel_size, - tensor_parallel_size=args.tensor_parallel_size, - enable_expert_parallel=args.enable_expert_parallel, - ) - # Configure cache based on backend type cache_config = None if args.cache_backend == "cache_dit": @@ -490,7 +481,11 @@ def main(): vae_use_tiling=args.vae_use_tiling, cache_backend=args.cache_backend, cache_config=cache_config, - parallel_config=parallel_config, + ulysses_degree=args.ulysses_degree, + ring_degree=args.ring_degree, + cfg_parallel_size=args.cfg_parallel_size, + tensor_parallel_size=args.tensor_parallel_size, + enable_expert_parallel=args.enable_expert_parallel, enable_cpu_offload=args.enable_cpu_offload, enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler, profiler_config=args.profiler_config, diff --git a/examples/offline_inference/image_to_video/image_to_video.py b/examples/offline_inference/image_to_video/image_to_video.py index 28c98990a..5635ce16c 100644 --- a/examples/offline_inference/image_to_video/image_to_video.py +++ b/examples/offline_inference/image_to_video/image_to_video.py @@ -56,7 +56,6 @@ import numpy as np import PIL.Image import torch -from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -503,17 +502,6 @@ def main(): } profiler_enabled = args.profiler_config is not None - parallel_config = DiffusionParallelConfig( - ulysses_degree=args.ulysses_degree, - ring_degree=args.ring_degree, - cfg_parallel_size=args.cfg_parallel_size, - tensor_parallel_size=args.tensor_parallel_size, - vae_patch_parallel_size=args.vae_patch_parallel_size, - use_hsdp=args.use_hsdp, - hsdp_shard_size=args.hsdp_shard_size, - hsdp_replicate_size=args.hsdp_replicate_size, - pipeline_parallel_size=args.pipeline_parallel_size, - ) omni_kwargs = dict( model=args.model, enable_layerwise_offload=args.enable_layerwise_offload, @@ -524,7 +512,15 @@ def main(): diffusion_kv_cache_skip_steps=args.diffusion_kv_cache_skip_steps, diffusion_kv_cache_skip_layers=args.diffusion_kv_cache_skip_layers, enable_cpu_offload=args.enable_cpu_offload, - parallel_config=parallel_config, + ulysses_degree=args.ulysses_degree, + ring_degree=args.ring_degree, + cfg_parallel_size=args.cfg_parallel_size, + tensor_parallel_size=args.tensor_parallel_size, + vae_patch_parallel_size=args.vae_patch_parallel_size, + use_hsdp=args.use_hsdp, + hsdp_shard_size=args.hsdp_shard_size, + hsdp_replicate_size=args.hsdp_replicate_size, + pipeline_parallel_size=args.pipeline_parallel_size, enforce_eager=args.enforce_eager, model_class_name=model_class_name, cache_backend=args.cache_backend, diff --git a/examples/offline_inference/longcat_video/end2end.py b/examples/offline_inference/longcat_video/end2end.py index 8071dfc25..6e0c5593e 100644 --- a/examples/offline_inference/longcat_video/end2end.py +++ b/examples/offline_inference/longcat_video/end2end.py @@ -15,7 +15,6 @@ import imageio import numpy as np import torch -from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.diffusion.models.longcat_video.pipeline_longcat_video_avatar import ( prepare_longcat_video_avatar_model_for_omni, ) @@ -236,14 +235,6 @@ def main(): dtype="bfloat16", enforce_eager=args.enforce_eager, additional_config=additional_config, - parallel_config=DiffusionParallelConfig( - ulysses_degree=1, - ring_degree=1, - cfg_parallel_size=1, - tensor_parallel_size=1, - vae_patch_parallel_size=1, - pipeline_parallel_size=1, - ), ) audio_input: str | dict[str, str] | None = args.audio diff --git a/examples/offline_inference/speech_to_video/speech_to_video.py b/examples/offline_inference/speech_to_video/speech_to_video.py index 964cd4874..62dd29720 100644 --- a/examples/offline_inference/speech_to_video/speech_to_video.py +++ b/examples/offline_inference/speech_to_video/speech_to_video.py @@ -23,7 +23,6 @@ from typing import Any import torch -from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.platforms import current_omni_platform @@ -250,17 +249,6 @@ def main(): "scm_steps_policy": "dynamic", } - parallel_config = DiffusionParallelConfig( - tensor_parallel_size=args.tensor_parallel_size, - cfg_parallel_size=args.cfg_parallel_size, - vae_patch_parallel_size=args.vae_patch_parallel_size, - ring_degree=args.ring_degree, - ulysses_degree=args.ulysses_degree, - use_hsdp=args.use_hsdp, - hsdp_shard_size=args.hsdp_shard_size, - hsdp_replicate_size=args.hsdp_replicate_size, - ) - profiler_enabled = args.profiler_config is not None omni_kwargs = dict( @@ -271,7 +259,14 @@ def main(): vae_use_tiling=args.vae_use_tiling, enable_cpu_offload=args.enable_cpu_offload, enable_layerwise_offload=args.enable_layerwise_offload, - parallel_config=parallel_config, + tensor_parallel_size=args.tensor_parallel_size, + cfg_parallel_size=args.cfg_parallel_size, + vae_patch_parallel_size=args.vae_patch_parallel_size, + ring_degree=args.ring_degree, + ulysses_degree=args.ulysses_degree, + use_hsdp=args.use_hsdp, + hsdp_shard_size=args.hsdp_shard_size, + hsdp_replicate_size=args.hsdp_replicate_size, enforce_eager=args.enforce_eager, enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler, cache_backend=args.cache_backend, diff --git a/examples/offline_inference/text_to_audio/text_to_audio.py b/examples/offline_inference/text_to_audio/text_to_audio.py index eff9c9d1a..3a67c662a 100644 --- a/examples/offline_inference/text_to_audio/text_to_audio.py +++ b/examples/offline_inference/text_to_audio/text_to_audio.py @@ -38,7 +38,6 @@ from typing import Any import numpy as np import torch -from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -306,17 +305,13 @@ def main(): print(f" Seed: {args.seed}") print(f"{'=' * 60}\n") - parallel_config = DiffusionParallelConfig( - use_hsdp=args.use_hsdp, - hsdp_shard_size=args.hsdp_shard_size, - hsdp_replicate_size=args.hsdp_replicate_size, - ) - # Initialize Omni. AudioX requires an explicit pipeline class; Stable Audio # is resolved from its config. omni_kwargs: dict[str, Any] = dict( model=args.model, - parallel_config=parallel_config, + use_hsdp=args.use_hsdp, + hsdp_shard_size=args.hsdp_shard_size, + hsdp_replicate_size=args.hsdp_replicate_size, cache_backend=args.cache_backend, cache_config=cache_config, enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler, diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index 406aab39e..1bc2d5b11 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -10,7 +10,6 @@ from typing import Any import numpy as np import torch -from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -462,17 +461,6 @@ def main(): "scm_steps_policy": "dynamic", } - # Configure parallel settings - parallel_config = DiffusionParallelConfig( - ulysses_degree=args.ulysses_degree, - ring_degree=args.ring_degree, - cfg_parallel_size=args.cfg_parallel_size, - tensor_parallel_size=args.tensor_parallel_size, - vae_patch_parallel_size=args.vae_patch_parallel_size, - pipeline_parallel_size=args.pipeline_parallel_size, - enable_expert_parallel=args.enable_expert_parallel, - ) - profiler_enabled = args.profiler_config is not None omni_kwargs = dict( @@ -481,7 +469,13 @@ def main(): vae_use_slicing=args.vae_use_slicing, vae_use_tiling=args.vae_use_tiling, enable_cpu_offload=args.enable_cpu_offload, - parallel_config=parallel_config, + ulysses_degree=args.ulysses_degree, + ring_degree=args.ring_degree, + cfg_parallel_size=args.cfg_parallel_size, + tensor_parallel_size=args.tensor_parallel_size, + vae_patch_parallel_size=args.vae_patch_parallel_size, + pipeline_parallel_size=args.pipeline_parallel_size, + enable_expert_parallel=args.enable_expert_parallel, enforce_eager=args.enforce_eager, model_class_name=model_class_name, cache_backend=args.cache_backend, diff --git a/examples/offline_inference/x_to_video_audio/x_to_video_audio.py b/examples/offline_inference/x_to_video_audio/x_to_video_audio.py index 9512ac65c..d7571a66c 100644 --- a/examples/offline_inference/x_to_video_audio/x_to_video_audio.py +++ b/examples/offline_inference/x_to_video_audio/x_to_video_audio.py @@ -11,7 +11,6 @@ import numpy as np from PIL import Image from vllm.multimodal.media.audio import load_audio -from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -197,14 +196,6 @@ def main() -> None: }, ) - parallel_config = DiffusionParallelConfig( - cfg_parallel_size=args.cfg_parallel_size, - tensor_parallel_size=args.tensor_parallel_size, - use_hsdp=args.use_hsdp, - hsdp_shard_size=args.hsdp_shard_size, - hsdp_replicate_size=args.hsdp_replicate_size, - ) - cache_config = None if args.cache_backend == "cache_dit": cache_config = { @@ -222,7 +213,11 @@ def main() -> None: omni_kwargs = dict( model=args.model, - parallel_config=parallel_config, + cfg_parallel_size=args.cfg_parallel_size, + tensor_parallel_size=args.tensor_parallel_size, + use_hsdp=args.use_hsdp, + hsdp_shard_size=args.hsdp_shard_size, + hsdp_replicate_size=args.hsdp_replicate_size, model_type=args.model_type, enable_cpu_offload=args.enable_cpu_offload, enable_layerwise_offload=args.enable_layerwise_offload, From c3f80502e00b564d7b74d43a5e21fbc6b9da2aed Mon Sep 17 00:00:00 2001 From: WeiQing Chen <40507679+david6666666@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:52:24 +0800 Subject: [PATCH 073/267] [Doc] Update vLLM-Omni WeChat QR code (#6369) Signed-off-by: David Chen <530634352@qq.com> --- docs/assets/WeChat.jpg | Bin 266492 -> 277429 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/docs/assets/WeChat.jpg b/docs/assets/WeChat.jpg index 04818fe0484bb41ffb910abfcd9dad1cbc29f902..31813898a527baa17b24de6ddb30a5bb2326e91d 100644 GIT binary patch delta 236867 zcmagFcTf{w_%9qqMNmLgq!Xoxf;8z}MX3Tx3q2wtCDL1fz$yqRRX{*$LWUJn3R&%``K6E4zIVOb^Ml;WSlqe2+7>jjC!#-rxZ66*{~-RIz6TkX1Zn0F_R%H zYC?FKSEjM?H;4Q>nECVH5e2I%*eoWx40HlY1*H9?_gFIkTK}4{>w2IS|B$MUi&3s%!E7`$<#NLfRQ;pFf=m zf%GG5`FAF=@KKuvsu|@s$q&Vj9ci%vSwf;h2{onIyklIO z!bpow*+Vl0RLOUY@SJWCWh(&N6+AvMZfE?2`D)q;NH6TO-v<4{wkDR!3@Wn?dE= zrwshgTKa06mDGjdfAIH+FH1!T+!;eIF-9+>SdJA%8MMfG`fYQ$dvsyPbA?-JgQJQ7 zbJI68>=p-ntz8~WyoEoag~MvSI54jLVS8=Sidxf-YysJ zabE&?>nQhkygnt=k$_xSAK(EdiAT4i|3o}V zd_0j+8}I#z=}t{Qd^H3S7je@nfOxPeo<&VUFVq8P&@lA7?{F=H+OZlayS>h;uuA;b zr(J)xS@(dUMf*74#t=={Z5#Wd96nXFSkmr2&yLrDtln|SngxdLfPCBN1>NRG28~n# zYNwoFYBt<LgSXk)(_-h$l^z^`z*fkCrc7 zh=pjs($^slTj_Uj!Cbv^y~AFv~)Hv!=394DzVs6an9P{>Xzr%Yx~YNS<5 zcRX)mTd3hpz0UQahGXoW)CuSeiu#2D;C83`7Yox%;r-p2X~>Y`yM~^=Wi$O(Ki|L6 zDXz@;SuKY4PJ<1eU|{IE?J*g(q;zKAlhzu_FV^>^ae2I8dp6PJON_YI^)E5EK>z(x zCQt7I49TQHnb?jbpx&nh*iElcU1UE2#Z1l98qu_HSQ%;d1SD9CqJ~M)ULW)#gik<8 z6aV|1TUytFI^h2VOEE1VneNr^>2KQd*AzLmavxIrm*ZvZ<1})$Rj;8UG4hWDA+V#dUp_*~PneZh?<5%Bw z@N&M6T3&&d?e*(Ub#V!w84d|w&AIP<4#^CFt)l-p6Lkoj+w*6a)4^PnM+BjJi8sqyfd{`#c^;GYv>y(3>C|dOC0-KzGDu6La?+K{ZuLB&B z{-_b`8gT;B;y4KGG6s$=`-6{HsE-gffCZ`zfxUp#4*IDUPs?xsu6Kjso?LyJG|iLFn9R%!;S=hP%Jue1MwM8wv}8ynvq8T134|vIq0H_)5DsTR=Z% z9wahSEPf*(uTreHcn}}Z-eL<_9Yw8>QvSinw+e!KGL6S40H`_BrXkbH`hIQTH)YXX z3pIPG$^;3dTdjrTzvjmZ^-Gz4xq_9s{ODKp$&VeEKX4Q`vE8-lW`5^*lm#dsTik5D zTf?|2(Hx&&Yd_@RJVI7xxa_e+AF8PD;6>~MdD!(Q-u4v z#rOLs7?UNSfHA^-JRxx*3W=U_fDZmZCDYDq5qin@;1-%Pz9PHV507}`u%q*}SLdBb zrs57+PP2`D6~M}8%GR&JRD7+_EYCs#ZIme}y$wt%FA`7cQHPCPZ%r{9DXgzg`kd)9 zX^z2XdjZT8GHHz-dpGdu+qL-2r7i>Esb%7!0Z>%X!u0QG#bD<{wmbOt^2-*@MJWm& zUWjltKtqJ&Te#8Wnlb=uHq_SpD`YyZfA`G~?yWhrOox+FNhR@PFxd2yqswg{1Je~v z(k&^VAE# z5j6~+a>OM>E+H97B@$OG>L>DQE{0no1Mp=a&wh@h_-~s?H^Si3Gmd+*2g;}`Q5&^l zxx-dG06pk@fRVpVY~%#AwRwYH+heG_QWGph4JkgP%AWd2 z%&m&fkLmY9PQOZj5^~!1zk@We6s8FLEQg(dps^>Q&+0y!Oq$Dn_4Fm^N&lO<)%Jfo zTTpp<<{q&T zx`*yc20C~Nu&$}p!PY6D;IN0ueY`KZz;?1EXwNBY%9B97G$q6gGjb7Ja8C8@H1mD%t+lJC z_J_7Z@T0^ul2IEso+2PgQpIUT9lK8pk8IS*W^}jN9C|A^JS%`d{or+uCA6DO)p%}a z{}{4{qOeC0PXVMPvP^#|ytHdh44eLt@X%1s{&H*58jnq1$Nl&GvA6H1{s1fZ1fhNp zv&XFfcl@rNiYHv5JuKIvH3Cne8ECwHB9lFQiN-Wp;X0uZpjdAL_L9BVsb+ZuULFpb_k+mE3;-O~ltSB*f z0#(owMK&JJvoVtryoYfCy*WHgvTww15yeS?D7dKMn7-woP+&3PE`Tgh zH_Fv^WUAi-XT#6JPB1sbz=@< z4zh!d)75&V=Ns*8Wx%KS(i?p*F21-q1L0pnzsZ#(4?o z%D82{cdHp{VyCSBbnyg)H%N>{bJTj0ZecqI!Pcbrp*TYYe(&ea4+VU5fhcY^5p9(P zus(H@)L5*S55Ii;cV;_zmf+3f$U3l7N#@vuR-Y};IT*+V6YR6RgZnQAzrC$W+x2YaT@iHB z;9=-ZLAfFUH=Uj^HBw3eMVZuCGJ}w{d(eNsIw!B*;OK4gj5e#r^St|KqwX=hiCLlD z4Es%!CcOy=8}^!jd2x8CMTxH>!-6f0qg$Xw8`r7<7ncaL*{JuKD~r=7AjCmQaDB}F ztp4r7`byaC0#zXY>Fo?Nf&A5dR8@k&@M_4Snu2!XefG_>ns#K+ zf6IpJ(M&?E1MMfEe{!zSg9A=llsG|=cK!$-k2p)V-B66@)ltKqn>j75f8>NN2%CgI zTV}EVL6dHFZuWcG=H=BmDbEbql$v&+kfh?yN4ae9$(AquX#N#mu|d{IRQzW zI-^Pc@!EG=+)@yU3g+jC20Ff0-R2y;R~BZmGZPnBVY3LVwa;EzblZAgaPKz;$6{n? zz2c*AG;-F0S*Bx=kpuhsOPplz6VA)Cr$zff_ODsPzpCFZF?(Y9{IxVq5crw$?F59C z+7!n@Nw@0S)MUxhn-C{q_w>3FwVFFKeT;5o7>;vjpqrv%9aiN#oyIPaq4o z(l0^vRu*?&uv}Uaj&Izxx1uMH_cW%;5y_p!hDxhki6o7ln`+OTxz^U*#_PMG7uRX* zi3W*~TJpn}Iw1{TWVrPh82X>q1_vAzo5ZI6zAY~6yd*`rDz1l@$NsUHe?oo{7xSLz zYyw-fmx@{xEbYy&O3V&L_?m^YSXt@xSJ`d!#>d4PKzijS;Ax-VVW0?RJPr$S!<56PcH z{7~sbKDPE^ttubId?sg)DlkQ9t5?5`W&$NB|Ga(9?NIVP*EUruN{!Lk^ISvJp2=y2 zw7!RlrxiE#7s76$Azuh16qOXRzV5|23n9Pwy4So3MW{t+oF@J;Y={jmOZ<$NO!ve! z$NL=bLauYu%XqJlgYS=HFY*0Y8nYGIuzov|e9LW4DbcuooafiSiAjmV51#=A;9iEX z-xgHMKK_xuzLZ(L+Wc%&pFjTxWG$vglZ6sW`1oOJR*$I3X#IEmOo^k7p4G$H@J_~C zA)(?&>-1>)KyjbYP(yei&AunvYA}Vb>h4=KRwwfY#`*i-$zK)XK4vDXwuA8yBpc=4 zgqn_XxoesRI32?i**IR1yVwOVdGcvkp4T>_yj=o0s$t9FOACh`Qd63piaY`ZOD>DU z+pF~rwKd;I7c5_ybESYY{x!~>8wp;3q^B=@t^Dh4o?2iFvqXyqIzd7GIldaPQcd14gl(bBsV7C$}yyXiyz8xCpNbhl_ z{nw%mCeGHTmbZiN&_UOWyQlW&&^bX?Jq>#8Liu-AEI)>Z;#G<61jbIOsrt^q3Ga?6 z@HAK7=4Jy_f{yTu*GACy5WGpacv1DR@oT_tC6u2cM@srehpshJ(e9RaZCa;qLw6l` zDdO8u#`+9gt3-2F!~5f1j%XJ2tXr9%GnSZC-T1fW=_7^^OhUTn_$lx7*W;(2{dW)r zywQA0_|(SzW)omH8YTOEOj54DHf`?y#l9l?N`@37yw*DCs?q@weawL6#7`;xRWoJ# z^u4J{xigL^Z)5q(XrQc3V}e;0aG7Pv7hU~9xov}K>iGVv!0)bX4u#DVRU2&TN|a7> zuB)}gHpaPQOcQ}}L_PUA9g#go0Bq#Dwt&;*pD9v=^`3>S=hbI#(mI`~1$;E~+B<4n z#>p|w+VHr?w(mAMpp=Gn3bs91b7yA>H6@Nw%mg1x zrsNN({mFS|S>a%>0sZI`^fv@zx_~9zOfN3$gtU!SNJMb3sfzX8+y4c~*e}Ba+51)` z_-iyA*}MPfnk~O4IuKUBGg97qBo4283T&M>+Ig_;IDSkOUqsSTH$?La+z}@-5x=C) zX4nuxhj!2FLg0|z`Z#APA$JyvMsx}^l)_BfSs0pP=uC3sxiY0|g}{XCVh5Om6_l4y zEA|?)j1V>r{aDj^p#7zpjLcf$z6=UsSK@X_P8Mn`%&PHhCvyeiyuiA?ha$f$6--MHX=2 z8TYr|CIEVSX#slo0YdysrTPH3@}c_HQDYOem?mh*n1;Tj=FRlMr_;J|=>!BWbfGIF zQeB3ch6&sn*Wox1yHGbbQ|pMhFWOH1Xmr?rRcN&IMWxyr#&bCN~2FIEHup#dYRc z2Bah5S!X;<|B%N|Ky+fss3t|&YgZKDBc#PT;3QaaFVYNn{~YRwi#yqPR<5F%0L_II zG*3$q4w&v-jIStPGUnJ+sOPDhEdP5D^4=j8QLOaB`9^dl$1y$5&YDadu&@!8CnRf{ zw{|oLp6(nkYg6-5?|*#3Xmb9ry@FDIt&$V=GAO8!ZL_M@(BvjPff$Aj?kE6@Mx;B{ zh%)OjYG`O_2#tRduOod0kL?6*luz2poSHhX%*ee3O_U%&F?x4TK;6!`zvz9=hJCs) z{)XU)p(CtiUxLT^WjYF+0SNwc8<>|hYH-I!yM^trW>Phc={;GKbg<;^yr}cm|61Zr zO@*-!NFx|OuiY(tbUa*2aP&%oEaAS$-C%oT$+8(AX#LCB@jt+--JasG155GRv8Df{ z0Y+XzC+(4e$NWa@P|?qWE{cq{@im*e6)qiy`4xHki>-A?qYf)RhHB`b)bMlVC!q7Q z-K$@O_B7g^)E0G2s}wx398cr_#;rJoj$1y_Q$se zBzPy{*3V;tDNC{KioLl0SaNTLlr=y}Pg;&U$^+sq9xOF*j3V?1%itQto*YNpI47je1`+LYe@_9SM2l(x&e`u^*4yw ze`jL2%ezz3B7Q*}WPEcSXeAT0PTIx!h`dwM6Jor0lDuqQ*PSzCD6 zO*{Sdpp>7w)DcAqjBm0`!nk5PGHyiF{x`9+nP5kE^E#%Yqtujx|S)^ZKgjc;y?jDEyJ6T@_4)) z@rYbIyqbLbF?{;=z4JHV3>x9B?^Z8x!R`)gv$bf916>PHJ%M#Z0BaihaUVc)3n zBzHn?=TiEQPY3_L+?i(uyyl}->cGgg z+7r-18kAeGZiHgIs02vXyEMTqZ@9jYxO2!71*%+QZhp=E<8(E8p>g;c9SsI=mCT2b zO^tuGX_h+C$!-BB^RT?G$e-3_s0%C%6W=6%k zeqC=;QcMq?@h-L1@BS`myCll?JE>!ob_ZxzEu$zdHrw>GUHvok{HX^I_j#-QI=-JI z+r-pU>LHLRz!_NC&x#TxWNLVnzp~ACdonJ&qMCD)X~CDzRd8p?)AXQ1>S>iZL16Z? z^dq(4Wo#*_sB=mn1t;-fw6p2cculiip5(ceyO%#_JAYT^Sheny(~wPiASo0N2b)4A zG8{M*C4$nmu$>mf;t=0WPm*vH@&v@agl{KOXCazLtbjf>`acFY9gCj4VoA4GG%Z>} zg}G2ajY-1sBgpCM#ge}kLgqWuhsMxKF#hQ@v^K0SvnhDi23T*mk7g+y2~evLed_Vs zj;;5-ni^tupf%{5%>$7?VuaCxz9ZHi4oNHJaXci!F=a{8oPet5xShgH`?RY(JXDiU0l3y` z-H}>Dp8hWDo8=zE^Xoh{>6afE%~t9U-GuMY_!#lOrvB4^jw&M4x?1Xc%F@sOW(Zr_Hak#W|-K-%{FMwd{;?Q zFZU&C#*b(5$R^gS|3b`n*%7ZIfFEsQq>#dSSHK~8^L*QIiM5q_ck5dA95hYj@t2)V zJ*6AX7MedoQe&;LchnwF91e%Z%$iIG1Pb-Up_+@A_yjUg`lL*{_XRAZB|d4YPrN_6 z#Pr)wS$^tmF=C8X`Rh82yYr{l zy;UY{ZEGq&o8TW z&i9?$8jDDDql;*i+e`wO#&uRLner$|%#eV@B zJAYbUiV-@{UjoSZ-_$#Erz%~<>1?#{?dg4aBwe|t!>33#IR)Bv!rS^Js9RbBQK2g6 z``tUAi=ED-_2~Un;-{ldXsy(CkuyQ{-taEsdJ{O3V&%6b+!6G(F20=jL0ia#;oS5z zHuEs?4+D+qz-a*)zrWn(X2XdQ>W7%?aoAWhdo*Y0qRZkdtSCeDV8T=|evnwEKa_ZLMLkHnYUim8? zoZ-Gu28tzK5`|yFl{JOc7R|UGJ~)#spQ>LievLshHH_j0W|cs&2je#{P2KUX7!}q( zH^6!d__5kX_rb%zAg+(J$x`k_TaH|wShB|NRcDNIym0BcAkNGsW1fk?8rTly8C)TY$( zSLox{5ZpZFvW5j+iGy~^J83w=)Lw_J3Vr|}o6!``e#}3LH2o<&ND+_(y=%Jub}AZ| z1nQZ^=N8w3ebO-TT~AwezYayA!fw0v%W7BMd2!J^RJQB)JTmQR22=YW;uLr%Lu9a{ zSJuvpyJ2Cf*o$*LIy0UB1Qh`1e$AVf&3{K$x8v1XX7>?$f8`z_wA5$hyS# zc%MhLEn_QYX+nIlf)(e5&V+6f5+@U+BB5v7u2YP#<^H9->WWra`~Jt(LApuTTWYZb=7LC~vUu^&jd3SrMd0arln2{=A zy0rrxNqnnKzuup22{ts*xvE%!ct)Q=-rtW;A3aa;mc!Fw_WA!{cGprz!2gjr`jMRX zo0X=h8d1&0@pLU(%!WST)Cp*v%@*1zsbxvM_-ReaC3^t-Df5N0^<$NFF`VxsLaK`Qc<#%;I?#<#2~6i2P#(6g`t zUQIW8wIsauIp+^>E0Xf0Fv5sUH5iPK;Ob7f8c8a{ZFyb?oqq#u}IVi(BH|hZ@Fey=&HAz0Gnm z@M%@_NU(GE_P&NJ+6-`D%G|g*DAonE+=21r{78yO*|6O~_fWNPb?-eLg@e=b0ujv9 z<96tUg)Xg~`Mxie(IvvWTV}?lo+9(vunhRH>;ic}@B~yMb~h9QK!Yi0`sMM@Wa(E+ zsp-`y0ChEA=q+lf)S(q9dtC9Lkx{7eZ)v$8K3kPnVHYUU1jmis8%fJn!mw|BC1p+t zKH^)*$$uvx1Cm3&0dB{2IdeyF8pho7BFTsiAD4Tu2Cl?z`{Phtba2cu<)Ro|&^obF z>-`g?z;$o{oNJ32)_h^iXQXBL+5SQ6N4m+tLm^8K-OwY8+kx{|j$o;!iZ4CjvNUFt z^OB214YwcvOWsCVMx<|%p(iXsS0v}&wkj4K)(h0gg$WZBqY=7ug4wRehRd0Dv)?}H zaHpIR@1|GE^617*{(y%{K$aTZAlHwBL-1Mbuy$6`rUa)$R7{G zwdH+w3l6JAt+&Y~{%opzmm#Vb<|$@*$X*+vU~rm^VI+<>;dYs!y0QVrB3?;NZv3dW zk-R|&>@u-?81L!tc{bbSmqgM9p=+vDx=NzC&R0k$9-EASuSc7QwHQq?SkSLN``G?ON zs=g&4iEZ&Ox9O~a@${gUSMtbC+bKb!>jMwg_*=!-?ZMMe&H3uq5#<*aVGZR{=!}Cy z3OlM>s)YWDwc0qDCUd5(`aP68gG?a6rZa^vMJkD|B9s!0$tpdmqu3pBXet!1!=hfM zxj>I0W}zfsQR0Li12=Un^*DvDD#>4?lYq5YX3AHX3WSy_5wId-o1mz?S8A5&iX1G< zOw?@0Cb)I+{OeSyPi1<#Gw8hv5lx=Y(U@Dw4^s@2-|n$(lIlKY`EQiQm%G0dEG$XhiG9Z#iqSflRG42{&pGlM1n;D0+ zJ#U!iI%XbBUaUJfHq(GTqYRpIL!VP)j1Dz^(hK0VIch$Qxf*Y0S+ToQtht}*Myo)8 zhhl1~<6iz1N+fZimc>SC=O1#>=f#S~yr#Bg0&LtlpuU7_33aPLsGGuwWAD4tFnt+%9vy8 z|FX8d{L<;Adr9dlFP%cjR+z zlYZ&!=l6_$2NRkZVn|ZNZ{wbAtd?-Vctq{#(s_UJZ!F#0Ra@yMO#gJb&S@p&oJI7o z#oe$@oVdzo1F79DDn7nxyjUT`AA6i%mOruM64n53?#T2^jH#=2_WZW3B;Yk8vw^Yh z^n4?5i7k~*%|4Z#_i)_V84OdVu+xpCNBBhCZZtxx95#1z{ElC9o^YKMa3?mPLCB*g zvf7oaU@`O_S_w*dhb6~~{3*=$9LsVcJU zCe)JR38FM+27)Oruyfr{!l26}f@Nqbe!Mg^!Flof*Pc@hs~frFYZFNR7WY7};S*2? z6o;z$YsTzvb*QQK?A-1jbIH4=F-9qd`;D86i1XdrcT*9MpCz|pp&8_@KwPUn@miD( zFlnz0nv3&d@dk6XF~Nl;By={oE}?h8ceYH z^9T-D3B1UD!XLN{PqlE(YiUWt@v8dV{Z_L4dO$%&>5PwT%yDPu4G(|$vw`A?pA>=4 z+7K%XPaR$(lcK9ff&VmxepNxzpW6Xk#u>m}i^uP4p zMaaFTI%?jQ9~-X@-Do5+Eo`kgAv=R#^ljCYKFaCbu$4zquJkXYqGHhXjsOX^Xg;BQ z0@5dQUzQABSZ#~N9<)kzGznvmQ@i8iv#+OTT9}z*5Sd52bhsVf5H=X>k>~v~Ocego zwXB!kG8@3|SS;%45e(55xO#)L_ks{ls(8OuA5pHwoXm%iAoOGrC2W37vF5?zf_!a~ z(Z3q#-JWMYP&2zR*<|q+NJ2i@crW8MyCDqtwFV3IYYT(o+m@ikSwOQ}fWIY8^?eQXKH{en zJj>_uIc3HqdsLZ?cIM;l$_M53;bxGZNWA&bJ5$OXYTQA&8}C*#gX=e|hP0`dF}4}e zydNVXe#hbJiNght`egoev}9RVAt}n9MDsKClw0UiXMe$~)WTHh#V}fu)>roUE7829>9YGyfbX}b znqCpS8N->jH^aaeZPjpkW{qm|Ed?a-`d=`XUv1I&5hGyw9^yhRf|3Lew@b7a@rpnA zy=0#Kpl2><`33r<+fY-DDZkOte`g6VAau-u>OunHQ2Y}%PWWhd8h(!6<(uLL5$er> zKpM==Sdh(V5MG%t=>S!%WHFAAe>1T8L|;LG&8V}1=O8@654Ll=D;j|b_N#X zBBSyWdP|wN)*pXS!>t88Ca!Tzm(%Zh*$0@#`VxPvL%JAc8ThB%cX@&Ag8`Q`r~ zUzCZy5x@e=2drwdL1fK2`@3y{`mCmsEcXv|XXcd3`0glC>p-T8E)8+sh5UZZf&&%G?kV>TJGI z_<8F=FOuhKGBl`LYvOKn`n)q~v7mLeGT>M($1zPViq{#_2ZR|(V?ZGZ<44fy zNIp#oflNf=`W+*ycOCb8>p2bQXo*^jyNqFj0(S_37#s8%Ou;*;{ST`&)wD}hf(>?C z&Ae~UuqS*DS3>mH*LodX zD^OelgQQjV%GcEO1G3sB|D-;Bk+!^;eGBBM%&`-UApUDemXcJy*?&=zyGL1AAULpM zK%VIr-=IebNA)JDp>s=CZ`!~$aOR&UC#^e-Wulf_qx186lP%|^%D|Y*lGcbgX~Y6c za|5OT?z}ySH|_}IpnsVUf7UO%9$Kd+q;L#&H=vb06@(?O0J-M^zMnNN56$s)U?KOs z@X9syjPn#)<#Rp(mD}s!cEJ>Nk`P(D6PIIgu4;T|=W|VK<8zzs-$p|`y4v2a4U%<~ z__xxfiy)h$MCQ?TZ=B5T=4gS!a%OFcM#FDs*!*QJp`7-wyp{j5mr(mCeSEL;c*;6` zqsA3HK)IR*%;TbW3>N0klEg+|CWe-9dDdR|3RNLqPwbGoe&hNjxCdh~s)=&*-bDU7 zC(P-+(#Aw9lFcXM8J>gR)!t|RCq*&-d+mnB*Uj#3`7fbOS2bORrWD)ftJXR~wR!|( z$KCf^MGYSWG$*PM$U4lS8+<1q|NBa3mLi}jK&>bRxR!`RV+@~bS>EQg%V&GaE*HTVH8am-i#~!M&oIAlv#hFXNwFZIa4cF+n#8b=1MKu}n@?RZ z6y4R>eGcbi756bFi4L~Fgqwh_oB$>T05NjjAqm0{cZ^v|3w;ZDGs{XD3?|Z*mSlA!+I)1*bzn%SuRG1`?58tkjHzoeR1Yc=Z~-&Z zMxhtwPC%R_9n(aU5&5fgtZI$r)1XhQag<@*MsA%`QQD8f&QSuX8DWBmJ9waPi$^ow zUy6NWq7tdb2|PuSyd*kU3{QtDl~ z8ePafb4)lpK4T!~{N(%V(riF-HT%@i*o5LX!+*qy6iJ5A#xm>Qv_KYz*fvz$d)_Ei z!id~olXW>n6XYQ?j3Q7`?)Hsty41*c#@3qdpUT&6RJ6H3Y)5(-^Vw=p`kO@ek!+~r zP?w&1V%O{ygPL)P$!FmcQg>utCT}R-64jKd3-Ptm!JPrKEw^3E2bv z`+MkuZexP-@4L2bGOt@+>r|ZjQ9@%RXcEsA$ggxa&x;I%Al`Yb)G}1?-Q{?7akZ+; z!Tgneij6=5D~E%IOYY%<=9J)W_ncH>N9^ahjpN9H_zMOAa7Dg}Jv&)iF+rvsTkZ3aLYAvRGNPGCyfDRU%LnGm`@DQWK?7PrsG^NH3zj-%AK>B z%yo1lM?Jr*U?c1lbyo(PE8a%3B0sdFn3Lk_ye2owOWl}vc;;pYSt_nNJyAhSD*sr@ z-0oUfpk0{;3KB|IUJezUfESB>1|Ei z1X~|Fmx0o~y2aOwrkVfX;a`iLQcF!C$wvLYV?RsGiare&8&2u#4^?Z5wOG*?!+TPS zL&be+G;s4h?u==pq;QkL7byH1h0JWHGW@c!2HZKJYZ^b_i0;IZPcZUBfVb8>zG2{16)fij&mmv|1y-BY&=E@ z`s|iQ7EpebcUP{6fpq_gVlW>*1iPqoaay`;6ex}kYqnV)T6aSjCRQAY6+HD{iS=9^ zChW$wopQ@ui!WOmKQL&uF&G#QqyB@8^h<1;{&K=P19HJu@Lm2BBxp%0r9LLJf9LQkK-5>@o5D)UtPXuo|!0 z!|%TEZ{M0XoI{S0X{7i#%ySvwdvOB#t_8$mbQsBp@TBJWNs(^VQ?6>`I`$rol9JxU_~N)B$MM_cu5J{iz@h5tH%Rnl8xBx*h=2&13Pqa*N0J z`;2i}-rHv~?ab$YDWSq`i2c98IK_8rk67nPkhVKx)282Bbss3azthD~Wcx7R3yyF3 z{OmC@xb@YdQ6cvOdMl^#9dTtdmEHFm*DCQqbSTj^h)*hTe`d)LF+!g z!wJK^^YoW+HmPhI^VkP)_2~~DvAaJ3E%@#UgkXOQ$c_1-V{+NzYp5amW*<4@zka$2 zzs@J1Ej!Aw05W3IQ8DMA}}^?1~?{f4}KCnWlb!ZM@~egYz9+`Spj}i8dEXOCM0U6ym({l%B|TdW3^{Xqe`ei z5|qpi&!V?n_hdoAbZk{oxwJLN+c`oCF^t(W1nO(Z>eYjMXxXm3Pw}svs&k7nm(U9( z00VuWl*`bhf+y1kAkj^1F1&PJNv!^su~9IjRV@AO&L4b^;xAm77>&+%m?1ceSk@fe zZJ`GIT(pzO^52&>0Xc8q1a$3u0IzO@!{s+~+3{;XK51P4ITam2c|M`={XIt>R?tcq z5LNw$h{y(=SK75l6VSwm!?Y`I6JhLl_2u7oUxb5a`#yiN9Wdkn7N7d0l?ilNJ9P5` z^mX7h72ANOf>?XZ=lCI+_cvZ#MtqjB^(TGR+dH$C_`PEYkn_Ylg!u`#DSebh-7ZAy2|C~ zyWg~U+;_cR?`j3mpv|bHd>TbE{ZW4{_#kJsvfR!MZ#gT{H~q@;QtGomBGS4_J#6Pn zEdLnz`Ko2sK;?{5h)m-*2}3)c8-=740o#eQHMd{rzO(s#wOJvUy-@|b|IXCfh9`E!u4Zhnq3az)_#EN z8kgIr#5aca)QT{*t>YE!d&GV3l1)xFg*@T-yUI^8;q3&UpEtx({N>T_5QZLz{n59` z^#EgvNR#I?C9xkBdRwny>umA4N`UcMgy_V<1$Z9@=BSv$QY9!+|1-#(i-8}G>hR@T z&E+YoEX*!(cM1_Y_Po0!W{Mp5(f2OQO$dMlZYB=R@?r$|Ng+%9;L{{0_q@GY)mUy% zc`GvvjOXX(k#7ylz^NgRY-|H5&;W1TE*0KiIK7z*9y!>(G^7S~8L1Eg8tj@m`}JNt zcv!}j@+P77Xsco~zHAGEeLP6ZQ0;BpYmAk7Zah)11|?jRBRMTVnWt^(iaX32x;a%e zxu~spn|DfEDCw8N&Mfy7!<7$k)bIaA)|-bz`Gx=EBbDrteH%q6l@?3chI$K0O4(yt zWE)FDMvZwwmaHKuVhGvEp2%dEeV4I}86w$cBp%F6ztiXXUf1{Yy?%d~iRPK-oclTV zxwqH-I%iKjc=E2YpslZ;j-7q}aQLAr-Rt0Fs&V#~SINRS{#ZF}p%vN-d-mCX8M*_} ze+>6L_&{F7?jOEvqmK|dE~VtW5Vm9=7M}&VF?8ZCjb!O64mb?AK3bD?CEXHtivB(C zCwVbCWGRKQHR3deQv|RJ%>0+~PmV*U+~0oxFoID#^)2@8{lr*f?cZoed?T^iDrDnn zA3yBnP-$8Jsm4^(-mJ&T&fn~hGr+E9RH%zT>qRU@zT2!aRFv=@MZK7H+_uN5+wk)&w=lBP-4CVh3vJkm3-GN(@ZdXlDo=X^MUuOo?TMWY}~RpV{$n{TJhyD zhQH)6yHu1N#fpmCXG4an@Ml=PC^lb6$YqRFS(93C_TH^VT}1c9VEDKica+}&;gN~QoZse& zUb@Gwe#EKLg`y`cd?ymDy`i`%HT=J-Bue#94Ku!86hvyK7h65WC-SGrOYOEB6>cN> z=2dbF?%R)2A7!q4(Saz$?gZ(SA+%vcI z`y#Eq2u`&B45jbf><;COZ!`+0HQs+9-lBdc!bT(EzvjT=lm{_w_g_qE#1 z5B0m1Ph*NEY#v=YHQO#ci>i!kHE!T#$d9OXSxcCdp1d3V0^#f28HQY4IA7{C*2rr6 ze0Q8}YXz;tnvgj%$WY!zxh30{yh#};=n`_}Ov}6b`w?G-7 zd56NQOH$PlSx#iwSvs zBlwvhR#<-e0`sk>J_h1~=f@PaHLFn<3W`APv|w+6QfT|)7w#-zDz>;$wK<;)iE#7 zY+(V}m<@Av{i1uWBuU-8(z%d>Z#BN3zgPhyTQn1<+#1;GS;K?2$2gKtxaSzHr9V(~ zxv+*dhmWg!i7d`Ax@&GEctM%CVR!( z=}J~h=tm5N!4!alKm9E(fo=ZK;v{r@M~Z)cysp=oiuCZ(0R!RThoOl-CsEx`#|Z}zylkj#=TEtl@|-i2Npj+5B*T_;%ykGIQqveG{q@I z?m-c1U6sJFc-W=XsL``Lvitp4R$NOnG_`=s?eIr_>}%X@>n8It&dI(^_+H8GT9{(B z>_aQPRAudf2GkeM zG6Yra$da6}?A+e?fty<0iZaF)kN?!J6C#O@{9(FkuQ}BqcG&;M)9ci8YEa(2%v5>y zKSeprAQn&V+Zsw==!l~hakl^A(}mH7CKOjC$he_6o zXt#?~{EH9<(HmEx{{Q{mrv4LQ{2CQYq)>{o_?mYZq>> zAM?|C5jK{7Ic_)F#~Oc@rkhAVr0=2PFtqZ=L;W)U(aNP-@%H27patPI6s@2Vx&42n ze+z0V=r{W(!hb@Wn3|hYIGd1sX8jHGrLNfPKE`1gB+}%r8LHf{`7|EQ{!@IbM);2z z13F&J!NQ@|wZT&4pjZY&Kbrp{yy zd0eyewq0Y240&fp833&R*Yh%iEi^fpAhY>D%Nupy*h4U|%52`b>*TH?LtPF@3v)q7 zBl^kZ;mwb|wxL7XTJ2X!AxzA_Z~0#rn({Y$2u|XvmEn$soq=TvBTuIaJ@4G}7r_rX zu5m1~6e{Zb8FJYqw9OgpsUsnni|w*6r(!uvvl*fkv3kW10hr%c86B(-mCKb&ftorW zFG-mU{Sa2OHC8_IKvD6r&<%sVp+YwjLqr#W-cSCopH@hCdk7?E>zPr&;X;uJROGWY zlru7dG#0dxCU@;~xNT%tn}5olsowR>m`FXnm!Ujor_Ke95+(mgqc>-UP+qOfRC#75 z{y5asiGerz3+XBr>N1##lO4>h)S<4si8ZA{Y|NV$_88j=+H|~6o}1|7Y7r4{)iF_x zvqjgTMv3T(x-zy~e8!427U~TH$X#N$M|*5+CUbGuXidW-}RY>zV zOBR=(bnyMet-kQh`F8cAM{FA$U)SPToDegIu|Hjj#_?hU62(3?d0RAgedyHQ&6N|U z%nf+XQ^FmdA8eKl#-d38fO44TwRw~t{#+Fm>7X%Gj|`X%d8nFOAA5;?TL%5+LA+3r z^6m)R-0|!==Oe_!(9Af{CB`jgH00G6g{p{qQGcP-0eeq;6;-ubTr`)^-n^3oY>q=F7_Mr3%F04duFQn)G30UCQ=lZZ4}n1Z+y|Wpvsq;Z38?Nlee#w8F29xb+_zeAAVeLO>W@%FA$84I!lPvMns}Kwsfa7(=`DiF+YPk#<%(T@;IaA^`w|6l`gF_8 z_l%PxA8b~tkb?FlaTx&*%4JARvL7BHwxup#@(_RZXk?8@yNQlOilb%#Ey4GtO)rpa zx0SKLp4gan$v@?m*>RCTLL8$VW zsGN23mzqHY+8793XEVU{fvfL;qTh?Z2&nlZ>VX_xi*@iX0={;p$aWDmP9zb>2}{qz zyMN=}HmoIYJ@!e@5>5sQbv(ER1WCNi{hO2 zwE69WD}AKu6_R&_R_sHBZqfg+kTrnM1b>7Y|ND!ed`FhY!)ahKLl8qzZ1kirVVS&w zUhezTbZI`I;VRj`Yq#z~-;Kxx(0fRvS*d`8V55yLKPF|KkXagnyGePvR{m>RD)6eC zpxu8nZ%&DzkDsJ>2)8}vQT42!5oG%_B6LGO7}%R~NK_aJ~OWxc>wkfLux4^~I?$pEU?+MB|TP zQ^?Ibw-W+H9`1ErQ@cQ1NUA@;-7y(#BzX&Z)M}+bRA8w=jdPpo4o5#%!2z_(v)2dj zyW9PeU^Sd{dgu6Pw7-dRzK>Dunf2kc9_pb|X{*=x5{qAwMvTz5z}%;!W>)hlX{YYx zD->s7l=I(MuSlkh8f9v6bhi{0eSIhphaM%5*KDTJR6xZd?kat(nIP!OQlj-1)ry4u z8jo=K9{4jS<{37}Ax1Lv!KxB_ed53#VW^zpF?&*jG8W?Jbr54^d_U`_yIM))JaosGP*U6NrvjOj`tzy+*!lr{r?}2W!SvdQD9IA^)P5-``sc zV6Uq|CGiI=9rsVXDKqpeZ+)iqg{gSD*Qoc8IVqU<^?3}bV&Hy` z64%dZ)fIPzN77nEA6JyX^I`h(kNB@Q2kRy^;|GXUsAl50Xrnq_9!{oC$u)i!W@SP$ z$zD(MI?RuGIvPtK*~|XTDBBauCYGKA1FZ9YFP!{{O1(2ZvEApk8|Qo1$B&-~zeBo0 zpI6$a-&LY13xocS85!t`wHQY3;|lSH2t~Q3G<;JzfUf>Ol3VRCSwYZZSAim z^Ww+JT=jQ%FJcIs8)O;Sa{+KyjUcbn;LqGo4?5cDTDS+iu`+Mu7Mf(gLw5kX<=)fq zO=M#qET8-J%gH0_pf4c&kUgI7_Af!>z)y3D9iNb4XM&!gG(l<{NpnwzVpFMMwe+Y> zEd6m=DP|`uOT9KA({1NPoMr5f@f=t8wHGq?MHM-ZSs{N9^a-);#u(ti`fIF%Xx9m( zxXoa=aF+i$B%!SnP_RhkUB2~8=E&Hcgy!3Wp4LIUNloQ{5!u>NwLyMX+>#Yqt9p*udI5rYm6=cU~6WG97yU$4IS|7Y-ixQeA>AUTfvqW#vJ4`1n5wGkTy#Pq6N5K%^v` z{1wkOumK^KWOx%BYqtCoz6Ip9`=0u5pe;!6mi?#kbf3|N7-lGOW|4e=)d7`rH1~yB z5f9)Di|w(Bm%~k7@9?^b-T&;^*PD0$sD9r?Qv{wdfQ~1#fg7w6LeWBa1v?j(0vu2R zi~a{;v?l7(oR+L2f5`6YWRG!6w3LMfC}3X;UDsDR4%pLR1&~Q@-~0&73NR&&=s#wH zSW#EbkNwn4K?cHdGSptC%-JTqLDxG zhcF%K7?WdfbBSLJ&Yu*n)c^33Jt*lSn|3_<4T>&q6$HS4;n+eD&2d94HMA%gpc`ri zfZX_@nIgh|(7#Q>ofy+TO6}tS?c=$>wlv>qm~6VJ{KsHLs=7Y?RQJ@8|2|!nU6M#iOop5`!aw-c3nB6bU}s1c?nER z1uTh16evu5QKVQ!H6WK9G;}4u)M}bCONyn5i4T;w-|x4&ET3Jfe&GCpjtt&}HW2CTv2-%mqF*Q2w4(s76A|DAOkdpVvRQ~K5NoZR!h ztfh=z%qB;xWl9pr2K-S{eL4}SVVN>&z3jPqGNxO-;?FJ4U|NS;VCT$mNWi(8uzwS-3N@8;izhnjV@M}7?ZajyPI zm~CdA_4gPwih1VYrM|Pzk5p?Tcb$LHIU1~=dFZ*WBr}SkNKa{R(mgkNqF%qmm{P|) za(<9IO@^iA{qUz_qCj^l*GqaC$HNZG$pW?=wPK7VUUH!1kMzZPZe{KR4)gO=XHF@B zBd`6Pw4r?#dlR0pq19L*D5Y7OJtJ0=R}PheVB3SbF!pCzJvb6#7CpuYS1 z-JCP8FMhv$@N51ZvA9r203~kI6}S>zOpBt*Df1lkEQ*k0?UuF^`0=DCX3HSJ+$W2U zA+YnSS1tRr_<*)aXI#GkFTW1n(X}`8gIof6voxj`k4E6DsGJSz1CkYz@ zm5-xpu=d#i4u?f@SuR6rmqy;~XXY{_X?2%j%xnzuCOv7fJa~O`ek7yju4KbJS8GI* zz}GcBWbXe!B%vDy2)rp+9sngVlL^(REU)dlEIEk5qO{}|zycWHb@zeVhb#gr=1?G`^V^r0)vY+#l=p0cAufi(dkF1L`(YmlDSig#2e!p5v6AX@gzu0&tdrhfRson53C{ zt24HG-$CQL)rAYvPpl}!U_hrY%*z(1@twNrpvi(5Ge_rm&N!D@ z^^TP4N=5h{P3`78l8E}{UBkZStbOMhULR^Shi0BTXo9P(aN_;iuGBb5ETE>t%Z?`h z7fHSnNqG=?Y44Di*BQNu2&36|@eXCX3Q%JS5=;scH(t>QfJv%^7Y^)$8)S*ac(>P^ z$q(({t_uKx^x26o4TIdv8z5nB8^Pj+zwY ziOgVasHAt9P#5hJzl;X!P};1J)qyH+55kKz({7rMWzsGuioE(90%xv}XbN~Au_}++ zKGXIkJqTHdqFo}u3dk44nIv)rA(A0VB?$b~Rj8+Pyg=GT&R#LLJfEdrO8{D4jTY&@7v+taUO3=SAf&SOGAaHA7b%6&T{2TTM@!zft){GLCNL#a| zrHzF#44jLft;Cj%YPB$wy|KNJqP}UN}dl0=##8ocxSN& zANOJuG}G8w_4SItFF1RBKPqv-{piVtPhezGP$x*U0wX!ykJ{i)*+zmkuD=MS5b$3( z5byH0LOWKu{Tx3NRXvoThCN{C$wd9&ChUclJ^P6pt@&1DBF(68+km_b;>LS?ao`5= z{T%(a>G>Y-U28;>lMmXAliCJ*vb*!lq!t(1s3 zgAChW0@KSP^Y_NgE3Xa&Pv5cNYAvgKd|rrcC+7bITKml+aZm;<*})=>R;D3W?CJ?M z4v?YOf=Am*T}Q9RDeob<(P9_f!$W|!x;-$5Z1cClzyZu0Zw&8h6Km1 zD+mZL(bn3DY=1CQV-7N@EI|T)s~iq$8+EcSnbl_wr*#n;^)KR;c?~mG6U_?ozFo=?-vvu)klxOGHSrh0 zRC85j+o=Eo=9mvpoP42{XznEfA@D^5210DxnLtu^pw44YCQGWnq zMZyZsD?t>q7b0{wL97#;hH4Q}sZKMsYTvItUIqACHW|mSQ z&Vl?AvtQ4tgJVOQwng0^MMr;4LA_vTEa$y$kZ`U#+P842e`A$4loX?D5_B^TXn&R(MF^!AC#8T=kR8{yY@6 zx)kw2LVSMh=(~3^kepT@QZA8n+_S;v9^{O zI1N<#6xeE?e)&YZ-M#OtY`_&gC7(@yhWPAXgn~F-nbp1sZA?ATLl$oUlP??sdb%9q zY4<5{5%qVe+=Q~94#GpH)I2Wx#7)^X9x{=8WN2{HZiME=kSe6BZ&7j?f*e%6A!zkS zMJ=7TE$w}BoMig3(j(^#`!)jvcypSxOgfAaE`{{*!+ZlECof#$T;xGSj=OcvM1^;fv1ysmMjY`l$Vc{tIy zP@eZoL#^|5AIH7>nWdyF2$;M%1%wXB)uHORX6D6Kz!vWHwLvLCQC(M#)|)Dl>gRFc z_K8IK!-7Bb+sv;&+Rt_~7!m3vNda>p&hI2+jTbxnIsT=-)Iq~1-v%3wV*MS)90!;E zm?FP#KL7aCNacNk@9~GwxWT<d|^xeC6bAJ(P4(FH|czJv;nP!QKP6=~^$uxtX$UJzqbGW&Y55jy2@sry^}8_K)?mOdqL^O=1k zeq3M(N6prN^&d7y^>6xeZY54<&gzMDLllmBmtybjETA@gEF692z+2lh>qfiW&MqAs ze|YK8WGbg6b>a9K6#F-y5H4&;i%80SleF(7g?OK|;-TH#{;``gph^1YU&KMi z1xyhZvK&(DQTXFOL3Z6T_Vdu($&R@4ace)3O;-|v8uN)#C^s`U6S^ohxwTlMnRu|? znlE2BHQ&ax(EhD_Gk>*xKw?hezJMbyjZOFcuNas%Y8Z%s0xt1zJ4@m*`HTTd(Xu)Uw4KWPr-o=>Q#Rc4#2hhSzS6O%HsS%-<>sV zCW>P~q^Z02TcFo06eB{z-gO=#cQwSSXmsD*`*x4W`FUK31m}U{cs~TIO!pXz1aau@ z#}LQy>Tm_EuaWe&Z%3h)8YQiZM<~ugDHfjWDqd8XDl*omh2$0OYZqu@GpN!1+U0wYQqVa!7{IFO?A1`A>BfwBAgr5?~)-?lFne>=jLmKMW1a&K)n` zj_;I56FFr=l1OY!b!>C9rdJH^Xn_lEoZO)1}o(ySJ+v?75~dz0iIYD*Qd^$T|7Y4u|t6 zT9`evYy)}`-pcJyEwm{;aV_aXROPWl1?p7$$L~H;8(ipS z3~BpaWsHpYaPeU%V%l${=;Mbq{yfRI2gI3MUl4eZaBn0Kq4fl^6f}O82uwmrB6@%c z9Rm@L*BBsPrfe||Id68-HDVj&r9FIg>>878g^=Tq@BQ+s{1$ju^M}bP(FgVK7E1PC zDE+PiM*?`OIqNT?C61v868dr1IocpvfEgnh3Px?R4HGxsOEAlrF{rtIT_NCDre#Bm zLTU_X7HR}R0qrI^4D~;C7G6In%!9hWl2(!&;2g*-ymL|#SKhW%$>Gt=^$1%!s4`Aw z@r5&!Sz@3Xv5z+W0caVqvR;R>#CvwFv)^L&_ucAP4H9l3X)>2XDiXoGQyK*xR09Bv z__wH#hFF}%REUjnwtp3%wms0$ts(TTCU?XgOvI7T#^{sE@9#cVzEfi%R?^Pm1eR`} zH*5l4HAl29oM|oa+Z>v&<&YbaM0!|hU$g&)Ot%(DV3}_(#_C`ecMvQ$loRSZU_$WQ z6~`XmZ6KU%$eNl65F8R> z&{YbFw4vFytP*S#H7xXmgh5zhfRC%{Xt~3-xdB%tN9UB{Th1R9zfXBbvOeSdX6-d7 zHNk3e0vM|s-+%7xi{43Mc87C&?LAYOaADKJ9O7dAR|!IxS1Vu&#$F_JOS(8(?6x6w z5of{o!sqGc10JZT=YeR7T;Ti32=$lYx_VO1vh1s7TsT=^20mjYn@FBf+3mAS(#nQa zk!_tM9jlAuUOYyEs<*$rJD$hB2(s1`PVi9xwd(VVfMB(LBK*UTNEffJ1=;dl;2Toz zf|%nl2(|@MF@z1T&t@zUL1bO)8>pEKWFxj-jLn72CFb{Pjeks+iQQ>Z)pTr08SEIEijl942;rwyabF z&`E*_EEs4}|M>0+Kmcxwf~^SnDTwBg+hjJewpl>8YArnkld0-)CD@8(f@|gAe7T(w zzR!8zdv`&*y7iaGF5LeO9g5~*1He4u%rJ61d}b^}1!q8Ai$MzD&Sfm-Ies2iGc_}* zst(fY>t1sg@?A`KVPAmY3td$5@&a9%V%3CY?O#H2s_9%Dvm|<0;H2E6%UbS6Tz1+$ z-#ge?Ru?_rkj)GsdLX|iZ-RIZMD=RJ4Ts4%(GhL?4A^oK`bE0>#4jv|${m&Y^4_}o zTibg^NL@s)O_ILcx+sf_o3^%j3BKU19z>IDC5x~WFt2|H_+XyAd;K^?NKa7gzxU(L zpWLGz56m|5l*~qMslW;<3`tk|)6FS@DP1G($YT3+Dfx@2ejJzUvf~97)H5)&|8rX0UobX=mTD8f;%Q;F z4~hKid2Pedf&zL{Y2DLcgFA?2>;q?rB+>mrB65%F8p|LHkYB;vU`tk}uCeyIk{MBY z$UM}(1|3*&GnVGekau9CTZYV@s?uF-+XsLywZTU#k+esDN|%?4v56Fwuc-kl+JPUwok}f) z-|q|;mxiwjy-HBVI^OPi_*|fNtn+5<(K!SU89sTfcvlc=Jj;@~4SO{c_ppTF#M#SL zm^2kbRwtGt#ivKV^wvE@{d7jDvQ zNz@^bvW6_&ae8ty)3%vDM=skPzvK`8N9URQv1ixw&Dr)|=lG$*5G~w%00S$zm(G0aTMSXGmQ!c! zoqpdKzTt-yG|f0UpG@h(j$0SExa~HZv5M`3Gxo8(6}~+Tj$9kT>-JwnQh%a&*bVF+;!{LD8*r&0;*Qajlh;sj9W z*HqTD^{75B+r#co%zE9G?y`b(&WHTsr6Betf%Z>VS|N+;(v0c-+1%yBd1!^43}flB z2)v}J=cdNzt1EAXi(Y71uq=&jcQqq?nrC&?KtG6u6 zi~2iL9$Ll>2N&@@31%y42CNyozE~8Cb7J=lhI~l-u40 zkMA#Ynv#xR>3oLk+f-qQ+}x~XR-&At?;$hOFM&$}qDve!?o2_2=!&X+9Te>By7GJC z@{4z=9`>F_ynaH@LKOBq3qKx7f8{k%8VtZY$nh)+A+nNR8s?{i%AeAx2AL4s5mib5u(sKobiU{1Z>FLu z&!pOzb&6edRHodjorvi{gF!VGqpJH_LrUP%O$?nJT5pl(LgpW}IZ=g)az@f3jAAV| z>T=yZEFaDbD?Uj|(o^Di&9-a!e_BESQ`=V*&0BG0I}_i6{>UoKohXhb3$Tt(m{9~BbYG3$5Av(L+RRjzG;4q$p?jLdxJVNXW&~h9 zxTN5!AHq@pRgcJ3K2E>hBheds@Fn8tL!&*dX7PWO6IS`YD33Q?l22Qex5P-@J@)>9`aFVAMK&$;ul^LCzTM2yXYG`ig` zl5vy?N}Bgd!*@!7=N(GqtLH9oz>>5~qQKb^o_$w8r60PBZPUQYSvpP@Oc=M_lF6_- zNAZF+3U7irXS2Tj+qk1pk*{^RBl{>}$(hVx54N?p-O$;|Z}S1cRNaQh>VN>(0gG5f z&HO4AVaU-rmaS+kmGIAm=BSk$+8&a(6~jwTWPh^8H)1)x*-ln?JJdT%)6uq2qnvgi zjr|(9k26jvq$OpnaU$@E3OA{E<4MwdN{#C&zW;(R)oYZjV(9HGu0^1X0>i|aSLi*A z3$qZW@5QKtMjW8E`iXJndp^p`rvBS>Ui92I!k_aKL0c8+XB=90C7{Ma&GwbR-4n1V z)fXjEIkg`txp>AcKbIIfO(|wym^;gzGZpPe(tX9YOQ(fUy_yLk6DB!WjusitVam{0 zPeNbw)@H@Yk4~sFM%JPGD!#ERs=?r)KpY@(+VOiA1&xK({Y7M*P+p0HkF+xmu3ZQ7 zF~RB4M{5G;GJWmJvnO>)t@pwWJhVw~9Q`MaQ%uLFqVL?7URo!x_&_um90!Q*eljx{ z%9ObM03}-W8Af>*{e03*DlW351K!krxdZAQ%bhV^M-qO$n?=OB>>)BlAb!??Gn6%0 z0ks@JLA}^9Yca*NCIh-{;q-`2(I<|N2fkK}+fTk_FD#Rnyz{C3@QW3+DNs6VMqri; zn&$4cJd`o@%ktxr?Vs>4ho%nVTFGl^9g>xW)QXS0 zxTh-XIMfKf>wyPIcvNoao~VW|QM#o0K)moc(@Q6&=q2fLx%?=t$j$MLnR|a&f`#Ii z%h%UJYyf720z+`1kF7|8r@sMv(L|b?ElDcPUQk%|PT+^kyuFyH7^~|>^XpNax8%9> z-ejBn)_B!z{mde;l*O&#G>W8MLBWdA9^4o0nj|JuudH-Ip6X1-t{i-Nu2i`wT3T}+kOw|YDR|v>0VxN`z+O3G0B&d6(59jRfbk%9XYOcO`ro*nRwW5I?QpxN}_7Ptf}seQV1` z8OJ@>mfsJ#<93yLaqWJ7);MVUJ=^c9TcLXe2p~r}&N1^8-%T!0ZbpYcx^3W(3Syke zb@7=rNJ|!+m@pJdI2F*WS;01;lgB>bQuL1$`w&BmW=CNaQ%glK5>PnfLZ)5S2rKKU zQY5!^yQP!v96~cePh6gB2IZ&%4iH7IEY9q%_)t`0DgkA&>&|VcX?nw8?`DmNr%z2! zuy3p*Z&h`LEN|t^I|;-elG6KUT;NBpwmNo`WaJckA@l7OP0cVX^V9kq!3S2*-7>|l z{@{s{lFkYf1&-^#Yq@*qeGU@|INMl~((C!yRUASVDuJlgx^ z!=gq%z!lGl^Y%_Xet#nL4GjxS7FVL4*9B-j4pv>7-3rnLiK|AYytF1wRHl3-_ z@^mR;rONc)bgy&7Vv21zr(FJV8*4|N2S&@FswENm8-m>l&rur|c%bGK(+7an`}kHE z(G3-bKR4##`vpl=3G(x}_J>jT)I zy>rZJZPcz2L+Q{@Jd*BNp7OrOG4oLKv~2Wt5kgw-)Au2z3;UigJ zfeMB&|1_Yzc`d?LN>`RvQlEFKSjO;0d2C5+!ePGHqKq)}1MQ>bkR$SYJEeX3C@oZ* zmqu(Jfc={b{ZIaG-F)Hc^eO$KhOvA(RH0S9CvWnaun-SIi5;9QdZA221(vgF3AM1W zXc*8#++kzpdN@b}$Q|}C;#(N*3dlk3pqUyrR$$JJy~y>jwHK&Az?MOagQWU%EFL;2 z_VK_;K#Cj09nbO%zGN2n16twcCY`uqpEWTOiJ?VL?_C#|h`DI3eCePR;f#ze_Tt@&~W}~dGtf{N> z(Ya~qh9R{_0ua0Ejhwj~-xTw>rAmyYLi^mu$bQ`n7uYn5@dUm^joXWjB(KN7X z|L}wL2j4rkUPptdsFpZbgCCU7^R1UGtt8q~b0Lwk;T1U_9^K;u3LXQuTg5 zasQdVIN}Ty1uLH_q?#RS_rp*ox_*pmo2t66+qp=nUKNqnJ%+e0J@DU|4JTd4Z0S-x zWIsPcpo=mx)>H}~C4McI*daaDYN0_fX>xWg4czs`Rx2?qYdUe-Xd z-x_>jQMSG&@|PIv7@STqH<)~H6q|Lz>yM3qwM?si>0(1?r?h&f^qp!dH&5}T1nc|e+IT0$}x53NqL+_ z&uw1v@oYYJI%U11lYh5$0<|W=A}DVsGV{Qu;AVP`)rAg2ZIqg2@GqACgNniD+b@P2 z0XjkhFaib%Xz)=Xc;xf<_a?_ny@mPi!?eixbbqPiFaK$gHB zv@Zd`ND4FDPh}j?*OR@9v5|etkb?qTjMFwzJ!@Evv6Bw4PV1SLhD93F5cM;=5W%rYoZviEjQHZLV@DY8QEWe>@bUo#w@- z;d=-)PyXc?y6C1KjXy%oEBr}LTVV<8b8Xn`DEiQ^8!r@6NEhVq6rMl8{rcWR#GVHA z#1j9bPzK+jO&K7f+qi!&lCPVL+-KN?lG?}LL+x0h4tMWVjclcV6z8!%)Uo^U0aO>g z@NUMc|CLo3kjzD*7nvQrpNE{7H>sjACH@C0JhENA&x_jrSTw(19r1ML^`p}f;&E3G z@`GhJv&0W8bpp&<%En(rP=omJY+NgZKJ>D$jh50@D`!i-LNAWI^Dq5I zh$7Ig@JEu%4VR^9AWt9OYiIvtQB66|5-|w7+B$lio!9#sW=xYjaXN=;eikl8gBd)) zcQ~kn_gyfqE7}HhEoyd+l>_gP-`-Jen@7*G&z6{t`$j#&{}R@F!*PrUa`*X^j~YO) z9999jVQzfy&Son8;!@LRjb^);GaEfubeyoosvrSd`GCjf9%37Ny?&0jp88Dp4!DO7 zo&DL+Uv`FfP#;t{7_1ew|a3YT5ADPcN>lst?^MmwBAlVeBZ+_~+4I z07KJb$uQ2*ZWEx2+-B)RO$%+L;doS3X(jV4n?jsxb56qNQo^ zr&3o#b`P>9)8)ocj^s!g6&dmJ0FRBpr(GE355X2K)e8^zy980}2mC!Ozl*V>4ZvL* z4&r-R3e3t}di#n1eUBts+||D;5B3~t36bzD%+q?U0VQjcpRr7nKRj}j#Qk#jO*Ddk z3=L}l+LjQ-Wq>0tji|T!z>K3pEsZi+h3&^#TURQ?cTdKziYbfBOaGUUbYXpX?5Bmn zgFwCn438 z{m8KD#;EFwR!dHCPgPOx&vHZO^_-7RF{~RB$mH{ld;cPC%ZM@GvSdJ#v8`^lN4Zf~ ztc+gsBF%9`?PulBu&gJRYB870Vs zXrKj}zcz+~dyq6>@01ppQid$Y6%blP_jA0)laaKGDqsix$iEDZF6oO(*=o1b@iC?m zzB6{EQP+npsZiy#zDu161?xHzYi6Lcv`U6mty)Pn(ZsuP5n~c<`FUDa^Tf>d-Qp^x2A`Xj_uI zS*pG{4SJad7NRfjIwvS@M@-Z_oa5SU|Nnbq{G&VDb*%s&P#T2SVOHDCZr0A!>DaZ0 z4IV($vMp=Fj|Bf@w2jqrmT5FBpD@zn(Xbj+z0LuGa+);?R#*j6+NZ{ht%}>9R9#Jv zjlEV#F}Ql%2b|Co&vX6ya&613vB zvXftmd-JAkXr6i2FAl%a;KrN_dd=bf20zC9x~n1Lhl*!pIQYHXY!uZFoZxFHKHGKA_qO3g0sNM zycJUW?ayoh#}i$+9w{fgg0c2L&DV}O?Ru_E033(H;?x4(#h*cFCWTN&2*)FBw?!d` z!*3urZ0e0@sN<4(lGXpm+ndKj`Tu{TBV^4)WH*IU*@{G2hRT+t2qC6Y*+P;~V=nQ@ zzJw1-Wt6qhWY3bZ3rY6uV{B!~OfoKJ=K7qs@9)0PxzF!=9*_I?xbJh$ALKF3bzSf4 zy}Z`v>-p;5N4c%ja3%XQ&&_jIU#<`r0ujE~Oy?=^R3WU7s>>R+z0s@qj7!8wdEv7P z?nnA=hVjp$50*|b)7MaK)MN63itpZoP4Fi1`wYV0%fuoRm6atd133Q8xL3MzN|vVv z3)x+|!clr&l zh@4w+dv;f}Q~b-J9cf2Uq>BJZunq1DV}X&m)j5F|D;WYMhy>7vLAz)v^HB1ye^9m+ zL*BD#!mjAF7>?nIm(UHL;6oEZH{FX1hlv=R;*+I^Jp z2kJAa;|w)#<3 zUL67Of_N=7tgB!{t7wm9UBX7#x7Htw@(}SXs|Y>YOPUb>tU5RCaE2p+m#2UP+@EkQ zWRHvc6gQ#`Q{5--;EjuZW~i2FToZ>%Bd87!GC}N>Ek_6gd`~vO_-t6W0(98muRP1s zq1q6I0!5tyl|N)k9d2j&3ij%d%6(=mdY7cP_o`xyWT5Bpk+CE*o_(l$CwGt^-p2`e z5Trj29PBS26X2`xN(sg@hJNmGxnO&-YyU;<>dVICBRa;C4Ps#BphoXzV51R17@mQq zwIi8oUcanhogRcgmad8Ly_zQd6-X2e@RsYbrhEU&ORvmGc$uCw(dd)bjy)mypu7QJ zq&Xkhj>6>rZHH+_?{*F`dgyfrXgj>t*%GAnfsO;;q@MR}T^oKX_RJl9g{44{>fL zvdM@h)M$vMY4J%`>X3JJ6BRawqLbhRbq)K zH%bjRgXj1*>hrWbGi2cxE@+tmbb;k?1p{IU+as6G&;YrwwAHNFkJUY}rYXH5=fT!cO5P%aMt+twIhxy~ za4o(;XP=CS=Bi!!EBSuI{_ABW6;-8WRnAF9TsL&6IAdEB$MT^aP%D1*HP~tgq1zK^ zs!P?(URaajY}10s5B|~`TQR-T-&UxSUiNjZ;|M$Fvx0$Z8Ha?e%Z^YK=Z}~}oWFr0 zo&Tn?jR58%)5H>9-15jDX{4()x>i*_xgK+jomzNT%#f2-jE*5z`L;~K-~h0s$7sr} z|DXi%V(L*9kTQw$=U#iqnSzY|CbJ+f4^zT)_N`~3!JdKliL-=L+NMMdw4IPaO!@Is?S3Cnf8%p1$?GmY!_=paDx6Sny{*62jjX}TM{y~lF6XtC#ZH!8W^~jYY zjxNCtBMVKXcfM-IC1^h0yvTEs#)FudhC$1mt^oT@A9l8k->V4X3$_v{LDb;^4$5}@ z^!Miw<@HpFX}rN6$VWR;P0v`it}-JsbwN>l?}qTW@%%v}_Yo@p_mdpaGLLauVAgrD z5G5vKl_e50+@4Vtr{|mlr?gC-*gF)qOZL{@6Lv!@%7-V8Wa}l|T$tt9LD}1(wIyqe z0*na%`VnhIddWM=aif70^*95$pct&CF+DBCFK1%6Mz2aqs)#z%-7G-=xzYPRARZfm zxH=9s*7<{UIoU(ihH4*!_s)jknajxMl=HpWW&JMSJyw;~Uma8HPCG_QJC>Mcx=!j} z1mTJ_Ead}{X60;y@Ov@$0gY1D%QDZ0?Ut7wWW24ft9%sI)TGFL;w_I-(1L#9?=ocH z5Pg9SEO_8fQ6gqQOwE3EP?TdEda&CoXbp*lKq~5tAMi+&p%I8e#L&Fj&qi;+93f1;<8XC*TA-!0E_t}P^ul5e^Hzc?c3--q=7!-#-K9tJ$& zM*RTD3KZcjhXaGH=L-EGky=<%JhAv@>+8tMjPlVquF;m4@sTe<0vqXd)#kxczS_~J}X+8qT7Ri*0wN)O}znmsoPjZWf1@d;T&uBG3jiL}Gy{4Td zLS#Ion}!9D_IlK-f28w7y>PN9R5pbw%`T4_q0?-?C;RiA?4jF_Tn6>m+cnq#!TuLv z5|}X;&`)qQYxO5|*o}HSV%YLVo290GX+XG5pKQs;#$$F3cXoYlO!}aa6KXoE-A0Br zp|-slUtcRV6XRT@2aW@_#}kNRe_q*h6Cn}vYa(y=QGBknnd+LJW7FP*LRfyR+bXOX z#zaJvgOMZn8|6rc(evfH!}(RC2leh&i7R*p&NfY*V@I=-Ktzrd34K^-98=%H(xR=9 zBLC>x%%8Jg_(*<%I={1=fkP$M5iiLO^<*BKld zLwa!-aq9uSJqMVcl_zSN>xKy@>m~tpm$+`mfO&^1Q767Gj|#nBOz7*AHRw}*6a-~Q z@Ln{P@hi9-Z`g2=2gL!5qf-)glLr#CuFS-7+g=LIx{tCccoT}{fGEF-n`%H60Wtj+ z`gvB1=?XHT9a;FpAmXcQ22iJmy!zb13hjm{RS?aTJZiy!!P{mBSr!FE2dq9p4nDrt zuRf|#i9r{?xq=Lp89x<(Y6GDD0o*loIG%3~r2khz&32aEnZ2QA7y`3p+r=PpE5H38`yBA*(+}cq zjl`+FSG6r|d_MC8vI)f!KWtS3^kx62SSYit)W>h`m&!cXqOCPKHFlJR569LjX zW^v z$O9n7zy&Jdxv&i>UPMwo6SAj6G_QPIBOPCoe(P6-ANr}_+OK$#XBxvgN@R(VkeCOb zKsaoo3b5Ol{`*I`PnR$9tZko|J?s=R_*4(NC(oPq=<3v8 zV3zolR(8!mKk(|Z%S89u{_;nU3%%^mkCR{A-we~%V>S15kVX=}lMy~9Egj)w)7W*o zw5p~PE|`d!os7-sj{{mKFs=H{b5k#$R`XUjy-QFYDab<0vP-eoaDPoz^@K)7B8LGZ z9$oZk1XK2x;Hij=_Clk(&yo#BP%C%v4GGGC69yuKEoz9joSry6V+W(@Qax8HOxE`I z;;nb(F6%6(KYg2E(qq}%veSUBH@s>q%E9^!_~pPE$R*M&;vY^~)@{Q-I3zOc7ct4LGUhllXUm7vb!Zrtgh{{{C!fDtzd@%T5KDKpggQKp>EXsXT__f zM>OLTyCTQ$>Y}$uu%G1YMXA%#zu!~uf>W|U{*burHBJ03&TycSnpXLdt*+*wsh{>F z3G)Dps?Gcwzm$mN(-$-zydUJ@SEg?;Zn85F1rH1@;C=3l7a0{a3Jo{LQ+q_d*04rp z`#ij5GW54hsI&R06|Yh+|7ui0&Bm}PK=^Wk8UN`!vPN}#4Ed5Hv2uD^(;*g4w67_z zuk0JxSCY0kY(*}q&xF(#x|mU2D(7=!R@oGuLzehvrY$lI0lt_STs(+*;x+SGS`}r! zyT-42FnV4)YL0iix%6$oDOX}71z<&2=cL+oJMR)SKF$J93 zWuW897abcKeYr7 z(XpwFELpKP_DQ$+{~~wBj742f%qc+qi)AJTaBGkaz$ybzz=gaEXq%&WH*g=WA1)G+ zRI3(!C-3fG2^mRc&Ibn{|IRRluH+naaFUiE6&f$uCpy6}U2dHrOX95ns*WfO{OzW+ zdLFt<)$OYE8ZPnfN9}Tn$F}`ws{HF4*(zatVe;+<5FdaxpbI+I{M&I|+ zx3~bxtNPb@t>=g4L4B3B@Tmw>6B&)t?dmb3EP>1EcLzD{riTu4T-*Mo4F!=`rkE#L z%IsorrCKC^30KHsOh0Ag$d3 zmhUqcB439BicSt=D@04p`I|V#AM6&amT^<)Bi|uYNwWSL|?hF{9@cq|@&+Rc=(~^JPKxHj+x%y~0A7q38Tx1w*bRN~zL zZT_hb3TKdx6O#c0L%pWqej1hYPc|oyjdO$19mG6p;OKPehm)u0wSUg8~uq6 z0L)bIX+j5MN*z;PU7g*bJqVOiI27p6%+1@T2 zSZ7A-VXy16PNim((gVD9OhwI3MvK&RsJI?-T9`Dx4K>~b^Y|_UWQ}o*c3OSKEKD+j zji+^kjxQ;n^e+AfRpA)+B`DAM!Y2S|x{GGP7+}As zS6sq~*311sGzQ8ckyf``e!+^JL$nu{?}c{-NQ|4$hZ?K{%jyECn)5_<5Ju%R`Vb!c zKALZfuYf7A8wGx+S6&1oy_H&$HfqTKpGeiU0T1m@DILFZKp* z#HJl1T4w}RfV>#52kHJ(pHW{QiKvwo5C?zgkg3zSjMG}Q-4q9hkbv}B!<-B)ID|Gi zWEbNbmAOB8Yk}V^qyIv@`v9c+#)^uiM3H9H(^jLC!SJOg=87r|kJ z60*1DV%ZHtZVc8cybC zy(C;gPn>ak60qXZ&OA5NK{#SraBis0B<@_(5W_jfS!NeJeyQZ86avT_V(nZED!~(7 zSeTFSE6}-8o}O2nYgsty2}uq98LnDu4Gh#DEJ$;RHTzQ8Z}`mh_Y;8zls%=sFd!IM zsQ|weww~kJKMrd<(|IEl*1RidL&Rh=O7QI~H*PonRPDj9u}_aUoXHzG@ zKkjvB{HX$6OI{~c&X$>3|2EcCxm$j3I*g*D`!Ec8Z3U~XvcyjGAbf>P#R;`|C{Pm4 zp81&9uczu$q}QccC9)_eL@?vrQO$EdUTQ`<<{y*}^r0u(02^e=sO|$ji(R}jtXfy4 z4ZT&K=Y&oELX->omJGb-h0m6}GgG`R{dbr|#4P6@n^uVC3$i(w_DmVTE!~rlL8!>_ zyd{~Ctq`i0eQs!GlUk*FS=7*6OW0JNpmnZ#--1n0A)OkUL zi*mB$Zxz9P(&lQ%Yjxcv#wBKgavX}k&l!!LcPA0e)Llbb_YI=rvW3gNS7au5cgij8K>p2aD!-U+L9 zN?AWoy>aw>3K>zE#k;NU-LMyOjf~vnl}^+zTqccy?OaNOR!o^ncoW#yg9f!^7!N1l zbsxPkesrQruw+!LMO7l{@3R8j6EAmY;y@y!aSuePalwG;yU3EEr$1Rwj2vx`yG=JK z&B*Z|S-dn_TvH^lw@FeWCG{%^$(KNl{LppcDCE!3rHi)!ScSWl)MQZ2m`i2<%(o#( zYzQ`2=^v+)0MdgYV0-G!>+)3lDP4z{fJ0V?CCoCD|$%+ zGUFKGzG4 zfe=29{N_BUQ5AB7Db59xTz$FwL1A$ zo4xm|k4qPPXMO8(^HY?7^fKQ?&9Kv5Uh#~8P+KFB=0{p7g+c?6U6 z&Bw{?Tu0y$;;#lwGj_(Q=E-d?x(gIs!rd5q6q-0`@9a|b!AleG;rN&&mO zWQs;3kJ+>lC9upX5)jtL)1$^QU)?2M+~~J`+6X?E4{uG3?SM4{Kzshikyoi zSQ*_$+PQ?KohK(gNF%7yx(5f_A8Q3sBV?`l6jS4KLXQ;ONe)N#ts}Dm9>Hi(A}26cUU_2D*I|W;<}1# z+W)X(34=;x-~|b#!{*dg0jLQKMSv3#DR$PvRY2E)912Olqb#0YwzB8f*YRJUm*N5e zR0!b@{FgEOiU0~8P9fSs;b0V81PS+=`B+M^_3tL6r@8XmxWm`9J`Ho}cRPxTIJoZd zLZoP5!wr(Y7fz#BjF=ku3j_5EcrBV$g4QnW7P^z;Rj-F0Zk(eL*9A9vLt%07m(KuTB>?Va`-w8yxHfs-# zYR}qZO3_ZNJ4x5j=hMOnd(l1ZAZR3T$}6s=a!GRh!F%a@naBD<4KiRqF!CWbk${G0 zKuPHnf*=^tYSjH;{F1DN6^&i-X)v4eem3c%#!C?m^l7g#5b@lGn)7c4A_A(+z0Us; zs$a#x0_O2I=^A5sC0TDJHE@m}p3BQc*rJ6V1&^%YE`gZEwxdCF_gyIziiGH#jNsus zFQ=~Pu~rvM!i~lZQ zhI++;J5cYsCC|mi7w4;G%;n||IBV~)ZQT6(EoBxZWVKxY=->g+D67k$r7|@lt9Q?| z_u;vch%>Dd%YPYrp;9hmzW3fTI?j2Zh;3@kR_f4D_ah#8Ba{qr{CFc5q zfQ?vaJ2OLsC~vWqpNOnIYYb`naLe1eYAs8Ho@GbbG9t_z$Y)QK&b+ntpFeRD8j;Q% z+>{2+QVgMWG(DD}88pKP8_ItA^PyN#y;Se)^FXaR7IL-9W~UyqxVym0|9oWpg}G^M7k#F3SXmGf8i!MK9NHp2(%d!bdH56oad2)Kw zjh^p#lz#E~jywPRzpiO>SV0_{0Q18Q+}!ZL^uU{Qz3Nv5bw4X0_P5Zs%Ub<}s_?J4 z2cySTgvMgq&VS&d6rJ(X>G_g9Pin@A97~w)&%N64oRB5u=lxUl%~PXaNu21CeBww55uA`J~R4oBB>T zy}mHg_ie`$zStH=!T|0t(f#A)lTt zHQeUyAaI(Vg1FFs`oxREf9QX>E%g7nT%TnKmGY`U8sPe$QMOEPqr$Dtb@x zE* z^&^U&c7daFeGX&Y1u)_F*Fi=9Cu;|jk0!mpt9=Z0yFQZ*96HKtEFWlyh22S^54ZY4 zdmVpB?OJbnkS2fJtFZW-ed8F{v9RCM7Z^u)ZMk0ZBhp?QRtOq|tyAu?W3bEI#n4Z+ zNJC5(k-pbz8K~+r8_{H1$^aG~4C!L369x#}>{5i+9TEneX~KG1GDYcJ2Hq6|JiDYJ z8SB@Z*{SHx-VcH~*J6;}OIuI>fl%Sc$a8xpfs;_00#3}&oi6~T)!*AmJuBumTR*v! zv}{Umw}t~|NKxVf)M7Ck=IJquKA!-^lB81nbHyi;b{CI2@Qv3-7d6na(ZpvPk*?$Ljgh| zLy#{7AK_^PIGD&MPjWw%r&m?!aFrDa?}SwEa^StW9>2`k^hM-5 zz@+>j(6tefhhSNxNZk;u-;P*mQij>l{@``)SHMI=%tSchJ6X`40nCLb;MDWwJppO; zcBVSh6%hf=cY&!Br}=Awz09g5Fjmtgsr3iJxh;imByM5qEJ1KM^EK%IS4i=|&Y6N_ z;j-#KsM_bGCaDzbHVvpwodHxwco0t*OjNB&*P8e@e_iaC(J}L8WuZ6ZBM{|t&!!$O4s57{W7`7_fXC=w$T?Cpd0908;59P=F{#za z9z;l#oeFtRC7kkYChcGueuTul4lP(!4jzB;N=yCKT?qed_PLm&wT3On_}B%G)`vVR z8$h5w+Gh6T>iP6-8BvA1m>6sNj`FmJA+P?f+KA9vQa0uE0(@9~1wh5afkpn)cm+jhxB@i}PnBGF4ehF7ou+eH>cv75 z!FW|dQr-oPFRl%o)%x6o$RN$Ohk!|q@8>8ZykKN;OO51{{=mL6Ir>^nbenB&B-ieT z!!%{ws5oo44RUdT2}mBkXZ!wxdaaKN3Ha{_gN~`HNY>sd2C#m@d|1Z)cm=vuJ32oo z-MT_M`$B*D=DKq1m3uqNp(Xy3Sm?kR$_HyxPOX_tAb-N&LN}%)VAA=%{cvL8NcTJE zR`--C(2{m^XlGIrPnunvieKE^^C*4}rN_9iDcb!&Rz7Zq$Zo){WBQdT{3v7r@|qfJ zp9b7UG)0yfoJ#GI6>oaqd!V?*p*rP~$I(mPGl{Cljs)UESx{DHD!qqgoWc}^zOkvk zEX8#+ASEV3w0i*2^xBmHj_!xOV1DzI&#_dp(7&1)rtA;(vdB1?_VTGHoFLxVi)tn4 z`6BgT{P}y0^0ZzH>FYB|j|x#H#t&~1G=bkWKH+MhG7AjZ_f^J^pe`+MglCfV0OSKa?rJXY6WTtcl#1=Y&VxRp#1NGZ! z0`Nl$MNUAC-Q<#~2CG=9jqm_cw0KoURm881&fk{GXKDvjSaWUb=2Vk-#>=-y%mi~D zkjTm~XBK+7ilGJe0;u%sf~q3hx^R(XG%Lgp(vGR%mGe@zv^n>&->woc|P+u*wv zTs(gKEkeqL>`y@C0x)J7Xf0^dDrFE!cWo=q^~#u4_1{Une*MJj0GTr=^CPE=Tq4Y7(Fh9@P_Yt>q%6X4kaSS#0sjYK|I2pCuYOWZ<-Y5=q zDl5T7)y$Q-TzJa&9q!iqJ2CdSvc8Q)UFE>>iPNIh z6*W~)&3Cn2;8e~Q0%Y}Jr^0a!A&@tYW{TZByRi&r|K^wxNM7^0bN)QCZO?-r(2wmS znNou*vtsQbQ*-3_;OEfH+dFlaR6o=vH(U0*4dj%c*}$(JaO%3fDE7f@AvZX&nFyc4 zGBn^m>>;(FV2I0 zD%wt6-DI6-`*0Z}W+=GABi;Ta{&RCru1dSG$5`p00C;Mo4u2+E*633!s|i zx|rrmqxsN^IW2#h*A*Rc+)5B?a`~LBy>)ADVz9eFjU2dfY2?G{lR#ak=oiv={;yeR z{}W3J`d_vD{wMeSuih4V`kySy|F!#iEn;A$K$e_3J$#xRe|$om5mC}-pb&K7_Yc*n zD3$wg`Gl9H=`C(nVStmzOxzB9iKx%L3ycOwFBT3Yc|ZAv6D3%hOheYuUc4J+!n3`_ zK$0%@&c$Qv$;&V^ABF3UT!FH@dSSde@-?0;Ab-MviUr0v6}{X{kDT_&nR4$XNG+Wr z6y}`!t*OWNPIc;qPu8nKsg4_caf0d9e`gLd{QxZ2&jZsS3mTAMT%pDT03wMM1;_h6 z;po5&EpE#c}I&sX14#~LH0d{lK{=pR3iqK zuK}66rBzU~a9axpD`-c2KN8_9vLqcPL;RDlzU7x9JT96NGWPazJ4CZ1!P;)r1(rx+ z!vllk0~I0C?U_5Czp1JlD*u={{xe-e_ei^nnmPZ}!4ZyqWh)JBR_klHLqIJ1&p3cS zE0}VVIlD2+i{BcWVSB<2F|F|_@xL^oZS0g7+ym70T`(cmYGmKlF=ZD}e&QRw1^soK z9%T{l*?i?gGMz9KRkHddTe*T?e$Gi}-&!}}i~Vn)Sa^;Sz5%HAb8v^p@Y^>_nFj|+ z<%ygV%{lHTOq`#3JRskWu#s{9$az0mQ**aNw4@Rz3gf#C$_8o(~@Qh|ob_IG|b*Kk--PELQ_*|RTo<1{iO{lDOcqz1 zi^8-fvJ@R70qWcZnwkeTB@zK{lnPa@w#K4iEt?tp*#35@{-EsXJ2iRX)6RBw=01PC zB#MiqBwUmJE)fpNyCiv|TWb08k&?D|9Lz#vXh+rL(ZLGck@r3UmglI{=~0JUX>${E z51^AOEiaR;6f-Nf*g$oUlIfTjw20vAp%!YP-1xxMOzYlqbD2P>|r=GIkZjL5&- z*k+N8n$9_MFupj*)U@7WjJZqN zB4d<6GhHk4P>i$X)Vb)y_2lcxRstKGy_ZX0`&a9u7@1A zcywpjPTNkcd%tzm8I`azPa7yk9phOZ^kJOslJL^q&NC$;E2D-f`)cG`JP z>cVT(c&}+-;#*1^(-KJdjf{|pvfOU}N4HDyE7Bd(+#&C;>})t-*mIIJRda|F1X}sJ zz@^X>p%bdKdQ3P7Pz%(ybgH-dH1gJ$Dj=&3>O#shW-eAdKUle@nDi{zLVoA9olfpo zL(NGK!Mpx|8b4xa5W^}#4g(sHL6|XExz*0m(|t7A(=IX4Ex>z^Zj*_r6{%;i@r%eSWqwU4S&F+@cU2Uj=3@8U_aGPi<1Ijzw8t* zUVW9d`=j6pYPASU`HYFe`b4haWb33ExMTQL<Z(S8%x1&Sm-e)>>oc;rbM+C6!c50*^le#YFeO-0@sorR-`OmWyRx2GM*Kt;7@z;+d9zmj7~MB zZx~LTE;06%eG6HnN_c1ry3SvfZPw=_bxpxxJXBM*IK&0i-4Z~+lYR`vpf<%3-t7!b zK|dDorj%YLqE}c(+A&(6d~HH}od?bnZ0m--W&CdN_8gXK7rnN_Tv6m9wY~;aLxJ`z z2^d&Mmk~v8Z(W6z)7zr-%&8$xm#b)rQEGZv=uN}goyhYa9)FJEO}hIoTczc3+U?sn z9HRGOzONHNP^%Gy)+v!E0N>UWSCED!7AO!}1l-eb?`uZD#Ft%uu38lxQ9UwAZ+F!m zZL@tm!zCP!>mWBmqe|`AvZ_{Zo_0Atx=G3OlvvR174`Co)1&_LPM!6yOW()w*_m=f zvJR=?%DbUD)l+svt1uJY=QDOn14)LMuaPVxPJp?NC5K^Y#=u2x9bU0wiBtii(rl;^ zxUH^OG+wE9-Rsjs3%feOC1-hf9Z1iPgmu3*%WmoJ4bSZ>$O{P!Y>fBkE}0P4fh z^r<#X8AwB-+cNS`g|UnZf_$w9*OhneUHsJ%c7YM7@pt9{ypQG5?=)*^t~g2tVggt> z|FX-6K-Vk zVMyWfs95`}IUi|;Pe^U$g5-=&H-UwG1r)^4ND4(M+(0%c_%up1L|Z;pE&QI@FiF(_DBU!p;sA z!L{2^o?PV%s;gRYyYc?u%^$FY=){6#5*)xG!b+1Vq$!$bZaYg%GW4)Ai>eFB8Q9iEm z`yKYVcdppTc%t%25B%LiVOhC=J!NxeQrx_vuZ$T`SIw_waV(y}){;kP@7^w7b; zBSj{Lio2D!WZr~ouu>k3%0r;=;WralUHQPty}aEZ>Sh&y7x%v>;ht7(t^Ys&M>)rL z{0>-{N*o29St9YAlvirrUac;N>zq_iebwMd{j8X5#bJBqn9`fK`h^Tk6Onc^u{GYl zY0G=4kWOrAxMIKVHWF^>0G|^{@VKe)DR6 z8wk8>jBV3xKgnpTnuek?1?f;_bzalKs$PW;nVlA_qvLOt%MM+ujpD+$a3sA<{23O5 zLb&i2(_O1PL5Uapv@9Ho0`%49Ep%%=Oh#1a<6}-8UABG9Gj1!zltzSj<8Ocs2h_&7 zckjrwnL(ObOrhKQ zK?v#xkf%rqnbJ$zJfjo=E#De~{=6Yu7Onw01t}0c(+F}y+ML}0sf6;M(NOvN`pENT zY1Dc5;uVVZZ;Nf9&^>ddP@cCWA_>UU;;GWFsgPDN;P#>!OwNy*#9YatSH2`kzimFe z^G+D=zHrm&iP5e4CMEFSTG9MKG2o%ZE&U!nOFgNUJTz%O zs^$7L_ph5fg1Cm)#Hc;69+pu)h_`*~?4bv8`OKw#Z7tt(cj#_ea+0KAXtnMB zz}MYk;+6XK4!`*^ z$r1F2*%j~D%slOVVfu}ET;bZG_#K}`)s@1`v(TNyCqQGL#g6t<^IP<#o6|sWY@^j% z&fD{su5tYkOEWm{=g^}B2y~jYkTA=>lzb&KJoEwwy9hk2U>iUviS?KoQDB4&)JwxI zGwtaZo@+II6>Q}*2^OC&7zOVoid=Padgq&|nB1MpKYHMf+i6fUFkqb+dEe^y zyKiJ3s(>L<(7nkOjCFtUA6%(r;7-DQ^E0pFOzz$dCJo|XNC%4uM9*MAGGMubmUt2N z%KBMMU5$&@TFGU+BfD4j78fUfb-3JgZ2mJSosZ}EY|FpEwE{4HXAoI#CU7ZwD_QqJ zGzkMjN;6u_W-!|W;K-W+>ikHS^esAo%6)qC9S)>gwayO&fJP$&@6YeK((dPb`>9dz zZz?1Z$vo^T*^$_7{IrdFHgU}tQ9!XvunWxTDrAoW@N9>dO_vXVRTvl|fDm>gTF;MZ z8I<3&w=}45+R??Mra1oPq~-gHK<{0((KB3oM9dk_x|DR!edLkILeB%c0pVR?Uhx?2 zcj{nj<9$8$J&2EU5ga%W_w??Anig9q^0l?mBA?h6$ja;l9)6>WfgYiUT)7%!a0G7* zhK-FRi1+Jh`e&JB9{4~!Uq7JcbD$~-9|imrDHSFEmxolc_UeYQdP9I>3PXn1Cc72I zF}qf#18*m&!=~Dso>l@x&6h%&ljO4yWt<4>*iZqwtPt>u04)@I3bK!UPf9mV$Mvmy z%MUMG_&n9S)WMZ>tI9`Hc_)`}7}^uX&U{0DNraCmv1A{C(2qE~4m23#nI^oTtfFE2 z>3Q;_>6?}|mdLl7`pnLGr{MPv8;5L@zNH!(*a{?H@OsQn1yYZuK5B-1n*=C=ov$d+ zo*;v@OELX6{eKR!m6pY=TMz%J56xv*hXU6~v#-4mv$l2r5M z#K+F$B0if?KTrsXr3I|6aL}_EYp@#y6C>pt&N7q4;=HHdPI;32PGGhh7I!<8j_TU(qw2da>U!(Ss#p>je8^aZrPWBZ{&vC_7 z6~AT~2j>^~yc;da_fV^bn1}}$nkjCy*}6Vr5<5~mbFafszAX>8(^J@lPe=OI8~*G} zhbYuNGck4`cr8_|8%7xU3`%V=e_?mwof%?u?QoVZcSKI5|0ZpZ>CrBYLJIIVxE{xa zhnesp{zYX1v)4GFd)P8yCl?S5SUdp5vT|pxzqrmT>&jBE) z(`#H(^AFGERewy|SJD#GBew8}^&;4MRg0+rCdo9eH{Gv&37k-y4X~MKGLU zcnzR{fE^v_sMPGNWCmJz?#}!*>zR>*bFcbPReqGmNXTU2>DMfgyRV(zwxS5HFX1|I zEXag|c?0U%X!d0tGzApUr0+c=8=#fkJ9r7<)ugJAHapOqizW}!ZfbZ8XI$1caDJan zNZ;*P)Oq*Z*}e-2T^*>SJg>9ReKE?L7QV0^52HmXb}FKI3=w_}YLvbH)FQjU1c&{D z!UIDvEQ@uYUez=eD|I3KL{-iYjUcG-{LuYasHCNme${cb%S+{?_ZOF+vR8#0?wU?y zp}%`>$Mu!zOhjn=awh9M`0awfP1A11JdrX97pg9HFQmp>I1a8LBa)TDzr3?65@g;> zht$@dREiHp*LkgJ{%4`|jprBW7-Y+og4Y;7fVURr2sbxEyZ(F+Ac{J=j4a*#+2)+$ zDLnf5X7Zt+{dvl-3e3h$I68p-RfA|h)JLn3_^O$FWcOD3KPXAk(iFn?gz150iRvN(cq44s}G1$vh18cHjk4)+TUJJWYX7 z58E&=6))kn;8Y3@6My_kEl3hSx>`9<|HVoHN?xflSZ68P4`67<8IM5QL6Yw!&~mNikwd+FVz%)2j1H@c4pKOAwgi%bR*wz z3=L4AQ>R7$6aqrm!aO^n&f5T6s?L;z7?V=)Wct2C92x0mFE}M34d_3cq6M7vVFY~@ z+{qoU1JYS9K!_x-*qZ!o6nROetoLl1L^dZf4_6YmJO`CGyn0T4oz^fwUX)kNrb6lPQzX^92h&UVxn{kKp5#}7lXFgNMyaKVt9cuIZYGUT2A)gQbA8~5i8`96B8u*MpS7%NJeqtk8>#c6nrS+xj%4; z8?neKhmoE4v#g;iKgGDN#7v zl7@{4Q1x;km-Us0$PGD-Esa&%Nk7t2I1McD5{^r@7JC&@90$oe2rA$KV0tKj*QB>% zMaEqSom&gB?|*4Iy;T}k#q;FwYNcx)^ZLkI`jI$Q-zD!P)TY5jbUwss+W_Fq>C?tT+o>1d%EG3`(fw;m=XS&c#m)8OLi6|*C zppZM&JHQJ4{ii1UMgrfV7&B?%72N{G^M@;({Ze!>-8LZM3}ymTj$`c^rUb6|Qy=Mj zfksb6PI$q-pTboAG+&Zy>VN!_n=a?kuj6aLi;-Gho*(etUsV-h5gQvR+$!qnFna?~ zbj&6Pp#n$~Bv=n;NRDLOV#mFAX`_2D-!FCbDbNL)YA|tin!$?xmU*5-(E&cI(mW=k4e_1L`+#o-Ymz3_;RA_Jv@A$~YKrkP zMpgS_vZck9q5LxAs(aAzrQ*tGxib=*1IO>}FflyTT++Uj&pL@_fG9_{KWqtRoEK`U zkD`e;t?pu}!U1`(X>*2L`WqN$TIOufb*`-PO0jD3;De~NqbHuUOSJ3FhhoE|*x=6uh!+dl-n?{E%79_?pv^;J_eWAuoU&8Lj{ z8Ac#{No7N*&uk^@O#1DUiuCb}b=*iTNXJ?+F&?9-`At)i^?PVy^(FG%_zX3oENS$W ziD-TAR~}FqvmNQ>_V0TSkW@$VR}Sx*t7t9ip`U3ESqp z)qv)-(G#A`C-6yD5XC-G*d92>u5Hx^0G?W_wL8yAIw(i;1w{f+fKP9F3- z9$xmAP2eYr29nEg7hu|+Om6Vg`JM*96=`U`L&Cn67{ft{dOWA%hq&__@D24{R4aQS@i_c z^e*${1n05xEoh|t*tCZj!XbOWRlS(~V0-IN9PdQ(0{Sj%TaJQrP_2vQxgojw2J>9W zffM2^b#e3M_n?}@RX0*DrG0p73i%8PV9fccM>3{qo!aGwdYGzeo?}+HxP%XRe8kh( z8VvQIX=(t7KiXs=&nf3ARIFZcv%F8M`24X?8S18&H#H$n#2F`h;CWUQk?T*-8omYSjFPjlSi73x%m_l_q!95jb##74%*NYw+>*pdC`UvRvwD+HR46oj2s0f%BB*-KG5q?(g`08M z&qx5F5kRs{I_IN1S0CjIJcausbh}vEPI=jLRDCGhkYWGp52@#7J}=-c)jvTAQh)qD zQ7_dVAewWmfsw~k=Nkq=2iRjuP};9aSEmrf!1bJc1;;+~#J+WoGQHIEUN)Rh_8Eoi zPd8-~+-V7268sDb{gMepHKkBPw_Ih&uqV{eDdSBHTHo?V$)`7c=VPQ|zaM?xWL_-B z5qK_w1kp{)cLy;IM9_>Nh>&Owq+PRNq1X#Z17;VVbXg-Jw_a>DHh^5+Zm4L<$M$pJ zsuf13>f+aV+@V>7&b=1f)g&!H;%DAZwo&%(vg7TBz|tcIz=NuZAzQ2KNQvE6uCqgP z<~Lev2JVg4x0u{utBeWY{LV@6-E-EgO)}Nm2*$Ykx)oG9pzxc3If#Togpyq0S{t@t zNv05-JGilHe64O08LL`e_G9dzxMuCU@VR^b{IJGq6(65AN4uU)czr)z>d&tGZ!?6v zvMq@TSTvcVD@S2q(6o#xIJ~;PcCRVi+Q&V4Zkgvx-kzSv;p<#iPPp6h4LUJHAOf{d zd5P{ogHGbrYH^Mx-Wa3bdyaosh@LFiXFKz(oZ!=U!# zWXX0zIJrAoAwxYf3geM(&%CjjAo6(O!7~`d=&2BfP#}1Qw|9gmE*2V zg&b|E&;0Io=s;nL-B6~MM;EJR=RsR*^Y~A|LpQ8}o$V|m>A(W>Ywo@(Xqg3M?C-(z zRVlTD;ENU42nIDbeGC;N$z+vYjcw|EB-*BSaoIMXwRP|pe;!ND3znZFfW81iPSOKr zN&t|(iza6Onw(aP@5VvSUUDCfB?(uVmE1c2vwWZ`MjnK>3Kv63{qv8YAb1T>Mj&-l zu%y1Ic=8!=5%Q1pqM1i^U#FbjTNc)3d7obT5K>sm?>9~ona7eA5EFdN_!kL>CHXrLpPr+*qs{3u&QEf!Ka1}% z4Dsf}gqoUgNoVIPV~4ND>@gEO++I>lI1Gbvz_>C2w+hpPjR*D}Y_pmVK)A6MqO%LI z^n~jJzLWj5Ae@tZu3zD@>Ia^&p}t2#pAXzFPPyhV)xdqw;TqrQIi?NR9@ev!mwTDq zigS_~P|DpMiM( zW}!I^ew9Nh8nO3Q^qdW>7nMbaPO7ahcC7C^ETvp|IzP5b{!uK=A{2mFoQT`Ui|RWy zqL_T-6|bz$2$a+9AS*ApWV@(1Y)$H+Ee8q0AY1TY;5pZ=G$aZ2dlk(y>6_0psTX_| zUoh2Jn@GR(+mE>=@Z!&e!;kJS9|L3^R??!udT?c=G;4#RjXk^?-=5{XW_YimK0}K$ z=#1P^_v_uZMu^CV;~gN=X+1z4C;hreN{p?MR_SmRACuJ?Z*V!dTwG2t9?Rlo%X#FalJhW+vdb+0GZ1Sd1Nb9EAF-!0INZ&YzyNlAA2B@uwf3X19 zKO_eNIs)qnkG3}yAi<;VAvvj@R_Z=Aed?n=<@xHJFQ3@5TMAm_@s|7}te^u(Ej$it z`}`MzRb%Kvmh-(K+T^RQr^wNBZ>cWfor&4oa z2*<-EJBF30Icd3Z+xHFOn_WA?AL>6u;-GGQUsjb;UanMGAVfC`qH82X*aFGax%9`o z(Df32Ct!4CQxxlp{jYurj!yqu&rnrNbFN$IW!*k7q+8TN)`Y2oGv|)lSRaQlo+rM z6j1PgETGj5@Wi?}XeXc!FjIGmGW*f!&RdOv?`dWJ{%~mSO^rqc)(LOqA!$IV)9PKcX%lYcX&4vvCIrGt4c{g6D?lJaKx3&|$BF%GDKv-Rrn%_o)*~o9Z-0V@`B+cF{d8S{&k8?3RekFG;(5p3bMmyv+3yGsELC%bh-m{5dyY8SEVfl6IJ!HNH{wBk zy#$vRswmzf8$3A)73wkPWsl>ZFduRk4q&LhUr|W zBR;-pOoy}pP01SJiQh5g;7tTLJ(P43%&R@^u>;x6PeX+#9Fwx+Y zk5~brxy7f=_Vd<_Wa;K4#qyRi_nV?IJes`+t}W;tKj!uVcKGYYfYUz!+rzI*QK$NH zqwmF*pRsMMt9)M>39YO9`YH;XK?@%73<{K0E(y2T#W=!L$Hi%8(C21=rgqMNoYy<6o;U@DMy!PP6Ss+8YAxa?f1s#QYM zNGd2u62Q*JiCX)I4)-NHAcToa0amiVFBGcW6+w%T50#;&%&}J%2Yb#($LJp?(t7JN zaTZy)Bz$YOkW|?r|GQCEEvzDJVaHQG|Atc)$LSK)B+y6RN4vC?E2^#WbaJsH9)TnL(hrTJv(qPN9)6f*q&hGmDhgahud!G z+5N7`Ro+F)cFonrY)&)+?N4*_npLRJwXIKZQL>J(j{0@^j9dhoJ9dKWv+N&{C%(ql z0_#t+#DWTh2C!C9#B~HjWAVoC+^E5{_b`D|A%F_AIbQ^}UUyZGh>v2*k4+JCqDifi z_eiZRl-DF`;NgOViHjHX^F8A8?Gk?{HXSq%Sq%D)q@VVu8G^8o6nyC~47P=5Ua{Sa zrOHC|QUjVlG@knm+7^$(a-w!4=J2~9ay-c7Wqv}CAnO?vp(482ip&bENJvc`NLob% z_p~a}+&|Q1%pd)9_SnlkU9Xisaq-Sgxcx!SYf!c7DZ#_^>!r>(ny1$Tt)sJ665hYZ zc5bKJD@#4*DW{Cg$nm8nbbMx8a}8r%Kv1=?%u99z@SrQmoimET9CR^B=EdJM6X@Ho zVhH+)Lk>@X`hj4FN;VHG z`SV0MjC%2#J21oWB&h8T#EV61Og44EMeZSEGB*yLcC<3z8rb*GV7n)n2liz&~^veVHkI97F5Wf4~Q;)oHX7B32zxVcfrL&Oy+HP zmVToQ^YhK})L#Z%8P8RiR_txQ)y?v3|5VUkcZH*jLNYqO1|H=qB!DpU@KTb5NzHI=DeB$CW%sA%({-?AzfM@- zWqYMWLq)1T(IZJFADV;sy>g`9XI@83e|`5zY9-sL?8Z-#JSZoj7j5-0Fj5XN(v7L_|ULm2oGz5a-IEmD$tKK_o znOw`c6UXZIJ@$3|E_2rY(?!Aum;8)xV%PXOF;ux+h+c&KkKGfLCS)>#WR0e3y(RX8 zf#Fwz(Q|Ja$hxl7ct66DX?}rsd|~{d1KMovN2foL`{m6m3GWlX@2yKyG&W`Uf&fN2 z2Rp~j$K_S;l=kkCC#X*!%c$_Jcyr}?z42$yz=L*XcdhrruD))@GbQ0fJ_aOW69ot) z7bhTc&$_!)N7{U*7NBq&oN8u?d-nly#B0-~P+FK7)l58TkX2dKb zRvUejP$H>?dK2`QZbna=UZ$O}V^v7@`>Y)rhDgU#4G_$WMd^Y{EGN>n#aWJ6WOe6m z%C8`|=XZywGugfV%~iuN_kf+bf(;Ac(4G+ONGosv!{xQXE)U<*1%ERyB58WyzO6`j zX0}f;EA44IN`hr3ec6>&7@`M&{c#-eT>{RFSbZ}IEtOuz{LsNxB~mra=t3M9z=_M5 z?P7+kBuWwL4HbvmsWFr%(Ps4&3xE5WshWzP3qRd_sVDz^V5NOqm&%_mz+{Z@_zUw{ zQCZ4IK5mTc!O6>JszDZ;-(mB2*Bd!Fia&91MxD(-AVPBG#=e|8kaH*jp!^$=L_Q(G)$~ zAaN8VsIHpQ{TC)!RIBUcliHojo9caA;b_NKL;IICLuo3Tabf$q%sG}i(t&-a zerRs~kh#3MZl8Jo$KB2Is}>#}`e_Qu77HoJnZ-2&(lhiV2jvzunxOaa+tyT~WU%MO z0lDVn<8n^5qh-(cafNg5)w~&I>Nf^qQ_!MNTP3n?JgV95o%a}?-B-Kxk>RqlS1q?1 zbe^WYDW582^5d%fA7A$Gv(LWCEAtw`ej*4X_@|A(pFo5)=${zm9lT(3F&X+vv87cBAuU=lE8S00HRNl)D|+T{SFwxw}ET2T#^f4 zXO3;oU(MO0V?&S@}!yHCY_ zXwkA%uh;SXCFgU|%0GL1-pk8AW0II0x9U%AwAparU9A*HzyFAF*Z*ZcmE$PE=5ga; zvV{U|eA6A&V)ox>)ad(@xy=T*(N8vmZO)JQ4sn;?_zQC$E~*=bZYcf87?O-m3Xgjw zm;B)2_;v$nsy&fl6YazvtV=O5NTATEuV zHk3t0P@Yq@iO`ylQ81b*kQHx3J8F%0*EL<6KYrj>JWW`A)st0(ibf9orb~lVgY6b| z6yU2|5V;=*Jw11mGh>%>Tv!h;K?0tut()}PXr9`o8jUQX1 zlRm5^c#S@oaWLjt&c66ZD%-BDyHE~DvkBQbAYu#$Nl)9!3Vlh^_c9N1P*lJQgchDy z8rS5+)`j+?{e_&&{$8j2yk~rPW*8Bh> zgPmsFJbZ7-)1&@<=Jx=BJLH2W&dAnB2Oi5+^B=-${_D=RLRn<)M}oVsq66wvxOd0_ zYA#eBa<#ICL}?*w{!pL|$jqMB#f+9>;RuZdcVO zJYD)aS;&C>N%SkLjNkrur1z#LDsTwR>%+8V4+;^*58tQ6T}!Sl@692Kul{@QT8$m5pZr}eY=rTZ$y-}4H;;({^2b`C|n3e^d2 zascl`J+ynnf6mIogFKe;)|L9ScK9!>V;Q~r(7U-e0ScaVtENx{C&_1Gi@g-dE+09^05)Ww~g&ngEF+ z{*whkz%}ivD(z=bSLW5^nfvP*<+YM%p_S+O5JV062x+u^2Ah$;Iw0R*Q~m0PLG^KS zoq3*)@n75b@9PxYSL^JkvhmZ)Pyn2S&0{B3Ef$jRD~xZ0jFoM-es?tFJmVGU2|!91 za?_2P!90_BbV6Nbm)c`7OCs*!8?`48O3$?Oru#RA>yp=eC--%s$PQqIMBIdY`h;9s zs-p?nV?`pbHczLgH8Rr|T$C&KnE#$GUA=wN*X}kL%r4Bb1d+VzJNEnIa!FUDSrd=cR2BuWEQ=+RMi$LS9|})FMcVVZd;rICuVsQo10730m-Z`?F=9*)k9m5Xsdd# z6FhoHKr$=CsH&cM&nvb zASeNfhEvOvPYh*7<;E}@S_|1}lgq(QpKl}$pYSkAGb-GgeT4eGg9SSmVHrtPZ~)y5 zNBAN`ML}m8Zw3mqG(bce($VE7!u#3S^0G=-y_{EJ#t>pA3(9bOd~kE0JR9quAj}G= z-s?~wSA-aYcdAGAweknP>%LYl|E&i;SuS6PYHQ1UbFLnCeyu(q`SUn30zuxdwAS_) zb`}ootc{rjCPxM^WTCb_h-=UR2+SBGWVK3AAI^U+bA~z{#KxVzZ`YQhtxi^)tDlIC zh`z@i<^IS2xhA_g$K>T*0G$^>x|sM>VzHxt3M3FZ@8+bPZO3Z)sOPs)7yRz)S;;?#`*4y?Vgc%~Zo zt2u6?`c@oEO0tnK%W^L1p;(7n;0lF8MM6Xi?MKGV$K42=0bW|wdXJJ-8fqn}MIQtf z(Ju$vxJ0`GyIy0(R(8Ovk9Z4~GF%Fa#!hk*0cXu7qEQ+fpKe3i3KbryIZ;8VZmB*t zJu{>D&3nOo_7KhZG%LHk-HB@pZ@-(-&sfmzK!Y5kSQ1nSZMwi9GbG73x&xB-pNpmo zpb4&(a|X_~g5hx21)E!p zdzgMC^`rxHALzkja$TpvzERP{%!yelrma;Wx`VQFbyHQXtSoR`XALbCcUs}e)%A-D zM~>8_ekHKbqsF)5L8@j+HyYPBqbbvG4s7ptwrw7yBcM4+p*N0wdipvt_XKNNFQz=1 z?J1mpC-TSaFN}XEbQ->8yp5*Am*DhtWFwx08m9}6&jkVGoZ>>R4zaJT84R%?bK8e+ z-#hWcGNz`ue8@aHTjBN~m?UzI|GAj>Nd*|+aL+EEClO0MS-KnlA*gD6T`JPfz=k9d z5byNF74JJ&t%uCsPGxa>pTBVAgo&p4+T)oRFjnpOoR3*s&cVvjnyi&_-OIzCmRXif z-skB2y_ceMM!~dAYP04|{thXD{!dtl7b1+~r!2d2^oO)SHBl<(!0U4Kt$Vy@sxZf)E z4%UedL+d_@Gv8;<4z!*kLXSEAjBjSqyrH&t$XdR(jl|3!xg>1c1^PLX>iEUJf?M`Z zEk(_h&Z37-R0#gK^6gxSkz&x1saJ6XSg7kDG63q(%B|;M@IP_}`|=K0HHw4(vT}^z zT>ssX(0^KF;L!0#;8LOQihy&DWxT+H0BUJ$vSsQ&2Y{_j;A5+T{k0Ma-{PB!_y{iB zs!jFPhz=E?qRXa&AN%|mSzE`{qDnm-%aaeH6`obbq;glkKXE$WdNqu1$#u7uq5R*n z=>G4D8PZt7S0G9N{}`&qc>IH^D^r%%iXcVRK77Q-Iw8Z#$ix!Ta)p|8=Ug1c zfQI8e$7V#{lIWm0Q}@P&+cE z%eP1B`Ip9A4-836*_hq!Wixelc>a2i$oYhgNXd)@3+7uaI+1JH^%NsARO{yhwaBq| zWhg_|an;+m%WpuM{rB}7Lzbj25=V zm=dU~`}#_J*|8W=W0v?KqtH#wsx|jeK-|HB69|Vw!%+2Z^J{E3oF`NyxOE(WsfIGW zGA{+k&sB{a8olY|mQkVjuvkGEx?dxA4c5GnW9S52%VmAmieL zk=ifPRp3`)!4ov>^?uUKP2~*RCHb>ZpNi{VP4l<6Oh2kG^5%ZwiwZejOdam9KV_Qa z7W8g8^Tm>nw>L^44V7^<^0LX*v>WCyF2)?VNN1}UH5oBsMll)9Iz1YdPR!bGbAEK4 z*~7HX2r#HfS->546KG;&J}UX9^f~0777GbZKf44l=D|Qr|AvLWYN zu_Z`A-}~4&4&;}U9qJ`oQ;NNx2S;>2NPGLnU}V1_elpAXz*({=Ylw%ryAe+I<+ngB zmL1IhAp3*nJE#{F#^mk7;c0<y) zPX5MW_c3`0Say$kh^%c)X&8}oOl;%GE=e(xm6(bqG>%iO$ znbT=O1AR(ee=3s2MJB~V)Luh>?uOm9tD9tR`+3K>jgU}F17(T507ed<1}r{3keBSC zqG@M?%>ZEq`4kchZ7WoG%aFt4eqEiKu~3{o_Cd}JSyk7OqAX#es=#>xj1c@;;9Ypx z!~;OMmv1INy;X4kKy;SddD(v$CHp(v7ip1(H7`TF3Amuo0cmPv1bcdlb~Y5vSn!yE;v#>IYR>^g>4ZK^oosZ#~ALUciTa1r52_7$oU@?gXO! zc!)}7w*}d(WB&~o?SKYC!VfFC6K;-Y%0lo35zYmN*^81njt|$A&4UAmc-b*LrJJR zX=%1v!&-NiMXS%XT^2lTBH1L<^JEL)_w)61sTG<($Xyi2em>;m_6mU(vP0iOZRJ?f zRd@oDWQ&+Ybm#74YOd&n>d^S2ipaiM6}9m#ku$j|0iL%g@zOOHIu+2AeAyfMqe*S* z(qPkFn!~z7gdR8xgi41<7c^pA4<}lT+$7EBl76gMHjIqT#21SBFF&~b{^O9FSJHfa zcB+4J<;kD>VSaGZ1qAivIHkUdQQNBEZOBUA#^Uf?R8QCDfmN-C(n8j)c9xE_eA?d| z>yu5BPe&#m1Zqcahp?-xyCxVqu10eU(I2M!&#ypFT_KV2+9nsc?I5Pc>Qtv&OvRy%#%=?VJHGLdm-D(Z}hkMybFG$@9+-Q-ZjhIT($5Ix|J3y-Er4M z3PlM_dU+(!!-$LRx)La8{)`OQj~ z(vP8n0jE8XxN)>s`0>-O3E?emDytEEM3*iB3cP!xcA3I^m4*zJ zf^Fv6^F8MtAwQx#i=9v_-utsVzc^TF{t-0UqkudZt-PG~{F`i2$_^L}l1o3pAh_J> z?}WocEOb(f&GL|rOQNDXu}1A<51%eH-j{PcR-0`%@SNA^`U-Nu9RdgmSlS%QP}@Vu z3;C^=vYJR#0#_DvqHV4n<3rrzRF%cF@bJzq< zfa77Xn9~46z(~BQeOPo$w9&AI*rK{LUiGmT{ph=Km5tgb-0eG;38dgWKCJ$w_4 zYMOJWon`lLmcN=Lbvqx%a9YDe$J!Rh*|j2l+`I?b_^BVLNLw z{k1y`wr*1dp^>C>HXOI`de@{hzZ5|Hq8N|A|)pog%PX217ka?7I0Vc1!OTv!wJ72iS?+5h%>wP^cSD zvP;~*l4CF;X+EfLQ8*jyYhYZ*{ZJ(~LLg?*<9eN@+vJs-;VWI5X4{Kyk9gm?@9(Pt zqsFJ@J_}dAznR4O$z492^CX)sTMdAI{>@4J^KVDcVKTf?4)iSr=y4!T{Ua`~M zCT0D)unrlcn2{BBn3a`EO!t#u5z;b&+MKao8@!?8aMDF&98#cJZv0{}s#E4*+9N~D zCzq4Gk^~PP5_#!^R;~J#U8c$M34SVlgp(x$Lf!%9@&F>F&H{u!+><%gdN^v_IJ2MV z5EpQA8^Mk|BTkcWHlMv-xj#*rLW{e0Z2HqP##cj`y!|%p)F~&b|D3Z63OCvME^kFt zF}{V8eUDPF^ZiA^?Ice3p6VlQ3xcO4Hn7BoP#tLSIu*RoNI<^d?Y&cb9cEw&F(uRI zJsG0#XGp3Y9yDQ1BL2HW|F;|o{m);1jwwgCBb#+Hd<|?mTNTEeFT6Vgt4r8$7*TA^ zIT3N36W8;l?Tj9O-n^=f{}nBO35jgLN!CcJ(qaOrap7y(BS?#2R8PQ2FIe-LUkj7g zqt8~(vejz}o~D;(mzb-6yD%Qcx61lox4HWR>T9S3h$r?k#i-~urgU|zHJI60H08Ae zM0h5Ae9(fO!l(}^k!Ni+EkI9!FiRp?sC&n(Ju!M`@mYv*aARNuu^cj-w;QPdQy!eA zj3q5qBW*=}ip%cgr6jQEI}mI-fL35P#bwk`Ze{!e0Kzn43O|aEsGDo$HHn-pHFu3m%Byh-zs~=X??`I>zha%6)sTXoJxY8 zPBIG-gfD2I3>gEh7A`|O#%~5VR@`@MHc3!DmM%R?!cvYiO6ZOx>{%*mPUGpt;6oHE zdArf;Bg0ma+MGAN)_vwJ2 z$Nd*(qhm*p0XhuZV4%{VkvApc@loY#MA6}?&$W9e zE%@0kbY1zt*>m}5d+tk5^iPrWS07joKwmaM>mFT9h~P;V>g{HVQ-kKQk)n8skP885 zM%9khObd2m3l%h-)c51+8M*Z51!2XSdtN1dM7+_aW5Ci`fN!Sj!D;;Wu#Cekd3QmR8@KB~@$k9I|kK)|Zi^Zi93d%#tAwpCf0Sis(Qv3(# z?!><^r4E_B)QCdvmx4R}3(d6|Ts3!bvTbalruJ+yX0pHNvf~u6s@Xt9{^BIFA3-@h z1&}2hBbh-^Ezu>}FrWTz$I}&6fgW_AlWPny_c6p<6n%d9UM@EkX0S(g)>v zU`Dm9@@yVY^6Kdd3k7J?>Yhkf=pcR*Z3v_cDuZl+@Tm7;R`UFuCtYTk)ZGs7w}5zE zTrRu5-JB0s*BY^MH23PWPs_)SwD&F4s18Og3s;`#x)Uq|@(ng%h7oUf8$sGkv9}C# ziB^kYA;zSQt7Ku{MycPY!k2At>2elLvTx3v5%^ZtmT(zblTGS^UhL2unEOY-la21; zWbYRMf%%NT5q!d6cbUsWZSDYeK!Pt%$iP=kuH*{o9|!6DNT_n7ghvL^quxS(tk9+E zL{gzyjDRqQV%ujfn1R^>L^m;^9R*aUczDWfO&CLYVIiO@k z=tK<*1I(b31G7Ex`yrqRR;3snXDI&Dhp2`)`{9&&J`v|zss7ksLWPmdrQEgj{Zm}| z0ajj3;qPW6cDo{u>{cLS)Mp-nCeEgVZLm(8cFbsRs10}vW)NZ3CJON$xn_r07;T%@ROG62|Dw*pR^8hu7fabk5SYQg2 z2O?0OcS>+Pug&LqSfm7UPz|7b0elsq31CaP;SkBHH<2?~a!w3q9;z9f)xo9xty$0C zhhyU2)KQPekcN}x)563r-lA6qFIZkXcO`F9fxZriL&aDR_|7pb7lc0rzW70hOsH)I z+Dt)5BPr(rj=i-~GCtn|g971I<7;XJXsVxbjRcf&**_n`5OQU_H&g}illOf;HxP4X zy|U!*nkSzn_@1_1^SF2b_iu&`a5y~lPty(ozhv(L%Z=BN4H~Zi>*Q_e4y`@xSThG{ z5&RpszJ8`zl$Nw4F7v$akYZZ8yL5JAwNy3dyq&GZvR_}QcF1jDqhqbYR3NU?@|tQ! z)Oe=vM{LntD_;#DmYKa(FN+QHFjt7o7Rb^&^Sm=Kxy0fE%>@a4l!ieUfgiKRQ1$eP ziEE=Y``{p=YdLAL1IfFge_gh6a9ey^VI=PZ*5iUmb(x>Mslw+y)L)f@PKqLGY7NHR&<6Xq; z1PkxpF5jS+h`?Fu0mgxsQUp%)LND7hV+VTy_bdFI@}M0SPZv>Pa(NOGsfxO*O@3sR zuC&IeQcCQ|P>s)_Pd-@=t@rK=3LM;f!zIalb?xKj6FlE8Z9NJFYoTogzZs7P47^Kf z(AN070gg=IknS*f0HdJbA(-+kaunJ$2u+3B6cB{+oV{ROH`3Wg^sf^QbT5+D%ZaqM z>lHtP`hI+V^Jywmu6pl2W%JnQMr`~aABQJQ#Z9iVF}C2GYZI^cUl9Eg3qo9idgwP(k#H~SbKlGSQ8k364crgvtK zUuUQ-^shumBRD=Nl5Xj?OOc)I$~e(D9DurlQ9HzS*^1|>v?LFUICPPhlOwa`D&Okc zzpz71q$y^n57-ARQ>)z zAXfy%>sk#HEK$yuNFi6;-Wv1#i&`enl}n0Ypp^?I@?*=E;~j*%O9!j_HI8fvLgps0 zz3Z9Lbv3&BcT^F=SW+nn5a(;Z*ZfZ%3H|Rl>3=w+6#@pfc$oGlfWJp8O~?Hr5e`y}McDC2!DuQd#Ug>EzrO5D*}T<2jmt z|4`rwJ3FZ~g)rCx7~hPA?+TdxyM7rR=uJi30J1kb-E^T8ieS@-zE2@^lYWl+E;RFN zHCl-ta>cPDQuX!%PGtJouXDkBoz_Mcy)b)*PW?mrjDYY)PIwYG)*?xDzuF)neWe0G z^K^I%#-j0W2()qvn&Kav6jS1Ta#TND<=2@muw!^2R^QHHMUl&=pqNmtV2`hxmlRI9 zW>+{`pl-fpI(Cig-2HO-!DspUciKBMV4RvE)B%DVz`&S1z2V{6aoi7A5nSHydiQV!xRF;a4BNuGw+{@ zjhBL#`~Bp#s**m0xg>C~K6d^5PfWs-THW4=PN1v3^Lj8EU-7PBMW^Z;e!|66A$)zG zZDG=>97tUJ$~%tmeInF@WT5pL0K9wEQ0rl8^4M=m#fZNAeRPS7iD4vPhcv(0>rYkh zQ6|6271D2?J<$DfoobWuav^sDj2rgF1muN&L`q@Gd@EGZU?4MUT=ci-=7I0wXNN1q zUBvI$y$Cbrkc}HB^?Y@SJ*N8Qt_2WI8cn1NK=NR;A#FcsU4?J=%Y^p-g$Wl>r{>Ff zJFi=oZU}nsbhL^+^HV^p3QV5ikALGbm?3riHc!cA{m0C8MjohehC@}*)3PlnCRe0+ zFTvoFZP@gb6Do5=)Zd0*FQ}wxkAuDkHZ7N`9pT(Oge5dU9BON5_42%tym23l{4>bj zr0JjDe~Nu6TQP9 zo3dG~YR;NqQx1bJwZg!6sbqe^iiCK-q59(>nO5PIOiJacsVmfKq3x6Q~;(qmw(`laZGAG_<%cxbaoeiFm zcIN!Z5ibqJHF5vhYWnG0?vXkBW7Z_?$c?+nXP57mm7QeUb&{yxC??+lvU&kxwVNj4 z^tsrzr6Rv*c|6^=X4S1BTTSaua^oV;@}U>(CuPr35yw8ULd4Gq(m5!1$6%Li$jcXv z{lkv+!V1|XfXTRHh(z>N_DXv0sHOOa9ELFI*K)W55Y@u9|+#@1l zXJ;oM;s6BTW|34yD6TdZ^piMKGBc$eE?oW)dP3H8(SP#0?U5VmG}M&WRrpc(%b{V3dYP z`hocx0uX{){ORa((YbVr#|KD<=nCXe3D0%p>tKQWI7z0 z#!1fPUdLv#SQkbNc}vVecYHjvZ=rhn>barb;~B5gJZ@MDK1c>BZV%*L2)xeLc=1Y+c?|)h>wVm0#u`B_=n>;n;>4@u1y*4QP{q zsl=^vW~7H5`~^#Or=Yb8tb5_r75<8=&K8v7A`5%XFTIgGO^JQs#>`s~AxUENM+87g z92l94m5mM6nKm&@sd{hO)B=xZF4nV0Y@ZuXlA8*6^zC7Un+W0}nlA3<%`wKtjx6F!s zg_PIjr6H#Wbf%MW0@-!Ot-iTOIC41k$2ZVziIIRDFHS1JG5JWY>F%$oHn_TD6W;dr zxy#q;FKkBYFB;sn0PO_|Kd|AMyMP`c@Y_eRa3zYh98wFVucX>e?&048_X6WbU!=O9WRo%ehioVpK&;rlPyr6T*ODsS%1X_4FMgEsb-{Hhvm#y7 z_?_1;k~kr0aKQI!sY-zg>!(j_yE@nZ8h#)Pu_T*prxjO5xFH7~d3D+q?@EPOjdSMa z?(-z~%X<$T*Q>r+eM}+Q+TtX}HIENu4B;UVsC$Abl$r?7hsbCI&#HR)*2_1jsTJBE zdlbWUDQv&Lzg$VFxt(1o*cR>VY+)ji7~ny!BizwkEA@@8MYPlOgHZ<|n&WR9|wR%AGeY*ZS!TxfCU3HM%9o_@%cGr0C z7#WxZgC0Up$7X=!65hjWRAybS&g#n?Z>f)YlUjJJBe9lWFc`cX)-c$fFoZJzcKfLZ z7wJOm;Q1-fNg<&!jDnlg$_n+oD#u_mTXJ3eeN@G{uqvE_4yCeJnf#%vg4GAdez6Dg zW}XNg0PsM1Gy5*Rv2#|Mgou_)Rm0Y3XZ>7VdlGjirsb2kK_tX6X^IIVGMvVk?aOmr zJ6$(F)HT#U@78X9dPb$-2&g>m0i_&P2-yrLJ?yFvZhX5s{x-H3^SAN45; zCAURxs=N~AT4xn_E*=f`Aq;}FqcNexOwc3XnIby5be9MH9sN;w-oG$k#01Wk^rOY6 zFnEOs7^KJS?Gn&B;}`2vc$3W(JY~a0hRz;LO?5Pvm0${>cG`f~GO5k#bAybcF$?7a;T&uymjx%1S8snmen#u= zX$60|@T{a*A9q1>_yRor*hzrB&O@Hq2^zS`Doh)am#$CZjYbiHB+$hHNAJdvy&2cf zrvdeO(8ufmy9VV(oDD0YfIz;@cOPOYT-1Zp98mjn5i zocwjT6`TDX&yl;FRfX=`&D5UGvxApj25b?&RbVgd4a`0)^j>LAx-i6c5Z!t(c&fut zn%(LUTm(5W?YLBbaX{?8u-|nhcRiJUnxCyQ9x{-RiFBKJ)s;a6=j;Z22zV3c%crFD5 zP_HA&>3SzwxS)-Gz-OWh@e$E_IQ_L%#O@IKLY&^uza99akI24-P?S-&WSQ)Rj9n!uYxXfnWSLRMWR}kD`+J^q z&hvZzIoCPY`Gc$LiWzf%-uHWby<4^(#cLw*Bg9}la3t^6r!{*zE0$3I~@s=ft6pmWGt8~_BN(4 z{p#5Ta8ul`$rdCs=9$$HaRfXu1F(`(7@l|%-bD}yO)8tX4=o#tK!T6;R@>mYGvOW& zPH@(WdVT(EDUJun%?|n`a-~cVkbl0w+y}0)Clj>3R!}59XdGfCO7s#K2LMH775=;a z|J$F1{{MR6|A(&&{eSv&J7_99xZ@#^2M}>8O{id4%+2zzxM)(camR9px@0w)tWIx! zUJyHGmfjmEt+7`|9v&-KEw=403IkiOyI+}~P6ZgE6M@KC7(=vnUU}8mpPU2Q`23g~ zufoL#dVnxy zrGw6Hy9P#P9*Q5M?ldO)Zv<-FZ*e5`Wv~qF4c{*2Wk`8A1zd-@e$li*1?`SX8NiRX zo1j3cRc{B9E`RF;o9_%S%2@R6X+3nyU`1V;z{)o07kF~kZ3x_uPql8bWdo`dPcp2H zVW>BJs(%GRS`_64wFg5!5FZwdA^2T~TY-RXPws{HBug3i#X&>s@+qI(h>`cxT|pL4 zim)=a+p1*(72L-^ps-N;6pCz%O2ySIt=%ARgT^9C))xWC=+6j8cQ=M%5%02lZH6_L zT3YKj7vGu*Tu(K)cvXjw|I;d++PfHurC&mK?S|rZ50tO=b!e{0NINKGASuwAHH@F0 zohW7TF)5{>9~FHQBSHbQs}b$zpr4icP+yEy?diU_pWS)d;bZ6?u1T>@rMLcV{&rz1 z2@iSL==)3|5fu3^G%>(RHJP_&iH#8$w+KN5;vkZCIIIUXDcy#XY*7Yd>9HAoC~0oR zdnY^dwXg8|FRuj0n|~k7zT@{%I4@LXe}MCCEWWb#VAg1KcM@6$TwfwF|!`BB6=sIKP)@Odae z@N6xDaiGMdVYKGgZX))sDjcG~>&<%6*DGMRRP;>VGY?G?M=;1(*iGtcjL{Y z7e99ixaHTcJZ9@Jo*fU-uue=RGANdsa5fm zngwq>*N618$wzOb7VSd=9JvO898}5>zf8>_r!mBoNy?E71&U)g4$e>|Q%UN|R&U6y zg+@;@%c2hF`bJCtu(v#GMhb$#*uEGaV}V`E_Iv>44vWVg^r9LavStqt>m0N5%1wnz z#oy-KvNyIKknbvxxULn$;YK1{B@pM*Xa|9!B!O6rszr22BVzB3<|!u`2^5X$?%XNq z7?tsv0D^?YkU_d=Iy2?KVUDAok{lWkZX*^&c1BIi&p}=4h^~u_2-?$1i^A#RuWv@{ ze?2#`Zkf1sGoSZBMfw|J*=o2P5lWkYX!@QkzBEcE^%^r1Tbq|Y1M$*^&oFR8O*SwHAWOHH$vYty`((Si>H$|>eV()6f-}nv{|*Dd*vh? zCep(N32hLFb?Ex47x#5KS>I+Q#g^`*XCuX&dlD_ah}m|$`UBX~_7EVj<`oT(Qz8ee(VFG(f4|>$^;c-lLeH zpuRfdfd_5aPXw};-I|W_+P*nS!w9`-%wZzk>YQq&_2s&ep^MPlD3!FQ@pBjXo5lDv zjX7)1!p5?IwO$9C?6dVCKrlob=Z%tHFm6yvAArR;NWv#yWbk$x|4O5j^W&@8619$I z>*<6IH1933R)|Fi_u!AobmkxXM%djbT|awQ|qp zg=n%u#)*Qv=a06PKNGTjJeQRfd7X{<=n@(jaF`uK^!L2yYC1)A4(d#cVd;)w*o%JU zo^L%kjQn$Ap>{;0#D0b&eoBE$D2B5^Mf%BN(v8zBt(@RXY@!g1HU0|!0eEXXB!wc0 zZ83{bb$=k5V@e=1HWYfpXeKo7&7=;f-WjO-i)#5|NYKL}13#jpK= zP9HQ)TY;JzO^u<1gmVEMKYaVPesZI}OKDhn2`)HTH12cPqc=)2mSV>hg`uLTQ({}C z3@KIM@(pNXbSEG^P)uGT#ii6+BfRy+vn1lR>C9Vqkw9)G-|YcOVsCe{(pmrHy`z#g z@2=0jd(6flf+wI3{pXPrx7Rkvy}dY5@W|)Vco_y1&YZ-e9P8nsOYd59dw!_K`(M>z z_e%fDbt|a+E%6yU^atd2AZ~EcFd%G1J;DyV$kM9G8T3=bRe7dyr-uKe*sS?sKfN)m zF*m%`U=sWL={e%QI@w#nbZf9%snUShwU?BT2!Rl9Beup^3o*CHwCqEhkMkYbZUvoa z`-L7{T7}a!5Z?=_+O%Nui~7K|aYO!>oy8xq2RCDepeoCwPR|6p=38NSm;cgW-IX3f zaS@2E;E4V0dIl!%gHoUh<3TgMKllDlZ*N0`=ar!?%f+F^Z|mx6YpYq8^5lVT2PY<3 z%%7O-uRI^f0}ccS9Oz6n>AmKVZ7q3kTJf}Vx7Z>uP{qT+R2BQSyO?_CfocnXsQhCO z=RHrx0WkCAjAOoR-F3cK4AJDqwpkI3C3yqfbU}aZU%vbh|7?GW#jBG)o_j}VZ{ISb zmpn40iw4D~SB(e^8sa#CUKxKF-6>qF`<_QN0O@0{HRH}TCvF}`GE776dTW`@b@Xvm zeCv}`5J)Y)Cd?z34uBUZX!S2S8ySgWGlUZ@?~m)K7>X2J^Ex*DaevBs_UO!Uj+j#i zEqQ)NUeHR;{q+m<;oUY8C*pdcao4%hRGo0YwfaEG@XMBCfj4T+EW=Bi>JsmYAN~AU zNbAvyr}6tgD(C(IbdF%VuakHNs#jiZrby2fO+3s!gA&@^Yzf|gGLSkI-P3iCBu@+L ziT&1dRXF|&{n-VCRIUfC?@|HH_XAx^8K~ik%B2j0`_-i5B#BG02i?B+y$chNDNTPS z#i|5*V4Z-Ws*ol4iPlfwgPnxPjjb9kv@r9e&RqBI?3|M2obp#jrAourBDGCKv+A;e zTU`&5-O_gP3J?Ik0PJ`j_nH`Z05HmS7cM5uNQ|N!%R1;I>^s0LEYo-%Ttldz zZOdWFz3_vv#(+Q~zWgs3lSeIqU-yQ*^ANPd+ueZx_f48jHsqYc!G39lhgc37s+SEE zPd<^m#ZePKtND2$<4e~TjMLT@CPe_eF$0JRD6Ja>n2S(78R5u|R*`qjwj-gA{_e<< zn#Ps214r1>KkUEsNYm`Bq|VM$kX@-j?INuJ=MTiG$Ya-NuCKk-NojG1)(7ayHjf=T zUpY?i=l^i%1Ro_Y)ZlC6D`A|djtq<^pk{7(fI56JmB z$fGFY6NqdLxVeu}_E!5=hC^{{VNyLM)Q|D%0UGCOPf0^kC13R@nfb>OSyxbVo8T7V zmYmQY_2j?n>n$_& z)6uVyFNd>nU-fg*CL^b#+^e7jIgbPEC2JnNtP3XdV9F0tZRi%k2}S(20SLHzttZl` zRkRp#9w{(&D=pqd2F*J_nzSScIrT-9Esq7cT${KX>|`J=WFhlSU7CxHWM-9D{2PXf zc!gg;viLyUotT`Mvfv0$)LR_C zsAVAhx$8HS%m%iCK{3PmiTS!kgzm@C&Y;hmr?$A4tmMnp4hGhxiTG^mmpl&(4AD1{ z^D4^R;JjrEg9*9BA|z0EvPmQGR{!BrG?rcwW$? z_i8JDYUh=~t&=zHnno1#to1GUr?iAAaHVsx9AzGC4Rhvey8}h^7ZgJVba6QYKr#)Q zChcbc)7(`Cik417#^sBV^OHR93e$Y3{`yN7b5{;3riG{lNLjt*OX_$fqbGiojR_sN zgzlD&1FulYtYn}n?PojbW*Bs?ujJsI@KV>9olvs^IPUauMH_P!V!1#RkD}cc6W9}I zJN~M~4go0HSO+40Mn1Ji2I_Tz4{lAHsug;zWs^vwC*1axYl%634&oQ)4l$N}Zz|S_ z9hwyZB8{wmv=GC+xYIKjdL3@rsh?f%DK=5-_)(|6qE`NcSXQFAF^@n|GBLUz7N%`S zv7fi5X>hLv&*LOkaz}dC1St+Z{toMo@6_uygr>&r+rD%snsE44ckllqCd7FWb32w z&bpl*efWm79QU7oQ|Kt*)<84FQJ@QoVu)$=&cZR$l*`>{I9N^w*wtzMzK)&bo$K#} z9omC_JMEv<{N!N6W_aXvWZ@5WN)BiNX#}nXlP7y~%?NOyfP=c4W8R#)>{lOw{lV(k zxWU~AfgHOiz43I>EqDL`$(3Ak8fdqcat143I&+A9V1H9^{r$`F6+PZPSqX}{9O+Se zEco~j?6K$#an^x}+Qop6X+`kD31L5?$=0(;1mPBy20@ZFSGdQo%R|^n-rQ+V)4!Gl zs&Ds%BoouWn2RwIR?uXJz==uQ;Q);yCRB($X0G%(MODq{aC5AVn#I9OPDk*DMd^H~g}g zzW=Pr!B5Zjz)mPH!f9&Y@Rq0r!=OBwmDIL^lm5YyTX^SY_k`mwle5!{!XdjWZwFUj z*1c)TojD<+fca>c$Ny0w0|>3w=%i}HShZgGd26aGaQ9^FD8vJKA$Vv>EQz=@hOKN~ z*%^0`CRRGF+`C#VjZ>d{+j#fr&kvt9`I??-YU-E6M*)xu&*BTDlu}PqpbkPDfuG_1 zI-K8%B8v;ITG5Y9FqF)`SKL)?wWx^pG81?cX`xaO5Kt5egE2dD0?x8YK<{NG!F#u2 zyaH)DkPWj4efirW?On-a%3gK*)iBj#WAcNh^QO%=-U)qh`(jwD!qH==VvIt+!jn8% z2qfJeU?V?ZK%LOhv<-ML?N;L$QtIHc8Pg`|_L%Fm=j0(0l&TXLFQuFH5}%-kte;Fk zh8mn;eG_axle4!QN*2I3_xZ)RbDmp|XhSp&+fjz-V5r(2O7H>R9anFjx^>n_H~e>p zE@4tad~^a8Tc7t_%koXE1ZYT|&bWT#qWhC<*K7~POlKefV#neSrC;&}n^T17371+* zQqRosub3O8_j;XDCF!J;tMgZ?^GA2qrSBdvS72nyMF=j-KeBzy{*GS(207l&2^wJw z%849dTgXAl?l;%9w@ zMAMTOA|c$YI-zMc3xamL#5jf&g>2A03r6+CG#1a0p?iUktn_fbzu7;SLqq-hQwIjT zLy%87inX%#KDoZ1tDSqlJi!DchHH)mC%OYAH(TDA~IH?u48hDH64xKz#fU2Hp+ymVmx2`MxPR5Q^JE2>IN> zssa>Oy#HmCM_ofxqW4mI0$+NO4=8SK%bex@1E8=jG?97!z)4>)^X(zC3NV}y@qVCy zY2{1qdZ}j;J6U}b?>L?;Yz8QV?&W^Mk*9ap#(#U`dl(C@kVlPXmNBG>Xju%Rn%o*k zVw%aw=$urx%b?!(eF{zAjC_)>&84NFa&xcoiT3UY6JS;2AeO|fg7bWY7B*BMs>9o& zdtG!X;!m2G!(ojn?yl~oEw$~E!bfTi@j5MqUi)oNTl{?tgg5=(G{3Pm^KgxX{X&1! zTT7q|VH^g5-VHhq%l?)u`yP>EDE>7($?5B7c6SMvv4Fj)J=l<)T+8B>EMkpb)!*Cr$}3;FKJ}!u zS=N~_IHZ_yTJCA=>6iVY4~ir|RT(c_td_ zj#Yy|T+3!QLVh?t#$iftN1ioz_q=ycUa^n=CVVZTDlz3XXyee);-Je^T?l?(0dnVI$)Urs^W9RT^<{^WUU(-dX9{*tnn9ba9Vd|kz6UN?z%nCY zKX!?VNXM;Zpl<}4H9#ghrm>D0Bfi;+QE-fX0NU}p2)rC9&^^{JcR!*91oF}{J9Z+6 zLi@GPc{Ze)Usm)NDB82~aSU)yQ1P`YF#G~HYp*9wlC%+nAYZdi%_OWgqsSB8O^ zgRc}9k)*brF^1x-Xz}nCI1}%HB$?4hpdcO9X?Urc+mGOxRVWxOzv?DXb1-l^@i$>& zh??~ka>b9=_TWSft;7~~;8C|f>KJ^e6v1r1cIAsMu@pK%^wef!C`0Jw90yy*Xyeu2 zlg^H3zgY=XKhD@?rnvk;E{X@>5~*;OKrc<2v<9A~BncmXm4z5qBwvW0lkS(L+aGQi zmF`QZbW5B%rf@q({?3tiM_}raeL!dtcrYTv3+X{ewjucITdAALt%}c1oM$vg#!2>Q zrD~yrp>{z65t{D{q^Vl>;Ds>9DH5H3$a+b)nL1m~4OS&8BRuc=N}FYD+_7~_cev8l z&LR7bD-wl6(9{Ro1|%?iAf8t-?$=Yo;Eko7J?Pz7W5ybhWhVzdV4z;npq#FW*4hAdKeZvv4KG-(X8SSWWTG`WuumbDU)u)O84Hu$TD!u=02bVyK>*o-CnaQ_ zyUQC~9us2Hdtn)_*=y;#j!v`c9p}kr7JW6nvHk@yQw5LgPqE@&;ddjmD+_MhM1v^c z&LbkR{L3?PS0)Q}fL24lVC8yAZ@bdf)6-W$|Mb3*Yc0v&TFqcEDD-0i zeAq$t@5Gz`Sgc3j3%Xq!LO>n=q%}CHEA#0d&mR*4$G>PLQ*xQ}%uIs!Nq%YvGY=y{ zL3JLRF)5%;S|(koO{;p*9eh-aYS}Nn9@CmZf2H?t zRhpleOG)_cUE&Kb5Cr2m3JrP+{@|8h;s_y~ctZuUmie%umf&Q`)6g>C!O7_<S&`CJtvNeRIME_QtaK$QC)E+)KlNQ+RQl#FPFZR`uBwD{tT6@!TR&&O0d z#EN(`Uc__ED+1!e&&&h{TRB6SnPRk$X4fU|X+74WMSPDqH(Yr>L3zCDdUfn!*Qtl) z{xI}o=QkTBkQ0aQTCGXUeD*2#uO&O9G z_S&XrMY_vDZXj%u9m$u8aY+QP*~A_AgBEy!kI^BDXF}Cz-}xcG`Dyjfl>-<~+l!F> z^Z&S-*F|YF2-?XwqIt66DdIm^B+Iax%zANm4#C@9wtVkP=^oyLLmC>npC>Ecp17o( zDYw^bx=kZYGnzh+-UG}%lt6sj1TpsYwy^{q(L8!7hVJuap8mQ$zR|w!V(;H2TPxPQ z@(?im+41UfMi-e4XjlNc{lImoaS%l zZ6_;%P}^jo5ND8Nc=W@vtl>wludlnm>VahXzk4a{H)yww{o)>iAxa0!o7>$f@N+F+ z0f0?0qxaI#`reJ|na&5A3jbgS?#h*4caw#TPXH-Yofo0UkF=%oC=_Eyp!UaTHpYz* zx*z5oEw>dhEYYSb%kz%h)ihaPdAY0~Dw!&NEqp5DlweD0JKJV^8g^FqB-q4@5oKeX z1}AX^%N0(bLg1>zGOq8g3o=bQcZ$an>(#IF1Y{z->fSYFdnu$w9`leB2>mF= z?D#J-D?~nueA}~aLt+y-_d5VC`;jL{A0O1t#un;Ucko-c3u<{s9x}$`?nr%KSEvFPzxa z9u=Ken7Va(UGcEw62P>v*=>R@Id>F8fD-l)?9o6r;qx}Y(0u0*l3wCMUK7<;qm6@; zr>h5~-YW7c+rj4p9)>Dqtm8h4dxG|3S^GKCW-!zO znQQRZ`Knm9Udk~M;VW+x%Z>ymkFB}GAhsQNTqs=3H`~D_$MtTRkHh0-UvF<;cB_-a z(3f_bjjbYrNGv38qa`XK1)bvyXwq7qt=L}!HMh?-kySna#m zFNztuD(5mPiVg@L|K+lXRt7)p20l(MiXm)HJrfu|5gQ#s@^`cyp8r~>ehl#`afN!e z`0?Grki{N_STi*?lnTr(5w_Ez)v`owC+n6|P}Fm@;%q)#*zyU`musJ?w)Mtp#|k$S@zLZN!q|w@L_-yE1uNnBWi*Qql;GS~kqnVeYWJv-5KU_?f{=p5C?B{4z?>yT)P21n zir$*NJZEmLD5Uecre5GSK^rhm0CS(IZ=RL)7NSArM3)rTxpx8cR_B&3j4kG`^YF6G zXoqLnM=tCEwBC`ZaL98&U^ZNpsQloMzYV-*#o@w(8}4Deq0k|(wRv-=vLivC%ua!7 z#TSo{f`~OZveWEtdZb2h2Rah8AxrZ6ffU12l*=OFzSj9Bo<9EHmXBof9!GLl(*xXPB$Wh_{_yjjb5IHdVPhQIXu9<`6 zti_VN(#b60ne&(BwWo~Lg4+g^AM``QJ0)p%U#x_Q+ch+bhh^WY6KOXzJ8?bT;;NYq z+f|NT{tona+^Qw8o=!4q4GkO|Mz%& zZL<>L(JVexRrt<8=gz|!e)4P#{&v6lweg10iIuq*N88_~H<|5&-c7ud6L_|Vmj?8_ z637xgD7OF57|E(*Zp^KjpUUi()g3joE|PvI>rz^RMo(G$hVqxGDh8w^ml-%KrZlt! zlrv%AxZx<`LJ~w%h4)7I&Xpz@$@UTh#T=Ih%Dw$>V8%vsvMFNr50gK|$XJ#g5IS|T zJ3Njd3VQVrD{vf1?WHt!B47-qTFRa0L>aRP?n}E!KuccbVgKUrs%(`(J?7xq7_kJ- z#LP$@pn4CWCs$|^B#@n&Q&Y%+QHK1xj(+9{g{w?dS1L3&Ame7~=R1mzUL{Bww4|Pv zS1=H>Rm2WK?Q1xY1D9b=)AT69v&O?3hLA|9x1PSt3|=^}vF?2aq=Zjrx0$r2{* z3$x%5+!&t(U85!nt_Q59F3TFxkR?Bj;3G(zF`MDrDISh}eWIAxpB}aNC{8R|Y7o!G z_8u48)?-MxviOHw!LtodYH!lWBi7DCQsLc-?_)=QELW^nR?PdSt{o{qc54BGy|}l> zVi~Ye0^*7HPdf1_$kopGTnt?Le5Fqt;+(t z9I)>~6-Vv7<9O9l6m1Uwrv(@qssyNP&Yr-V{exgp$1nmEs5gyv#JsH;&Jzw@8mp>9 zNiEAB95$`NG)P4}|06Or^{Ff7?4XWewdBoLn(MD2Fd+BA(@q`&aG5o35?+wTP5go* zA^CsmZC-QmacHepe?Br7zZ!N-(Qlt+?1Y|3qhvd&3DA>(eR2uqHFLTPO<|6ZX0#8w zM!S5DGFr+N;|)rZx}3quS*K6wj@&8p*3;UlVowfK5rd4i+2{%XVcfxhsV^sJ^hRZR z5ymrgVh= zVu8V52Sl@(f+(Obk@yP#@SiT&Kp>DEm`E1C0y(coA|u=eP|x(uF{kSsMxP(}vC{vd z`FDTCo6o1Sr_>h(DhkS#gF0GQGv`{upaDu?QXLK!Zf7$Vx@K+t?eZA0Y#Nww9glw3Ci3n>-?duH*m~Y_}4b>C9{b zadLJXMLk8EEM!2Z$fXZLRnzu`M|^G&tnm50qDvj5J~ln5`U7cu^SEKE?P4n#8h+iU+H7><|8Y>)_w$v?qEe6rV*x{9^R?p!PR|K7>&7!LaUX^gq}O zpU~|v#%W8sIFK~O&BGZ-sz6iV_0AF%bV#H87zps)m!aKC#&-nrS0%WTM@L1u6@=SWo;E_N-PJuS|t(C;TvsBPZ zi^8)80C(o|5R*ZcCg$7_2Jt#PkyoQa4UqD^nzrCOdRf)fR{%(lCfMHHQ=S!6^w=+a zAPyq_u>Ky_Eh|?&$te6ahh{zo{p~-K_tBr%pio5p(rO5KmP)6IL=X`P4ubdu4D$8rQ{#w?$qy$i z3bL?XQ=YMxzvlO}f>I{l0=hZ7Bf_`=;{PsPlbA6=4+OSHgiT+<0m7;`=-CXK;ivdE zk>AS!%+ejC(Ht}yGLVRj!5)a~+!45}(^d2iJd~@3AGP~`pZ!i0H-y3P&i^I4usfa3 zd2BgbpK`i%@bxg4N}jztMfhn zbL6bTenq>VNQO|-Tk$jDC0Nba!9N zyEZ5I<`|9F>CVytV>Y)?o&&Vw{8VFRnb9#XEF0OnCk1IkK_`^bHJ*BgzFfHn`M6ls zWhHntH6$XBRo9EY^}PA|#nVVRL>iiWKaG*^*^Z|RgYY-F>`O5sRKsZ+Ih_OFd%F=l z8082$SgFRwl$w-jOL6`=wCXcx@#d*~ynW%6_VISiVN)f|Q|<#lBp62mY4S*>=q}al zEI|~k>dC;c!1uP7qc@|Ww1=pzEpWq@S9S;{<-nu6qxnIDDQtC#IE|W9yC(n%&JPwq za2d(0Rrfk~fef?0&N%L1zl`n#!%UY4#+(obBu-d(P0M3-jG=c<{Gw0c!DLxQq8&$5 zl*Q3+(l!x^E^0;yT6|M1#-x-MM)$8!59B+XZvNeWp%Jpz^Zb0quOe?G>Ww=LkpQ*^ zXaAlHTm_?hce@U-VHj4AZ$uF{fhOkjG-kX$n)TRa>?gz!HkspQ2-g$*Q@8@he|J>_ zwYib!b=&$f`2=Rw3ujTSKEFn?xyaN@E_a^uiBab4Jf8`>)t)ghlT^Sh1z16MQsZ7> zplf6>T&tYh@~Ye|axD2fMW8K z`jdN(HqVlOuq*iZvk4S34^x51{9wCtb< zDjaW<1sDXHjso=<8A|-q?||AvOWd*2SXZrYXr;)EbXT^gbaG^Ceh;7A?`|k`wg6rQ z5uZVuX{bn$FJEAw0AbkVJu2k$8#Q4`#nD{pTnxUv9>Aw(T1bPp?Fhy8dt``j&VLbB z0Sj2(ihg?)686W~1ji_13PJv2%mT*w$N3;QdvljviewmlWmn$bGjJk$?jLN;oqXAn zmPFp@*7r;_l&I($a`egm{83iu(Jz1?A>Eb(cto(z73m8t%!1x<`)AMwkJX{C=`PLp z(HA zcO?vIS{6{;xi{%evAvL@F%co4u%9dHz0oo#XGsrp(4F@?V`Bv|W_K37=Qz)g#8*u9ft zWDr%Nae!DX6vVNZWjIS%AEZnN?Y^O^k$?=Gn8pxw?QHr{2AFzx0#hm(Iw3MuWe4sw z>e^jJ%4etK^FNt?Y(_k8uj}V}AaVcWpaR*s|75P=X=}CM^bjl=BYsxMXVC7+OyQj$@^;4HnpbCr}=!wgvJYj;P)LR7ZlXxdw;x< zwtWg+2Q#@|6z$}F)}>0ojPl7H&&u7>XC~v1G4UwZfjB~=UEB2Kx!JA__#`c)DeZ4O!TQuOE!O7F?j9l~a-mFTg}MMv4hsUoQxB<@^KA-IZ}nRHsX%}mrf&A| zc8AC%*dlg~xdEf(_^ZkFYOU!eG@D|^HH(^|**d7->N^Dt0KM^f-t-sv4 z?+$fTR=%mR{=xGk(L^0Tc>$sdQGkyM|E|j7@uY(X(yoV9o(&(EON$PuWL&!IOH~{W zwL0x==Btt#7B5ir*R;J;to=IdizW~EZ&XeYlT94DJWF;&$Dqhoz8e#smaWV(KvpY= zIo~=sdOBB+`n0*JL9pS3#+Y^Nd08H_9AC|{p3{`y)~j>mKo(DX1ZAV0O!Mv8cF2i$ z8>t+#svAosi06d_IoTz3dQEaW8cJQ0mT?wI@8JYVl8Foa18A=6rhV8D`go zb=?^m!1%yz;{Y!2WfF-%-zMQ$CIx`4k^3L)NH+F6f+{lxyur63yE%8rAN^*m6F%P1 z$>RN1;Em#*Bc~SAjaDYqYKs0s!uBh=tC9ikE#Ahix{INC1AETgb_8~6HSrL#S}pgR4X#*hSF<}mgj?480@ zkSDK|lR%QJs>disQM185Wf=)#8aUhi0rNBt1)3+F{ipswN<9)9hhM9uNW{yZy*XNi z?BAwV770FosRL4;GXkC2Dd)stBI$_~6iv0a15Q&)WKHZ0&(`TB$dd=U%li!OKMxo% zW{nyoMSbhL-GmV~%Rx$Bl|Nmhx&K+Rek}~rZe{-mOJ4$~g#U^9b8~FXoFNMWrt)Pn ziq&m7%TyDC)}c4%?P)XxRx$ksZ4Xe|R5OyJ#T+U@&B+e6m@^xiBBs}SjwwVm%@}w* zYA~$3)7Q87I`)n&stnP%%@F=g>}7}!bVyxDqXYu~JE{7@Njiyt(lMvDwf;k=e7~%8 zUjXlU;v+#ABs9Vdg4$mZHXlN<5LILh^=CQ6-WdAd9n&8`{ns|j%j$=C|Df1uh%?pX zjTnZ3UqiOIZuMHKqRc?KNXE5I3&l+@_KF-uh%I%`R9#N%)*fjnjBbPN!*VifvE076 zy(|SU-Bx)rKK|e_C&zO~DSBipk+j?s7dNYlgrI$B9_W9&!3FSSh}eVr+k~WNp|*9< zgb@JAuE-z+P1%~oo2pYQZH|%?5?YG@S}xhqyR3 zhQIp()w1!fObb^-t?m$xSY6T?5fj-zpK`1|$!eP5e=~D_Jz%>H?F!PpZrWxgetV$I$8or)V6UXmM52>pWLIkXpHQ%J88NwkfK{Z+o9l@G15~P(; zAWIS|I#3%2o7Si<{Z6x8_juddnw)l6$u}u{Y~H?&74E#> zEA7xzbKL1s#n1hb{3<74h-O^QjU!-tBZ>J8%j1o!MXqiW(Xl(~BU|ffL^%9=_^lq& z-}I`|sKFV7@EM!L*G?7Jt8FiV@`luW?th3n|JQP_>H?C6Ob18_J&+zz5BL9r6)NEI zzMoP4$ng+MCb-$|C|S(m(Rx)T!iqAQsga|zl~HKz-OTB15(QPp!m^Hk`0(M2F*msw zT@G?ok6A#f29W7B&}(SfD&@}hEZJ>iO(>^Uy|Eh}wK-Y-CSkE@cs}er+1@mdQ!y0w z2p%%p3Zh~Z8`S<5dl>xV3hS$^Jgt}_-Rr;|Z)g!x^znta0~9APa5MY2;Vmn*4Z~ya zGH;(1y9k50NS1D#FctIXK+<6;pv#e92+5JPG;44 z<#|a78lG;Z2t};lmwgrzS=~L_3CvIZKd>mQL4YP2%8KMT`wCiwetWXyb(cO<-22UoRu<3_u=w(LMr7;ype+@wY=du6KQzxLVK*Sgv`7|&wVm6}I zylVhX8|pbq%j-ex-{a^TXZmTm$-jsEecFZjagj%H-t7 zv8_Y0dBCl~QIyZyyBZ=cyLVvNLyB<_|7)Jb8w36i`G%PQxD;lE0MoHJt>d@Tu+cF` zpTng0=4U4l=pygCsXh0@eE(}=`ghU*7KwaJ2 zbibvhJe$V1xkA0J7E%*Ug<0)KWICV>V-I#k_DKo-+`Y#xZH zCb0ywC?H;-sEv7~?XCetxUa*Aw!glw47U}z(^{#eGFaMFJI6X$z;Dls4n@~&|L#aD zJYD9$^ossND)lIJhTS?uDlmTEP$$eKr+t>u`^RzBFJIaMs*8(< z0E6;7;j}!mwG3e>)S7h~{|!mg>jeAa6bXi~HQ0-3Rl=O60M$0+DCfcB*%Ii+)Q03#$YcW`&i3#*c!GeyFM( zow~)m0=0Lb$brasAOJEe(jozNi?&n#pap4tojcdai_Exq%*c>Ng}D8?LhfO?L<^hH z?7L!FSs7E}-Mg}ZoNk!ct(9E7W1YcjhM znEtqyWs!3<2}B7y#{M-RSP@8#1jKD+85vIin*v8xtAXStTcr{b{p${f>rwkurb}% zabdEYz776jEfzWp`ZbQv#4OV}%e&mpXK6g-o`eW)_Zw8WmynEo)-0Pj`Uz;dV3Bfm9fhbgqs?z5%yKQn)-M$ZF;*_GG zfW8yXUrw*Gh5b#ScmFSnScqN(<3JMdGYz}ceK#DDOkMl2&MoFaP>gs2@0ccoodIHF z@kU26NLt*Sx+Y3m(N1dgIVtCD{DiZqXb^ht!T;;7zkyACwn4bEWl5}- z{5jzXJqrb!=B%&13GCf}?ZLK&HR~Hl{4W~;?(l#z(3iQw_7hD$s|AoC|Gg|gBi;lu zgwiZ_yY`P{HJMLZOy{pynO|w8mE<(Gs*K+L`RztcYja~1m!nd;=8XjP7qGtU^oMEj zofW|>Yp@mJwzMWFTNu;;iDCu%J!+0dSyY9p6s2ozW}otEzm_b_ev~^ozm9Dwr|>Sj zGW1oMA*9L@d`q!o2oF$GM_ETOXDMTGTN^@<`^Q4B9VqtmfXBfng;wgF0|Ei)Z66S{ zPXMWiaDZ_cVhM8766oi!lQ7B-n_d^@&OBwfHU8<}E z(fId)I=?$jmLpF5*DcsD{fb6TyRHSVUKwd)J~IRSFp42Y!$5esqTi-Z&$ zxbXqjiBct%6s?|1R)HjgHGLjG9sAYF%C)t1+5LrQU%H#}#dEV~X86)C;OF&Nd?5C4 zR5D+*O0XtpMJfV}gc!!73M~||^-|x6mtJ-b&r?R;cWr22$TX2Y{ORq74{{4aDtmCC z)L`-Lqm%=nqA0{ar}QHSkjRQa-n7yS-!{}Ywk+`eRh0o%FJrxC4%!s%xk00l+qFTJsyA2FdN_70N{Q@w^2Lsn8%cV4ia zQc8fqtyfbcclS&G?_VeoZ;0!&&_sFfY^^dwq_i_A9NrZ+BMAZU-%0q@ch%65ACA^F zPG4Ltxcg<7Vl=NrnVFlQKW{K}db!v7f4r(7bPYcm77JnC9D_xo3NVlRe3}ZXEgi z#)Lw<(UIXg-+!O;Dzw3R1#8CPi0amRo6m7ZZ19X&4Tc5AI*~?)khCw zI9tS#Q=Si=;B_HEsl84FPjNCnbfBV{nIxrx#xHzW1=<=%lefDXXShbN%M#F|~2!wEu&x zFAs#W{rVpvWZ$!nA}U*D4Vk1YNu$NSOl41Yq71hvYX}dDY)Mj>?2(LJsE{rD%pgfJ zBc_{Kp5NvByxZ@6zkekEn7QxkI@dYpI_GmfpYxmQuog%hswthIjvG@8tCO_x6j$!ux*_E7Zt8;rkCV4#SJEm~>z@_~ehLF`v8C}{HEXpwa^Iw<* zAv&fM=;q`k8R#smzO46^w;Iw+PPr_e;MmQr5+~xXcAC;){U*RjLWG5k??g`17CZsW z*8!EJiAr)Yh_@N4`m!K`urL>W!lL}1OUgnNL1%BQ&SY_5 z(PbrrO#TF5FI1!-h8xLw#EAzj*DJ&QJDTH?)$n2dr8D)R3i( z1m@C>bHJYqL&10gCGD#^X`&aOgx?n>A1>Z5`R5^!?eqEqM{Rm#2Vv$MX6iOBloAr{ z=?!NB2m+jP3lEvV06O1Cp`c;DUHzZ?ekEW=!i)6}n3Fgn4u0IXj?#oWu7?3Gnv=!M zT_Wdf{LKv%QBKc;@%3+nGu6*v@cp}QtDhjq5J{duisHYQVb6fdUMeV70}K*+fTgP&*h7dV)L z&1rzfW{c<8D(hWXx|U^4 zuXn;OBFqtwC;x*3Qc%aa+qY|; z#>Oh@B{BO^g!FkC154Tk(6DIbhJW|if~#7=VThs==lg3*c~Hf~&wcx^Chne#L+J88 z+ycNOa}CmrZiWp_8R_bhkpKAd$eFZ;;b0JGWH)DygjNM5J;9<d8AgbkSO3+Ji+vI+AjwlWEYD2bquG(y~0`_}%ZC*vN&q_Vx`mEjNwT_%F@1n!``P z>(w$!4240=G|(!i3&Vwm#m}7IKi~KoeshC5ZeyG>Y$FIb2dqu4wJVJI5uzW zZ49i;(TUlAbDkSORUzK+y#TdnJ8EF`+{ruOxuw76(dsAZ*gozLZ>{(CwJDIIH5*Ym zEP!Hm59kZ?gZ>QRoBEVz&wNc8UC6b^}Npvr=0UV^v9z!uWqNI<+FKG8e%wcGR zt?t9>l%^~JV?G1sTuWieF@L3R?qxO^~9@-UU5?$V%FYl5;6V-o$T%+4TCB3r( z_Uy=gQF&boWC!vb|@SC7a4#FWcrPOAqKBjrfm;iNqv=0ATs(q_HC-%PiexH=6Z$___ zLh_@458Y;figX>nTL%F;)X+cSsUaLm5_?Y`>7Ek?;pv_8#dRAzWp9>YcG;SP`_j|n z9g62pd~nY<=gxmsVJ^Lg1t7xO2cX9I?llYMJ|acF;~4!k#UztBi0Jcq;ng7NQmpz= zd)VC5KVUK8N!UAcd7)34(Hur#R{133JyQ#Ngz_z&WGPN6gwn2$)#<-oSCw@R`sOu$ zRxahkFYBCC<2{@>(D@`R;$THwuSFrOjO@Zleg`--tUC$EgFPMLi#EfPnVEsF&JOo_ zv%h=ZdR`|-E^AzZ^rbM5I9Ls6sRAZaJm<#-e;2*O!Yd&si>M&H{|0`eMU|r30NfRP~#lwIv zYur|utlS}9$luBd>llr)xmpcv>Z=+qnoK{k5aE(qJJqwKt2Oo?(EDCc`Y2)UFqaoPzX?hbA z_oHnw`zL+^4H$2l8}xf{ntAP{MxcC*$KN`SHEHP#Enps?*AVQ{>?0r3`~H-8Y-Cxe zNqJvS&t-!9EwRc_85+I_gId=E8Q>D7aX*rod})jfgGkzSs9nW>GcEqEXm6qx4&n|^ z8tH1(3&nj34fOVB@3&a&IhlR#cUFV?$96}=Q&u`Sy#Fp^Dq-?;GH^_JIs{u%I@_bP zC^qG!6k#G`E;+3DZ-n;+KQ)yU(6LW!LZ9rr{jk2`mT2NWi}aKpyst9FpUE@Fh&MdU z9MAyDo>~AlfV>~-&*2Rk^lFaiLT1qA9)2KeR5li5*?!L7>+T z-|bzCd=(A}{a%ZbXz&4bCcdfJSEixm+;1zQlRRcFPnJE| zU%NLXEv)}%kFU{EpxL?Nec$ZmC-+RDX{Du z`(Q9!HA1oIndgf-6t2p7t_8e$q;8MK#tg3OZ*#rHLbF5MLxZZS z+A2Jbxx*7VCrepQLM`)5o@Xhb-Hsy}ulc?vfBr7ki^#XV(8l0j!sOr^u)h3j4=DEb z_IBaDwjbo#pckJ`BWnuZfDWJ%C6n}LEtG)apj*LV;?2LxNN?5rp1qBD{Kd|-A$5u* zR4hOrl-K@ti`^=(kc8(4`@ag+bvL|sReVieCOCh~k?Hci+*;RUsFOJ2ZyA_+`h+#Ei4Dz8n+7B*ezCn<5iKorf{&|esx=Zkxkb+qcoG%J zqYJPCz|O>?eiKgAq|Ml-DB@o^$$yGQiY^s4EU%fn(uA3PQHC(Cd?hS*EazTGVyL-_ zskz7x8L_NV~2L~vGJgm`zz%I_==NJLrJeb_I3ncW66jL|ISL;M`QBa zFGRjrx<1RQA?EPbdQ-8*D@m9uSN0_MpqEj@ zpcCXsLAARG4{y;T4_bzuA8k?R?SB6xMm*`{^#bL@Z;`Z86I+vZse6~A(Hk^O1(NLX z@zy-q=F_E^hB5Nqmt4K9I@)$c{G~wK3q=kq>t3H6p%MXTW|@fToCmR){1CJos^}W3 zgHY1Xb$|8sb(Q4|?!<|o6OWz;)Fqrbz%zYSA$qz^?a8W}p!Xu!XtOMKM zA*5JXr4C+2^8f*hA1^cVW6WepRJ%s`6V;{{bT7Yql)t*YyD)Ak&%pbXMSUsG`U~8@ zMky!T_Y;0hoJDp8yO@(bwDT4?9*9ELPx9>Q5G)xG_AeHO>8``TkTDoBs9j!ba1V?k z&d1XAap9faar& zX|d?~2BP;0zS+#=^X_v)nIT_4r^<)6BT%Loel8ymh)*XQ; z1Vl&!m#(G!sK#9u0{|dU3P1=~h*-sV%BFtv){J+(1XMCN3 zk{p;-c*0P^6bhEl&S|m_3vaZQmw!2r)MVLt@Gqbe=yEIqZa8Tp;NQ4`7w1x?*aE6- zLrGY@(_uNe#ppRMQH$52Yk1Hw-~YyMC&ykktLaZXP>Q|CjI6A0-9H)qm5~1G#}njB`YL5A8ufM^Rzj|ox`Nl@XC$^F>24#N z`Z(Us1)n2|nHF!Kc={Y6zRjL;6FR)-tWc+7@6pms@1ys#WM?tZZUePeunv%n3p(B4haBlVKzdy08wlt{yfefV_K@}mhqRyw@{)+ z!J%yPUZ=^uzB2E-RNmPh%Ky69uqXOm9@PP#nFi}8X`=zVe3C>~Ok=!e$5-78)lO4L?~JHsbiH_xy;z))%c;mdXbH zOI;r>n{&>v@W+AlRDO#Fc5iRX^ao&M=E&*HoBDTh4|TlYm_vP?m;v+A&~F^j9azQJ zh0Z5$nPc_I^Mw4kdxvH!%N%d&-7X8e!`|e&^nAoLQpjA+X>Vu>%gF;*0}O#5ijf2x zH;fSi)2@Y$CTV0>QsX@=|BX4#`er?`GNOLOYfSZw!_;8d!P!HfK29%$Ukj*|xLoyW z-gZtP&u2Oy8Ue@JFIO}D6 z{3_o^?(M+85vQ%`#z0>8nHP|8Mb*8kseuY%hbYBcf~kD3gN*1#fLi1+^D~Nk*RzMV zo?0N@0L19#iOzrSRjq%Sw@d4^5vSg*>DacUZaBFNh;4e|HmYuUj~9_+X4z>76GZ+UKBGMIko= zP+7QmO5NP4VNvTRxl9B?LSZ`@0~VPa@zEEMH(HQSQ5y!~bN(hj0}Ja@53>&9Nm`hx zLgsyBG!m8vwXkhZ4DPNtT^xf3nPrT)GxO_6xY+MC$;g~^^?0F`Gv?)xmtwA{he5Jy z(2wa4Y6ttK?0iulw4xv~d}9ju_PzSyiP*+j#tU3ZnLb23(Rgyl_34fkM=JDxo!A8w zVRjx?{{vFfDp?iEU<{~*qB@)T9~vlwnqNJD%Pr)}Xn*_AC#|TIbf+}k!(LjZy}zd6 z*aHCc=}+0cr(^_HGnx$5TF0R&gMCE}x*W7K(@BU~MkSD%g;JtK7$sQDbF2nn+iaL< zziU{c8u!$|(iOV%>U{O_l;=;@)^8+;yL)h!Z^VcPr@!4QBwGSdoBp2#j^6cYTjDx! zsYwvOp;Y4odSt~E9O!4D>>1$+xrT)q1UP=Q>|Pvhv5=2msdw~-RHT%v!4{#s%%+fO z1-gL!#yN^EU;^BdS;l*yXb~(0@@@9xjQoz*$HCnuk60V6(Ke=<$T-ZpBLuemKAu1V z*Bz++2-6Tq*Fl{~cG%W|B1%m|CTLk&bdhJA><`t|Sn^o4i?)#QPntPA!hDN-S?sEn zw|{#B|HJW`Icgp%g(ic1U*XZ~*n_4eH$QxATHLFbj!2)0O*8&d0~W<=5+YQ{%G?0n zS9+yLI>4-TG??BUb>50&YnFB$9E`BaT})lO7+slmH9V~C_v}r);K7EPp%6=xfN`qi zgO0(t=I}`eGkNWe2KlKCMDpWCG4u!U{cwscwG3c!IS3p~&Mk;(KLMJ_rj^ezu*Ndg zd#}#NyK~tewG0399XpA7OkX9xh(=ZU!#W_9_QpZ&Hw=Ztw++0Br<6LQ^kAPbkLHI{ zhtoClP3CB8ij&2u_6pf2#eV{IgU@Y58OP-#$nW)0EZ{jGroTy}7DrYJRHr=<(biRf zeBGi-ev8t!Gq0NmLP)T(?Ez6JW*>$FAf*{Xz0|5Nx3Jd5KkjA>lzb1IuJ0*R`EYEc zt-4P-DM5;lIC?$i>Eb>*h*5E$!2tkvHq1rw1T10v%!}m^0fx3!pxSxHy?Pu2>5aL+ z$pRYH2LMm52Q3X&+`^@5@}k?dk+u10aZ?i6;gGAOb3uV{fA;6_{nAY{H@xiaXXpK- z-wrN_)3w9sf;r3tGzjlx$CuDXs8ZYD_GKSZJR=^>e4ZN_B{uYr+^`&@1fYfRNlXC% zSfCyHuI;j~xv?enPnV?D?Ca^96JCOsl}o<$DP&9^@i2;4ma1K{>gT5C17kZ3F#tk= zkfls>3vWdxBl)I#3k6j@93zA6hE&wt>I;Y>%F5=vAI%lYQ>xm{RZiYGhrI`m@Fj4% zEST~@JK@B;U4%mpCTG3@v-^?V{xs=Oh@qMrjVdaH71h^v>$PtRsUOLejqQIU;!_Cf ziQ=EB&9ap^yP0!-Q0Chyd2_|4EFsU&qxwdDZPzaHe^;W5#4@=Lkky%7z}`3+_-@On z?a)GDzl;IXFRJH$6;nfJcn7+hZX3Tp&v8RX`%G*-^>2h62Hfll(e3^;*-aEf*JxKB z=WZjtRHqG0oY$>0^jAe#cHHpfaMaJcA2|vsx^4?fose-FJq1&W;bf|XdoE0Lo)YeH z;T_)ucXb%kb-O&`#SfdycOK22o#i|QwBO2EvhH1GVdE!cI9jqjVq_bwb?4 z4VpQzTRIS6@fgh5l`9}HlIiL=yB{_Xg04)hG$iRi#;(3wP54u?*Lk>4U%u*ee{+`n zE~VrYNW;>Y)5u2gP5`(LYqJlpR_K$Z?WHN&dIhWjd-4c zP;3Rpp3nig&Tj*%Yru}PD;?JR`Xin|#cZ2Sbl_=+LEu_xpyHoQA()H!>-_LxB#15W zVQ@wc)9;XLotfib&aU}{`3530BVT+3Bb4ODziu4s%-Am-bEmi7mI%@a3GQ7jPRanY zClAR`xXEbyC$(=82F+_=a>_99a%|o3IiCe&(hjtl>swWJTLth!em)RfezV5Se?#tr zO7-i#VG2iMSoEKWzW;@10VcX#2vdN7`D$4bl$~}OX$_Lc>!|8=Iu!;+cYg@7Zg91Nfn){L>6rU)jNB?X60U;firyqb+>4!uGRgCMS>nnxZrqu?aTW+d-g2 zr^PUX+8@tvB(4iVljfrUI}vLMZh-!H!Yu#S*O%Vsk}%Ox62H}Co=4`!6(wf?GUxz= zY#Ty(8gd1Tql9TkeH9zAk0)|;eiCj+k{{_AmK-Xv(425&;jC2nyk6(_sOUkTl7zxU z{iV~$6U-4j*<^k)jXD710-=S%L<}cZp9G4!MxynfU;K8O`V^F&QvU7EiK5%q9lx$Q z-h8#ZugIMa!bFTrR6nv7)txWiYsdvF6Z6HpU1T25csG6{{aUG5n!0LMu342ganeRP z>zahwHO`%8poxQGa+lJKnY}13L(y=w1*4+Q0=3_9?gwvh@%IKQv>y1%qOPqhidGNgu-ryeuBJ@8Yh@=0~ z|FG?Y$d_=iYFHZvKGf4x{QYl)@Yb4n{n}max3T8Kx4O4}9^<*ES>A+wJ9*DqVJShE zZI9`A)_)0~EC09f`B*YP4D6hGshqtN!j*2jz-$ZaqlS&ivP!19x8JM_>o zbXE3`jfK~%ZMtgvgEpWKzw8%-A2@&hc;?yR1J^SJ)1{gnbx|i*D+UkVa0*de7oC+eC*_)^5DYfBg zlIp;=)0MZoi}-&*R0T#;loZGy+t0MJe#M@JF;!LdMK1bAN6lKD=koOyLc|5#9xozd zV@Hq4Ju<>M=d>39#cO3EaypjDr^R>-#u(74e+u~wTuJ-_H|Gw2A=!@JATx8dIO|rV zP;ayTytI!-)bOm$u(W?bhS^epA7t+VYU=Nq!x*Z;59H3VA&yO3vL}-5WF1v(!dy2jPTvKCbQ;hiK+YPMjGkN%=Tm~4Kh&>7|9ceWLP-yFyzz5Hp7V)p_ z@+I>l891=gm{@${U|dL~`~pXG2u9i-3XfV@*PL&-lm5wFzN&Xp=_ zW1!pW>OxNGO~Bfdl+*ZU>z_`g#+D?{YJWQiEJo4)_}L^W<@Y9Eu4btmL$_hbu8j~q z3qK{rY?lP$Wi7al^N8Kiw;O0!_Yd_t<9*br6>yiK9{`5wZ3w92JVvf`>3NM1N8K+U z_ZKN}W!2bozPM-YIZuPg_A*`EmRbZP^knEJ%KoXJE2&O zufK-lN!OFzt6iffa5egB3=57ZiEqHc@`dzg5mY~59lpTeR@gMbM((;B=>PXOg^XN) zY6e->--znc^v;%;C z3grpH=U3%!y{aa18Zj=G@X6Dm!BKf&^^Ou?lSNF^i+-CeiM(Nt5!nA~S>INc@sFTy zIKZ!gWWW?KVQ^)VZ}Uq6qR{6fx7L4U4_tgxo8WZQq2hT8i8F|sb95aLK|OR^%5o&( z;D<`oQ-320=7sU(TqON&D@YAvp+)>60E+*OSpCz5+1dL9GZ+ep=Ai1{wJh=<>K4pe z0;sLramKiCPdoM#h8D?Oqx>hPNCw}IWXcE0cH4>B*cg^z9^uLEY4m4}y-4cSdL%>3 zTXbjZG<%c|@C{DTEi|ks`-FDWCpWCup9R;Yp5$OzA*MH1S~Q710W{YmvV5x!#gGm~ zB>Z)>!T1CsJ$vt|MA`vNmpWZEk9oO}W&urJ1zM^3RWDhowNyby14PJZK-MouFTxP{ z3z`-18k?!bSHZF9rZn`3U;nYh*>BC+y3>Iy3T_={AEqBwZP9X=+6`6-{kguDs*@jZFmL#1CqznjK4q=ZarDA7fQtj zk};hQ^GWsFY*Q(4!H==3#eL!*L*lNUSwz@3ETx(&WcV0UJwWoL!TgA-p6HIiNYiyd zb$pOFyrxG&u6{JzC$HSs#;E?|4R7a5NlRgedE>4>JNNjW8MY&xJ4&4c3z5FQ{Lku8?-v0mICg zpV$}xN*TR$LDL2N&qhiP^>R}qBTI8lzq5dUxZWJ9*A!%R{GlxlgqIu5er+7jvbvR? zAFFVkMb`c-O(^F(Crb?Q1$43BOwQ{7kf{>?04uLhaIA%le25iQs9u?RS)=!$cKU+V zC!TDkTSVgVVq3m_S--eMw+BFNL+eN3X3#haHbQ}}qw)?9*fI^+6Kt6|j3HoPg{40~ zGecj7gXq&CIh)6ewl=5dt>ALQe_1XVE9ymmdMB>%ZK+X)R(_MoB##a56o$0`{fhhNN7Kf}djM(C_5Q{eR=yCq3N(^YL1 zy~c^FQ2nxE#h5gBlF#Mj=UiM1RRb`+c^wOwbPx|W> z^u}#ciU;rRc#o-U@t$g z`}U-KMIaEHQ~#CA?rANqaPU45H5Tgc`_0ooPtNk|>cd=|c(P{G(p5_fbXbt6wVyz? zQW2{by$S@C_BrH!gG&oYSlOkTcna+_s%`-Esf8^i)gEtvx_1w0{D`NUQO}C>(&MF6 z5zl=^QJC*{m)7G#U!StDY-t|Zk}(Wvs&8({D$bp{daduE-lP8hHUU5o$7aPr1+Iu4 zM93J2a6pgtiFO!?fRnq#U4&bQ*C*_UwO&G*pVKbByBhvyfXiK|$2EPl+#RKDyoUw% z0c7GaeyAP2U2@l50OY*nI-WpCo5~@Hl>*PU#2j$wDo`2AxdK;TC$^JTA3r_2L(V(n z(`0j_2#eEuT~o!2pn>zDj;{wCJ+5a7^}jTu_Pw^IW5K6^ zqYFNcqQT@Ar}$q0^e3N;ua2}JBl)tPO${nuA5m@n#1p5R^DO4qNAzX10_8Mvq8p#2 zIo&s|8!9E?7X09BzWrkw!e;>mtAeQ$*G>%CUj4qT_!nHur3gZyc(~XvpA-&<)akVb za&Pcr@=K3~xtMe5*Ail6>}9p&veZ)O5nxt41&2LInahMoX*@S4rht@_)bsZvTihov zT$#EMfAmLZF$Y;bwi^m7WDk9Z6QWCzvr8j2oG>>fweml8RUl4$Aj@k=2lnkgUo48k z>QWAzhbp+D-xmfQu@1ZoMEDCsw#0zg)P4l-%1|eI#%(#S3s_Q{0B+o-8Kd&!?!$6P zOX%pij00)6F1<40ntFNSb9a7g=06ztBlF=gy%u=`P48|RFC}J z;!aSleCCoU9aw}woUk}Ppdq|iSi|4r$}fUtXOy*YlLq&$=Elck&plt&{^SF>EB4)( zl{3Yd*qFGx%e__>Fk%B7Rj&abDUy78kRNJyJw#4zLZ(!;BH@XS0$@GiEq>N z!_E>X44;UyFDGiQ84`=%;%j28doG)g*?AV+pZNU=94&L~HOjf(9{fWW)=Vcriyhe- zySJV%W5cf@Zgb|jrQQ)*#c+Pv{u#vQfa4&ca-?6QnD*kQTzf&+AiL>hcyia>*6M&i zVSj4wWDL~z%WYa;wYG+=_Plxt92K|@H}qnv#}qf#aK35^tbukfY;WBqNYVi+ItKh} ze`|;YzrCg37&1Jtf*1JJDihvmIWl2}&vjP$(a`+h`K@&U!8^nI7fsTc*Se0~5`4tT zybTe6quGzWM0REFdrw2nhTA%gYM!iLm;GLV98Ianv!OxgmQhtgQ$yBWgCLVd0i*VP zojk8hsnr0!^*)}ye>yy8sD5JYh|i%{A6~d6w^d}E@cg!~haPSAnTw6}>n3gnKN&_1 zCa?cYrWt^Yrttpw8djfBsjaHkyLLt5h1XEzvfod0g-vNP807sQ&bXntVsb?v5Wt4| z`s{k+qc47j_MqPz^C#84dy=gS!l=Td2iA`FRk}729{5ZkVn|z1Pig^vnLQs)Bu08V zk81Jr!F&_uHDqSKO5yjZtG1HoQf5e+(t9)RwEI)>2>ojC)>WF3bv@bx}I+rD!?e_9;%0-_;K<@Ko`!rYq6$k9`4YIZ>kH~wa z%eFv}!V!l{Eh+Xn)=qs{dFYyub@u#Su1k+xrT&Ux;dqJ4hW_tS2R9{fJ&^tZE(&Vz$uK@4+_9aCZk(YO(M$9PM=(P0IKy&G>fN)0pjF}OUY^EZY@L(B4o`0V~#Op z0&%~p%~L_NuAS`imxk3SD=l}Py(t-2tT8j|?9~6|Q?PQiXZmcShsO=`8z80i zVx|2U*^02V)0y@++! zv?X7$lBl$!UF5swmc2K`z%Ql8I*xOIhnRpGi$p~6&?P{IbD$0|W?UKC;+MvTXLh>) z=lUq=t`>H&i9l(`lV(5eOtjbsJ%LhARY*6)Xc1a_y9en>Ag(`m}Q`gu&m zV^Xf7jD(EDU*&89V5j5oqm`R)&*s<)O!xG9c{?4^1*Pk;KK1D)%@3Q)pVfN z_3onxCb+c$DbzQ}IEe59S)-I_C=^#;>xuzkW!l}9x{qC`vqnEq?S@5hAPz3kAG9^MuO4XM7wlt4CM!8S<^P^&WQ+UXa^KB2j`MBSgAf36BkLjNWV<@{v3|u~9 zDxARB1~j3?jpK&VojcfElikGD?tRyRZy`oK%hyiyDn#!E61Pb6a4?#kSIwNOW)3mR zHQ&8P)|pZts~Q}vcCwTuHo4x$2UOY09XCNdx?PY(>n3Xu)`9tZpzlSAkdg5=7HA$A zY1%McM+`ohPHJdOypa~ssPN!Y#@z`}#4A}9FOVIp4COWaLF(D*aeLd))cB}dt0lo$ zzN8%Jr0?OB10|^Mj!bSm?JD-{C`6dzpUiX0?4Ei%##A;XPWy|^_R6I?-#$`jkyl>w z*Gr`GCm{`%=&$Yl8JrxyJpX5|@Or#45aU=e)F(ZRnhm+uKYvCUf;n(2XeLh?+)hf9 zj{1NXrE669e}afXV%H9Q zJt2iD1{M`h=OI%wU%?FvZBdHkQ$KU=%D!8StQw#7!DQHG8@l6OcO6Zh-FxiUFGcx>~+qq-lcwu#5P(x`Das*w5Tw zjTJMzg;P@xhtEl8nhL1A>8{zIE)TK4dCBE@s+?uE_XV=F>4`$_r6~dr0*~I#%&drXFad15Sf90CacRT<2UBmo>;C_C7R_mHj+zojtNf4W9haJtrPLAU@c#*OrfS zQJ`}_V=rzwhshrRfqhR#C873Wx%rxa+68S^a@DTs%Vo=k332bhrRSeZ)=w16dCIvh zu;n7sm~we(%+%UW;A+qPjrh}p2XG!jL18ujzw%e$xbC+K!6v_GR>O1+o`(~S8GhMR zlDDMyw^KRT)Sn^&MbWyVbRFAGOCoxjm4ro+J)qmw3jh|IAl}p9Wq{iIR*ir9S&gAW zJbL3kJ_6--JHaq4I%xkfAcYc$A|C&O`hMgE_{#H8ic|Gy<(3(Cz$d{^p2OSgsowI) zL40&BqYrGtp{Qfw#XyW8xnPfPu$BK_U|O2FKUR1ZSMAmxj0!M{uQDc%S6{Z3igtpQ zZxUE{H2$;Svy6m~7J~jkCy;(ePO{p;wj|5TG^0Cy@?)BjHlue^UfdK^Nah(7JDqD_ z_G36hM_KyB8yn=Ke{vwmI(!fXhUhyXvI+h^Q)T29-TKY~Bo1%91nb8c-i6<$uCR^4ZLJ3vrLz>4^u<9A&)AF?zBm1j1y_lY@aWcMd6sF0^V6`xE`| zQ@kDg6AZ!|@7~|pT5Ac5`^@c;ctNoJ;9rWyhYwBbaz~#CLLESnRFKm?%p{%NP@FLW zMg|a&DV}UUILY6QYD}XGE0KCT3fUS8T8~oBrMzA#kI>2F*_d{jp4~o2K2sMn^Q`u| zu^yCtAjov15`fU<8E-)SaF(eE5o!UG6J|7rF38Q?#|~g27Q0qpC=3^q@)di{Dbf?- zdKaIo%sf{6)&Qwii3S+pHP&w}9#P`h&$@aOOlE*BvP2OwkkWWp+>|~^sT-Zt%HKlf z%M2qn860sfyV==4>R*=eo^@LZ0Xa(=)OOp-`3>T{@dXF^ZU9$OC#C{LED$aGy}wQ z|9f$w+M)JZ6xjitI8VN^j=)|Tj#8dpxeMha8cIc`?wkth7pY|q$E>eOUC-bgJy_>4 z`(`zzpCHR6xdeIJOgT;6 z8aXx@l3}!8#pbo&X_Wi=F8y>Y`o2?%oE4uSY`Hq24D+wQA za$#tK-mQvoglpLVX>dJjaV{`okunK%j8OA*CusA1br!WkTWL=Our$Xkd%60!->6X05hYM2i zxO)5ERRzD;C4EC1gf#0@W(t^R{Rjk@!J@MnFiN8opmwyO9QF)Zz0=p9#Qz}b09x#| zdstC7HLb=sM%O{2z1^JE1RY?!hwB@@Oph6kCp2NmK@)4jwBu0wSp!cpj(Fav`R^O>ZK!`ZbidIy>G!h) za|wA}_mdnP1@T)YOg=TRa#M5@@H3b>0ICDOqC{7kJ#t966v4qLt>h((;bOR@OQpq;$naR`*eM;$8Q`l{Z4ujGyg3(rz|-*ouVlu%v7|K{&S<3H z@HtR9EdfW$8Tg|Ze7(v%`7-LEiyXPJ11|vo?sUWJ>5Bl28 zOXjObQ(r$iUz7O_sxhTYjnN#j=RkQ`y!0l91EOCd%<5U%NL1BV-kq8D-oajalRjs< z$QYA58e^Y;5IPrW3l7I+a2QKb-guzyZpEJz#RYqoheK^lK{$d0Qq6si_O0nY<)6Z- zIwz*h!xUo4*JT|oNA>TU)Czsuf01K*TdS}aml=Grc;3@ zm+1&Bg`zD=p#&ea zfOoo=WK>Rhn);>uFK$m>Y-~>pAPy_UnCo(IAhV%*Ao@H4N>RFu=LYW2SLQidz_rLw z=CZeMXyx9SnU$UUmvtUzn9z~UokvfD(MFcN_#1e_*FjLd@5aGL8k3;bgW>!n(u&^a zYNF%*tpU}$pU3sZy|NO`Q zZIH=_{u=?sC^Cl&C4oV|bWs8Gg6qUzl@#HQLLpVn?X`KMhclL?eV%1gc7444(jp#S z2Ho;U`{^?MZCE8hf-5J!mPlF`UCBL9XQg~An985AdG>=j*W)GRM2}E2Q^M{o{Q;|lqg)^xmu;0{j2U-Z0KLQtPN~Upba9uwOb|AI>M%IXBf(-ps0x_ z4O9F z|1NwEgGee^LOn+@{V5MZ+5ik{Mm@N)#-tYY>p{>|#-!!4FIm*gUkW1Ktlr$6);ADW`E|38SGQ25L0{1(Zky0fU&OY+ao3j=FR`(j{gAeCXl7a_|q>lq9J#G z**W^vVg3hO=!Co7@0SWv0-%wf#D-L}o^sfx_58NAR0O6+(QM<_aG|d!K7m8^d7X55; zwpW#!sUELVdBy6!V5_{!rHb@l5}-(}N3-Ji(MXrc0gpZeSJNUPEE|l$VUG+UyC>#( zx1=a9y|UwPmjsLcnG|%tr)MVbWf;ImsM^0rG=UCI#sA4+f4(MSQaQTj<7A~Ma`iL; zz|iAUo*b<7Wbw|hN}kR&zv*`pZJ+h_%@y$vB4KR{>Ym+UX|@fpT(|J?Ag+GxECH;8 zE=tW5|NlqZn}q@BGzE_*?2 z(s8Osut(Yv|Jmww%+31}mI5;73!xE6@+#f>16`WcjTUX&3v{!V8WlM{1%!r&Y^v`s z9qw+j^Flmm>JB=XcBY=uz6eonI*_X(UwpL|N{ce^5*Rwz!-*9io=^g9Li{H$MuW)1 z=(;5!?q70BWB@SFU4Qh<3@U|4Ig6g0cY>{`G1hXO{SifjRll0wRwWyTOa?a|sA&J( zen-vWH{1J;J+3A4Lb>Rf3JfKrjC_F7!{Xzho_Gh~CV&hOmTE`uWE{IqI;5>S$PcwMDuPlqo>Gt789v|c z_rhH9WntdV!83;Ugw}MbIuG#NAHVs-{eg_b_2wTo>sElUAI~_HMguoVao9gnlre*> zw^(wxA8QARdL0exEuCXNQB5;Vnr$cz(0jb>^gZF6ZIyX>l76PNnDpEMuNB@M{r>$9 zOQBIy;An<18|wk;8=wtwvku+bVrI4D^*BCfryh5&usx|?zq(8luE=Nj%Xp@|;*90y zzo*$07JxwZ+5Bq^(DopnfZeb?1`v8aM^b?Al~Paxb>Z$8h+;Lv&3?aYFzillldm4u zJN9Ey>RbfntM06E@DAR_U=^((w#hGyUjSnG{~XUP5Ug`|Lxk(teGuU^R*7*Oj-W<_ zaP^;*A9Bv_8U83fHu*X`^1Psc?n^t=_aADIB`;CNYfen8tb9j$8HAmejLhN{Bidr8Tiy_!iqH)YjAb?A_L z3-)7UWucXAh4Uiv+JPRmW{s2K*2O8mqL-A}3)8`L*?MQ&?qd&XOrB#?V}mCDjaL4N zVkiufzMTM#o#xuXm0E$QY#XZaqlgfHk`Gti-Zkz=w7d0eSGZCUT8Dq)&y*J=Fb20zj(ah3% zn`Qdya(~59(LVmcT~O=&GMmQQPFzam;%l>0cOR&pGn4kBonR>(p=*JjlseWR2xtOz z(^%#>C)kqWK)@r6q=e!aJR}gJt9LZ48P)jQ+L_IrdquA4>vDB_`JKmW7BcFYCP+ZW z{tN@#xB{J1E4jA@=Z6DS>sxD^V+-@gyuVcRDDM|Yt=s|aHBpKY{D)&%hO$yP3z{C) zm+6Xm9uY4r{buP_O<9xN4?6@rTQjQ9-YnA48VCoD2_LJGr2-KQ(O&puW4#e0tJ&2% zXl};&S#8J8?B-WEP`lisQDb5SRfpO?e&73|U+f)6yKLrxugI%>w{NIlEn54=tP~X_cfAqSQMIBz?~~~uQ?>?~McmJnrTKyUVvf-za{kpZaGlP;q0jY3fa0-Te>N>a)y6z0LOLFN)CaT3%?0~ zsh!1p8?3?ZC=p-;3K_REN%^rQa4kV`-d{prmH zmq3y&2gRdmD8HJ;)hOPzbLDr}KA}+m- znH(D8&WXG)8v=-P`45V{Lx}oqfnrHaDMYx5cE`kF8VnLzk$~Yf;xqP0cn<+jRUI`t zhPtPF+iLknhiZ%!o8F3Qpjv+%|C(m)onGHLygM*-hd}@22AXV5;>QTG25w0aK<>Ro zUmttGhRgdVdO}D?7xD-hdy=D<8J0EB%C_s<+Rtbs6_(Hq%F45XdN0Cl$I3vdQrhn1qt_uvpde-Tp8 zscM5$e?W%8rpR$|N_77(+RxPA8yezSCKF$+_R9z0K49Pk2+88bgMexmiO4X>mwG1n z(pUimqe}6qJ#_wybgx_{jt13O{$wVxL~C!*s#;y2P_Y=P)0r|l>3;eePHy*pK9Qj&BpkA_46=-7 z!*ujt2tjHcLT@TYqlNJ^y||r=LiBSC-_fjY$}d~ke(W>uq#!Ny$*a*ZmEHQQfHvVq zhWJxAhd+^*vIzyzKu_RltNQM{lnsKbC`B<|t;3})kMvEEXjK>SSi?u)tn#kw634E2 z#wx}%Pl6V>HejY=UIRoDRvMhg-Nq88y;0_F09+CVoLIx{4IRqtMw%PqsEbbjaj@rwZ(i0))#La^bQ59W-}Bv zUiz^#hK=~?wobnWvSYQYqJm2&t6r(93J{&2s{kXbx=TFBs9I-~K40W_6RY}}49e3# z<@)6ovLYi$LDUa&0u_6JJ}Hkbo9}ji%uftU=rZMJA#lvOBw+W7_wEsjnYSska!35e z?!>FQ0krZ`_G*&<@ zxG=6JJDPTH{?oO3sQNgdcR=J8Co$j}V&40Q)h zRZW_)%qMv7L2lL<(En;ZKt+;RFBR)&z@_Dw^j80u$AOiz1IfwBZ06?XX6JM&laDxr z0(CPsmI`>|nN?xpFWI-TKeH=*!bBaXCtOWFU2+_h6VKSA;wGbTQR0f*_7#R9#$rXW zV}PzGsD~lebDmkuaNJ!(-{*~%Ue{QSN=lQ#-}$3CNdB23iPpQ?+&QGgC*rtOz6ZGv za99FaoxpA9mPm0(o0wM6@BV!@hu9+<`t_wh$LY@}#M>>exIKH2koa*R-0qwusM$=$ zVvQgIsItwx0rcS;-KZ%JD~e4AG8!^8c!!aJyT8>{p&jGfoy@YCFI6n%wD08f?=$MQ zbHlAkrkGDpVay=H1@z3UO$=~yPC-exyrLP_rEpodt9EO>)Oa)YuKba@7Z*81o;(k^ z)T?v+r86;hDU5N0rZg5OdBdT3p?D58~7 zEp4h}Da(D0eQ58x+fFm!AMm^au3lEXS6~>WfA(XZU$CLM-q-Xmy;(0Vaz7|P;~10t zR6}(x+8X30E_Cp zeV;KDnqJqmwl;pA84pPWQ(5L};{FeJ+AduB5se;E2NVoX3TmM= z#`zaZIUCD#in;UHdeRdVYMuWtg8f&Rfx~cbS4ozmEBnXg&J55oo?#t|;lO*^-202D zl^83A3_36%3$c@>_@PqE2U3rVI_yMY}HwE;uTL2_zQ_e3ym2Y=&3#{9tW=x17br zMbkv-%g5JFEeDmCAJ(!&u)W`ne~E&%dLV{mE^R;aDdh77uu5TIWJy#6wJes*{;l62 zw(hMnOJr47D`sk-?T6z?#iKMO_L4Qzs5uHhYbQv6$Am5rWus7`Wvt$2WJm5osPt(? z^`v*M$UA@3zapAMoqt}L7!QH=lTDC~nA&~0!Nu4QM4`%6?Vt56^^~zZ0YG~hcX(H5 zeX`E$e7ifG!V#Pf!I(W@#{!0n>H=QC0$ETyAJ4cwCzo=<6(eJDpY%c9%rxV-^J%j+ zE1ByrYI@`omOs9W>g9Hf#>L=g^$Vi$e=OVfv7ExdrFNEw2sEs=t&cVss}<{$)6GXf z{(Vu(RiN#nzyB_u=Z}}6j}qdd;hdBs4peK*S1*RxFi4(IHduUD=sG@-xUu}VOI9V* zr)UN=gW%Z{gj8_6m^Su|iJ7JB_?Tq0nB)=#8-K>vh9} z2GP1tRCCN!Yip%-z`AF?-yMz411+$bASMF&O0FxdBUQ_RkUEGkxP147h01GvntY<# zC~{w3V9(`BDOE3@?qG~GL~w7_dl_!C4VJZZS_D$axuK{|ys)dP!C74$Be6xXLC$}C zOwVl}GeeX880Z1HTEL0Eij2e8?#sLF1?W@e@3bjSEM{npKM40bRzKv?5-RPG?muU; z%!7a$EF?Cqz;HlrA!Z5N4lpgCyT>JrfEU4geUq9SPi~dGdvaubGfJg+5t{hjrJ8>= zU;S-o%J<&EH>loMD?9_T(dgW?d&UYQfZZR{VQ8P}5mD}&lU3p8@8jjeZyg+}*;fw4 zeEAO^6lRaMRA#VKEF8aYarZFJZ)rOUH9<; zq3WWwZi$AfYOvYbk&p{!mrXdwr$-fjlo-aAWNnC;8;R{owfKJay>faC?+znjb7^1{ zII;zPvtdCaFJKAhVMtBQtTIHcx)qU>o+RMIiiQKxMrI>OGS&Rim_6v|B-)oPTK6hJ z;+w(+_KS}^lG**fN7}$gGiT&CY)cCGSo==My|I7rD!+Zdjh4xq_dTwMp9ukTb+?mo zGyZBO+CW>yo^L(!-Fs&@RM18y$<`8*TyVX4l^#e_Y15ti}PQ zLms*>_G3<1Lfws0co3-5X{)|A5A&^^^v$}l$5J^*hE1SE(%hW$_Ka!7D#I|0<_J=1 zKFk7eTTJ_aZZ{KTI+5bx5VtOe*~uD~?aaBoe9L-$Fs_n}bK-W6yianm`*~lrd+d1l zSh&nX(F@ixN<5cX@c3a$8Mt-tVL4#M@eX59+K4xjBeh6|H4KGl>nj*`f6MD{vT(OE z)mk2~;8447l(Vp~oejDUH4zgr(ne|&gZ6WNEPbkZd=#?KRQdQ;R^!VPWs@pR?Ju*g zUJ&SsKA7q10envbW-hcqG6udWDGVeasauH6->jX%pf6gMaT7pWZX^F9Y;N6#6i1np z*db)?IPhu%1}AbcFfCIAk!7lYDW;NlR&94&_!a0xaz*9!;^gG=d~=?)BP^x+ z+x!ZqwX-141prd*(Z*(AF*E?h8Mu>0z~*}b3e>;Gt#H5k+GM~*xZ%@{SdU~56NM`hv*6fq z=`nNjVsKy?FZmzT)=@hns3-y>lvH zJr1Y z%pq^8%-qlUqsH>oK!%I~1>;DyFO7H}oUAw{* z&**q_tkq_JbK_8hw)nvNI)KTpLjrxxY<%5`DBBt;>VE6(*(o_o?GVd#@43dblRDj0 zo4u+Rg;cs!%(Hv#yx#SDdV()mK{Ml+yLgH%eiF2k&OU6@qB=xJ9eZ^WKXu@jt!ePi zGQ;SK!=m#Z%Xa6a?$*hl9y^Da%q z$onKv4WCN5dG#vA>^Q2~0w1Lxhk0mmLpT%*3C|H&hiis^NNzYNBFFse4^@?@1etp) z#MH>oJO~EhG)WA_KD$4kLkYqR1Eo}8{>dRD*Q&{|}CY+!PMa3xEK7=9UAUP4KHUA);~YbBmbTs|G}$dIl7i_ka(gH#g6X_b|atN3HsT0uL)d>q9?=V$dB zOTl3?(nCE2$XKk#!(VsY*z2i6j!%H+gQsFDExiN#n>>>FIzd4ym-PYvC#Zk}7~Pg3 zkbPkyB4Z^680zr&3^5HW<$vQ{+dj%*itdHHD(ADG>Mitu9_jI{<2EIl+vs0tF!BpllfIs91#OAIE4a6%6^8AvmudbOh^h?jf~dk9Tr^y`3E zy9HGzt-m}p-SJ*)`s4$PrfvzfJCP2*MKIN9%9WEVc*+GRK_t#yE=p@eCL6s+#bIc_ z#kvgo3E*v5`6n(#v4os~SFPp&wi@W}_tUhr`}s*@p8ES^pTbpH_19 zDT8tx4)q_g)NFbOFc?9HAvglRJO#Stcqt%?;(v<*8|D+w8wUyxW_rARHHjq#VjT@I zw_M@H9c}wRT}>xhTb&#!9MM+zL>jT;*O5F=km<#B;3x4c!B@<U%DfwH z0gsWH3xc~+dXJaN;foW|a@BXf9n6fzg19551|l?Jc(*f{GP=a(INfncMs z01?GI>wm&@-1u-5@^gLOZW0+)iE6!4=KRg^bh6!Smce5k!)m1p1mFyEF(BY4lSl<* zDg8oZ2QeX^i~2R^RQGs4iHse;;CEp6Oq?xeTD5+6_p+By&j}M`@9O{5nF5#G|I(Sd z3K6zb_%mOScflz-7dUxtb@owDB0zj0Mg8^{`JFhGsF@JzXlo~W)$vA6r*HX9%JEd4 z!&ISou>EWZC|>xt#mIU=qA{Af`;mK?{D#wov4%0~qDTe%Wgp=HyMTMRLqL-i$58kH z7&IK{V1s-CwjOZJ>D~e&VPrtS0D)=Y^nt-9<+5PrF%Hh4L>r1p_oDHn{XaRgWZogS90^IRHzuTvT;r!J9s3&#cKJP;xr zVE|W$lfcXl83eyhXs$J@TZU*0y0Ch_L7|nz-h_;AmT8Y1vouEbMwi#TnCHwB1YG@A zbi;+ol&-w_p23)g#+$Eo4-yPg{`7v29RZEJSNp1!VuX|u5wvpFF2GesTm#<1BoK2f zyASwn$f1}?a{_Tv9aMYBB4w_wQE2M;-KbEn=e^b621#V!sq?9$U#^@py90gE6zS#R zOk`w&p}v3!ZNUBxb`?w^I~Db*02WXjwS#JqYED}zCDA`XryJ=014I2ki6vnfT8Xt5 z@=vcARNnJ=fI#O$Y`}f_T?F8ze~?=T{OCV%?SS)=CD_c2C(`mLQ9!`l{29ZuwYJr- zw^hz()>_$7uR|aZ+Si`-`pFY9ug4^wJ#wCRGSL2QnhbAB*gFWUQ~*5Jl`liNdVgNK zkrvldeyq%Cvnr!zvsUN%9O&lAy!*>UK~Dm_Bn6!JMpmlW48)SUh53s>;(V*|ckOE8 z7RS5;`~y53-E-81Uu8iNG;RGdqa1GrEJ!eItHr z_V_B$KwiKQS)v6nTa1p8(8wq^xU7sJM&-H(eW_mBaH`LHc&nBF=c%j`m*(z)N#%&_w{ha;ouUlKHKT-ZT3TPf09fH{0IZZM|f|&*mvjA|r zeIoa6W2w2snT4%9RF9btskyWC6m&-oY;}S1r*Ul3|3gm4jj&LKS?e2nO|=37^7T(w z6k7C(^4t*S`R#$308rK1*{bspAdWG`M`urVmqHa7$m7z~?V>f(e0EHvJK z%1_@|Y0OCGxsk#6n62*3p+UXQMp3O}tg9z|5;d#OkvJr_#X_Nl|3jNh0X?%WB;lJQ z1pwKk^_0|7$jhTDoF;es_%wffs^-E__llk61z}aYbdh&%CZ6Xm z=qaFF2>jw_OD^XPWxMS-;>yv+>YrA2Gq9fJ+XTtsRG@1%3qD`E1DAx z%}QdX3Rnqm@hnLvDPU*;(gy`%4KOwK%m>B?e8QX_F+b*(z~v)fCar>2xt?T<)a9QO zt2cPS@6Vws@DqPc^LH3a@ELe-&|n2VU9j)-_teA1y9>N#i3zgpzM0-xnQG%St)TA9 zCo^UZyuYV+Urcy!$7S|YHI@k}{}*VJz0DSU{bzW^1)#$rikhR975h4DohkJv9ZB20 z(nliCC*4qP7(XX{#qkc|yXcYe(NPe<0U_pztG{Ve%y`yrhWhZjpFhu8>R7yfu0M2x z-&(M)OdxjD;)s<7XYvDHC3&}KNYL{q-WfkL@D~B*ISdpAk95i&Z7YSiGhf1@tjKH8w5j8bIyiPcClW)_o#k*WJ;KnU)5_JKq;Ftt8De!h9cy7NKr z2E=XU<9evrMpc6|>+*aoA3p*+_YV@A1QAxSdqP+@a+PRmv*HK>X2mM5 zCB)}m#H6Z5Vrfl=xWEh_!vy!MpS3dqOyKS)5ZJ_IhIfz#JB9AvYz|D+LknEAd`5(C z`t|D4)Asy`7+X2KqH6Kvn3@bPWG{2Bh2US1Q9$KKkp}_!n=5!terAE+35X+cL!U1j2em){Z@d-p>;{%hw+N4H}Y&|6Jx|6;HDvj(jDtUC( z@1;7_m_BQ#%6Wn7F)8Ua!%!eCNcpaux)?w1MBEr@Q@8XgjhbHVz#aG&_&JmG$o!S<>b{GUo<<00gl zDH&|^nosymooKYNH?9~9pgsg;$Oku((SNR$+DH&e$T7<yOf!2D{-ux`@#KCXVhKX%ms43RG?SvZ&(O6L0*EDVzh=9X@reLMWZhKjUxX2=F6DCr_l58B0*Ndu zKe{p4r1PwIkRm8T#XJJ9V`$_^86-4k6_kM(NxMEp4`rr=&lw$v`O;?o=;!TE^SjRI z9LPXFO@m{zY4hNsLP0Sy~(4zTv$;U*LABbm0|l5^2_;I`ohm}Y?~~MRd_pSL&{-X^gzV! zEOWJd^UID;?yF{4s*bM%XOO!EBFGwRLxcl=(D(B-VZzQ_5;L-=XxOHKLVi19FziV1 z9cnDl?i4k0d@x{gQp?1|l9P-CzL`9FW*9joMVjw=3z<^VVnT2Uu2RD*lSfgVkbT>( zp+5`jeptR!{wo0)7co9;b8g4qxZ#vz)1!^S>4=fpwuosAiFsN?uH^{sPt&Vb|FSQG zYNZ|R!_FTx61^|JB@a{%EQ#E{dY+eWE^C){V~Q#ZQM0>7Z0 zttymNyZFvpSF5Uvz8Q-;kh29&;i;|DB@4p$YgdTA%9;(1G2I|tRRs{nK(Xt`Uqr)3 z0*awdmx^*7)ooe&U~pkLN}}@H#I)1tPhXv^6_N)s%ob0mR3Hcs0XxkN*TPF_Jz!x9 zD6K5O3`Rf)Y&W>v>o95i#G3lPN%W*!w?5cxbNxYV#FAUGp+nu~P6(D@z^l@><_w>b z44>zh*DRJD^X|V_&9_u*Cm1%dnu|y84tw1CcJiEY(Wy#Pae(c4rZQ1epu}e_zQC ze{#Y19hvulr?_3SWr$_IL-R7um-}orCj1F)ZX5A@eO!Ace{t3-`;PR}UKO2dzxTJ$ z<{|Ho8XSv>|JFZ6XAi?M3RP8Ca%g|7RD(0? z9XoBg-Pqh^y^ffncirjQ?t8qE%@B^91Ar?rS~1WB#oaXw$eq(}L&kSF<;QKsHsszY z+Z4NWI$KkF;H7*Geo}@nMu897(-EhJJ_sPPqkj=oSwB|}5ogGG%@!mlMTI82bA2N}YCV4YLYGii84sJ6#B{AfLI>;C&{qvVdS*2*QXi`bW~gkO|D z6MJ`-2jMU-F^i^H4^AAX(`1>kZPE<2QN1LW$a56qNL`wYE@aZGpfa~kjkm9;2IZ85( zIXY8bA!F&0ocGKDNK&8Fq7fl*2WdyPBa783aVt% zH^+aaOy0DL9I!iMVm9T=?Xl35-a(hLw^%^lw5~>htXTXuR0rVg-1TNy(XBDg3*e3u zW2nn{SAVtUr5_z3PU_QmBedM2FYz+ki}?l~FIaCAeRdwlH^ejLg;LO~-*A5svbYhO z3DC0V1gC)DFW1$G@>EfP%GstqR(c!Tamv}uAUjL5%JJKiAyjKWExhCbLdPzgM+FTH z-bO%XNQ$pHLlQNYO>Uu|05DevYGCK;T*VROPJH4Yrqh0~4qfJ%UuB4czNrHM;F30_IjdzX0~eQvisB``mxV$^{3;M*aL>Z*_ zd%kB+11_;ZJ{HKTfytd&B@a?POsQr`FT4$3!xgqobiMCphXu}keXn{>VBOx#o-^p! zGa-=aoXG`XNprHhA5dD=83vpP=nK^O_$c#Per*tsT8t^jFRyil1w-;&gcOOalf)*A zqh>ihdY#x10}^+i6m^bdTQuuM`_K7mR`l`ZtK}@9MOc-Qy^H zC;7Xug-Q-14}L-!>_l>7mHZSXsP=#&4^*HQ3~an!_-xcjm(@MmDdT9fa7*}@45#cC zNE~j$F0&}m0qrn&r;5asI}7-DS!xYqdio6s&0iebCo&?0Oig>gLmNt=02;Ya#*hbB zBiCDIA^xfjtl{6&4#l?VHNlajRi0n=*+ToW#r=cJ`@=YI96UXw8T5p&{2AN(?>P*Z z0(P6c1Kn#uc0Gj(lV_7iSAyk#POk9#Xs_s3`)6n*NSbx;=2eNIoHKd`I{=}$G7aWLtic4r0MIA zOJ!$gR7%n)W}aCQ(yz&i_$LUuej<4mqZOYx<*R^ja+n^J}=r4pwh=u%xwhkd>Az}-LrH6S5SP^ZO z=jPr5arJKI!;q#{&B}enlCTV8hu^37;nuzf7nEW@|65!q z14#bOGb9W^qor8O46z+26s0ygM`?RW)%krcgYKHRe`Czn2jqm@#pUjp{&rgWd6{=g zk=sBFQ)LI+fTh4k?WL+i_l+fhdfvX_KQ70AcNj~)9ly4I`;@d5;d%c}U1F_PrSbd| zT0~4HVNde+Xp?Qd^4VROsS~VJp6-H;m2uGM;P3|=J8lpo5lo^Yz!s7M^Y8Bj*Up}# zHOGU_Gu&6$>Gm1bweO=t)6-)Y-aIky+pRq3CxJL%{e?3JVxcRxwwvM5ZNR*efGqnN zAI7nwMOh=DWn9W?EXjRy`YNl7eC@K;J)TTCBOR`QWGNM;7!&mQZG8Zi12j_w;*p>- zvTZRKf*A`1Itd}sdQ?11iXxR53UsF?UDL!}&v&_p=~d^OrOV_v-jwoqz-_|8r}+W$ zgz)fTDOxiHMOK^naN-hNRS`aC?s85V6>{{lhIM|SK+2D1bXMXF>ApSBu4tn<2@;UC zLd8xiZ%-lDC1#;^lqDFgb~ld7acOM%+@cC{ayej~|33MZ%2CP3r+O5^L+kngDWwQEH>9}b9C=p&ZJC4nfc>H}`Tfzn)*sKN1_ z2=5aj7lJNo^$ms?a@M8`$=P$6rSaLLwsv@$%-`d$_U*fV+fCxEcW2i-Bhg|bhu@%( zB*Mp@l0FBoZW11-1w_PHXG<-yfZ z>Swefe*L(qTvN}X^O&A01au1A%SeXA@SFqr+Z;vMQ$R;&XT(pupXTNWSWQ)MStg>=&@x6>C-*(em&oiG+d6rx;-qP1HsoV@ znvW^dF4(#Au1q(lDP$Uweb$Lx4{|Xlq{U#hnMGJNAM8$wL43W9I9%+M-S>JpWIC?1}8Qh>q{&l;>>g%n=z=c@UAdQst{ zq-rt;;QLe?WXSv?LEJvK0+Y}_w=Ef#uMD!U=Zg-#61a7{klTPac>m^eBSW3%;_-*n z5#!5^?+PZ0NgK-@BG3y|eC7-L}ogd8migr(6Er&5C;j z(x0&M`0^5flyh-?4#@673WX%O+`6GsmY^GM-*G7GX(Znb`cLd8u|y6Jht5-nTn~AB1=)|B!_kz zG5notxF`c;;<}(yx2_+3Lan)_-ub$xyn{RJ zhTzfLjbJb}+wNH-=tstPuNEzGv>UW>#h-|GShKA2TL=|A?xWrKM|4NoecDseN(%_I zDa>{upmP`nXr_)R5rT*H3p$W>9}5Bte*sUn)kSup_RdRySuev^yB2ARosFh%v7DqyZ3G4@fVIsq*=PwZ+PxR6Jy zjMV2^Trd|i##EGstBeV^dGk+SR}r-q%~~*s2?A&b0CW4Fs|Mx_`e)g&zrib1A}Qz5 ziOB6Ns8^3Y$Z3XJFifi$M@Pxaa z8krZuBtidNhAngE%W|`UamH-0)$DH2daEXRa4Br7tHgI>@Q2k8Ext=Sm0oITGU{O7 z>mD&y!Jt<#{86SisF5@HZXUnuKxk76lyuP;FC5}3zWv%tWY#-ypdHm}K4Dwh zeXuSBXv}T8{|CG73eAkzz*jVh-Gfubvt^wHaRBVxdTQDr{^XEU=PfKQQdWDtMTXSq z7X^%TLQsc_(%p};_}p@4Czy7%?P+Jp2F`Azpld?8C^@jBZXo%*h-6jxc+e>&p-=d% z*jwU>HS_|=NPA_y%a+O z&j|!`^GFzmH~~zUgB>6j2qVYS9l{N0+{~;fN9y@*Vn8D$?asAxkqIGFHpa6qL#o2Z z1S0H+s;^CDSFV`ub2WW2x)W8lM359_QSb&B=&55zqB zYn>4Z0$Dt;g8-Hl^E=82d{!+fq9|uFDIeN|;o**;yldVK<5n*&E;?Zs-2F=4q)sDh zRbML`*+?I8+HIb>ktEfJY(z5@-cxExJY>t+L0&kL7FYw9d61HNhDxZ46;)lOR*^=k zJsK8@?O~8eamC)+Lt5N7Uvx#)f;9)x#(%Hh6bW=E0EYmE6xf5D#J3yskK?_Lpknn| z`&z9Pr#hYN>m@7H0M5OpJ&o_83Cp?H&Z;7#iSwn53T?O(D>~HbaW!kJ?Mc~UmIM|5 zINjA*b)hQ80P!w5$YQVwN2<-P4g?z6&vssj5};V1*VMxufPbv)(&-bg*Ljw2XU$*0GD9dynK1_zF{{NzO#gwRzS~V3r;l zE+T1%e2^cV5O5evMT26)l)=P8l-GCvA|6R>g@S3{PM&^_LW9rhC`grh994w~$99q&Fd6bewuZ+$g8$m& zMHSJcWuQn7!H}T?DGh8jcW*TRIx0E~&x30P-l^y}C^l+$;!_=)&J)9OR@N$~X+E49z&C-05{Y{$1GYy5@7h*lq_V z26P3V*(NmNtYBumG76nT34IN3cezRp@t5oPJgYkv@J-QXuj3PiUCHKhwMR6A-tY|^ zFX%I@Y(i^7XKYl@@)P|SOBSO|vg^m5b zY8t$V`-X;%Y_ip0HX#UsVwl;^6m03>LAr-@1^8HdHP<7zMK@wTMX50XzF{WYFrKWf zSWp3&HjQ~+=Lm^;SNH~~e|kl}oY3#ZW@`85w3XB&o6{X~AL(~nO( zr2d?ip6(`1Ym}VJcfTS)qP?_!jh3SWkApCS;fwO4n}we@r5Y>`x>Nd1$533!tldNP zz4gv1s(pN_1zy(1JmLMXE0!~Zj(2K45W!1>7q3LFK@SJFp(!{}2wM$xH3#B2r{-g! zWfbs3iZi0oRYX`jn<3TFj-i-yQu5PLG3!DJ`G=_2^_FFgZpG8=>gy9F&E8}zbnO4? z$!4#YoEfM3jKAgg21^Ajd~?TUeH+#SUp2Vy>=xLZ37_A5j=hnnP=6Cx942jyU>qT5x6f={kXxfxznw-mWqg(k`DKhu|$uKbAi`=hC0 zwL2bJss-e$ZH_RQGtTqOY*3~XN(I@Wu7Vv#2g<|CeOSYx*KOyleuXPr7TfcY)1Ril z|9B$@u>eRt@rp)1^fM5_piL0om{m(2Gd@T)h$FAmqxCrb!@m}}KxDhKMYDRA?a}P_ zqviRwWGj9-xirK-cb;%`?(p;SlI`FWer7>3mA12!2fev$LzQQFW{3>MP`V4_G;Dbr zu$OGaGR!OCNCIp3@T#7IUC6JJMuXoydzH;q6U=_t{^mToS&4=ZYSKQyFR2Ez1)ccV zHf1Vz%`7T@3l$G#JJqm0XE_%v=XTt@|D#JqROE_d=}nv212G5g7EB3EcF^|07nc+v%5Lpr{U`osM` zV10O4x0C1Ol^Hv;4B@5NoTH0PP>4Lj$x`XNAO%<}-;i!qS@T2V=}`t@%l@ z#4*4VJww5<_}oAaOtZWQ^ga43kXs*okhK=*v;91Ny*WK*yDP-^Cz@mmtF!K{0>eM^ zT!^LIT&}^Mf0y(D-QIkC;@3^dTmD5g?p&4p0&e+bzR^06A7ySXo1wT$LUKo48vj$k z#^4z-3MMHSldJxGz;1w}`(e>s_B2xq7gUk9s5*J)-uahW1j?#`)p~6 z&6iUwNCQ8U2(4uS{n36_A9vZ`q40 zM#8rRtxXZv@sy%%ujY6dFmC6;1t;RdomMmfM_MNi+SJw8)JBWQOH#FLyLsQYzd&5K zoExk97vm=~6QbB7VZ&Nrr>$|55dJJ|4$NPyVSFdquOrwT^nP=VG=pN2LR&$wvg%7t z$NHxqqsr{PHZKGqB#$(}LLC^`7-=$&Rr02}?q%lT7)MC`6>nxoLak78f~V$e?mXss zq@8l__~qif{L5YnH!zO@HxY{Qd}?l<8G8-?XTl331f&)xNGS0yFf`xZe6_xGAVBxq zM4VFbtmdwPFaP^M%YV_t6xTG^*mE7%iz(2Bg-MeZ^`P#F#ap_13*}~ zvQjAvO1LcJnWpZi0}1N2*GQ|K*w2&FQ-v*t2;i3gAw;Y*XWnU z_)|-pz*)F!g#LjdZm9Yaw-o9jy9)>hAVmi{cK-)sZvqZ=`@W5j$i8RaMhICF*+Pb7 zOVVnMse}+JYZ>zq*_R2SC|Z=VP4+AqyCTVwWo%<8S%#U?7|rs$x99!6@B9D1-`{ck z|3^nh$MGBw^O?_m-`9Oz=XGA^$rnw4glH|AoP&cp$4bi5z6krqgoq3r?oM&~F+Nrb1Ifw^_5o97Cz4jl30ezxcHO&)?A zTaFhS*talNFSKD*!`5|UFpgAg-u^L$h`epAVa6xhMbCzS7#z5_PN6*SDuI- z*+kw4=WZryv;z`=f;XpXHf_hX_2J17f7|{Q=(4?Zh{@3T`W*hu%n~p-T$R1wC1cp_ zWOE$C2qFpyP3f@Wi=}boXSHAw)r5o~jF^7HA_(b4;=f;IBQ?9-yZt|;-8g%zGkuGF z1b6DqE)1-SWQh0?LKvbkDC=;*A^3(hvm|Czd#&M_pCg+K@KqyP#4P0u@A$hrPwFtJbg22Ht8D`+d&C>Cq9PeF97)&6M9hidYdw;_I|`lQd<6*?iRy%7V4 z&xuv5uoW;>Uc@4dfz>9>AcyM0%rZZA9U;6mfRK%b_pNf+2_05AOmR<#ZUag%^J>1%0UD8V^$g;G_2v=3^;*s=qT zE=Wwe>Tm=MsC`1`G(lOEt;2GeSOxT2IQ29$7xF_+4Y)Q#%>4R7xXe#4#W86EjOajA zK0bp{eNDUd7x$t@7MH2=WEp$y<$Pa6?;x5A64Fe`9HTVoIz3b?aaP+5q15ZT?yU1B zu08A%FK0~BD_Bf1C5I&-#mr*qFwRYwot4LR>u^^!MNDLrL`Aza)Q_yYHTg8Ake>8O z-*|AZp$fYB{K8)OUC%ZE4veBd&yzu~07Hd~w(5ElH}^#q+Gc=xeUQtZw0!gBM5FqB z&YP*yad8%(VEY~%iO1-GL&cEHp&p~yeDH=|yRyD2uoY~deV;i#D?_yF`nWD`>1$D* zZL4J=|52Rr3w&w3QO+f(!=Di%m;Fg4;;%w_wWhA7l%cLJuj#dHU?GMM_{v2c26wwV zvk1+`8Y!;MPu2&_aE)ch5tjVQv;lk7;O&M98M(#p4r%_W)%NJC&-Ml_^ygmghTy$e zS_BZ4270iW(LE6D97tOF|Biu$1>S;^`3Q1Q$hwtEzKhyq1fY#0k0yqrSAl3)S@l*w*5QEKAs29hN@}wqwGjK>sJD1bl(`cuq5f`G)zOYGCQ)9@^ z(!u;mhb~&zlVL$VR7gA)-7D%YYbx*1z5Q-};DvIcTgEdx$lJ9fSJvA;j&1~6^x0fH z6D&R}Iv7whZBu5AeT~Ye#?dl>mTW1}u&K0S;^t3$U4&jBcWJ<-8Efht+?{)Mh<>*V zei3Z&%5VngQ1>+EfYPMHzwKBG2oGu=amL%#L8)4mAJ6_gY9{0kV;D zEpFb2g0>Ku%}JRCj}45iGw)SIr5aX$dp-NhQZLQ8QQ43UscRJm)PJ|iS$sxx71jU* zM_ej_(O>^8GNM^rL64*kWjdad))+ z?3E*czLuWJ(~6SY+{co!vlyD0%enW^6c(atr&Ot4-_Vb_>VKwbTDh_+mHqHWS2Ur> z<_8$0=VF*Tu()@pCv1-qag2jX;JS-J_kb8&4bIFI81Ge`%Ln^;p{;NM5&<22h1B0O zvrZbGmPFTAmdkTc-CMzKP+6q6kseSg7e`VQ0=PQ!msbA=sU`(==L8p zDD zE7v+=m$|^%fgYF#wHryG3UzAY2E%6oDNK9J8ZEb8$T-O-Yf67%pG_i0sdSIzJeIZ- zY;W2-HPr-CwRx7nQDzS07m{HPr_?#OnNec;5;qdQx8)KcPv^Pc4)1dn+{Y<<^$+D0LsP25dk5h?hJ(IAsYUAn@U5QgZyd7{?F8V zpU*wsk3hXWE)uI)wb=4ojrFg@A3%6s76A`yu`K9U+dE)^sX2uq#gl#K2Xoff2rEk> z!KUnx>=fILu-*F@+YLJ7)5TPC<`dk)G=f}yI%z?KZYm@gJhiMwjjHJ@rf$~xF@yhs z%`;y{`~y3(UDz0kOfMz__8Wi-m~z{xpt)O;2EfKYrK;6;08QeUts1_{Y=|UOyZKu- zG8g2E_Z%M&%})JVd~LjfjMrylCO}XJAK=C6ObJn=0PITv|0OO$UPy8v)Z=MH z1b2AoLPAmTk2`Hb#b>{6Mhfi&A6j~EOq~jauOXjb+|>c>c#I^U;X|^c94V#iOrn78 zr$)+UneD_8|0kt*qGzLUZ|TxG3JG%7&3upjS@U0{Un~!Zu6e8(yb$m_zyNLKiJlL4 zyqnDBY7V}dls0Sl7B20SA9zDhD&tqp#^uUt_9l*l&xDFV*uVm@16ij@W%ps01jB&5 zRJ;#y9WHy1Gk&4v@-+|J98IwaHuTk&CT?z(g9GXVLZZw}@KdvnPK%A*Z_q~QS%pI% zXj7Dv8^(jUB;RJW6T{GcwkIBX&n|}BtN03ci^8T;MDJv{BgnzFjD5emCDGE~5P?IP z=MgDw28LI7bvn$shS4%KC#SEd zqc2y`@UL}`obi39M}tJiN4^}OSNDlsR#IOtgNeUI7C_)6F(9DYXNK-V)GJhyv981B zN>MMJ$+JU~f67U`z28XsJGfLu-}f0y*USth*{i*K(DxbpY4xB2!NB=i5sOYdt7ijP zvJ~rd_q@V8GPxefDrfp5-tm`m?RDK#B5VO>rGBn73fo4q_h<#G+&*Y)E1TOvh%=2p zyK0irmQ^|gMTYA~P#dc2Mje!c?6Pd!QjWo3*@3^d?ZG|Mgz%Ylp}t0fM$I(}L*`n+ z+u_lkUn|tLm(pf~aaTe6!ptLw;j8tF=S zabaG(tq>O9^*2XoA^RKWq*NyRaq-Y^JO`3~Lr!!zF!*53Ol~s`gapwlKP9~sC^iYI zQ~MtTr~WwdG^ca%d$uII##+bHO-puq9X1vsEWl|hXpsHxJ<7y{^9`zg85c?|Vr$Gl zN1mm{hc%uoZ}yP5^83a^9rh>2=j#e!EQ{-J>jlhXR9Vp;Z>}{q)n~oPJ17}wvwE|i z)AlO2vbaEgr_3H>NWq1E0fYb^=qj9lla&FK2D@d(j?^SOad7aeMVqG88{ zYYz@Low3tu0+FAV%tbu{0!=&_&SVq=j>!bEn4!>e2mH5=lWnrobu|X9NPf~ayEOUa z!LooWj|5{Ml<{oV-Q{J3J`>s;I5ED?aCqq+;2ty!RXs0V6@E2X`K_*8%ShKG(OzUM z7a0B}a>Q{fRl0OSabIsIsPGaj~#BoM-^_HMtyO!hG#hlVl_W~F^{FRBik>=2_f~mP zrUBhY&i1p|Hg#0TLJqutpJ%_J7OGP7v8TvJ4d@NJ3 zUNm{i(KapA$*(tFwT*-ghPq}_ZbDJ*mso&;)vwL)u+NV^J&8rY$sIP*rn~!>t#&g1eYt{ z&oevQfO^|@Nn+&zr_*g9@RDs0+Y|T(+Xqw_y?zCT&{Q#9?-%pz7T?oyu4k83^J!!p zId@qqQud*o0zqMnjjF~t!M-aaJlsJOVzEG;+qa|Ixof9>_INrSui#!f%=O@|a3IYH`*VoJf3LLwIW@o7t*w>8Z@zz&|5o0C zGxc>-ieb|2LsffW=%w9BlpTP~?=5HELlBCfeF*<)_&(Oxc@eJ8;$-}Ik;;2dxN~N2 zP_W8=QR0syA%>UO-q}IN^De?Rcy)jKt^WqsIp})E*{vewTGc{cGVYAE`jk7ASv1_0 zs3h7AW>y}e)bPaCCO`C!wSCm|<;Wl!j9h-o|2O%%!7|mV`J~kK*0oP6Li;h0oI~n}_U=2gv0xx4XAs0u1syEJ=*62==>;hZw zo0(H(r}pVuqUL*Sh*ss_y2Eo$;I{2;NQZQu|!y*a&8DVr#qkSX=@U>D4W=jh+WR2buNzJ>lCF=#82 z5Yi52-x2Tp)~8FOY&#=lT;kf`QL;{vJ2Tb-b$0wYmz-F^O3{7SON*Rc#@S?(>LA2$QBO7f!cgzcQLuv-<4!bzBu_*Go-SIL^RU&B4P19-duL;FTZoG*wdLRH!5!xd&ErRjLAZFd9ODWZN^LzK z&Yv&r-%#5;WMZR7XD{1O484~Vd-|{-0;E3pL%)KosBx6B3DHkxBT;e8B73RYqx2I` zvhNuDoF_Y&eyRXobGX{p2Lmd4@D#NC@!@J`JpO6(yN~M^9@yTrdnUAAPAy}$u!LA$ ze|>ZAQ2ZdGP!fudvq`h76nKB1y0|gL>2dXqmW9-(UQUZIc3A1%BAkKl-JUqx1t;w# z7-su%#||9+q{g13fH2v$f{Fj4qY89G>KER~xXEy%<^8{8?H<^T=kM=*-B%yFc5ell z4bH(D$_nVtAnwoe{Lzt*%IS7LjUkX!yP?&{yFERfX~vZeiJ~ELhZQqgZo9r?SBXVm z`EM2weHLtPyYRG65Mc;6RgYi&Ua;&o^5SuN#i=)(ua7sH-K&o>L%sa!vLFwGBEfP% zBx_(;-$wHjN^g^)1)A}sBC^Jcr8UeW!1a}xv|hSWgx zktx>bI7vSvWFO7_L~`|HdY3DIdl<+4Gfpw!S8smHAC6|f1?@gyM=!W9FhFDJl0Lxt zH@k2mKbw*1W?a-lgLmU4Q8z+@s@x4DetX-7Wyu_9$6In_8XN42B?(_gj~8#r(ZDs9 zuy^?%*#3z>+TbX*fTI`=-gh*Jo*Z4wc$6mjHT@D&Bw$<*n8CQ*>Z@T3zsb6m)ot=< z3}TddWOJTAa_^70*#G2#?owg?)xPq>5uSns77Se)btC!0W+M6dBSQ~*@OxkcsoAO!v-?!t2hp3Xy-z-y9~ zdSfBkY*$>l_3*5?h6@4zb7kE8fUElZ)0ej3~$ zy$m2B(D@I{8OXW(p+5(u(S4}H%3KFM#^Rt&EWlID^~WJFdQK1<{c}v=ADCxz9vBTy zsT!Ow+&A;FXDMAaOryfy^E2CYEDRB(jwL1mZngwOeM6g|I9#H5ZuVhQ@!Gurw{a}} zX|91A*&0xca;+pie|*mPn+1$5C%xMD0A>_PwglrCNRljIK58x?693@R6%drhBvfQY z*8)WHI7K=AA+iGhp>a&LVU;V*DxWHHUKoS4KALsCB@!Rh5; zoLw*|?3M|Wa33!uYCgU(K1h0D_aaI`y=1N`{)aQH{Gj54Nn|nu)D@iGtg(rLKj|j< zDQ!ZN{qx1W8}eh+lPYAzlrK! zY?s+Gl<|H0wtw5?;R8pWhb7yBJwNbM_@O*t0S@b1lLrYKIAnWYfjH&H(qT;jG4kTn zHf@wT#U<{?C@N}rKEM#I)LPIVWWM5Umuho0(Ne|7+xZjTQY94GItH9d_<$Tfen>s{{U^k8+~NyBt6Bs^t8gn%?x`{4B z#Idwkee;}cf;5K(Ou7M*y@R5#eX-bWY%4HD>yNW1VqE~@*S*x+v5--)@|Y?-^!>#K zx927H-@jRkUzwGNpGV1l09N7~Vp(A)u#+JyZU@Br4tT0_l>GG$Vu%b6%93*2^n=x< zZ~8R$X@8eSU60b-U@6XQb=IKNOo!W--}gB*{(U+?3K2ImIttymZ_C7>vQ! z24YwxdQ2u`eL@sHTr4kPzvM4vyZGUpKF5=%Wwi-se~Q6OV0+OOP)ErwB`Oms3v?ku z_Mvnr+ua!P-PqGyoOmewtC{25p|V*wPmh5RnMZ1mr-r3{gFa=Kls#bkqzJ)>4a6DX z1$1~L2v}rYX)?Ha=VkQ=jTVi%ow$CY`6!9y3!cg+$++YY{Xp4hQ@$zE3sv<4XMWYZ zUe2od{P_;Ah~xmM{SZJ!5nNMg)W_u0A0+z_sR~LG&2=IJucI*C+xWKp7_oHa?4y;X zxmjZ`-+f^}*mcmEEtU(~G-AkwfF+fp^fAOS-RA6+(k`UvbrX<8s9pbS&|6s5Po=s@ zV3ZwyR8;pV^N4>j%-w%aF77A;2X0nJL7);1ADNx20%L?UZO)Or25gG z1N*X}E{sA{e7lK4b%_I!k6tW4{52{Kqx{ zleoiRcO-zJ_IN%zz`#3i&g0Yks8|wEqrs@rZNVQW1thYT`EqFuH?Y}Kz_|0@J=d0! z6A&lsEv6-i1FImKT@bVoqz|9;gD%mPb{?^2OBY+7HL9ff{EBa+rl zkMUL_=Y;aQW4kmPwTox_^YQwi{(V8Sr+6S~(iBrm3MQdmGef!COXfq`nbA$ft62W( zcRps}TvE!XlOIO;EbUMp6bIK2>ET<1 zdEok|JK&o(zaY8XY1S8Hoe=<(eL~Xz!ZTU&fp|1;WVFeq9Jt@h>g8LF zkL~50z-9mjMb)|NqZ`n$?m*?^Hz6prJB9x)tNPiL*D9ZI=bc{`nKUV>r6S$FlcYB= zY!Bf36m~n%ty!b<)tl>`&13kO;%+Bd56?ASl`B;3o*9$y%3h&!ae?cCNs$J0dt@?# zeADROW*wYn%m-0wOFo zMqUQ_re3%{!#+v!9)!$g5De+)#Y!XUU}F|qB`QEC-Vk5lRY}yUtl|s53Ahu&;=i^R zmmtx{+&_DSX0bn0Q~wnlVSy$Jfv`D+b7ba0gesgA%AkZVYHNDXMmqw}t@LZL-r=Wm zvxxeZlTk>UCh;#*0$gI{*yrH*#!aJ~uB&GHWOiAnq#nLp)$H@m$9RK#SA}-j5hnhZ z2_EAWiiTxPP zbU16F00Ife7&8sz#BovRE=UM1&{gF?bfO)v@JS7_Ul3+i-Pt(M^cc!zGd=(Lac7)8 zucXS8oQ-gDF+(n#?nT?A?vi$0qPhUbK`=aI9lp;CId#2yRlTsZCiRDvT=wzMUneCu za0l!^F6}|&LH})vvCV`0jOiVs>Cvh2Bj%cG<3Jgy^$K~yoqQoOh>5vV8308}xZiJB z-g|;#d*f%#p@{4XtDu=Z%h&%-v-qjt0q&!`nB`hvbO=s#$pCh}D{X?QUBFd7e7Y&* zkGjIjvYU6{W4EV7W`C};n!k1&Cya54Hnp2zFw4x3GF&HGJ!2?~#G?3HSkL<5QPeSF zAydua-TBye)6-UZCAXlm*k_lkdw=*|1(r}30euNc_TTQsOyy81WXWz^Dln?lE>1$j z14IJJRtwWB*s6NI_bSeboT42vebPs@cDxhtFzqJj*Zkk>m1rw_IGsUzjElNz?IT zi0%^fHic7V4DcxCi{n-6J zS3mb$xE9OBQT+8J0Vr>Tq?jaiV5cxdqjde;h8T~|Hp#I+X$RgPC)ZJ_*@IHy9HK1` zuI=HGUOK~d#~KpZJwI`+tg$&$O5)qfIrLXxD@dxU4{QoiO>?4TKKBmX_pv9nheNO8 ztPM}+3yrdL7z~oaen1c=H)$Fe65Aq@rYRJMmZdcUhs)MG;2Dy~n}H335Amt*nk)Ti z{b=y!mAt9ataLpdMc$s%6SkMzAEd{^L3P#<1SA^aX~hkjq-A1A+B^^ABH_~wRM@-O zfIRcvbkfoMhq`yHmW}!~QJYVqje~XZMbFYv9S|5Z0m$E#H93ia%5 z&izqbHtRX-F}=Ke{YBt4mzz!Phh83A`-^%Js|%h`1D=Qz+=2^wIBNIclD5`j3=jIS zqu%m6Ncr|j@3%CrEZemiBqPE9NMa-EOnI<)zEhHjI`>QJf%&Hpxo7U>>rbIVPa(0; z7pRI)SuGsPTg@EG7q}<$H2onKZy;rK5kcCl1o_f@H&)``d}%)EP<^7)zJPq0`oY|u+7oZSI-1t@ zR;XP5C^#B>Q^??#3q!&MXrnRdC!=y$ytgQ&4x?y2;OrKhM8HS_Yuyq3V!o)j-!|2a zweLO@&8c1|J=%lylvOb&Z14F#+ z1zcu5kcYp{QXCC77(Bk#wO#9v&m9{c{H`SX=)rK;$oityg)JBU%_}1+3&#KY@@YI!FP~X4Znv zR7h8m{uG#%BGUnp<7f(9<2uMG1tC_@cYNGn&|Ii?#!Bu}lxo4`TW0r$_QN}A{UNstW&M#I z|NZT{1+He!@8B(nw}OO%<37!a!n@d4q%;zakwrPQ)EG6~wbE}!7lwFKpTq1uMV7Ky z-;|GPJVT{&?Je9&#eo!oU^ax92BgUe5=1UsMBGQKMRhp?bGWX;4@%26WA%lPbquUM zt>_km{Ed3Lcf6hJY#f(rLc(WUu9mL%{ENHgL*LkVA>>r%8nw>o{{+P(BAzz&zq8qI+Q-JYoJbe0io zT!Dw|(v3+#i;psn5Six$#AbmaReW^dty@Yxe58Wo>2WrmUGZ^tN2L<YxJU$&i<% zLngFQYA#qSds^X1vSIukqT_k1ajo#_at8~UTYe!wq)RS*L0-Rn@{KmYZUxZE6N3n{ zGmfTS4>X7uscICip1==+i`|iVwbrlwBe*-13#c)jI!9}GyLq#FZg=zZ`zf;1K+2yW zxd{E;hq?$psu06c!o?s6pPi!I6Hij8x39U?y z0TV2KWD=q9jHdzK(QspY`O7aEWo2mz%OyK$59I_!#`FdD9mt3Fi7tV1rR@cH8rd(k z4Nfa}1R9#b_%DBF{6|i`S)27_v$f&SYtF7H5w=sD_y#A3%&&)1tcU#J%aHVW9wOZPzMPTRtAa+I&w~gFvE?Ij9tu*>pki+#;ZEW zg)racR#*Q%ea_ZDJX*f1>wSnwtzCbWi?LP-dwQJlGodDM639AJwSDk9AgPDxGHT=7 z^?EaA2oQh%`QU4+@>{j8mVV$fXNVT+%k%H2NF1^D#;`0#AJmyf!7+K-Ab`zuIQD^6 zMUGD}!~m-(=LNcJo_wkp2KZjGjU~?V!l@XfL>9JeI0Qt9BbB(b*X%Xf4EI-b*yU z3pB*uZ$qg!wua`XciTDnS4d`k{Pu}@hITemG3yElomP*3U!8kSk9*E_BqvL*>xIEg zsZ-|P)n5)JjdQ5ynkuQ4ijFO*WpTGH*Z~{t!A~p%l6)mUhP*5m>aYcN19YKygowHP z&oxbhaw)3|`b?~KMkf61tGWj$gKJT1uxU?UzK z(*2UcXRDw`LlB&3*6t+EvaqzY`ZkZ-=un>QSXn6bo~Tz^5+(A4wD=@#u^53y3E&ZV)O^5Dyz)) z5kCfN&%Vf#wR_K>UdjQ_d83581a|Eg%#8}y#e$vvcJWwz!B7nRQT?TcIb{_oj8QLbZo`4j!mP|DZ zsfa2(S;VK-6i0Vz!xa(nVcCcKt(0SJZdX)PupeXxccM3LvYd$P!{U?L+r-JZSO|3% zstmZ>)0C+8E}Q$?+3sx9Pl^{#oN)H_RfLt~AnW1eb6~`ByJx;ul2p35nTQi$Nk(~J z{K`^T-8PqNZlj1z@P*%j{PiDPYSz-*>x$~ucswB7k~upqnZkU8AZ!iEMd22=6T&db z^E{~iL4;X%mgzAYNN6yW`7*c$l221P?tdz%{;_Mh_e~DYbCxg(9AOKZLJ}o2+YY0Y zX(`WX=UvB9aPQ{>&o0eRwg>O6UwNPHlQFC}(0t@U_hr&$K}&W}42U5CT=UbB0&?SE z1TB@d1zuJ69R#5mE`=is{h@)~&29b`py&s#0==?WTJ#bXML(MF&V1E+zT2f~8X_Ydr@6GJ+Lt_4zdc+VAVTCs@1 z5O2U=o%=dlu3@qs^q&Inn_4~YKYn=vbtLGNXnTaOeEj9#7;$<{7~blw^tndCR-+%fp9wkSUr__2OM5;Xq3XtW z8Ue_mE1a_N9%V%6p~&Z~$vlf4qRRR_c3V9iBEQQKe~m?D;a@%yx}y1D&uGPYX?sLR^1jztYD_DpoI z4W$*CA>}c&a7wcqn(_# z9%UJ>x+he=y;`&8Zc|)7+h(s8(=#S|YBp+Snf=~9FMm3?c{{GMj)CgqJNXi@xj%cQ z%8|_4@F0l>Ix0<{&K_|?y&yHu3P0J=Rk$lA)o1YQj~E*h2k2|Mh~OA745>}u7bQ!~ z%Qtx3Hhx1?J9uO!R-bh8u*47f!kHI4u{E4m=RBU96>P7vj)7AOA{^NRvzrqiXfv=6 zd8@shY%WSGx~8BwtaSNWV{qKYq%Q2_%d#adIZl=(|)-^W5!^Q_ZvWHS8+&vi*jZ#gl8hXp83&#L`GYmXjvNA>krPl5D1I4c)~_+b zb2EW@W_N>L7mW;s$S9`V$Cx^B++Rwtw%i_I6c-^VvLU@-<_?@2UfkxNUACwK&>R7$ z=IAIjl2T040qJrkr6`5q$@#mxpf~@mYv6e|ci3p5hdT@hqA;C!s3B8l!%(c-&aPdEz9JEG|7Nv&Gi$n#JDmE)8oU+4Qr1odqb*Uux zTWLPm=x>j+(Ju&^t?bCUa!n7? z&AP^^#$T%Zoa>3u7hFa$IobuIM8jfj$(sw3@g?L_kv9s=ni&%a{CSUbHWi}ui2zdE zwB&xPy)%(_m2NyFx0Cl*s_&oi+xG4MFXo&7SwDS0&V68bkA0MB8)~m;J z7G=5*U|-xGU)Td8Q}V{J=P5vj@^7jyax!t|A2RUU$j7-=v4R0B5wO`P7IXO!3 zm(Vrnrrz!mmAEQ?d;O)Bh4d#ym**f%b#i4Oe2P<$f(&HYb(4u$sMWJ{O-6gB!Kh)x z|Hx)zfuZ`(o9{nfwp~CzN00-S*OMqm+`)J3q;d785kB0V?wl723U?TcKG#wo=&E&Y z`Q(S%SYL~l*&mN#)&rMKy#sXN-|EHwx97|6q1}wTF9mTO^zBM}hd81TjxpXn3aaE+ z-}4HuW@SJMaI%*Z0*as1x3db;;~-?#aT>qR6gL1*~9r4?aLS1hD(WIx<`Aw>ZInwpg; zt5!0)sWi&8izV3#5LnG=--e7s!7}^APO5Z7C>g4#8v~0vamo-wqK~RWdrJl+lvk8d zdSH8-aw=RRYSe{dlXWZP+V)}O%EZL)p&+x{Hqv(2Ziqa2S^g5%-;Q2{I=CRhl0k+y zFs-Ir1tR%c!wK?goh%j1sT|jl;H`ry(MPY`e=#W6>8FK#EcW(N{71!b?05i-C4d|{$}wvGjrSaM`txvv%bxf@4_?9 zxacRMYLC-G$%UOpUP8m+JX%`8W>yA6)P$jj5oX!k*SK4rx+X}sD~dW9fFLxqzSSlEH zcs8AM18w1sf>CDwjNP;%;3Vwzzley0rKQ*Ii{<(>GqL>?w0`8kliA@e!_z=o2ys?` zR100iJGKm(pGWnX(cO2=_8w>%B`G;9YCb&YpnsL9_p`}I_7hCmdk3^R|L=18gN8ag zz;IanmCSrp3^{ZG(dQ%^Ro?j*py)>z2R?`nS-x*e`xs-rj;Ko*RS8O(%m^3V)DSO> z;&^x&aWm(ZSQw<)4S~!}y|Dwtdo{H#z(hoOQDSfjTcmSK?rt^>zh-B>?8kO)9o`{-GlS!=9%xaL-C02qi$`;#SUF$nT@nC4YII+!yS$LI(D~%ep%vq=? zxH2x=E$XS|eA7c?BQk4R!E4S+^Ku-wvYAWIJOvUjUbh0yRJN`l{j&m4{?c6-Qxdyy zggOAkvT-!<%e^x}2KTDgh3ZY`1aI%0ITzI7pdWPmH(@iQl5uiQ_VJamiXZg_z0^sQ zxNIr43lF+y?kRe{%E~`_PV+aI;JOHYNr$or^%EtMacDp`5SzN^nhc|WXQHR+J;ZzD z2jpkmcqDAjV!rqTpu@4TI^e%z5PNa!(EyU)TG$JPh3{e+99`oe5M;+wKa;Tx^%hwV zr_>uG0-TOfYd=JEkvl%jk{@hGr6(=sCqnKrrJ1cb0?2#W5rhKwhd`p_Hie`_DXoY;)VT#k3C6rYQw_{! z)?%5NRQO+c$Z=j8L70J04lGXeXovQCsn84pK~3(^&^D%04J6|Xd0wL-&u)or-R^`; zw+6)l(LR;zXrak%;4VtnW_92sCHo*gAfbO9t(7=Mx;%EMK9G6N;Q}gHm#^u&>&(5z zyRd18LpQsHg_c8Df?&fu9H;$ewKwaq9#|s_4VZPT83fr50wX58M~R-Y?zArQkD(r! z-gtIA9@+E8%fInBB%!zya-gDG{|)y6=i?m?hBLu8XpKo=r6+|ZBfH%HWWEX^pFUbT zx}n^ZX^ImIG;-ef*b$p8eF%S4IIG{J<{dDW0_QpyY%w(S-5oQDWVM=skou-Omu7{W z#gdfiN(>`rRU0P~Wk5?=0P@sDJh&947(DE1R4NII`+e@U*{O_Ex4^bAfY$+q+9PF6i#rr)e0#Atlrm0C7hR z1U~Ky0$(uF`_o`Zyy;N@khliYKQoIEOR-V1tRA`$ zQ#BI@Y5`x!F_gvKP+SL(FYGi4=pf!JYY%&D@Pl&1|5Nz;JHA)${+JG4hbs4mrZNfj z*XQtt_j|8s@cZc2RT=jh_a0N%4Au~SQiY)Ra~VfmLsvSZ?0>A>YA3!X2M z$fg9Xhfv>v9}+kL}er@!-D_$4si-2Y7`}4{N zZlKprp!o_!5T;!{`T`9|--ls&K5^oA zcy(ZHv(IYcZd;>3+!JwQbtASegku;g1|al2;{nULTD)U9--G?j_=?;5?SuUI6pGB7 zKnOy?LQ})noYvvVZ~?TUe@y4B>uN)ejv`et{trilQkyZjP~Ctu-(MwY8%}cz#O>Oo zAmBEeu(Hd$5m$9+YLWoTS2K#QcN8Mb-&=2mdsoh+wE!)7$wrr8 z*hQgZ-PtGmUv4~)DbG{f(`P>a|7WD>zb2X>U_S8gNYkz!`~UhYzKMIM2H;4*w%q4H z)Nk8QJ4+nMkk=_V@^;0DkD8(5dO_@Q<0MI8ThfFq^c z`YV3HDkkSbT~pkuj^9~X7$n0oQ3q7#8~}62r-J)WfXnWE3tYZc2X*;4%eq2R&r#J> z132YSu*E|R1rhqeL5iRKR|2Ll=ovEZQ2kCARzv>S4zG;TocBRR)IhsH(KkjVNZlu~ z4Dk)5Ac#~a00Kq;$f-0In7d(ZD2BPA3ytHXkS+SW-rU9Ns7;@YhTrdd_=L1$O;eY6 zStjM*5WYZeJet_Ip2mq2>cYscvBc{EFVHZz438fwjGn%e^)fu_k=Shgbl~FceYuLi zy_^T%Fa6Q~H<5Q)1NRO9&JH?Kwwc+Lw*TbgxB#02(uK%(=NTo_3lq#7J_OTbwVt+# zoKYiv0yk1WO%^~dN?+@aAr_5dD^OlVvQIbk2Qc+Bu3m2Wo>ghLf>dtr1pUfOZ|J>d zqjB;e+!*TC!oBNbihwdvtA3t!3YaL&W9EOo;_AfF1+gq(e~OlH+1^h-A7x7j>2!i7 z)4JWcx=CuCCh&IVJ}W~>yiZSa$7}rLiy*tJ88EyB%to0le(&@bO^Ojcki{MP56q%i z7@9{P0nFZqupyf6H0$I%3B-WQ8T$ZX)XQbz0OW%tpLl9ZwPaz^6CJ zu-?AzFe7FB;?}P6PLa-ZOMN!;`Sb11_Au6E#DAxCfy$xp2SbB6AIzlFBEPvg&6`{` zdV&&OS(n#3L+l}sV$H{WzDVEui~q0jyZ?uCh2}>n$Z5I- z$lvXoSYV?J^Am&@NC~Km)<^{vcWo#V>S`NYlC-_NReRw(ZCgZWhf(mUXZmP zbyC7hdxV(jv8HWm*E+`PL>}tRGTE+@2x<#uWFGC5+Hn$Dm_{u)cwQ*qcTA_!5qj_?dn$%&0 zl76*@lQQ!OSt#@bMRddAS9Oi~kvBM9ON7hqE=D)ZzEoDrvQSvFMu$P&&QP@wQ(WE~ zkhBFrJ0fds6CRI~YSpD%pmc{(K7`qTzHTpRia?o{|KEfsadyGVz-9i=I%K&dY}xTl zuwd72gwRI?Ic7FxF@!EO53QWY-JQso-?4|*K@Q?$)CK(wZ@{ijPysj>e+L2|h0j?O zY+M|YM#1Y_DYnV4>8D?YdRML9z2>4vVBepJCmP2$=zSKJxayj1ujW9#2o{b94GwlS zvNs3;8i2U_5VhGUt?pr1HLi6$T>kvJPp|Z%!`tsCa-{cIpJ_h=vrtonUmivtWzB)^ zqY!vHvLkZcm~}Xc2p61-N^W{r6W1SeeB*-Chy95hk**z!(Xb{(KZD;N0Q(zQ+j)7~ z@-ohU6M1#>UJ1FVEVbmxwd%%ph2?t|@D~rAL*h{?J|&D3Ggi@bY+tph?`(L9`FxoQWVLa6cPRreI-%mT6}=huXotD(8qg z^AGIZ%0IBN94BmNKda|o=SO~&Gyz9bj(A1iKMzEM-;XdX14dR&)_LQgYh&(2(`x;* zS(t;tO6kdy-?Qwr_^zSP{pTle!9h>~8RP$#nj@~Xn5LM-kR4WIh{_VV(p&Eki{;&E z1HEl3qhIb_FSsNWQaZIlQrXvZMRD%OW*TLXNND2{X4-+pKTMJA1qMRB=7(r^ zC?Q>cZ4*&v-#V7{_*6|f8GNs(ZOXo&BaH8tFZ6FW?!C@VG0H#peUBQH$4tNx1_5uM z!Obi;11;iYNTN<4u&YgYY+)ST_1w)#;M=#J)-)~6wzKu;KVQ`DP~jJBf`iUzG(?!) z!4azF#ZYhxUuT<1JvlPN4Dm`QDdhZxdmH-n564uCh)Qv~A5`6f4}3c6>;exf{mjOA zKCw|u@&n3sUJS8+AlN$+j`cKLqZf#dsS&>{v@H5{eP~*i@mZ$1toHSV(Dq~ES?XZ1 z6!f7f9Y=~yt>2XekLy2%{|qEJXVr5r<=xWy$jR@dKjI_y{bDRf?h-qVw96~k#j$KL zcn4N13c%Pe4K$V@B6o2ba?8V99^AJLzZsfazggT?2f^)Tf}|;z-b48tRugHAD~nEW z$KZnd)U)~~F(2xyAGQN$yYK(NV)py0J&EO#R)jzMpWb}`J6CDU4%_3)p7Zx2KcJ6- z6S;(2a@^HUSljMK{^fj(914f7%#*!@Xi8fMvMVmmTznKnOK}W0*sPJYmwryToRzNI zdD-;K`Nz>=8%Az|w6qidBgMo6yRF&(>!H=z{a8F);N|6^C*f5236P@=m$)qWfKPb0 z%y1PR=q#=&Wr&9MK(z{$#Gv!;PAO;SJfq58>-Wgb&3bfkBz}{U&r z*aFVodGuGAiEdzs%IfQ3spdQ8*p5_grQOk9ClUS!TW{$zAo4d%q^Pp6utf8^*O!h3H$Yg{uB)gdklUbhk@_c{4_x=Ch-`jE2 z^Bf(Gx$o;buXFia=lS{kU3uf@pO^<4=Zo~-GfTg~0youd498FC`0SO_>Z!pb<;C%S zQ=Z-fcpulzr9O3BNT-L&FWfr+Eo0S*RD!^zXTo(%%;#Oow620`^38kbr}iplmzFnn zfmr}*hs{jVkCl+%ak`h09+u=-QhaZCe2fZZx$TSGL#@+hag~8YO;5#FF=_)bZbQR+LAX|y`6AcGZUtXlHT)G$&{y~_D{eyUh z9ylO>&Y(A9=v)S%kd>f<9HXY773jm7K>1XVBqW?g$w8m)lR;F9o*gHmzoE(J;y0z< zfEQJRTw!cf@lOC&4O-|89Fgm%eK?A>Q68QgTL<}Z=B)S6D|kTE&;dlW5A!ibY7_X> z3g&0784Q}a45*;mEzm>u^hylA13~n21CVRkl2IH=5@|G;>kNZP?$1g}W6S$19Xk7S zJ{AX=$eIeI_}THUWC$-`k|=UU9S6o0b5{4_^KvQ}a>l@(XbnF5zLxjA>BozEu9tUT z!LSfneUffIbf{4E72plcJT^1DD~Q^%&F0tiKG0IX5UqQP9I{O@|3rnAo_p#40I6TF z#8R*2%fL0tF&d!B#CG{(xbOdNp<#z252Nz|`Ogw;1}KuTk;(=N7_sOM1wyB~SzQw# z^!?!QT(xzoTF%|`Y0UQzck}h?{%^perOlO0~nVd&+OgGz~@yo)q z#`9|9z#u3bICK1n0uC|QhS>@uX)TuQBsE6tFnvSnB6vj3uKVk<@195B_sw0=`Q!Pu zSNY9-@#}m#iz$Iy)PAs|S*R}v?k)}bTp_ROI4^R<`Ag$@stI&gIQ8MDDz&}+lLp7G zCpL0(bL&|#1g;ySu^6(71f3&?)Vt?TE7g^RBCSEM$bKJmoT{oGCrNzy5|daZ{drgC z%CG}99l zR2}3HJzb}}rW-~^BA(5A&PYe|Idwa-8urRCv7ci64pIXJTp2;VTa0`j(ZglaLx+^G z4ecX4ylZWyBb>Q8^1kwZuj7#SjJ~C%3uRnoj)}gcCqZ#VFYVAdQi1!sM=RG3ri4>v zC?4HniOyV;D)5uGKKXHI(7lWBZ1XJvG-eNm+QJA{@>XhZ7NlP8)AlhfHX`_5_qnWL{D*Ui2%(J2=EX%^(S`xn;jg~{ z5^dpBK%_<4TLK?+KtAt*xEgY>9i}DTA>jvW3FH<`+-!ugFlWrl4Xe6ZB7)^ut(Cag z2T)6-b3+pmAoF}b1Cig-!O-?F45vOQc7m!w>V4m~pV(;X3iTB5(1)fTMsN;kf1fOC z_vsFBbsvd^&MMssnRs8rP|tISAs&B8tH7+{AWs-oJs#(arKk49kz4pW4^WuwM|9e>LTGIghId!ep(;*{wtsXSq zhK&LyA`ephlidxGTP7r%ga(?4X-nGr=k*1Pe2wbT)ELQx?z*ZfMwTFUUJ55@9RVzr z7rWq8kXA(hJr`+Cg!lZzAP{+tn4a2@d>-=k06e7+sy@1^mUIM+h)kIqj&n$|8s4q_ z(q%3CuJTX!;y787ZOe&Jc;f3?2oSU%y!^)sfO_PF;uHV2?)n@vJ^A`2U;@vYw0Tk@45pWaBuu z%qAh=0>i_Tbv4M7gQ$yBjURpb3nag&rn+xCY;26YH1s;xUSOjj`K})fSoH&11yjfn zLG%aN82}}E0I`M97T2P`Ffy=%i=dE}k1Fz5q$xYw-QSUe)Ym3wit{r5R&uEP3mQzPUt;`Xl2GuH6wwqJ5ALhhW0v`8ZKG1lR{F1 zUw3ZIdtpP&yS;=-m1gJuG&g;!Q%P-Bs)w#+0UF0ZG3rUJz0_$8{;e2yWZ)w7#9503 zkaN@-M}x|>@c1VUGM(O+Gw&qvB|-Il1E;SV-o!GCN0-C}F#f3^4(?@xf832uu!kpg zC@#UcJ?Y{^8qWEvb9dA^N|^a~Tc+!}_-GKVqYa>ka8{~*C;{_Cl=q7&MftIbnkh@YMcFtftTt+l>;j%NB;t_N8Pia#jBB4&~<5zU+@dgUTUv-k?X3- zWVO2RY3bH{-n8>m_i|k)0<#}M_28*>1zLzkJv&^h&8eJos|Qxqe}4o#quPNN6)p87 z-EJzeiIg*=x_~i8_fsJc5_}UpJqw0ByJW`djTM9?X~P$vZL4fuIbxkVcEw7)6t8zj z0KX)TPdULAC4+#2SG9tBD4axL5yYDqK$9COnrn%C(?2hgKWLh^4V^vhK3rE6d7`}u z&ogj+h*^>WI%Tr}x<4xBKRyT~n1)9r$9uVDnf$RUcGrMpZ*CiFuV4U=|3?f6f@lz( z*-1vdt&vt$j${P~V$OVhSn)U0AA0+_zZ{HCZD6E?CEpbn2|Z@4Zw&l10{|(O4BFd< z*|q9rT0UJzCHJSweHY^K3fX%xF^$tqRXV4QS%Nrt4{AYWlLK`tbZJGh)0(Lkq+103 zUWgY01lvu|7N5UlkN;dewU7kl<+FEf>d&06IhbUwl?7>X6)*{XbT_+Oh%l)G9~*${31+vYEn69pAc z<;Z&!5ArtV-Aekg3R-bwFd;n|!`~&cUU2?PqXMg)XU4`f&S|RaVF@ zOEk<8OX3IiqD6a51qA`iEUjMqE)nC@+(|k94EbwS^T}HJR+Qq`=RS}0j@p(oZ!O-l zasEQM%_(Bz2GS6SMOR(5uKz$Wv;5?G} z=|OMJN>}i|3z%mb&orLL8PAjeZTKuMP`N|*n#gYm6djl;?FCB7ettpi>+%<>Oz4}L ziVZ-2S8Q&mGY1CKM;mEcz&1gj577)1Y4>QE9S)=i{UPdtA+@Be2(!&fho@JDj%RyH zDc(?GRw$JGsiquv{gvM?9(=-Bfvmz~IiYP)irm1{GNFbLJc}6d@S1{#bIpEA@KE>#Lh=Bdz zp~G&_O0}bk>0jZ;P^V|el{U3Rx#pN6lI9v;eb@=(3X=%vK33^e?{$IzH-FgKOZUE> z8alVrpH8+DJ^POZ!>c_|+B~iUH>g&WPatE8fV!1Xmr7I*RH>=fd%c z3w7x!a2GMM)L;*&Vc|`6wWRpC-;}bcEnh$D6)g>U|95_U=M4UwUb6clTy$@~y76Ve z;ClGKbDogQpFm*g9gr7V49xG|w$tX6PF78zO5ka%Jy8-S(GR#fi<>VcN@sJVj%~8q z)djp|1S3b~>vTU1xF`ZC*Wf}jf@l#)y80flULr_5Jt+B4_c!r*J^7|%wSlgxRaZ1l zt_VnY#A_T&F`Q;uY?e9CK)pXfS0uOhYJPzVO3ILL`m8nM3N;;Z-@3~`3OD_H!)j#6 zmE(4YT_VE(@)aZA06vN?PTl~^vqKhJGUHw6szg5TcV_(z7(jO4T*m>D`pd<+SIPl> zs7t`B4!b!-IZ72HhY}3y{7)dox^d3&6+#*}nt&|T^yT88-(qFk|Hfx1 zX_>Rt*u11;yR?9+1bql|O53!^DJtHP_~S{V(=D^spIM0dZ~7jxwYq~x9DLrsxFo*B z!rdP(6@Y2v?=`&&B!I?AJlzoPMsCe|fh=*L&}N1tmancSb~x_L)yqQ+8Oq&& zO}5u2RaXbOKu}2DH9O%`AWsslsP?f5T0qP@dY|mT-ulo#Z{+)}UQx0zoCe_KnA;ey zg#=Lm97PXM;l!xIRi1v8dnr|!^`Kmo zJmIrrMpiSTb7)YmBKZN;4(O*TP$J)2@{jL=k^Ozj=e%tS{IN@GINww%(ZRt$IKsk;JLvl9P>&GV?NO-yE&V!#2ZxksxPbHN1y`c}DY&L= zqANix4Up!c`MiO7?gFGNeBYrqe~b?8MI?M$jz0IZEIrKlY(Q1jGod%0%K@-7;Pr9s z1f$Je(3qq(;CviRv#6K>27}^TcJoFLKi`)P(X7+8y!O!D(8%&#)nQv9ea(lAq2+-C zS=i_?kd71T5j`D`hxaRh^2*!WB2g9MAv>6V`kkt&qI)mf(aG#+v1d7y5;)S@bGE(i z4di)>!KNP~2C@yJ1yCQwh(_>~c?F_qXT)M5&&;e>liESfGtNL?k@*S_FL-XBNZQ2s z#dt?ezGEe}tYY5(Mg}MJjPd}~%tk$#N5J-)#W#ht0fvrtAI@scv9La--Kt$72W+e( z^|g^n=(TTG^4yL=!yI=QsUXYzzk=>J1+r58UKVK~p-Y!)PkvBNeq%kQ%?pda)u-rX zWBKQkK1v?+IiJ2?r+jVJ_5x#5DIVO>{S6V<7I4q$sbz=P4ZUHN|1kX2L4Aa}UqL|D zClygIHXxQH#-l5RWb1>&9*W$$1|hg?amcKmDCOIoEtG5OTxS86A)QhFDlCPmUX4El z`<0PgK?$ILDdeW>qCk(%i4F_G&gqSVf!3&ZPDWC_b1HWvEgJ#{5}sRdbMso=K4Sig zLFN-2pSsWp??c2Oh_`^#Zq^kdrFE7Lp^=lm0aK&)Pv4mNx2;>XCe-5?g7BNfW$;h-a%eBA^|j0Xp0d#mHQ6fp$X&1ombnUI!jWQ z)lWW_C7lm@0Vt+Yl}1O8GqD9T9GK8ymuMv&5x$g2(Cs-YdhyFvBu_C}6YJFU*|+rP z9sbF0^9Hh8^+&!99`jbuZeX2F{a85IZkFVHuBI&w8ENqDs*1d*y2&4_%*P+ZxAD<3 z`x5ly3*boKjeJP{hk@crh7-`en0Vdupi?DcoYR3EbQ>aeJqxTvcY6}7+$HXsG9(q?3RLiKi_C6GMlMfB6N&#-zZ@C;;Lu-enf@wc6gSa2cgzwus)Xk_|=!L8X|Us(Ti=G7r@0NZwDsg-k;U9zFHI2jI@Ve6W|e{pPX4u?c%$ zM@L8TR!a%QZk)R9Xo>}bh@b;f&0El?2Ks~;*8%yO3eeLYx$!C)lwlEWf0N0Jri-cn7L-uuKN3FtXInCU}M6Nsp?mx8 zfr5WzM=h^ix-1=d>}bJp14z1&0`9_CFRx)a2t!jeYis`?8A2$NnTmK03NQ{x{yEHPQ(Z~sB0enoq_iCUTJ9*3Ld1IWa z^rKM;ah?HFNQn+g=fg%!?~f+^_BjLTB~?!-m9z2yOv8Zxz3p_Y^qSub=T^Q z$(B&U7;7NnQy%q{WzW*#ZDSNQxUn(L=Xqsed>hrg_auLojjtO1;@dQ1$wv&!*@#}P z3)bBN645Zw5u1d8@PGsG0la@_c5rX+6a5&~VIoo{j4jPq9$Z;AiX8n|PcCKdiCc}?M98}N;dXqwj3DVx3 z>wLDgh2qs`ZdVB`gblB#O2^9WJ$`Xq`EfoC_^y~>IDT;fKw!7vI7A=fXqSwqvyiW` ziWd}2Q?U9Zmp^vdHI|Te-7WFuKA*CT43lCs`x;`j|6f|Pd|dFM@3NM2S=S0Hz%9Jj z%rd!Hq}ccIfk%V#Lzb=k%viS1k93-WHG%Bcjfy0TP(mY3$PlqzpLjEU5fPui`y)1f zZ~DdlekfMW=A^@TkdvEU`F6ZF0~GM;-g3reg&9{mVCH@82YDjcXQ@{wBAL--HHt;j zkG5XU;i%^p}XVAW3#h zT}iRnHOVvVrt=wEvlSg<$73_8T$Arqp;Tw?#)jtn^_3<2`s+opoI?`O=}E@hLB_wq z;EDz~Qy(xyCvfKi#Mt~#BH6m($5eEV`|}{*jhTA?u$sQBZaraNoXQ@1#@f>9g`cbg zP~OqFOl<+5=!bm9|9mHwfxGx2=+D{C7&JqrsdcWuK*PU!{ z^J9ofYdMQK36!6qWr!^Q5+IS$M-;Ze?G^hCXg1Yj&kVHg0^Bjxh+;xfKv6wD{Q8F> zEWNWnt&t3l5_b}4pZGPnXh!GB0)f;Z>cZjdAo=m$WmIq2kErG?6e44`qmXPe+U2MC zvH&sOUON9E&4+0H2wt&6R@6GxUZVBBZ&xS4e8-lv zLGlpN?rFTWRt#%5OUkihbhOQ{P;mAQ*05}QjpR?EdtqI}UEEDiKGyuX`;l-0yp%Xk zS_>KHPgeW6ZGSHm-pVMaM#ToX&qtN{q^uv_9_>COCNOZ^?3tBwcjm>Q|j- zcDABVE=j~7f&pvcC=3jAYB%qtVd*j(SQOKQt{9M8s&{JMGlUA?dRZczXJ=E6{3^A} zw7;4?R8UZ}N zF)+w!gfPYnHoI^*)9as<4~yRQ=b~Pjav@U9UVH7_4KABAJ6mNVly7uA33>0mq@wsT zLsT;Txa#4VPp8X3-I`#~)B$<|aWp6#(^pF^)h}N^C}{P>zTSOjuI+VFru2RCRF(-i{vW`h5HM{S*3wSI#q+ z0;|Q7Ku5^%h<$xN#f)qd<$7#&pKqwhgwxSbfKQJehO@or*FEqc27v~|j&QR4%?gSn zcqy?iLHa}iC-H?6GI>cw^Io_}X^Hf6|JNfCu2MVDv&)vpUSB2zR}?K}DRhRWuu-CC{tHsZzT;+k6W3p?uO$TVws zee6h76o}tKV5{;`mrx+IHm}4JArHcI?4s5PLd1S9RoZ-F6W9NHB-z9ydGME2F{Gp+ z@i;Gq*S9R=Cj4b9_v;XAYwKIl?A@#wfJsevY7!`P&O)N~b>EeIEfuW$%TJvDiA9Pa z7HNW5NiW2z-nIeya^S=LNzJSu+0pI5RB%Ks2wM3{4VLX;W&U2d__8HMHEP zkNx6{{i`8e{3W*`o`1zPivgYnkPZR#Rv4iA?gGW+qK3jh43aiq;l#+0mw^hsJ2L}o zzAHYqs<0CVlHD9>>d4=OB!^eB+#lGoYdg6h9pG#4L)|*SdfIUDTfo{_tVsJ?as)J6 zUza2qx~}kh%de-$Uw+oEy`izGNc}2s`u&)u%|s#5XyLfGI-Ka0zU3tx!k=Ya;iD^_`^lR1 z+!||6$r;Z9-8-j6f|Qg}G-D7kzZq!?P|$IE@@QF-V%qe`^p!{7dE^1_q?jVs3oVH{_cEzMt!p-uZpdxs_=g zPWoqx$8frQZW=wrxGb2!m;CqbOT_2UJ$GDPC1P~a0}7srj30YLl^ zHK!K+AkC9`1@qtOCH()Sm!MPm@UZ-%UIBl;C1VE*n8`&&tBT0%UNug%-8xQ%bsFbz zOEy3S#;N>r@!O?a&{rJyGL?cAJ2j#vDSZEsWr_}l@Hl}xTaNaoa$9oI%yOO)s z66?ZddFlZ4-o62}v207}RAD#ZtBz6u4g`eLKB4%?HpH?k#GSpy=|uj7MjZ!-CZ7+* zab6yAm(M0TsnvLjJWJHM%{Ybq;Z#NyRi-!^1Pht^5=%XEx*v*)=9s(-J57O%PVZLx zleIT)w#3tqRxUm-YkubIrz^5X>Q_F){b4AgobTuyhto)grl06*jKnJQ!B<*62H$oN z-wh|9`4I#1Rs}Ay8ZSw@uN**+(l53?e8Z zaKGqbk>CvU1R?0x(XTOhsMuL}N*wGp9P$7XlqAWzX_bm?-dA@f3=NB!TLy=V+wK(T z2JiBc;d)}zc~l{A4v)Mc#Z!4l7S;Pq0N((xkNOz3MM1vOcU8BQy$FOYKAY;LV zH^bYZ!KdArW%YpGB|#;9#*F@CA#H(tb`t)_6>->TMFh!?|}%mvZ(wXikhYuMYGA;hSsr zYFLGVXGz{xwq*UaOuc)z1nVz*312q2D_o-6g-U8>Xvsk`JN5ZZ_E51>7Ktt_NP2VeVoo8$PO%`ULkvsU?y@xPpKO^ zmeYDxnV%kAd>?0RbaZunxZCc!SdSd!3U02;s6}FU1zHU7*}vDq^h83bm(%yOL9Q7} z_5i17oZcCM+1#fnMQYJ9Q6j)D6^IKxny|-IwQL<|*3K(r!=EFwHB4ygMX>a%-|zO` zZY=y{1_kCj7#JnWXP~x|$H9?&gc<^KA{RI2`I{TbMN8Lz3pLl%XNlX{Zhs*WO5Uww-aJYPy>Rd1-~{GQAz7n&@9OFQ%m@rO?ApS2dXD2JxOjyo4@j^zxSZo0CLUvx zAz;X75u15S>8HHyrz?rfi;XK4RZb=mVOn0^)!LsmLW39?PL7%}9QOfC&BtOvd6(@( z{tjjnOw;LEswgl4mPQ|i19UtZstrL`dW#+H5+EDBZ^kWO@P`6R?AYB}Qw8eYs7rgg z++<@}(s(~O3dCVBWDTvo0I*^Hk>PoZ=%m?!0$Gw(3a45bTdb@cTH@1OtF51?&(kZT zx8uIHCq=9KN;2=$Ai=}XtUxU-1hoq-9vtb-KVw?O@Oix3WgW>11Di((8J2GtpMVX=exsW78aK?aK5>JBvQCIl-hW_3j zOV3(;$hccbHO!=dm7jP{ng!No56+UT&u@o4Sz1GvAn*)E822b?f>H0wn>r7Ay2Mkf zx2yeLTeVI4F+J8vUDmiOJk-MKSXWoqraZ8^MODP^o&m*{D>N+K0fM-jG{7o9bE!d( zbh{4G1_9Iu&6mHi%Jgvrmt_7T&RAgWk)OMUG`a$9PkZ_ggA42lREsKnYfrsMj#z{k zd#kV4Eu@|sz2f)GkUwlaK=T-((=rL%z~Ol49R5IS26QQK-y?gZcVR{vbjAVEC9%D`#40!_qiWHy^Yx1n)k<9Ib{2Qm z-{!@h&9%zn8+bo*1~(E1*vzB8BR+z5F7Sh;lSzg2zFCMSpN1)Z`|mZ{`PWGd7e!1g zn?E&dB#(?F`1;DZdX*J;a1S0mFZJvx@hyX7_Isw%JVgaY9_TrO=U>~JQC}SF)A7)6 znncMV--?KCn;3lg?Zh{1s%8E)!0mBqN19Wr1 z(PYK7y1TnsS&T5<7JCf+-*Mg<90aybkcZYqg5|A!eHf;VPhwu!GtigecK}C*STCG> z!EAHil_UWzqN-K*(gS7e=k4ZR_m+F^n?&znEo3gG>790na|6HJtyw5;hhsS1+0f5) zcoJgILJDl5Zz^i^|ELKZDheFF!%E>6xWQzzM~y&rVuL#;BDK2G31X?LX2;C-Ou^3g z;M`u$KNlG$Npi|!1No7&(07-&w=an+iWA^7SLw$)0l}@0LOKc*^ZCS#3X1(=R zG-LPsas70DV9DI{LCkNze;AGdAkOX|1}wT794!S}Imk9)K!pUhIa`U`AGFUYjHU7{ z3K8jKMbGALO zV=TkK)3im3fIn-IY9+tj9Ytdcp7 zg%bl9h0-zgY=cn3L31)_D~2!bbrO8T{TtQ=Q^)d4eTD)~wPzUGK2i$L9$LYG+800Q z|3?ODfYYKO6*HPh7~&Hb1JZ00*fh%ktq^k$AqG$C5{#4}n#E`v?xF7mi>SRf3OEx~ zGNCxcuyuNXL-^sjCXCn4S7rRd)|+$L(*6E%bY9Z8e1YJaZ0kB6o$^S(qu+#unNtXz zK}igMY_5Vg5~Y8b7o6VQ6%w1-U##xj26xP}!;nhNjUWkD>u$*aJ#CT$hIo4cXM;y< z7E$=f+4i(oat~={i(l5|pB}K}CI+h~oC~-S5;DDc+xP;zZR9YwtB1RR;oaZ9C=Fnr zAS+7sk6YNgB?tY+R!p#gxVHy7A=7Ds>PcV=(?JgGhCoB$E)O@j7eadrXa`In2lPy{ zNj(uA^INFZ*N>{OcYf3Vp-`NAq@!LM{xsXzyLapF@oH8%P_iIB8G|3(`6DhS2H(8% z8RT;LV1QnhluqR~+PeurnPgSD2Y~2~I8U*jmM`w?XpNG)YNnE6_wAA1kjr5}?>r0? z_yPnA+*RbkttmP$^%6u(e;h30eyXXn?Q_+WTaMR?>x%qtetUrrxT!p0QwK%~nGW3o z_}DUhW}MDZN%~4vCp{pK_oHMRn;cUA!uI6yVUyqRGZfpvRQt}jxt-aQC!DJAZO_xy zAuiBi0<>(DIFe#CvE6`4F5r)`ZcqEN>+5>0&J=ETygL$_!3hmX+F~_a%KMH7BH|bR zUz*2{No9DOUd*f*{ts~>OzG0I#!kuj(HWUIkr3PQMyzw;!gBDEYX?6|D;oOiaj>5p^IxANbfT)8;FvFHiC{2q=D_-y8_8yIKQ1!V_x14P*V1bDa}=2xTtCbr zbIOB_`Ey6yB%R%XB5pfpqyn!W@4c4Ge>|1;RM(nKd>37H{?+l&K@3NvBFSViw*-&j z_SCelUDUDvu-2S!`mwIFc&|U?kD<1!r@(uG0anGw2=Lmy3Hrc6jvuH)aS#~iwB;F}`*!GTF4rHB#hr+Y6ayIzj#x%#zmNIj zfp`li2Kx7*Peg(*h*pSvfYny4@)kT^%U5{UOJpkrOv5ED-L7wbW`)gJI97LN@bUx1 zz#IeQC`P;jW|ZQ4K+-y0vIG-GdJsIQ@?!ktEQL*pceAn~O`2{Am&W!IAr|G)Qk$H! znai0&xXg>PuA@q{H3B+K_b5@m28U{0f^n`MJntFA)Hk7@b11XYuwcdOu1_QtU{>7(JV%q!io&Kugp9oNp zaFR38R2y2WuB?x!)U+zZ_271PfB4_5>pxv(3mk1MjRf!gwlc5+nqjYCTfIGnLLK}| zA0538#F9G{su6Xkk2I!L9_i~hx>YOi;`k3<;twj{8#>oI(TK_?G=MrqN|{&rqAP;@ z)vJa8O7b-TGVpd-xAzQlPn+Ay_skL1D9xF^edl_v0Vpck6(Q&n0&q$ug0vf~)p?`t zm$!Pu2Z~;M!IUA8beE%CNZ`2t%s7ZeEic+X;puI{-4et_kgsdX8~ zSO))Nj>f@f9QHhkh)pTVB_v`I9n(Khh+kk_8aQaZHPpUJi!*KD7alIu`w_PC z{O^L-uzY!g%T|%f!*e<;^H~JHj47+^e zV%~+8y69BuSL#)A{5z^WX{?&y&*P0f*iTR$$?KR><(x^u@;}fR(W+g}@>vJhQR-j> z?Is4W3m&BqWhiX6Qz|)FZ8P8Ebtasv;5_cetXs6+<$HIhL@$f}%}DtJ>@F7MukMlH zjQ~k;(S7=PyT~hFx&)C>osVSg^qdN2E7{k{#mXcUG?H{Aax)s@`W;!ctYVa@WsBhZ z6i0fSaJwn_Wm`hk&TE0+46(U;U7TsSu*GAX#X*hB$wGC@Y>F+G`WL@;)0SU9(p3S= zkO6gzqPXcZg~Xd%Z1}W_if@s+L`<)iiQ=Np9VeBMMRYC5|8x2NHWlaRPT(}`u^Bpc zb2su5wFs2+S9QU&e%eCb0=`KNL9Iu>#o&L99~x>j9RHloG5Ijc)qL`Pq}dX(fo3n{ zmEu$GP@OaN`hwzq_Gqe**V09qKhfX_G1J+Hz>jDogTu#CN6RZE{&*Z|p^&!SaSIu& z;|lfl_w-f{x2D0*NoFo3spvNxahKed9eji_@Gfld@129)YN_ia{fN20Bm3P`Iy%Y^ zr*S9K*xJw%U&+KS{*xWuiy*5Zmc~;W3dv{Vx5A__iH!*hi`&mC=8>MXqVzpU%-4OM zofvq)A0V|T6z4;fu$`8tUag$-=HIvBhnlgII{hE}ex8;pFp_#Umo7Wn_aCLm-AI0t zNiY1Q=?CIYLW2l$W{aXXi6*rDRQCY6tz6o-t@|GDCf5*oAol|D?OI?azyiWGTag=I zQ*VN;-4H!@Eh>DHB1CcLq!gP+6O^!u0uKbs?+N{#!*XO<>;sIfDoud=6Cy!kd8CT$ zTC`?AR#di1ztk{&zp?$iy9{f^F?*e@QR9mb?_^&UPA<4AE%J-zKf$)>-wPuv@pKo6 zkmKJV$dyaejX+?+Zmt-YmhH1>FzGgR_g3x`$EdoieA}_nCmAnycZ;E*ZYPioB|u?M6VFr?23^B zZpxeuA5Yc8mH?|50WiiQOChxeJq?kieo%QLy1@;>&WyA%*#^Jy@v4pOwir-tj+j2{ zRu4?%a&<049~!@3iK+07(hNTejuGKQvZwK zpgzeXE_mci%INd~s^ZA#+eeRV-zsY9i+czxhht3PdGfjlD6$AJe= z4SXAUb)q3+u;J-|U)8%tcJbEc$f(uj#2Gs?7O@iO{(o36tl-?_@NuZWOBC}tm>u=0 zY(3l8_Gps9`HdX$lnM#)Xf3PZ?8p2a4i zZg-;e8R#G^nUTuqTs?qXXy=tUv1)st_$1?&Bkt6cG6k+}Wd%hJ4SBY9^}}Ym!Y`GVPTgT#8$F z&s%o4vc|Q#+$6BVpP31e2d&AmgzBq~SXIuC4z^$3iLUSMt>2QxDz*OWs?F>v5p=F{ z5Z%Z;bWR1L--zL~Dwo=LU*5i8Nac}^eu*hL`jB=3bo=dPxA~;6I!6JwPkVURKp5 zhwf?zW7(gIj25>wWcxB0gtL79SOC9Yt$U6fKL8Q!cM=aotE<*QfW=ds1V^sb<-1IH z1&ly%zJ43dls15AfBx`wx&Fo8;19p#YG>pF+r}vsN@cPT<#ZMSN8mUX)P45E@M>?) z9X;_heQ-6I4|RJ$_*@7E;VTyES#(86<4ICP5p1kKyx}**6VG)#Q^rL@d$r*DQmzKi z_g#O%uh&!iiVi09HDpxdGvSz|!bSU%-V#VV#g8GS4=%OWZ3z z_(viPCjj|=q-FZ*{vqv;=WkL;Ei0J90|vlniqC)(P2nH~?Un~+p^q6066Rbuz)|hg z)Le;7Towu*X}*>|mZln-cD^rgASq$+2}`}AY}AN3tr#V;rvA%*4oT^TLa^4KRc>9C zZ@!;(4Tu^KBgCWs_3eE?Er7E3nA8LgH|rc~TJt3)Nws_z7Oz$y1z<9lr7hWe6CItb zT2K31;Q_K^pBXe237EpDtL80V&?{|i`977E6@qr9IU_?dwIj-k%UQ;f1;#3jv@M`w z1WIp7sa|AgpP;^IN0ZyNA2%^ct9lMvp<(w!ez2Y1h*>j74lco-phVWWambi{UEzs$ zrQlseu3E-T8{Nokk*SdiV*jutj{W&1kFA>N67(L9#{hU1xFsMPGoH3EiyJ7agzl1G zZx~PZPYvk1d2wzfv3D<880lT!*Z^!dHt}6>Bu_tXRM0Pb3Blo=ANxs$tGcw-U*&TK z#~BO9+V?C(O=c%wh#7O7es*Jlof=$Be%>qCNRJ_K9sRDLLc22T%GF=|A?RI-1`Dqk z5xw)kG_z!~dM0hfb1Ez1#ZiX*Ja{!z1yl?9!pX|ed#GW2OPMe(hBR4XttIZ?BnAJ@ zFBwQwh##nF+PAC?!)f5w&wc%=e94gI9f6tq(HUAA7?(mY(#N2N^jI)~vXf#z*vF;N z)BI^Ya5aD&K@oOx?CV}s2xtzfG02eN1zeLtkH#R+@_#$(U;s#qPDk}Zx*Xttl2IQD z&XW;Kvz49722)E8EsZi>)C!g7PlHHKL04*0Tfs$}jJ*CJ3rY&=J5$I8c$@b+08kw* zq(HaBt|F)+cMp9#fazF*NU4MIjj2}+TBdeW)o--rj=A!T`>M>kJ;@xyGP8v8QZoR0 z#S8wWGWfh2bU3RXZADch8Gy)#L|s6}3``b>=Owz#a8-6Q`d`GIm&mup-VK%QpP703 zli{fDPxKEUBohsM$GYaiVER!_H;jL(3-H+JI{I+47?-vKRN*bi)nD>Ed?3DRCPp&ffa)dbqZl7lwd$IC z+w*Hm)ssqz*x&;fpxS%le@3+ffDN)lfFF02tWM`(q`=pmxry7&707yLlGN&yDlL3l z*_T#ymetl;>&|vYabT^$vC+{{6YfTvjqqRaS(ARqKJPK)T9~Z4k{+h}veJ{zHLXzP zhxe_+jq|3A3nxV=)kVX~UvX7C>?M<23PQKN<)gC%l$cDjBK#zY#S`rV|7aE(g zmqgUJJ_)^q-2iB*{H|v&(}dB*h2c;2Y3l4>Q;5n4MzEzH|6w>h*?>mN1Et_~Fi>0W zZ7q7bwSMyV^RN{KzL$RY8OdZD`HYO9k#kO21!5N>`I>57ss5Py!%n^cN{S%coa8oR4f-@m4DUN@JukTH>g zT3*$kGzOsrMi-11IQNs0Q(O^n$04bDl$XL>$efB@!!lYX--f@Z0-o!I_H#osWhWf<#j`Ff z-OUNUn=!1=od_;F4H&u%yi1X^uL3_6SRvSI(rLyt$UG6yvamCaG zmb3wk6dVs2dp(UFVeSJ}=l+FL9Wo?WTe5eTMn|^om*1fin5Ng7FFtGM*>L~0Z=vm_ z-2BuVO098wQP26CI+r(_V6( z+v{Fe7o-=TXC}e+LV=ZC&^|&OSqhEoFj(?Wwa0=%#nULkHYD(+A@^uTYX2&3VsKaZeI7hNk?;+(5kVMB#ygH*`{!M6rx?CZ_Klc6zh7lPz^@*W6uv zQPa&C*C1(*3FGhW@{2=kpdhA+{x{19YgxrC;8N)S{=JVcHzZSKa$F_)@?xsK?B-ys zIiq9&1MdLTYkugwUnk-bR55^dk%pQAUZ>=&hNO^O5W}VnP*i~c$vJPymru^baH@gkaPvh?()#eC{F!X8 zi(el~-DDTX6ZLtE5HSE>8SlfK@R?_zz1EhAsPg7_7AAhj#-ks0YT6|`to?H0GI#9w z3wc#~W^pd9^inu>hxcFN59pAnFY?ek#n93~10Tq&y48SGk_54OyReL$-Jh#Acr%xr z)9h7^mfN0Izr$f6o%wyc8;7_o?t;lPBggaXeFZPeMgWqBYYWs&#LN+pSp(nCS_yRv zpfpfD@CN=qv7|zNs*c#>#VKuV`l-nVpXUkh6wrbu&mohtwo&=j%=69cO0&a{jvUav#C81jKVR({uq%fqCrF^PlF`fTN>rI*E*Nz^opt3>lf%?_*qF9`@dd#}CL zufOjL6i>JP1se}e1a4n9GeL^yZl{fR=v+QE|^2~GYz;u=5A$0 z!gvO-iz1(a=7aM)Xj3<67-}oQlMO(t08ciCSORZTkX_BkkQ~cszKFCO$J|~OV}Ccv z+)o40W{vx2)~v+T%L3uO-49XdqBp!hgaKQkxDa^yGeG6W5(HdJCaw(L6k7Yxm{9d806XPhrv&@;%f9)9cfPt$x#IS zGf#L^lg>A&7>^ID(XDURy>rb}q)hsSbV$w1@5sGtj^_@u{zfx-9)SH8>KO>y z9-%4Bb}{W(a`)WCEea|$P#5{k%rHequ%YJO99Haj_A}k2fSE`V%e0=}BlOS^{c<$T z9U5k(Aer3T8(_HWJKzs9rXRwQvM$FDb>XB^aA)vH%_v2J1R1OikFKRTz`rA z{?<@>O5=^0ScWi01B!X&0a~3DLCo3|piJ%ulLbRyB{mNrg1W$B?86b_At96_9tMz0 zV?>}KjW?WbO&Ra|`(?&BIifR@zxS?MidI{EOy#lbG?N5D9&arD?k*#&3GD&)K465+ z;%(D1vx+iqOq2fhQ`W3}I4u@4AUZ;Bj3TDacG%svy z*W=f@_bJ{zI#n%=X(ar$Dsp73Y+bhaC&$oZufG{)96VU;kUSe9o{?aPpn*xk&j4c| z%E27QS7U%CNC!$7z?09nNW#F$6l|AB<@&O9b=RsbM2w|$akfcEOf6W2pp!?xPL%}nd7K8e)(@ldC2F_&4%lz>0LTg3G+s9bf=SZ&#bIk+7OrT*-$ zsQ26F)d)shkFz@c7s!Tr4@W2gXZsPfW=8p_{?`dIiSfzZyw=aoz7!6;jXYSvklu#LzMIgb4 z-tQvq* zKSyy@oX1%gP(UbY)sln(6I*lz0S#hBdxhxj$yqq-Qq1PYO`7Et+IJt=ugv#YLg!khj`<-y2~Hu|Y!} znHITHIo!_uH$9cg95O9E8Z$D26$Wcf{N4upBFTovu(}4d81y0pZHdU3rT{V(09OUc z>@lhvhaQN%e*eTx0wcqhJC<$6xNM=&U&6^UJ)hNP~VVZ^f52lAS219J}qT| z)u5oWI6?-&FfBzzGx?wQ05%cbH(ij*OmCTM`dJS7_SQ z_Gr$gV`DEgALI^y6`YHg2}VYv1W<%2Xy!~SbWkl`+!)p=!oKaf4mDyGcRVyqs}429 zJqWVC&ws64D3V81M^b(T=z9Rm$Dmyc<5JXj|nhxjDRkH)?GdQpb@Yq`4FLiDF=TdSF{aM1UewTG6h+e zZCa7yx~er)x~k}&c_J;~AZP#`HYg1b6%9d@+Y<5nfjI$Zs_tEkre&dPHC=E_73<6O zu`D0gJf^@_meLrZ(r4=D@r>u^bm>DL7EMbKW_YpFC3I-|%-&rMPrg7&Sc9a9BF=@8 z*T3~?loV}eNm{I+KQ>j?HLHBa98SG!kmd`;^Pl24s^{doXDE02ouH?J2h)7FK<_lK*TGrwqb<~*_;asD%92v!8FSA#T@ zKz*=$DL@!?r~`)>r0}-m4*FULUUq$2Xm5L&YOEFQ=4u*X68=ozyf5k2;w#>zK3paa zR^+8iyazdh@BSdUBwX+ zC(Eq}Uat~YroZr3O?8QCG}UnB+~&62v?bqX^57WNPz#cN>uYU}VBnPz`cvX}?KOwd zK&>alvZLLS7Cec8F#{=&<%$=Z3xGhA(qsU-&a-T2e2_eZ2vS!4qj!OD3s zl4>_*z_iH_#h9obu~EZ7b!sb(+`F#r_D%OwCf7?(bBVZAp1V_x?4ESOywJ8|;BW9X zO9oU~i*zCaKH@vS%%R+UH-5VEB9tK8Enw!`hVk`xQJc3b*FgT}90h|(I5CDH*dKN{ z!Y8~0z%=7)EEAXlbeA#kuf?vvgT2v*mR=vW)1J8AP;KSuzUENvQg$!lgfymBA!gxn zuKwQ7l2wGLXK{Ut)tXSAvoLl@E>L?ta7pP4E0Hl?6dyxrGXj}e()JVr)`+|tt z48j@B-+omF-wbY6A|gZ)$Gfj%Cb^hANsRJxlsh`A0h8GD=p9{`obybr(GGcY*prSb zSbBTqQ|gf|(Bc8zkloX@^zEU)b$li>JiKGOU`lAsMceNH|fNsf~-@}|9?ibkS>T} z(8`m>-AJ+t3O-gJt1Suh7%9V7E!HQjtLhs(VuQxNIQ%Qcd06oA%bA2)`gpp|$2~$& z0gjLYOsgM=CfIfhqyB-X;Wp9%Y2&XuLn2l;rX#Y$8f{9czaLE-P%!-`H7+{r?}}mP zw5G13bODwGPZ!Gp&yo!9hUOdJT25ebq@Rhx;TugpbvC|fQ8JXF08X2x0q5jAr#ShH z55*#{ly01fVT8ZMrQqW6WXMl=ErTLJ`qEs6H74=okZBz@5i^E`SU;fTc_WwSIXlZy zn(kQMp!^-j?3eqzyUc|j9B1EouM4RHi?#uu$57wFyZ3$wb|au->`(2+3i_l&W$clQ zRUf}@uPI&ET6ef&^x3hxE;_aw3i&N9f3CgK(s;@HU&vw>&icuYJ(7E%ogBbI4W3^! z5JcL7ZzIfp#Qh=m+iWCwVVMQ!TN(HWRhAcb|*t^{fx-_x;X< zyz&wHdXxRf5M4kXV(zb`Tn1yp;@}0F5ssD1F47u=5FoHBlk{*^Q&AslR^ zoLSH5=%}X?pYOSspH~a#R6G2k2a6%*_$Hd@;7}i^l_6r;61K zv@FaaXc};7`kd#z6d?<_N9Qd(5#j+dtl+XoIvI7SU*L+s$UgSp174GNNhB$;Gx6C!2;nB}*Idl+-z+;agJQkB1bm~aU@#xQ8FhZ^mq$Z&4+qlgE-~mD8GqIvf7Xuc8ts34f--yGB6anH;w}+82<>~LJ znp(=CY?9!Ix{>m{xIv*t8_l!B`do##FSL9mZ0rS{AXe(V7^WHI4l;{GYWo-!3H_7}p6z*1OwYcAR|kAxb&>#7yhcykizWMgRhx5r1`(r06;yU?LD4+6%~slgE$@&4hWy=Iuy*c zu#O8ooioiU^4D1k4a--Lj4)@pt2Nz+nLFi7b5jGWP2<6kzk}#hi0tN17uq5LnFw~G za+jnqY5Q+bBWUFbKlVw+CU9A~7mnQ-)trus$XCsoUYpf+^o_Ola9`#r7ueEK;9WB5svy=2y$HW+X9`bfqRO!N z?v`V<%rR!qNP#W-!_vY+w30)nuKd=Z;IpDna{bn0Tq97 zNRIY+CXpV6u0&Vi*OZCK8F6Y5xuO%l2YWdBx3uy#hTnRd^xlwHyf?v9Jc8QBL2?TAQO@-9%dlkt!B`+rYm|N~ zs6WqsZchIx+X7MAv3mbkK_&IO33?aV5kC+?`r!H7X0;EDZ6jfYs2DY)HRYStBfj6+ z4byKlW`56$WJuQ}oDh?f<~el5LDsP21bZ?%<5c@^{!*_JLz(cC#c^qJZ}#AthGOv5 z{~&^y8a?-3pCcQG2RdY0`&HZd>(`I!=00s42=YEVI3uj#m7S->=KWW0^#~c&VKZ1* zPzcdQVY7G3Vu8Jn+^;=MHpTGw$0GZQzw7V%6g=V(Q0cxr&vR+#T*V?2S33mD19}m8B19K> zPQ5Z3XiYJ@BBk)nx3@diky!m8or?rSfY{BISGqsgQUkUOH>i&p6x<#vYil4s1SzFNqOO01j#_?=nZ1=~A8 zf9O58rZY5N@)Y7&*Vkk}G^B9Mr-!vzBWk>y1^IrIU#4f5_{{puqmxI|*$_YKE^0ju>qLFPwIPMweoxnx zk^PDG?~5O#^V`kY+rc;5-C{rG-`h|~QPs_!ioQ2sD3fvn@RaW!(pz+)Qfv;B>j^o} zWW=XT?;nIQc>yY{e^NXcZ5@gOArCP@Ok031{|g?PI3N&N@RPfS7P5R#Y4TDyfs zwzlw+-5!LVe(a-oXyyhDo!;o2@#F2rOf#kN-0bA?10)1^T`fCu5;7yo@bN&_U9MZ* z;`B?8!U52R7lo7&kW$0VB16 z@=L`>kITMHK_DESqvzCHn*mJ&EKQiv_tX{NxL*A*nAm_XDDaOY&Ld z#MSF$X9zgCX!o5mb|SJvQow4&LBIw*iM8pT{yEm8mvH5o%7>l3ViYtikE0)wCT`F{ zctzw1Bk_v3{(Tp%wjpfS>-OfCSODK((f(^rKP^&24CbR96VLZLB@6E1QVD+Qr5hv; z+!m-g;(mo;CxDfs;_03^5Hb<6P~@8+*JB0S${6PeFlOl?7S!=C%g!S^T4yPzz4Gwg zMI*Kv_uba`rP*(Y+8MK-d?U59i;tnm0|ORA_N*Xa;FI_@Nw+fB$C1x{eZe@u=-rsj z@3{dUSM=lG^$nx-8Y?3Al6h? z)<21jO+Rx?h)>5!?kgvgqhd~iSLV<2u7kdA6<@OSJzBLN96DXo+-DEI`lUol~^LL@saht{*;1ON%?}4 zS=eV8A61rLqm~^YVX#WK`L;(Fn$48&$KB_h-fF|wN39MK(J^zZ4IdlGL$Qj}R@%dS zF>V3Uelad~#+06u7bG0`!$b8BJdOw6Qaotb{=gVhiXKQQjI98x5L@cn@O@U;PaTLO zd3hYp*es7@l?Shz)?3kznbBbKcrudQQLZ>)>p|M7UpNuj5znqz2yUQV9l6rbaVb{x zE;ZIK>)zUA#FwjYUmgE*k$v8u&Nl*10GRwJ$*GPnk1bdhs~xN8%w{#ol-KajCf3!B z0`KI<$-&pf?QO7%y`^{W;@*iFKqr9-^L?t${+!Zh-;b$fsG z^~#{Z4{0}*fas6V9)r10%kt|v0MtmAaHgH0J2EQirwMkfRR3>5`&Z^JB!g(D^UwvS z*V@DWnw6!8Ow~#`j^=-JziD_<;uwK_09TPC$lM3Z&A-labDjrLl5O`{Kb4HKHxsHa z)%LIbkY*cp(&Oy(N)7`}B`y;&ea<)4r&|%;*Mt6o?FU&hd9l?X1PQx}a-xYRa<7o> zv6d#oY6l*0D9Bo?W<7E%>OS_-^izz=R%>Gp>zTt}temOk{nR}$-#h!uUKp}5QMS3O zG5Tm4ihXIB^5Hyp{^cYcM5z1-{W5V4)iupVeO_WUwVY}H9wS?@kUcA});z0nLO@!S-oDHP{S~74JBX6Gors`of@roqhhzGew(!?& z=)f;O4d&oGR|}J=Y1<7y|0%({CjJ^{Bv_!dd&V+f_jT=tjGw^}fjPjIcOc2O8?hG> z3JM;a6@#9w`DlDU`IGd0KFTyi)lJ+#?a*NkKBrKZSfz{D?8om}x4OT4@z7hBe}c`= z44BoI@Otl*9NmiGF_eqrWKx*&L0>?XlBv-gi=A(6;h{Wtz1x*B;bZG@Wtx2k2la0y z*LhB6%*3tT-WK|T^{0x!vP9m4R?E}IZTN`C_U{X3_5Ga#T!|X{l@yNwruXE}mHYH9 z5Rs<)5A$6_Mx)7((%T_H7d%;aXmUzfQ={CbK5TpNLcbmVSmkNIC>0^w=xA+WsOc+RH*e`NGq&isQ5nj#CSiGeL5G{Q>+s3eQ|x=C zE~@?f_w_~4K~XMDu}QoH#xOaW=lj<#8;vTs*W#sy{0oP?JcoNA&dI6HZ&x}I%QEp_ ztA6a$lhmo0w3(+{lTiDBaoMxu@nBt1z1^M_BJ(6_=lRt6-IKRd>YhH9b}6Z|>EzMM z>DZYNbixHN&nmS-Zt1aa4g)l~^YNFBBS}~48lK*p-uLO*X^ta_cPFhM`u&(c*9)e| z>c{A4u%#c@vy^> z=OTLtRKGQ`=OGJ3Hvk!iCb%RQl?D$Nu4LNh<}Gv0+u7guc;fl>s&vYxwe-_cmg&!Z zt=mUJB*aQ+2&}p#;7;ix$6ouo-F(?!R(DyA)RyP7WbYYjCIh{ zi=EIEK+$$93!JSbZA7AY0Y|r{8y9*-O0v48!2#U>6&l`mn%3YZn2=Yfdawx@>ZGjL z^8qoj-|f=_2E($Uk>4+#R>-L6#U))F(;J>;j?((TNR2Gf`ez>FJ@8LN$@Cq6A{~*NQ@Xls3*#eyjgZGI; zvA&9|oxlp`u4F0>q*?xb%4nSVZfz_sgw2m{b@08TV*$(+O$s>VP!#w9w-x}OY`fDW zN-|31S`w>|2s9#!-WMy3Dms^GaOcH`NZ$;Ep5^g*iG#N&aRS@I3DA8Mm}(ZFoBLN_ zvxvqWXg;^h5Q~OUUlg~s-S^w6H)HIrk19=?n~Tb^Tm%y=pq>cU?$GWo0O=c^`ZP-a z+s*+~TZmT-XnytC*wzeUJh=O1M$X;z>k*SbbDDTjCP|)lwOoMczxs*XPY7x}`hs;K&BA-t+G{|=c^rekWxY;I@M{L(5_tG{2Djr*M39mC=5beIj1Vz38y1p2qk5S7#r4r!{l}gqcW(tm%-_BW#dEt8-VF71cwYPr5bZ^- zzs0W8$Lntndic7p307W>kdiYCE#*BNdP=9_YXuOf1+F<7B7pb3f9C+oXjHj05QQlN z6w3!~Ym!~RYR5nItQu(L6H^_zEaKFvd{Ws0!RVzpGkKyIiM~*b)X|225RvD<((h8^ zuLhv%F;Gp}5l`Q^&pm4@9EwvNy}!Twbn5pu{Mk|nmhc4F69JGDz%Wv?8em%jD)ceb ziWA1Z9xNPb>3V0~gM2nv<2@*&M6EnFksy?&yQPxPaZ0E44sRC(pHBq03V6zDQDREI zTFN&^)^-{j#n-fLv;3l!%l<)BujOssQ-BaYc=7Uu&wZzs4+JW|bxD6MI}YHh7rv4H zGXI)~PEXIBk*WUN!}+WvWxx=@(t8Y71cnpd(S_%!jtp>c9z?><^XQOFOpme<_QsT% z$;alA(rnf9vbRz_lGk&+yONV%%7%z;b6F{LT z-nY>EPZqSblHK+ThJGk^SMz>^>O-ziRGaZI>Z-Tb23s*4tv>=iI#zbjDnIR!eEDQv z!ioF$KYGdzU)t<~AT2QSqW-@X7lNTSRfv$My=!#xg8IcUT}YO)owCw&<-!Nkj--2l znN8YY&eqx8<2jdtXL$y9%85TW%%3trm9?U`VrGTOGY#w0gn~wyVC=R4X755CTn`lX z{%c?Herm4gK87b(pz1bc^R_P{r_XV6s8vZs=^}S4vb1(r!=`PWd0=d29^ZyMctH8) z1%5b)+PaK###F^7)Jj>C&=3gy7TW-awpf!oAo>kdq4iwy%qNN_@bkkupTD2f@8f$M5E z?s}dzgXFqxt)t1sZEex$_fbFkYPH8rmiwO?c-|m;)}V{_hhTNm5{l6K^oc&23l6Du zed*>kbwC?kqZI=eiq|!8)O&^7%3pKn#w!{vx(s?FDFXGYPyRvZ$rHg`i%SkY7X1nO zu(9SwfKW62R8vsBNQ|OowQEb)obGT_t4|sC5It?=z7+o{ek^INFT2U%COf$UT?Y=f zLt>b30LrZ*PU^6o@=cF@NcNt0y+yHpw3D}+S5;(=UGv~q(OVThe?(*+F|nac*O(y) zGe9K`SW8$B40Sm*0*|&|QZQqwrp8NfFoZdu^~r7(JXBaJ5KA#!I^KG|^lHNQg?phdrFrxw;<)LALkpE=dB2z_e?RM3(OV7 zg^jLF+(%aw^GF_hIFt8*sq!4o}9sB)~lJV!x`8c|lD@Y+*0?(!` zt7Q=^<8vckv3r%zoYozZlR}@{e!~l#aaj9qAX3ld5Uh56Y@hr7@>}f8wNW+px_VcO z?#avkqM4>|R^tp6+&+C!H^@7G6qkCExuLzEexIzrRzZW*f3=(jVe=H-kbru2!8lvc ze>@|BI&OYK=FH{8ZS0HsdV3}KKCcV`Z`<6$0uA%Rz+Z-Yg5A5T7zl<+ACm1W3t@f0 zG|kqXUZj)zV8ULas&Z7O-nX{t^><&r*T+HH)8yvG;=4ChK>wAeC}X7*4a#vmfm-C8 zcGA&XpxIi=jI28l* zODb4rX15(^3|y7po^FDuJOI}$#_R!7W%c1ryyO%OGlHAA?#If;P@*SP%l$^fuBN0j z{)9$94-dArwc-vaV}%dRC^JFtgjCN+xPW7(4Whqi2U#=6z%F-XU!9?u1wN$+Cq_0> z#9My@bx^mNX5|iO!q4?xS;v;lvuay|KD+39tRB7R_5`Cc6DOsa#_tIAjKs%C=BvXM z-_Wp@Ez>lMfTYRV_%pM2Vi@NbfM@d?ehNo5WaMgUQ22a@p1nTTOm1$Un{;hC(pde3 z?b9W(6B&2wqcigEi0XKJLL@zu{|@Gb0HQb%R9VTET99pr%|xOMMJ_7PocQee3$cx; zcah@#u}AH?@>_gNr8`0IP|4D}T6;eaAs7<#WyC{mbvc{iP%w_ljRRB{1q*?10WTK& z>Rc%YoKC6z^b1q-1=_rdGiwo`D}Y1B8}vmC4a^ay0G21>I~b=^j%f9yd%T;`Z!nZi zgRFVI?dJ5)%VtFP9t+mu{Qp^{{%kOj>@+Z1Fx(q>034_pu)6cLt4Mb33ow%Uf_xN? zZPivFPb6rnd&6#dEsq{k^2vddpPTVvid9l%Cc$2p-u0ZFiS zh;M052F@ikYCTvh&l^^?o#A`8H1p|n>#^G8ajmH%MoFJq+k@Hjf?APew|A>Mz{D~b zdD|5+V)kY8IB__vgV3AbMY1gcq{uU`;_Htm>F}u(eKze1{+^@`#JHDag%*)WoI47X z!KyMwP(qS!FHdKPPeYG zT6&CLJ3vO?9IXWcS{{lXbjNqSY6IU+-HI19N^`o0U`;Uqtd0<@mCiKTAy+W_B%r?@ z%@sfWaCM3B$;DM?vL*F9BXOsKm+m`S7N|k^E}L$S=Ie!o!XVh&flc?DvC{oZLK!#6 zw@Q+9_TeNUVkbaSBmZjs0-J;ggAhOY7wZI$@EQF_4WEFfs)F!+N!oIqZI_m6Zz$uL zHd|09vdZIiI+@p|mJ$`wr2Fu1L{CiK=-p8V3iq2A{F>wt0e99r`}dTObQ{Vo$NY;U zN1luXNkqm!h(zR66te;Q?vKk3je?ef+FYSK##4*b1xALV@fsIN-WcK1!rN>cuhDH> zJhSx3#qshD7t7oAU1hTQD(fzHU^QY>hvFV zri3oI(KZ0bpGyR$0vQREig7LwArs1}LboeVrc+hP72!;-S?gsf*32niZ0+R6{U&Yc z6qm1cd9v++DXPUq{>KiUEH1ibj$p;RRAd8s_i|iSOU4tm+MYFWvL08gze_=av6$=f zaMddz=|Mv1>6q=D;G9}a2XZ>k2-gBw+PK=Fg9_W9t_VNJKC!AF&8tm{7ihcPJ|LL< z1&~s2`~po%raMSv1XlLkfr6faIbHN-SYkM8X9u(`mzMwHE|>H}j+l^LcA|S5Q};W9 zcwtB?`z`%ExjfUM`sMAY@b;-pqx6@rPA6w9xN`i7eLnXQ1qUKm^MG2lEE$AFTpk(# z>+qF*c#O_Rw*jo+B_s@u%MSXrSh1c*zcq1T+&3JK&``zBfy_WONYhXq9N`rXH>pJZg+&$lgu6df%hT}uqbM=FW+7_PB zt&BBDyJfxKKNvjtYj_#i3 z(mQ7J_%%Bq?PWQlpsal?@Ddr_LA!2h5%~pkaOLdZ2X+_YQsV~tL_&I5YY~sxgtnRi zf8q$$atxDOTgS-L-amW$wn3$?U=XA_GWEmn948_@#gJ8KTh+95xFkuSacuZg|rp1wggOwa+ArtF`vt80rFg0@kC}}#S_Ww>>?*luKpvOpY9=g zmvyn}bZuU$3D}A$usBftsspaC$7K%$%jeMFmQY^+6M-ffGa$8DONtPB?wWrP8>5W5 z&rY@(A4Lx!1FoOj<0-vvUXttCrPep%C1NBwEaQhWZt(0shLZ=;F|2~4x{lS(%#@&! zfllAnJl~~QxxX3p`5iMX8f}%FU*ml8O$6K2%aC5VW5Fl|i0}=JSY5Xa`3JGe4&;fr zQwcyh5A!GeH7g4bWV9;^4XgVToay}4r1gUNq5X5%#;!TVhT5|hj}xOk6@NZGdTj4i z#qs_3ZhqS%lo57zIQXIy`ywcLr_=cosR!ULnjIQDL|W{TM}@q;Krt_VAeNBM8>?D& z{J|INQM?F7+HZE@YT7kTHJ>4B4sm^Kv~HR_5UA zeHcD0t*b?hjEXv(tTf{6>Bz_ z8uou6B3{&0TbZ2wy1C#%W>7)y@}d4G0GoeXp|1X*%DT0wuB==a?uo^%`i9ZLbth%r`@hz}%ZB0BKW0)H+YL> zOC3-ZnT8S3j3~>05T=rsa9r{FzTB9|wf) zDJstd4Ij~Pf~>jvzJ1cjcvRDd*4slBkrE z`Pp1e@}~>-PO#H3am_?0avr ze>AZ-BA=mB;Ulyga07;yejaWl<9i-2G*{dAzxxGJpi!m!A=c&q(6-vfdLeYORYA*b zya-Xzg96OvOp`LeNYv+8!OYybz=*4O1*8m7l|5}0Gj3GP34pYueEB<}L$-LS2Rtpu z-uhuHT;yWBwZ-Q7g#oXsl7#3 zzxnc&9UyzF1J;Pb$c7xD$>J?AvAIv}fI3M%m=1(7))Ei)7U>j&3Hcak14s54YDL4x*n8^XbYj*& z2vKZ**EqGpt>8N>nf7zq%!ECp6{lSP(!WjqFKa1&i16dA1I8Yd( z^KDV37;k`JMW)fuU&NXQ(|Ay2g1vt+X7!S8)F zo}Oq@YknKmm*5tBmv7+Exj4^K-p?0#iaIt+{BqsO21R2n9vJ|Cbn4 zOcehCMa@*&(Lf~Q8DMqg#+pHd?5@`Z1dSrSs?2Tw$~pCD$4u4lMZI3gUTK48IVyMe z96cFfe)GsEu%G3qsxR?Hv(|G#!#R7`Jj)zhoFdVppyP5OMbt{vd$+S$fOyJ_?o_4= zWid^2XhzT!0K71>%S&kIVRo8zpfaP_U5%?E65`VoqU^yNCf|da(>taITk`7?I!;!J z*wv++N6f2{F>8ZhGZY7Xf$EWKCrG?8!kP7nObf0(1vO}8( z#z^H9f;Q56&Z6~fSzoUprOq*B>E{=T&(`_QUpVePzR6xPj0SJGg8&G2kbloBhu~ma zS3KRX`98sNO_ux{AQ#HIKLL9pB|!NZ>*nqLv*oLrm9IF~iL>;+b9BaKD8lia+_}3e z7bR(6J-lxM4=yZD)WR%vfb2+HSU5ec5aUws%)z<{0dZd>*%bPmmfQ02;WZK+gXnOrjG?ZC=%!|B-P5fa=JOOAoSOzj21YiN)gihK<~ z`*7h*VQivh&A6rxMytEzy;ku(F4uMR0G6+ME1cE{&+44(mR~vfU(BHLTf9u3_uI_i}463_nglDVN-l7Up}-vIOxsLcx;;MZaRm9oO* z*di%M28oQ7Zs+jJF0|AZJ}ra1o@mvP>ei| z0^RJt?=OLgO!ud|@1QrSw+6-!TW3c2!3 znd*vHow^md_x@h`39#2-RW$YUnA(cC?GO*FY*hX6W5S6I8H!p>w5#~311E0DzC14b z;mz|n#C60|u&nHRu^YDmQ^yR1nrtDhSdDplYU(-?1Jqm{8?&)bdrw?)3i@>dZovH|2)4`P}s{cbnW4W+`$5xTjHhsTf9O!<4% z{FW{`KsI^<(aWkv^FU|g$ez4(ez$Hg{|8N3c1~kDuj^k=@0!ybdw#2N#QpYlo|H{)dke7(6}BWK(vfl%cw5+v>iTWIxiRrtQzJ!otm`c_(9ZEQ zTD3Mt@7kKPZswg*y<1la_q~QTe1q&(6nlU6&WSy~pvs#IT+>`?ab3>c+fh59x!ssH{=HyLm6>vf2e@@ zJxCd#Vq|c2=?sqD$%6Lc5ujW7hBxmdMEKn7_XkM)#@pO!Y7ldOEm+s!rXwG4;8c@2 zzu=ee|FUv>^dHkb(uMxnBX?m5Q14`dI}oRXMrSA*n?n*#Xi>0p`5W|!RC0V{8Jv22P8>uw!A$2s9k@f< z2M2Rk9vQy9{N}-9Eyj<>Po|zpS@Zj(U9P*x^fGM{I(Z@VAq#zKf(q(EdzB|JyZqOC zBN;grwetOCmo_o`X9CiNRcfkwL;F+)49_8Wp0J>Ybff5ES~REc=*tpwhsYi^Dh9<{ z>M@E|SMmv!h#!PUA%hTjmJbm7>LGMWIRpi$S%v-=v=R{Q0&t>Mk6lY8<2jRSvLnFv04OcRV8niYmvxN!QBeFs=?$?i z#o+U~-Y!Vpm*E4o^?(e@Fi{+8vp_k5&M&An?Ho9Zf`qZW7SGLUMwDM&<|%!Cv~y`u z;Mvs0S#@p+z9g?4aFzdQcY#TOfJrcSo0xtFLKc#2+1SEZS9um?57wRdA7)b_-!a~> z{e-{M;X8-!J${a^tR)-M#hbyZ)d7$%|2wMms3u5bjEtKo5GHnKN>jIJYKl)r%t-ZM zYaQvP*9opYm3I*R2UYeN&y6B1DHsqL%uZlxf&ElqU*nKvER1cZF;$vtqswJoxo>vQ z+>bXB2sdXj*GC}Og&6skTLs`7z?+8I<6IJ_*_oVN6jlNW> zaHwzyIwK%kEfc@BCVga-m>P8%_y(Su>z==X-0SOt!Z|zFzX`iyuitI24qx{;dCDPK z(!ddjz24E;yGQ;vC=vQkY^0kovQ&qk_^GbMN_sBanw$wxxfa=G0W6hL@j5GCl4N9n z+Lm)$4y?*fEu}zRK#!ObpA+7MB)T=$Lf6f#6KzA8MqEMVOM5*U^jq7@FDmF=jZ6Of z4%mMss7)vcI5$&J=pV9yZJ!q0hO>x6)2mr}qsMHoj4G}VuoYNOu35y*RP=TH(rkgOJ96Q0ca-sOP^|HE+Z&R%QVXPwByiz}e$Gfu1@F?^Gd4m6x zH20%T6#bGB%>~fK=YWWhezFV&7cH&gjn8L2=j3S7%FRN5Ww~le7u6?m(EzL->U^rCG4px5aVXN# z-|~>#nf-E^ik3!J`95EuaUbhK&Q8ZG<@;DA0O@jDKCbQ=YV$Z&lLoAkx$6*93tfmO zyMtRgc$qXo7eLJ~CTO3e+cgaPWE7f8^>bJNx{|^h>u}@i!gn z7Uf*bXIbbBsO?hi%m#nZ93-Z!CJs*Qy0L`Y@sZiSZ=d_pq3WZd1BUIz#UYc33!Lw1 z94qy6Vju|Tn_H?+w|V+QTjkp&V-I=Vllz*YZMpOLQe}`g4Jt-9;Nt*C@RW89uBLtl zX+&gf-0U zTfX85aZGvIfrYW08BLx1>Dd7OnxH^M)S&pC_|Fej-i5_5ues3-fSH^IW>UMh0zLtb zDZUHMMOQDGc3~dQB;hSa?3d244N$*0SV5B2o9wj4Khs{xaT8lZpj48=oJMcU?e@Mn zON^N(gS9~~G)ziuW)TjdU2=^N@R@dh92D=97HP4`Z;`TQb1`FDqPR#gF6oV*5NufC zHZgZ%LRsy)>E?>*Xzhu)<@NPtr_3ksvtu<=H24QrJ5xGASUs_${vWX$*x%Z+P@DWV z>KRiOejkuO4L|R}IaVg>jx5^NR)vlgIHoR|)l_FB_OY|*-MGQNkO&mcK=uG|(EeRd zpIj%~RfN+WiGY6BPTuCZO_*^CbC!_T#DDhJENkTLBoT>2A~LKCNchOl#;3ahm^^NE zLos4uEbckHc)b=7<1V;N8x&;RPf@uQmzSqf^;+aKn4F>(puYgTG%xdjH{)#!sPMJv z0KvA)a6Ty2B}TMJEhoXxdR>bzzc(%#!W?b#doSU!xuIp3AY|U~>I5_XIH;lEVJ!{2 zmT#brpuM-QwyB}6ev~O~-4-BdS|vN{_U-|Hu>ng*5tO3(Q2emUiDj0G$&;`?lSlMl zUbKYCSwiEShBaFMW=EvQ2Fg`*kUY+gZ~uAtU`VpL>rZhNT0jc zQ5>6}N~wT$Bkbw~;6rly9*MB|pq+FI)ZSGJnVH)_4a_5|IU zU7yk0XPs#3Ml2Cus_BO&Jv|E3=1AYitqP+1G;h2+@*)A)R-p9Qf+X7kwdSz-jWrKo z%{eK~T@~Rtasr%p1r9#k+*sc{dT+$(n_SHPvH^6HyFqCsJ9kU|!BZT6%!osDF=3h= zHnAsDH|m*c4m7m83OddjY@ykIo~KJxEH^TFIKe6%fw!2P zk~CzX)K{kPikQdmSZ&Vk@l&qT(YEJyW&%hxp@w_AV!C7?N6YCc?$UWbLN4wwMB+!! z7%{_vfd{UXXb#F#T{qb$fQ)jElv^4Q2dh{!SG_P^#y!nbLE?I?{M^BXy8)tG|f7U0-s_R_KI#VC?b>FHp zs}MuX@<$F`q9246s9iZFHr=rxDeMSQcP{a11y+09Lu%#9sM4W<>wBvLa4v?kYkeQ4 zZw5cq*}I8g-^8O^f`ln^A0ZssLGGFIzUt!GkL&x=s<}RCcGYMrgwD=HehJLuQq|R7 zZ?_b`AwU1G1Qb+v|62k|LlHONBZ<$-P=vYIjVVq@)U^`sFsTt_qTK~cX|Pg1%$&lBs6t&$4R&WSY(2iRZ&I70QkCDBCl&go zoBOjo`^?kvsUin7?f+}D{O4Y={6D0TMWi87T*^;Qp-c+`b zB}>R~i;#T_MUjvqCS@sPvTv258v86LS!W8}nVGxq<^4RruoJ$5rN?*3K;s z6~6fogg)gROO8`YP{JHIhQ)2FfBefQ-#T)O>n26HD8zCGZ|oJ3Qh}AiUQWT?Lbk#7 zw1Gj+j%jm3y_Kci(B#9iG&avxQ*O)h;Hgy(ens7TmZ!ACgQcKIH8udph1CeX>IRh5uQgZzRY3~m5 zewW(YIzbr3z*;?&#$<#WYQyM@RD=_cf7aJ~;?tB@^2}-cLeK1b#q!Q^%AE>3p}Q9) zIb(&6h;7Io+Y0S2rrQ+5>PkzF)97(ah-T~!Nw!>~yiu_vt!P%sVmb+*h0 zSg0N@krK#|Q2{KrJOn4yMn1v|g1W6!?8=rS%q0j~Nsv>0MwY|hwK zqQ*{p|7Nl?@W{x!gY%f67n#N!pWQL7SjtiWtRm54J*u}}1!m}rsZiP)HgQ9sNF=eV zF~4c&pzo7;35_Y7)9tdmJO6CUE6EiVs7rOeKC&Kt1bA3O{&2!{4@=aHvYbL;^Vl%B z%YA8+8n#6;=D(wgzK%i5`zs#wwU>EIXFJ}O2xYU?r4Q3RIw^m@~I9zRYq58U?UA2)_RgMQJR^x6D@+P#8 zjnypL+4D*KiMKn@W-IQgiwYymC9Zd{QxIiu9Q{gIM=_U6xCAbvdlx!f|9JJ4$LQ51 z?M~bU7tI$Y58jq(m@eO)ifNcV`wYc1gJv0yuQ!3>W-9y|fNP>4qO=1sVOu#+ASR~b z8ld7BtvK1g1tc7NYpPmmDt2F_9)Hdldr!J2@KS`?`+5ASOWMGjgzHQEtU=PM zFxbg9cr@dB5+x&JO1tHw7QZ!Lc#W+3sZVQ1a@>#RcGbSoC_8cgdB%rSaZdklD)*~U z9Di;yK*|O%V^{noKvWf8h)6)`_=8Q*mvNW9IS=Q9Mt(E}?Gl&Veg4lGThw98ogOKN z6gJ-l*J0pqCZmU)$N*`h2HXNeU-q4dA$>5TKDImY%i%imRH@yIy^va$Web{%tH`w7OU~-SlAP zO51M7+PGNk0|<_4gf>Sd&7>H?VD0yVK&_Jf4cJLvD==Fch+y5?n;H`}WsRMl)2&nG zziN&8x&6Yvyo&UM^1cei=;SHnavtQ`clAEX9B~CA#sJ-kI|E!>30agFpgzLv{`was zTK_}e?kOHi+B5vDKBc9hBuwCwRPD9h*cRnczD|g{txbs&Qz^Dt1mt$Xi!*Q?#F2cO zS(4TykIhgBRt5~7KHyvahE)qSzjX6aw!_VthdCKJrm>=P9K&LN-gtJHj{IA=ZuE| z0JfFRCKyWLjXtrC5oHlM-{O(q-XS4NZgzP;+gJDZC0}u%PG4%<#o8X|%iIM!)9$f? z=>r`Flb-{-f}eR`Ck-ncyA6GsO>&K$$%%28R=96ZkD?Dy^KTn!u#SIXDN(HeQx1-y zbo;a$|u2p0>phWxk*!46o-)q9VQ-c(? zJ$ua+7rTu@i#h5;$HL0k&6*47kmp@s1kk*04RwKTycP6&s3Hn#kXPl`wzIp(A=9wj z%SFl5$jXfR*$N83vG4Gp4Xg)nkP~{)+hD9&6NzP66MiB5s^)@?04Zn-p_W5l)#$!Z z8U3(1C?P*0Lv4YO?|Y+l-vki7tC$Y!nV(m?apjZ6k@hFLXXbcN_Ve~E=?t)Le()Ml zJ{m*z!gdftqo<&!6C*UgOZqj5T72Qb@=6FZCBko-U*K6#ZPUB3-M#%!C65eJa@SJF zJQlx_4h{rd^Zy^M+uP(;2_x#T1_pSewPJW*ezJa)-^2MG;X20j_-|QJ&mtoIF0~Z- z+9wP4teE$SG<@kpZ3*1g%d@&*0{xdp)PJ>YfIs5~i&?V1;2s0OJ!(XcUyyfo3A5CcE;d=66o9>G1)*#?x>tqJS@RE${ z`4#Q{>h)^kKfjL28QpUampv0?4)j3#9v&2i#0GV@t>ClEo*9A<_yTzbj8xFRclrtO%T)WmjTZdE zGQK281Im{M<@aAg-7mI!T#nD&!nwsX0*LN%1HhjGJZ1gw+x6pG@B#CdS9ih_W>@?* ze^kpwlty|>elr+RUg&?WpIVtkE2g^?!)oLuol>anG??7CZZ8&VIYYq+z&s9x9R;3+ z)aFrDNG|+hkp2&&2KRe?iua3;p=?A!i$a{yX50iw-Omf^bvs>%djjSM+ivWA)HL-Z zi$*%_Tj;%~(Jc68t68tD)Mc^sgU`%BGk-f4>^pie!neezv=+T`G$93@qHi2bm>Zff zyS=8zJ8`#W&!yZD-HU-i&z@BXl%YT-$qf-H6wKHHWd#6)o!_U!()Dec)W#+Z0W)Hd6sbD-mE?NlGGI7Z@E!&CPcYubLFIui@ zW3ra)?PIs8=@+BTM5HC|ABb#K((i!Oa%Vt_9$ceGP&jbkGag=*^cfb0+;^p9(_~(bP_x6)dMv)vPAeh3^n1j$bic5QtQwD ztB_PZSB87jt?ON4dkeFrFUx$p$W1$JYM}%#5EejQ;J6u+#A4{?>U?A2%^V^5O}|g0c0C<+6vn)EmT62VJn>*UMsBf-N))Ht!uo~^7Gbwo5Z)>$91{DaKO*~ z$1p=Amb4x?&2`}(=2d{ci?@g{&UNJ%=_a-v8kxW0Sp`+MDf&f znd7=2j&uJ024+M4TYKZbf5+Y>!nzp+L!t$x}+-rDU#E`TddB7nm@qqdJndlD?Y354%~kl3I)ju4Zl(-s@+i{MGjc zi3y7aS%BWavJCi02f|m)NLhDf-;PFM5~k!p?uD_12Ltw2RQoiYh69#NTEsPOtCdfj zI@6M`6d@HcBv2vH_F5-H0NG}_UFrLTYifnZl_L7#H~ z3Kv*>Jgk2%j4#RUSjl zCq^iu&hlXxAopnMw~1S4zG%9t+qE`&MV(Q*dbLKbfX=dd+v3TURe*|OKKlp1y3SKiNI>lFj!LR8j($QgsVlVuG`DRB-6 zHr!iZqFVj7p^D4GQSz<2AMy^%p#%HtCJiOMk|g6A>>Kh2Cgb+Rd-OMHK>8bbh1W`b zbi~8*emPxmGXFTp_37?R{O8Qx{)qEj+mFvcVBWR`OZTZ@i49JFUs7a-_~Q5w#c-wr z97lcQ)23hyx=net!fy5<~Yrw>n;jV{p zFl{gyzm-blfUC~Z^K#2z&kbjOyN2g21(S+dc1hV0+7e-xEj-(oOiqBtFhqa^ZSL>w zP@6?PLDq5Z=&dO zbaQ)}aT4I5jZ;t3zxE8yKgm3KNlR9H9if4yNEeR_I;QhwOCBM9HR+WcqtWk*={Vxp zI2F6ZLB(vhW>j^z9%zJy*Y>rrWjdYiG;Vm7ixjKTio`FyK3O}ny&p(b^h-v+fZE1E z4b2vphN-Pci56!W!j97*k=rr?=UBfYX~|leXH~E5pW2TvOZ>p2BF=lcl6yKe8^!sD z$<9N77gz9oY7Il<_NsXJE(vE-Ny_^NZqzYYqEm3Xd9LU7#gAMV#=*Sa@N`+T@I<{vnG@b8!hZ>A z$EzA;qI+=v%c0Ls0>)}iq&x*d7zEUTuP#qr-#7mmB219s!}cJc9$vo%zh(&5$r@p~8WTl2emv8U^z$I< z*qrwJltB(n3e%og5-6;F6U4!m_yoEMKt}<`pUoZ-9<}HR@^b_qiNAkNZ~qpUYXY0_z2DB256+lqWd1 zHwl#xpms%Cin_4R4=>C;MWEaMX3Q#R;CJe^dp56auGf6iwe!UEEukQ5fLDOTok07( zM4}f?^TbE1gcnSA2jR7vhx|3Gy==~V=G*3Kp&Re*F1ieb-~O8Hrr>`@{^{isG3>=E zdKDs&04vf4d00yD+I01~x8{?Ly=^<0PjQq4B@2i?W4y5IlU4JerVyV!! zGlBaAUx|gnI;Ykj5c~;u0%Dc4K@ir$_5+}gEEmnfp#P#2Q`RFv@md>HWJ{M+=tVKu zS7>-|z|tQon1jt&jHD7$fZF^@U1XHoyZV;rP?J=o$}4C0Puz3!GRx1qO?72e$SS{- z{+Ict7(SziMR7mNAH{6nC!08X()7xs7?I0=Q6bTGzd(qMM))KE=(-oZ7(qj)?Akr7 z9$G#;lMa&7>elnEZwmKFj&7LViglz1a=cYSvH5ZNSbErRh*d(e0SdSs=19;-959v(mgK!)Kcdg8K*Ql zn;dl8p?L4Hep@k>iS54d8eu8}K14*!GR7>C7Q6vi4lQLAa;rb~2D<;wz6+m0-`gEP z#OGuUA0cjbWjP|lAgJWS>(D1)(13AW`w-bTdurbA@wg-JC-+$0dzUl&yLdfyIf5SU z2+qiA0r$t#>3i1qt4bv@c+};nvx}*^LLI}cFEDdi=AF;i{%6#nW1cprIpl(IvcLz8 zLO%=0yPy#XFjO+Mt?HL&4EBxs%|k@V5ZoAsQgI6(MI{yC&@kELC8@5bNM1Cmyx$`XNK?0q=#FUl%kjHPYEv!)2F)csJ3+s4!+uP#yvoFHrts#uq!kdDk6M6&%W9m}L-f}-DeO;w&faWD1_Bg- zOl3EC)i=kGK1>5T4UvSEs9e#SwRA|R+%Rqv%5K>3wS3}yl0H#(P4%#+LKYv7g1EA| zOrSM}2|Quj{BzbxK+O#%QhrhH?~cgbxM4;8YyA$=lEmnaH-YMR5q4wjoI zMlaO1jQtr=%uQUhECyYl<%CWkDK&sW_`Ly36D2cly%p&rK~xc6!4C7EiQ|Ukb>ee& zJ~j;F`$vD!wYl-z4_qFOu@je!otq3nvs8g~%$&5q&YA(^Sp+uMe2>sB4FtW2lmr|d zr>2DPi>Rk+stj@sP_WYBrV7ODewhx)E7*Op_F2lI6T3STxALvZ#YXgWVe_x<;mm?S z_pRD**3QUsu#Sq+8gm$-5L{yQG+-I@kq!caqptIU6;2Lgy&p1{GR<~pn_eNy?L(<6 zDexfyz$|l!Kktx7E5JI&gEzb4i}Q!tW-veV^)SrsAw>4RHxJQ|0H^{3I_T9s3+thu z&W^vsX+2_GP22N0`trlE80hrTEyo70;VQ@t5-f30;{lHa4qUDpU}{UpJsvGcujfxc zQ7fT0WG<{n5Y%ff=h70wW{jn8D;|LT<@JnrJOr+_o>>9R6NHkvCXfe)DKtmiFs`!M zItZt$cVSljn|Kg$V17Rpql>^(0wJ-w4Gc@%n6i>w59zX0f+N|k2lvl zEaRtrH8qybQor(Y?zi)B*B4Xnfu{*`GKc`ccr&e+Kr-Oka)?zAB%asy@y=nDZAu~m z0{o9I20(DPZjuVTxdOzbfyobAGCaN~EVsDQ)J1U*bc~1c0$v{1zg;rx{djE3sS?^r zFUvmGa~7+N{de10fasBAR-w{cB2!u&tZtUdpDrLasex?!4!+~uM`*KgJOytwv?@d3 zpC2=Qv~i%h+*0X*>;qcQ&R5AU=))R2YoGf){1wZ=-AWiCPGWMSHKBh~YKJzZ+EEe# z$8n%1>>Tj${sZe6b}jx4y{fI4*NxE+?{#}TKbBYiOuG=df9p?eXNo{!Deg{yg8TB@ zZeI3%0<96@2aaQZB>NF@G2%0J_0mK17eX9{9(n%;Nay(mdT^X-cmt|TReg%>=heaO zZ`QutQfteV^9|YIrwb{DZL!_yPQiRshYu*RB$Q0R+Uc_Z=92{*1?&tb_GF*hxKQwW zZ~rI1hd)_=JnqWp%6~{zj#tQAe(N$eU^zZJ?GlK&;tAawPIzyHPS=~2(0?YP91+-7k#Q{!frF%y7RZQr#2qQIPj zckwn@y$8pAiJlint2RRt>lh5L^8rT4n2eG+Y)x*B|6fNm`Y1F!NQ;m&XMe-Weq|F!6Oz*Rx~75i-t#MKUD zUbtOKAER@Vontdx+N9Hv?R?tBST$JG$l}l|59Mw0uZ|E}eg915+t5L&RTE`D z8^98;LWC|4LzG;o&M%{#3>sGru?$UZ;d$wSDIcrHV#=hbuQ+6PxL*yX1JVAp6#8Cs;>6`U;}a)d^t`s7 z^m}$#PZ|-ZV)}sR1iQUoiRih)+&hLwGQlESY_{wE6JT?Ig$7AqT;(E`@&Kwws_7~1 z5WhrEL1MgHn%4NF@p(V(eFxmr_L`UQizXbC&*F8vsg?KkgCKs5K8>L9uRh}=OUK~~ zifnp5wXbH{>f|qLhQ_3;(_3@2!ZocX9*><%s(@4Z&tQHan?npjL6?Yh2O@mH8XUKj zJnTz}oBF<9)I#D;U$SVedO}Fgol)lj|0J%L@AbzWdK!*Qx5s;W1s+6sVhS+-5%<|) zDa!$W4iiT*khj39H;$K*s2(WD9an455@!NiTe<{Zbpu0bNh_k%ch{~V2*GrP@g%Rt zTi02*Mf!4`vlK`luKcf{~_-o5zj8P7MN3gF2*dkIUIBJ8<*(FK!7$ z18~}91y(oA01tG%ksh7lOL>hCo3@rz&Chfg2)}xDX>r;wL;jucho3OVmZBLDBxI`t ztLgpq=r4-2irF~iyIzW*BmP3DoTo4b%nz5ZAbEVKIp7U4M8KrYYZw9%-5==B4&qf{ zW4K23L_n5O{DS9J^a1!nKhtU$ZzY)my~;*}zwz`sXVK=&fY3a%p!a>3Aj zz{!OHnrgS!2H8hlKJv^>bs=-Cr^mYftK`D-&7LJ8f6N zJ%91_chp$SX#;ZtH`p2Pp;^@;5Td@g_89tcy%d*S$H5+t?!(lP6Ny*Jqg4QlZBfpz zg>|6rF&`i8tSPhE!R-oF@`TD4&!^*3(*oNJ`6bgJ=@I(JDhxVU8sy%XO{ zAyRJsPxtJy>ybnDG|%tj{Kb##-VRtB=C)3+u#^=+ql@n%8dmzv649aZ0NQS9YYDaZ z)|9N@Ub6jb=7o~Tup>=G;dzli49jOL}SnbLF8YruiC zZwoc&%?9v+854a}3QJiCu!m)~y+KP0od`yM8$oo^HrI)mGXN z?sq{c-g&pQPKdH-6<`Jfb3tdZf6M}U2AN8RyZ@q~VUI3I5Xi1Mf|Ela8f>vPy%rtD z_b%z4#VVR5AiFdk4k`1^|5DzPV>7h)pkKe?X@tNjv3psH*>dAy-Op^zIgv8bPvG3R z8^M7e8qLGjT49^5DI-MC&?vxeg(&xlyhKVlaa%P@hT+yx>Nvc3AaA{WdFosM^cXt1 zZQ|mCXm_DfUxKM*Z|_-y4?yP<%x~U4fpx=eaD+3#Hco=z0_My0%YefKw4!5)Ua~rG zk*!0k;IM*~+VcpS*_Lvyuyfivz2lbp={JP>QM*bv=Zx?fEN|jEIQDB#2e74N zQ9+GldJtjJxZ=Y50HvM>(9S;2x~py<>%F&U1_loJ*d>;5#eEjJu<%c;ak3Tc#Wthr z`qzd+_!C2TpQ)W!NM{oy52*R8ort;PCf0ea_ssKSf#pHlJ{%Wg9VN{JdyoR_us`54 zaj|&5c;lSgF-!^Qe=8mZMA^RAn7(lA!ZbP6zKtvfbeEe+Kx|p6eye@M>2`U-9%;ss&mbPV=6z>{`| zwUI|iZ!z0i?4KaLWAzzH&-}H-8cFpQWkpu!b+$?CdN5C1dzU$_8h6oDuUFM}nuZ$( zeZL;ySxrZfk!ZaPFj`sI4sGXtc$!!S-+-d57sc)4MSDy9{L11LCM>eVS}pjW^1XfX z=NBT7`WKbcwb|Nn*^h|io8i)k;>4LsxDaV4;Wm!Uac-Oh*R?=C;OLgl&^WH+Hx9+R z*m@YQt{5K-D{=50_7Qlha8bkhYJMKW^tbRsFW2%1wt6>zisP<>A6&)@v!PpfZ!q)f zyxu_a(^sl7-`n^p;HpPchHdN3h>+0wWD997Pc8}9%J(rrfzUm$0tzdva-DUMZNI3Z zLTtUUf2dV?SmpgaNT;ki;QLX(!a1kLv+fVRK&7?Wy_V&`5E#;&6FcA^+IjN>HD>l% zd%QuP9LN2vuSCDrQwNbw3@~yZ^2l1W@WL+fkD=K3u#Kw+f8-T}+a>fXTCYZ?91&qV zKir*tLVspW;5Z_^z%&9E=Y#qI)Z@t*mdUdH6e#7k7M$CWRSe>rEK*Sb8=ERV{9*Ff|z4eb3Kkji;Y zBQn_p=%-jDK(v4VMeXgOt0UbhK$KeHBy#NvCo>Ryza3s1=A@I>)#xqQm#4<8ez!9z zzqJd}pu)QBc@6(@h5Q3lcZ~$sX9^toSI$aEVodF)fpDS@AXE7TgE+}!2ac^hLR#!X ziEZZ0FTnxEK;B^+2YX})fOm*qRP9SkGZjrorcZn(pSC(q(k-w(o^U6YBf4zg2WzEe zAn*07f&vP39ZU>M1gJV0rlzq9M=MYDTx%Cn!u5(LIzkm9%6xb>1852j&&TMEt>PFcYv)A#L=5fe4ktxI5t8e{(<<2LBxv$4!k@Z z%v1=+Si|B?DtPs%qeoUP(+;tE32v9(efW!tDEIvE6HK#RG)9^7(7SM7dRWqeU~Pgx zIJ4;g1-#OIPdL|hYz`6JgnS@aJAeHQ=oixU#cju1P-O$GNGxNyB6f(C<0{iRiI{wD zM_S2vmGI|Z5J?NO2+|_%4@e*Rg;)2INDm@QPk;RskVxh)iSt&^uRL+;(4h~^9vpME zR<qj@t971pnZO-A@JArp-Anuzv$qEB86u|yw_Y#GOi+6!@`$4iJ z^xQxpONN>(aj-IB$OoMwDTY>2%5+9ztL||2m6-(hR}C$(wrIikExqQko`Jeg=m^H#(M|3LL}oQ!oS9pi~c|s*IMC(o2NY!E5eIAB>XN_ zIvs!PBQf9|_=+r?H#KFRQu4>qct`59>V)&0i$@XE94FJ2or9&kEPjbu>SJ8O3(^~D zUsGsMfVhRX+TzhmFIs8~zUk84f3^+_TsF1SOWv(k?^;{$kWq-GH2p!r@z zCvA<~DUSkw=Ly8X6Tg#*i!Pstt9NW5p06MV0N;_K5l%9*gpv!ozI0q@mrp#?{Poex z+|-o#doy*m4oTE|qqh4Mo=a&liy6k#*beMS>3UFM*9C>In&>SoO?uQVZb5LcT2ufA)%i{EVjq^$0HLrzL1Se}=ka`T-S6bx z^nlK4OG`&t%?OBab{-kg9+_>j)b;rDN;z%sP=A>~{Peft(ZO|3u!P!Vd^kN7=mBL# zRQnG3G%Qp&_~tv+-3Xm+yEfZ%r0A<|gT_V7p)b}(zQ2G3l?j+bAIC?6%1L!TQmDE> z1wbhEFUmgy-ut^91MefW$pEi&n%)9BM#=|eyVdWhkh2tw)_Ov4|+$KR1BB=w#s?_IToik~E>< zRe|&4OZUoi;0LcM95IzVT}O*ScLGy--YKoChb7;t+sHhTrj(0L@^ql}Dfy%Zmg;g2 z5v`s)MBf1!QV~QqSpgIkrwk&Uz8Z?dmw%F7NTm)+pVyqf5}To9;l5<0OLsq|^KYF$ zHh7W(T?L7Y%DN-_wV}RG@echjoSfUJDLmYJ{s>!UjT#m=1^4SMDD%79QDO zX``edsK3VW4}RHGKTn}37}xFrxqzZN*js7xEDiAHi(dalRah3F*fNiRHYE6P&Jy8$ z5KFj&f@O(u|32OdS5phC08e-2Cnui**J+2TyUq&jKlcrEOXl%S3YmAGj`pboN1`d# z&Dr_MENFXxdd^1p%eWWqy<0{3&UMIdMw)%(QOu!eS^UwDt-`Q`1-&Ohvrrb@u_5-t zam2x~b5HD7QAt}n4DurAv_*Ag4yENfB*eL%%YNFr3+W;92k3hq1v6>(4~45c_iO}yb)>m6U|re$f>~jg6r`fveok$ zf4rq9rc4f=`=M}U>fxq5mqaPTS58C2`xBR%X-UT&+aU((}LgRl-z+dSsQS2DM1} zmGP+jI&i&~tMpRh>0{VeCsjD9)}8M$(`F#e5rjKB8nb+k@+8+a`?*;)XXqX9tJ&jt zIV%hm*gJ1}Ond(EJZp7y!RF29#pPwcxGd2eqr9_a zXAk?Srl=-AGZ&)xV~0WYdowoTIs|ORwIvGtX_?&^vXX`o*N>pEPkwJ}$lp($RBt~w z;bIncE!5ATY{<1Q*pJs?I>Bo!E7o<=p-I(5kzM_Wm_vqj7Z5%j8Ac@&;d{@}X>Wjb z=sQ1z!`|AT@jhyhkoNR_g)!IOsr?Sm9JLYP z^`9Ed?snECxbR8%i)mX1xjjisjlW~#{XFmTqsG{& zu99Dluk*~2A&TDY^QTa@c^?X1-|RFgV?Qo!z2_G#kLNA;;(kdWd%oaV&{3~h)l%z= zW$ESuZ+9KOK_UZSH-N~JNtsrM6$6&G($tf2B6vUQgMiL_=~D&bIgjDX7T;=}BZ`jf zEr>WEm-Mdik%_IzTbU%!UVhd$a`YrlMoVS1W-r{vFnIjxI+WGRFqxlC>eJnEF?hs& z0n3u+XNIwy;W*|ExRhFvgbxGNk#IA6diB>v#hRB>r-}1#eenq{e-utZl z8@HSOsBu`4zBPcL6aTG!@I+ zzZzb;x_C99qVXnIVAE5-r+LA7o}LP;oc%;Ma4N(5i+cuRp}LiuEQ?Z*%1m}GB@njU z)>8BNh73^^uIefXj^6S8<)mDy3Udr_-R+XI|;g?F7AU7XjNn#ZlY_KKe31?$B2OO zo%v5aN*~-Fs4dRkexY%g$CRTzVN~jJ${d;Q*|XHlu$;E;QcuAMvD~H&g1e%nhE~Hv znVMggi`Hphe^i(|lnw2mN*%iM;V{SaePqLihq8&KA&?*+T*cHyCNOA21^jt9h5n~S zgTCTW&(IVv5@~8_Y6S%2-_s+nI82P4$a?F3Z0MkBm(8Qf{9p1LN{G-Vjv2`E0D+p# zKZLH<)Qq&2LyzEOS!Dx$`(eDbDt%O|MNPg#bBA-AN|MgW`QC+C?aYB#nYA%MwBoiY zM+G0wSihHzh&91!bY+?ISto`}p3cAMllJ&+DUiv(PboFMp1mWwi}+%_5b#;| zU*Lq#mHF6@eJG|_?NL)!S?IDWSn;$p?=SV4P1LBly6BcK(lC!r>aPD-6gp0cCRSMxQnlE81a0lmWFZVV!oz4i0^mh&0 zm#yncl7H2^(}w3)7Q+}3D2CU386r*@ZIDQ`FfQE6e+KC*c7F5OPfJSe<H}!emQ; z#D=lUxX}(p3rVVuhV~H~AxJ2CFX)4aHo!ZhzH_N3pmMWG0FSjq$%-WSkas1B=uNSk*GZ_e<4LnZH;cwe?4F8!fGyHwy~b=~m( zGaA}GXz9!WfFQ4Ou%U~an~9zQ%K4ye^$FC5gFMyH9f%CWq&Pv}oASn@-axEgjyxKc zU?Myq)?u2VjE_^Et4@$qHuivIAsQbv7KbEDyv4D0`jA;QgdenI6yRdsb_Q&L5)jkG znnZX1f@K^Sn!ENfa7kKTq_rQ5pVJJdW*;RB3)Vn0j?!A`qJL&M?lsCj*>+|-Z%QjV2PLnJVPiwbGtgJor46eL9|ZV%b7sc5A1Jw>ZK>y; zePbGhf?UP^fW(1E5NZ=#Bg3kgn=IZZ;YwX&)(K#^WyATdpBzabsf;`}DVf@CmH_Qn zOt(8|+S}#`lp6)xNPh}o5F>2Wf!TCvt}jJit1mZ~;JBrH%ufB6wvO!UF`_l8AEG<|MUClokzZJ| zc2rG+0fH?)xDiD4W5_EB>sH|;=}#3cjSb$SIx_92W-aDM44!b#_p89?unoVv6OI7y z_=aIjLQD$*IV5NYr+Zyhn!EI?U2*6u+`-Pt#iWvRQ15Y#v=r^wIIsi&DMjI#JV#e0nwx*H^Gp{54VmR z2@h!Y1!{-CwRr#vqbuc&365t&^CHBuva(Nv?tXMgSm!&Mb=-<+1AQ`r0T2)erGe+m z{leA<_<#y9nFV%A2wz9meYCl=AIDgZfCwkA~)S60j zDF3rq=&Z8E>S%+v07c;`maM1uZ6Prwb>YNAGcpOBHn`_SWjjB58Ou0 zfcHm&!~h>f3lMfD(vC55Da`caXtGL zNp<>*p&}mra@hTW_;RGHvF+=kZ7G5)JFVQLd#OQ26304Gj=@(1Ewl(jqVG{iT5b_qz|Acx|TK=Ca=uaahgA zm~o!pf-TA~HsllsjCRO2kj8R_d?r#h;j)0+bvTjm$32~hHlzDdLBdORelt_K(}=M1 zkaOMb?7lmfKP*>ew;9DW7q6WB1@==14ag0qtUJQFwD)4P3>RA5g=1xr+8y0H^2=~J*=!H9!swgu_6VQNbQCN#ZQ^4P zvomPS{daCz9`Vs0-FA0(evd)d6Dc5zqx+CM1I9bnXBa%)=y*dzx>c&7u6d1CwO&`w zB;>4>J8{Mg#nc;jKDBFax|V|O`SUz^0@|zrz|0Ap6f<<1y$dUew}P>5RL|yApGIke z<KZ&C>;3*X1! zl+~`#ejs|N&pRM+ktU8iG-l!a?J9I1rQ3x!qXUTnP)4G9WoTyX@WukNH9^Z0&FMeK zmkf6|zrNM9{aE-+X6e&zzUOkl)laqVf)9|W#F!AQ&`94kt)jS!whHIIBZ4YD0>09D zawvy*skfX4@mKBl)8)Gwi?A?;G8uX13fbmsor((cvB_AqYQqotOg+JOQ23IoD@xRU z5Ui73Kzg)1`yN0a?n6>7QB;}URUk?eZr@G{+wdFcR@i>(DFDaXx`Q;Ph-nY3*O%eR zs(E{fFpQ+3l0nz!YRy!A>l-Ea(k zEQ=q^Il&z#=qqV0GNMgxPT@DpGb~$P4*c%=e{D_LT|aQd-+ z#AF;7K9rt8U5UecRKw>w3clV8uEEu0g{+53*dGFQ_H5H_06^fjIHJY6 z3TPeuSyuqE2D}C+%2jQ8jjuqax0Y!I53g@}^OJZ@29%$%2BZK*%ztlOCH;J|W1`15 zLaW7UHztDhiWD5`mVM(R&v@Nt*OxuRlIaJvodQveC;J&biZD+UV}<&LUaNL_+pytT zAAHF>K=iEUnVy}8jy_U;w2u;;dcVjQv~Pi3O+Txi4uOWW$C#~dakRuVEj_GAc*jZl zRK+fjDxcrd@zs;LON-jSN=-CgERW4pq4-|JqMz`rYy5ZJdppEK=K-TW+!b+nk!rjt zL#-x-x+HH)s!yS{PxYLxkE|+;t#oZLjd^*=?k**e!(eL|8AyD{VCORMz(2Hu4?t(^ zSHVhfhao*cbTxkwt+MvPE&6uh~nkGP7mulQ{P0kZ`}FFe|JeM%FtPA=Jv~dnF^K9Y+oD{ z;67Y-U5mY9{}w|J@>&d_V8`qch}V~)(^;fAbf(3EK5kmrDJ@jBWzXc;d!F~R0UmBM zy@zY=1AfbMu_$b&@+Room;oyQM)AgVV74!uL0^WJv;D_h(B8en8ad<4)B-JoYV<** zpt@@ABx+v`ZaT5+0hXfF79bhEa6b0u!b$ptq{y17jw_WU5o2FYXmC%rRIX*N5z6yS z_TgjSh=pVc;3XG8f*VRs3AG48Clqt7viU4u<4)IqR@KrQX!gB4)3tM0an(XKxzPA% zm-NdmMsj+`cvipQ`u?I=JFkK5vduS5aF4KbP)vi5%F}pQqQ>h}!>#7jtGQ z)wc|_m~m&xTyS+Vm^gaXruX?nb%kh;vgCIUmsD@1?l4;h839YV5pBb{hXbktJMkP2 z)+5n+7H64(fa~ZQy#lCWyrdQ<-gr++@M>asl+R8gB~f1bnJ({$p3Mom)Wf@lO9Z-X z^$zF%oTd^z03{v&v>}mc1Y29d59lOnJn3e=fsvkp9bVld%Dd`B_Jb4!^tDJ;ldO9# zrB8%JYv2JM8}!_AG7D-3Pm&0Z_tsV=aQv9ePjSmpMAwNSH4ebnD?S)t*7f>F7)DrdaGTsTl(r#>|eZUYJwbxyha6Lj+mVCV5DYpNA#@x zlOSy-N#Nr`?p7Cc1>826P)7CF_k zo=@Yhr9Qw>z=(m{(<@;CUV(o424SK89}>BGRIl9$H89>U7l-o#PxA^#KTS@{$AJM? zTtc*JEAW9UXgK(-y@ttY0S6`}|G;@pZU~IY+_CS1=tdB%yhhrTKhP=eIXal&$5N*a3i=E_ z^!*5k8JQ`$$@_18EtxEH7(0?PlbLsZLPwV)W9$yf{wH`NTfq&*USkE)!Mq5LwqB3v zP?z$(p_5iW$}%p)&5fPfNPF|XZ29C4dg<1{;{s{1+g97KBsH)XAVa(~_}PodN33k? z{<&!MM`SzeSc)xU>)4oc-;+7}^V*p?eG#5{TIQdQsXAt$c2#U4HjZy|n(CGuUon$@ zx^eTI{Wn1400M)py+D;DqlkyTx0nL)UO^NXgPl_To^{18ju?yQ_4NmnQnNZ#gZfmY zrfoUCn7!+KP$CI&t|qtIf;*T(KI}Tct_JIp3rQJb!bPl*5r*fR^0#AA8F%G<8)AP> z&6b{>2yl;Yk4vrPd=o7Jb^3r&Z6!8HGh(o?;ukP0;el%=(#^>5zD|g4TRh^`0f`lN zEKwhvrFqxAaq7aoYjhrus#1A7sy0)*`wkYI3Xopy>lkdtIi>*X>bVT~99i@g&2UhqX74hq8VDheyi3OG1n-%96-Z z!X#zMEn4g`6|!Y5Ys@9GWM4v2hLGK4OO}kWFG;rSjHSgsQ{u`j_j9^G-{0@`{d||# z^ZfDrQF`4s<66%1IF9o;-piVWua&7Eq`irKSa$e&Y18Y@`(Iyj=2mm0UPZlT`)AUD zc#Q`wT_WJ`oOS5Ll8n$#=h)x}q;Ud9biCPVkyG(_h*7bdy^PVD5yf(0+qghttbEYx z^zW{>{hgg}Ch%Uc<|t_YgU=MVU_1;VFl`W_Z@&6_0zjI8gz`5c9yGK-#}DKmI=0wq zO}~wi@@HeCKuIYB-?b3LcJAy_7X8NmkW_K2*d=DFV6+j*naV3@M8nl$54f3VJl(Ju zze@T#JFGe7=J7!Bh4mY|!j05G$Ma+6%kfG~y&}r2e>nf+T0%jJXL6%=A`hdSDNsDn zJL!)2oy7VBqY!O6yj?d|ku$uyYU}*o)k;sCizld*k%!@2_Z8XzZomuHh@-RTgRzJL zb0pQT0A6d;L`7@)5M}Bc-Zy;CFx<{wGtNl4J9%ulSHZXJ(!yi^>*~q+SCWRafZzz& z?-A7~=(bY4{r*8cQ=`L!5eyWcHk&Re3A*@Oywk{%!Jf){XKVGuw_C2GeCVw-?|X!9 zkG2o(jOCISRj{4TWe=na_|NR%YG4Bru9eO^=4W@tB>LL8A4;Iz$Hz-)>adowt$~LR zJ7-zymejEf#0hq=NU);ZI}Gs3qPE+hDO3Y%kS`houHM`sFZQ{v@=YMtlCGLY1aGZxP#8dkii{PH8t=exk}T5l-Ml+bq)Rr051(B&YmZ`h2HJmA$`2GN zcxyw^tJ1o+6sV$#1ik6jRGT@{=vn5WMT!y`K4(L}oY1$-to;7EuG|EH<^p<);m;w$ zLl4No3zq&KhIhfSz?z}9H(@-}%N&6bzwjgTrdd0E#vPBgj;!ujE%{Pcb$NxHTOQh-1y%Y&NwXp^9d`t%3;)AL zgPhRiSdu;pKGlIIT{E4~TSRvZiozGQnLDsdciuOUr8U{o97@_Ue%VP)77W+qKY#70 z+$$jxCPC^DXa2>wWD3hVcjKbzN@J?p#>DR(9iys`8tzwGAUWqukEW{2O*YztKRSY zz};PYro>Nl;N$dC;2m$A(DlSpw)e@kw~<%?)k~uChh_vnZOO^R!%Y2ac<#-1uiHO;Fq|QnBmmz5`jTEqGu#-{m zC;F{i<-uY4F-y7%CtA7!u=aCNZ~y5b#4YwyQDfX`Z!} zot*ktzE01S_ss8SigJFBeL=O)3)@f)NgzeapZFd%Xg@+6y`AWNu;kcKdF;^x`a~d1 zrvE5w0)VeTj4Z_-pc+sB3Wjh_=S(1Tn|jJT3;-2Jn%<6h|B?Umtqi!vjVbXLcBadqPmj!M$Hq9e?Y$>;xhg=Hg1j) z?3f65G+(n3QHd|mwoQ{e%5(e0W?dQU!<~$8U-Og;jO5CUI_?`WGA0J$R!)IQHI4wm zTG=}ia1pt?8_fcU14@J}Ag6o&wX1cers$txMf;s^Ulxs;M`t;1w{)3dyp@3`H|+=R zBMLqXOd!V=(XjOSKtV7l1su{mIJ#?2CDzUar*$#4zi@K$65Uod^We~xUuRMxKA4-E z&dIVdP)bo#P<92Uvf-{J;D&1lFS8gcW(P9mDr?dXF6*Vd6zi8ac60R`6g_#)Zd>S>h{gA zVR+m4)x}SXCmmfKQqR5C*S9dZLFoC!0=NN4SQ^Y+(EFVv7&-|NtR-OEB@dbYcCpjL z%|<2jZhP8YuiRaooE~v@Y&yLV2=KTlpnLpz81r>hmD_)0$`%uFt0Uk@ZA)i0pm2cX z&Rp2Zc%P7l>GeVrhQZqC(1fv49n$sx{TxUAH@`xgwnKVQb2ytSz-ydZ-8Q zlrF9BjFi*FZ*llwo@Pq0Gy0|tY@CJ=*U$oBWdhaoqNyY*U)m0D1NnP&?X;b7mzeli z9_f4ZFj;6ryeZ?HlS<#=aC6CU3-*+T@JfAO)Jt%6@VT^|xeqN_PVm=aTZQ2>F?Ta7V*)$K00rFy!gE$uz(NAv1DEIy3-!Ylqdzk{LiT007RVQ!gO+-2Z?_TLU`ogrlAor zksm5k_HbMySUVBN_3DAdi#VhEDR$EC?CMwKBu`w=m5WC(fU-w;9$fv&FEDa{wD}u3 zTDC{U^?Sh*nH!+vesGJ7>c$=h?V{@Xs0ykenfvk%$7cNY;5EcgKT7!ujU!$$wd_`2;w>b{!}d)zX+X>PzPB z`SUKvTlee$gb1y;jPax@gBVruH^KoQi{hz;xCo25!z*XjERCE+zm??~xjT7&@)ou> z=81dhlmyD&BpbWVVw=l8UeXVARam7`)qyjv5C;n}{9I2i>;F-?q>JPK-^i}`8c+08 zHl3w}a*95M74&QlBA;2{db(&Th}ISO@a0W?eTK?xx$(O*C+f4*QpUWSCXP%MH%yf> znYN-}@!j<_%4w?IXKD};8@_iWZ4u8oxKp9BXLpfKS%2Umex+*0RQl2QISaWfr#T8v zA4Fi(Gw=KXT&8wBu@Nl{+;XBJ)@Io^^l@R6q*#Sbza#HIKP;BavI8h+3!^29Rti8t zvwaWLKhMB%(}UW9tOY|RH0q}T)e=~5x*ORm#52zaNhZjt_6C?y6g@wAy6Dg8PtTmo(PYx6siu1UCE(Vw+ zDw&oGlF$#YNs%2NxE}{2Ey&I>Y6&;m17tpJuIUX;?R6F+ysW-iuu-hij|a{a9ZYm# z+}4bR^>#GD3j|#~C+zg2r7vk<0H!1N1tcDR_vthflK^869rl$d_33UYWhQ5+cPjc& zOlv<6_iGSNF74wV8G!oNUQ2W#;6W%Z3y;U}zIiviU3Mlk=$KTJrr~IKN?J1Os&1FJ z3*&03{I4URd|ja>;I+iYHD@_JVLWoDLr#PPB8b_s$@AqV4|;L}Id zR*`QrsmB;7pkyLk!YE@A!0}kWggMr+-aXq^+v}=_)+z($=LIV$d6W6$%?DwIpEwbr zCHpaAVe4NY>E8bgl6I$jpuK^daMAPcOxxuDMlgVJTxIqWH|whDu^KCTPcv!#nN+Df zYbozwO_g`7LIiPY3DEH##MJzj0jRH*IaD!NnKZv6 z3Nv^u;8<2PkzqPwu=*$KjuUi~iYM}oM)xwTkET(t{xs~!G3Ni$!bxVPiZ)S1Tr_`g z#E_8*xX713FP!qw(oo3Q6|t&>C*IB^#78_%HaHN5GC6@ufk3fJx`mnq7$Wzj54NBJ z;Al3%gcz9>Ovlx{Svw+Y`o1~t)!>0Do!ftLw)!$M#ybKh&nd8wVxW7-36Z7r`F7xC zT4vQu*3V|^g_xq>sOO30Y+z^lRxemzKs;LxYd?Dq!MK@u`V5#FjqQFXU}4R;9TYrC zHq|4&!p4Ody`BpU-?*RK{H4~i`fZf{LdfKc*sj92P1#|Fy^PYKNfaA;Cz@?v%{59n z5D3jjJj9Cx`im6c!{=5C5-Pr#T}VB7*D>`>xbBH|-zsMDlncnVy8nR{i?acdG{zNm z2N3$i_e|;BzahFYu3CqWdYm-g3baRLBSJ)9PXq?E(T~*{D{7cjC-p~tIsPNpPx`0v zv5bAop=7Su|1f&s{-#9d26NMA*&$$CRo?*(n<6_4yyzMwgI0qg=3tpn#5^&fp@-Ob z48w4Um-E7OQ4=&G{LQFhQ^C=1QasT`^!QD0L0`{_Z)f{YD(9hd0Lx0)bONHs=zvrd zK|=#r><>n81lgsQ=mLM%_A+qb=gLDzrqjL$))QsJn;(#_e_v=RS+HYUYt^u^8bX6r zwinq)Ko4uHsqqp64XYu8m_#qV88QG2KarRMX;l2>-G&)l-Pr7MzbbLMz+ zN|TKK#VWaiDg<;yIt=0-`xn9(%%AP>chS34B_JO$yMe(1db-E=b>oFuw?VUdm_4_y zGk(Q~=l$64^D%K!Od?CEUsEsle6_psmKe?|@Y}LGX|lT&U;CZ#%5q^VDMYyt&L#wp zvU>y++UZ|96{~AfXy`I2bhAvt;$m*eLB0k;C^4^NTP`G8@H+`HF;8v%J5 zJSQY5PPhv|nEe|8!uWFIUjEJ5f2P-TnyrO8&4TOPwb(>X@c z{;tvwT4&Y=BCowXfnWsKW;~uaj^3BW7_0fbMdmo@w@JVan0sMo z)Gw~V%QN9!N%3TxQ%c;so;&Xi^KYd+xom!|72#u#rHFv*1dUf`EFj>;)p_xRNWc#H zFzVKq4U4W#+9^-mDPhDkweZnXN0*`JC!H?l^~tQS-$Fo&Xp*lI_09CsJT3A*etXgx z%{~5mzx`E1%L9}kVm2K~d$T<&51d=ef!tgAo0T|`mZU?=nHEQuHxuQHYO|EGQ+ujC z<;o3PJu$Admw>+lL_9*1x|3K27Pg`?v&24#l`*>RMKw5=_7+)xIu_)6RkkrzF?K>W zSS7Ilba}hii$Df!0wd8*4^UJYNmlMP%kF@f7oG}HCBT=5Aw0F9`0Ci=@4WMRHLcsL zu-y^w-aIzDgWYy{_U-abkM;Ji=F2bBnU~OT+~AG|sN@DakW?AaR9Xbgi373Z`N>@{ zJU0efp?%3zHxe$swuCN9OEqp9xbp29^N#7WrZ=%GR^Nmi=+d4+2OU+?$ok--n4t%l zt|`w#A1@kysVu3cXXwvc*v%sG7N5q+8I_`9oo5Vt4=#t~gBlB5U-&Tf0Qfs+ZSPAG zslH=8%{`05U2E@N#l8DVGEYcOPmf}i4-=I|S%ubaac`u9j~B)Cqujv<=d~FT4y4T0|;ts7{jg;bQbkrP3u;?T1{ci^E;Pl zc^KtDh@j~l7owZ*W-r}%#G*d1H8DwE+VL~Otd;T`bN zXeO;QMf5LICP`0g`#)S7%v-s3S+y}Y`6Rb0+Bp?Emnwd`QZLe>mZyUQ0g3wk4|a7q zdi%aQ;qu?bgSSrepSzZ4T?TQ7{pea zuJ@Dk?MbI<76%~9vF39=JI4J9+y?FxiPMb1(tU(mC4Db_-t(q?slz*>;B!(tmH;LK zo&T|K|9*1ugO@2#&t*+Av3qdELuc-%r+RDZ8G&DDl-x^CDaKR=1m)8{J*Rgb*2*4y9rX}?5_UtB1g=0M@u4WHV~(G!R%Y#Hgn7VuWww}nwo>Ld{~QEs z2B@qI&?Bq>UDy>R3`hW(gsJW819@=GmByjEql?L1KW%Sa{M;uHEqCQSFoOMs4%aya zGLAS-hHU?4Pj-X%&E$`YfRk^-DQy*Mqw18wjMTztWh0<^Gv9d8E7#D!7JDz@{ zf`nSZ{f$^fl~_uFHO~iFE3R}Dxa?pgM3a1n18_JgaDv$=fCy*s?zT|8m<;?1*hzDv zNgD;9VgI>yV-JKS@mJy)4>L|H*WCcdkXSNFzyM@NUVx!m6*@y&yiOLWMP`7wSS z{T662xU7-z3lLT4i0WWWuv&MV4$9iyUL<@&dbC`1i&L#|Y#H%2KhdJWN9a!1Mv>4; z7L7z2jNZ@^TnI%O)o_X?rn?KgWSI7`S~81e)xL@Vw!J5c>EnXb?Iil1fVfAyRC2KJ z{}t}v_u?k*A|W7<6M^12KKM7HaCm$%6tctabMeJ7aUcLkW)t>kxI}3ZcY>A_|H?|J z+6#P5W0k)vhY3rniZtJ`XwJ`X#mPxPJ^%4;2Xfn22V|r+9aH*Wx!gnCu$ucmDJq)Y zQa2AsQI0v2qet>;<7R{G&~Cy8Pd1j3?#-bi_NJ{ zXFF?l;TYSF8$uqIg2A!mxMj!u&XGpIZ5u?-eZ_FnM^GfE$fOR_B%KrE&Ftj1`L>oL zU%Ya5`VGfanMY6EKZrfk~AV z6A+T7dhpR3=vx1{91i4UoW+xHwRiGg9cUoLyG(FA8?v)wYGHN5l zc_Gx;pvB^msLdDK^N|))1IJ>ng=r_UehWiAmZMnGIY&V`ITpeIT4BqeMAdtME=2L8 zh&@7T=orn;!6$U>sULLV_f0{n)1rFq+(CY>4Ckzt(>={V2YTkPv#9($ikJqG-1M;^ zlFxb=2!SD7Gf8!d z+fu0vc3jLlmGAIa+A$}>JD0WI)Q=I(07>1fsW7b{Ee8u zSime9#wLfV>--y`4T3;Rclt-K3- z*L2VR=MM>!ew`xfc&m&_EO*4_o9cbln4-^h2%T8BtEn&L%R(V8msE3bb-2-e4|V-h z`LZ-eX!mo8&^w^fzUhK^olm)i{xyuE{J1gJUU@T0>~+|Y*o4ab8xhjC6FVa{&t;$+ z`zF>RK=bMxL|jJyF{ua4e6~qwrTS9~61U1oXPX5r;u6qICtZW4C>C(j0xA|w3R|_8 zOVk*@58<|>#w6c2B&t3={-k{9IU(V&!1*XkDWP+Jwt%6H5QBR5(tzYrCkm0 zyrA5|Okh;uS43<)FMMRMyYb1?W?{iLmn$t@PidO#Y2Z_Sv8*crWdX&m!szIE)5cK&m43cY zp3*-F-c2#r{WlRM645gd=@t$aZ%G30Dh|l1RCp{IHM1_6GM8s~qkHdpt#2wUdu;Ky zLZVzGSfZVa7|{jrHKnvU(jc{XWl@R3E=zyo|67)N;p-xL@9Y3Kcx{sqkh-~{iiRbf z+e6RLSvh?u;=QG7!+}?Rwma^Xr!?u0bRr;OwS9d7rYlhI z_P%6p6%Vx1Hq|r_Kko0qZ~N$FA?du;pulj#L_h=%5IVy&QB>X~b8_@krDh+Eq-KxB z9>KpTj_R_eHMMUojI`>W^h&r2oA>(80vzQxX~sX7Gvy!#NoWVRQwl1CQ|lNx%x8=o zS?Mud!xlDexB2~Re%e*|P553yXWW&5o}9q&}-rBK(Gl_Pm7C(*LBo{@hnJ|5N^t#^*;uyC`ZB3Fe`^cjj_VR!? z7lW21`PbqgEWy12EpQ9C+5CmA+3F};QEe*m5L@gvdQ@ip(bKH?`kU*9(?XWl^$xH| z3nPH8o$A(7K%9;!*e^~Y3=7(V_WQD-oFTfkWA~|--{=FahQ?a|DU;u`_~+454QLhXWzXDk)CbG)8S!&wNBn@DW#9-js>#=EEj*jo!&b>7(qK z{&$0`w9+0Yfu=m@q)Tyc2!g_aEeV7-ERW8yH7plZiZnN3ma~cGwbg#s_$)KaZ}yql zRGv^uyXiOO1#_u$9v@?8>Z!-T#y)tBR;UF9fmMGIj$8s?fC7c!KXUpxiZ*S#?HNhI z?+CYJk3@MhIV7t(5=puT`9C30vKS%iUu0$iE=Cyi=H$!KE@Dz7+OYk~ROGWaCpTpD z(*5pG8+ns`Sp_KLAU;OXhjcXRXo)DNd5-o)lPmMOaDp>b3TO2(@m{WgW^4$BMxOWB zn7$gFa`HjW%w3DkNMNJe?f0j{4`If!{5ldYbR_aXNN679K3iBd>X)Eme4bsro5{NH z&Xdc>(l%OU58$1_cK9&}AGhpMnws9Z3Nr=)+l}vCyI?yJ8%NYCq>N`}@_bJ%jx1Vl zyq@By%Ab7eeVTZs0;(--xWN6*A77RPs{&S0EL|BQ96y3Qyfzip4Ek@OThXcxL9)+x zgdYO>qn=wi#^IQU>SIR!sw=r|a>evV&_WMX_njt!?Zi@fOy~kMS%^@Be2eW+nqa20 zW_vB*pJv^+K|T$!UX;fPaGq#ze|2hZaqi2Bg@&VTrg4z($FNtA$85QTZZd95{-18t zcXvAXeXu0Tpacl0Xm?kx@&=w-(#^KZHZW43HVrxD80qY-Ls4*W+w8gZ?zD9cd<{L+ zO%-hbZ?zg8r#Mgz;5&ei;2g0<4J!I!bF&3GsyZvF{(iKg&^71b^ET$w$yYY7`@bbX zjB=&`o^XzPAGwwP0ImT~ofY2}hAQHQ)t8nGVq0_|f-s#2j_V5E%v&@Gd{bETfwrCF z)bVy|)zad#Jl;~C<67mne$fWdQKMwFTm~O-7}MQynic_GNya<}MhM=MKks$Km3MA# ze%jXLyJEiGys7`&n%j($3Vl#gy?lf1p(xh9)i5?XIC8iHgs7MtEe~>?KL)pVO@Uu( z%C{!m*={#KXVhZke=gzhL$;XM;^JJnQXl}tfSxUcMDcZ*R384FS|BrQ2oLJl!OBZY zN3Ew<(~LaI;YX&a6q8ZOl;)1qlNaN|%T-u7isV{Piz9-zsJxKu{yS%Z2$n5~*+5gU zYfm=oEqxGd&%#a?w5&Dzv$2xve!#av4p&ADnMgMapj8D&O>F!?>O_ruYcffjKZNUTIGid9DlE;Sa?G9b=#>lUrb^xaTkKivoQ z0B2`<$5pX&4k?C?eoh=mc_3XxEG+{h!weKL9#xC(1m?)5abT*)xbUfo%-@FMQ&gqI z88x53zgxXhZN2g(L#6#;*gXwJ=I2G}j>&iG&IJx*;l|Y4@E@?4a%otq1d&r_ zzNkvbrxn|>)JgTO{HJRc~IIR>EU$zXpC?rEUnz$`7B zdV+B`8=`Ye{gZpJBoowOSB%uSAE#S4K}RK1|MOB~+DH08HASGjZT9G|uTPIBm!@8f zYHdYukF!xh&m_VIx{Z8_ehZXb{zll0&`-F4f;-W@h^V^)y*RUlzBoyw)cfc)?q9s)_l8{}CP%xU_Jm=J8Rq!g)hq>)?=bF6A&iV9Xr zN8GHM-5rh%bubSKKWDiw+W+CIaw-D#qLdy3IYQH@#W$d*$MhuR68Qs!PppHR?<@#r zx9_Rff(8~aUDD5UzF%cDN&bsQ(h^DHRluDnv0Vu0TH&8;^H> zb#c&O1amTQ!D(D!yQvJX=n-dcYgbsXRe%3M^>jMny!ou2N?7Lw)N_^TL{sVCYLsi> zU=3`|JjhN(8|`HIg+}+OXsUQU{SW{$P08!5p*K$WajMUqFTAlU{nJxgUp`^!-o~7b z2bG2*T~e9?JmKu3&ZK|R&fXZfBXC!s9M{D3V(UyhxZ_ZVL->IS*XLzY!5m*r)t5zg z4`Z{2oS$9m{l}!8doI<Um3e57}^}c1?t=?~oeH)~6--Ra^`;(^6EJU|wjMpJy z-TPEQ?x#{iM@-&DU(cYJ)jxW3ZFiZcEvB%h$~bcRo6c6p9zE(G(>&?j?)~(e2?vmE z7resjlwyQM@ljHwR~v~>MQ*KH2=dz5+p#)6;rPRqDWX>*VWR?xc>dU>gYxLVthSy4)R&T^O!7|@>K^ux;ZYqMAM;nxQ+YYJ15h%bFV&1)%DDW*5l7J z?n1Ao=-3u*h;WRqVh<7Cf{grfeSI6GmDm>Eo8z9_{qBRxvUiGS=0Ci=?%)o6Dmib8yrF4>Q2nJf0N`<^C~3=L8+0@Vit8TFf>eLg=W&Wec{02QM$VtUnykxH3AF zH=4bUJKuYz!=P8_SJ?HdRwZYzkj{Mr3AUZrpmFyU-IBnZq4ntQn zD1VKPgxBbvxYi(Kcvf5`)jZvb2O$p@2?Itou#7lJ73hbEL8O+ZVwbYTB>u@j$kI4S zS*);V=2M(nZK-c%LYTJ!M`=4pslhOw6o>>VFR+QumJb9Q6a&l}NkvZ{a_V>>ued6^ z-#PwK;(Lmj&xZ+)s(zl*Ghv7bI!hXOhUaNT4+r6s^=;7Mz_JC?r&PuTEMhHVPG(s-Qio3jxGx|(s>U#1VWgPL#Jit=?;lEjmM`2a|7fbOyfZ`9|qC;Qs zd}^T7N!xKBFChDuHx72bZw4l+l%5xNGwq^6k9=Eo`6|bB#OSsD?e`oEH`0%rE_+6U zYg44*>Neayb)hD|FNQW}))cTSeoBz$>9P)w3KC(9J(K$J0;BXxHX7*shWCO;E=5-g zu>=2R4&jFNh%}w{P9Uly!Ue@WC|#Kfsr>2odhkk4pI^-(-IVr^$K*W}iYNN-7&8B2 zMt_E=!V8qE@ZKPum78*-t>wgMLG*(CtANl2>F4GBc~_=9W(hy_A|ij~TTb|9*B=WR z!1j9mx0Q?mE!ZZ&6da64)^t<(xp%@y-#Zi`G2}rwVkt6xAS&UFTj5gUNH_2aK4Sa6 zEsd`>H5}2uaozuC1A^{Kbzai8T~vyzGlA?!adodA*fgf@(dj?$Wi9>;I@+Atvu@;k zPdfFgk=h=JqD*;I7qVqLU<;48oQ{={s0A#{@C66VXsOeA0*K; zdyJGXAFo1f6|G!oHtoWabO)y>1RVg;4AMb_7>uHgn=*zfYd21AR-C$`Y)8KG!dmLS zll4^=@mNX`owE!)R29q+mIWe~yY2I=A?NBwJ(Xn10yUFH#oBYP0-O%3 z_vkx%k3>7Pl}hw`I$9v;8O0E&H}<8ABc?$=0tkfGY!Oh0Ry23O_1bn#<&^1s0F=6F zMeFj;xM|j^{sONv?=SjqO>`W&Bu`x$|1AJ=5eO{=4IM?yES>>$=OY$Oqh-hm-R+_J zRJUNYFcr54ogs?6$HT=@!Bw@$v&`OHM^Bpj;v*o^GHU7<1Uw%W2k9&_fEx4+YA0jkkz=9$3zCSlPR8AsjMfQ-*xrEiz;LR{8wl(o?(w zmcK9J&qKrsXhMEbmdd|HXDySBrpvlkxnQKp`!<_4Sw_C&Hg>f;85ZPDU*#t+Y3VO> zWpAhGL3i3E=}&3aSsE>}&QCN!DdPD9@o%B34BKDTQzV@2(V$ ztwSZ&P9r1imC3ggBHRT9EgA=-AO2CQU4!pEkkj%#glr=Uh^r*)f?*vvmW7)@1UAUc zb`U;M-^PjVK(bIRiNIPyqtnLg>2Gb@4u2J^smopt0i5k!<+@7pPbOk=QwP!_Ia(R_ zK4$VRF@$6e*fT0$rTT&AHIDi!jO0SCznOwo%5LJmspCtwV67X(%%>R8%_PXzzPi!A zcYiM(7e;k;#jWXF$$aFlRBdmoI(T_1JNDBt~OU+1WBgb_HJ-a57tP1GpAf~@a~y??|0(@^27gMU_S6*(w67= zfq914h9Ef@1qR2Vm#v=x`0s+0Ra#{p1~N`SUAi z@0>5%>H5-aUSjXsB2My*E3%t4R;$D!qblAQAAQklXuf=CMlw0?dB)jZ>G5p>IDxPX zDTSC&7(={A9!{UH&@&s=V!+wjOif>ch6f(VuZ)8|aYEta#|vx>(#eM)4$lOUbSJq! zZ{1@N7m%OPRFOX*Qb>7lP4@lz{U0F2IiZPe-|#U)bCjEy4JX49?1w0ER0F; zQS~)o)Hi=?9((#WqdXn9{0XW9rb-wx19;K~6!Yf&b+1z!+u6X7aL?)vN;Ix;R8QK?l0$BlM#au0C+nqQ}>r0Z;nz zh9txKw-l)>WuEQ#Q;l~D%szU@;(lg0Inp$71SMF<^a8$tgk2viHE;-$$#f1_JZ~G< z8JW<`e8o;>$Ouen;%s+DfOq#`8w$Q+4cKC86nV-)u&^F# znUmXrh{Y0c!sx^t9`OosCFOZN{BUt@>VcO#Zq`=s{@R0DrA@uKXY<0#o(5AlRX~qY zwvGbIePGZa)({GwK|E%d)c$&P!~__R&3?fO+ZYZ&5ZknC@$e|faz4N$NhmsBCn61pFp{<4wJQmYef1{1;KrhCSTN| zqSv%3JsQOVhV3UoH0c=MG^IkKQn~Yiadehwa=55Y-{zi+rwSq$vDCxBcLvD=#)9{z4Myt**(MDtt=BS+^g6XL zr`%B=JiuAR5x{&@D_NJA^a%xUn_ORMC3W0WBAwl>^%wz?*OS|$VukNURG&Vy4ZUb= z=%8aYuNBv5;8FOT$Weu02-7ysbV*`70d>tM?s4u$wrB*ln~bhkZf{)dwrP&~_GR@* zlzY$-XO6^pMtMfUK}R#@by+GO02{uh6|~6H=dtYI+BAWDLVFvUS4+=f!DyqRwpev? zy|ASrc22%FL#v8O9iE-~gMi%Hdc;Ye^KFhxZl3>|RF=?v}dZPRz_}I;%AIWQb zIaQFeSnlrYfDF~gQYrpjCCeF#y?5JlnNgsiBN=KlQ97_BbL?dG{MG6v{QIQEfN0Mg zc^4Z?t2?6=-?AWQ*0w4BIQ2|KVmfv9GZ6mL z0^1`M{9|l(lO=rt~3rh{{~)h9`MgZoKevf4w@cAtv>Xe9=|;va3`}ad-RB#e)cr zzQ0O)pA3F1434)X+99Az^CZvj)MGcu?K@kv_t1hF)!vS|BcnxT

2x@7R|5qIx3u=6+_+ z@mYhz>RE6=6H7Tue+e@g&85#=sc(GjgcV3E93$R3;PBl{IbDfc!il8wK9UqR^55J%LDV9FJ2CT-6Jgv0-i+0!C z3Z^!TaL>eo>L6LcYY8w=SlQ@MFBJT+&6Gk-)p5x zW7?kHt**b5;L$X;yjr)}+7_qtzCTPe$z-l_WgjXbc#K+42~NO6i#bkA^mu~_8+*^t z%gZR}FGBU8lRoepp;2}2>whq6aVWhl-ynNM5s>&rQvO!=Gtr1~IaL2JCPwJC{YgSS z_=s-c?Lq8Ikv1Id3Q`>cIEHJ2To;w&pau%rG40>D1OL7W6t4Gi0ifZYS!LmlD&2Hf zct$*EqO?F)_*QfTO`JCCQikDfxLpd2OtOM8ZK_LJv&0 zeiYf4OKFdKTrTOz_N}Yke=hd%l$@q2NqU=F306_d9aFC1H>qwgDB@56`WjjWJ!Cj; z84vS{S%E-e|2{U1>jT4|XS#GwBhozLo8JZ)_)j3>uUq&&^j}@zdjQSz)VMwI(Qy#i zDpG~(L_d}0p9kt1<023;ze$jLCVkPpqZ>_?xBkJv{<&J|wu5}ld82%q(T!YNG_(zV zTi=@OuI@dMd6lVl4;-;N^E zt#KXY(ZO+m6hSHQN=4ys73DQM67Z}-`9W@xSxdcbr_Gd2zuZjvH55o;#5J!~^^07L zX91ukZ8`E3eEs;6UUv>!7Ox*{WP#{j74~17I_LM~#9~5NjN3zE=56!;I4P{seTj_S zyR<_u6^tgZ#7DQX)y7N?CLg%=$m)$6a{_@n&);uW2Hiv0vpBQY$ixkpWVZCqibdlY z3r{vC+PmT=-&zrG{Ym1}Qa*dWk#gc#eM`J$a?k_)A?f7i8<#r;qg=p_Dv9F+?sl@B zBS|c0n({e)@P*xEBhQe>JLI?Q3D={v1v2`UV@K-$E%rb>Xgl)GokxXfQi^Mu($BSi zy&kXln%tVv(fA<_So+gA7xbCyNfR^en?QWua8eB7$k;ts&AY1^VSagwcQxa4hDSIU zy$+YIBx!IlteYgcVE)O)lhclJ*>f{}14(Ien2gw`$_615?Tqe!Tj)+JjIQr?^189g zBpTOKS{FD6wla}==^$uiqoPaxg4dr#5|=9_BNUq*fQH8hgnd5s{yMw%QbqeEFm8$B zkjqj%>3WH#=SweXd+Qn@)T3hCbCja6@Kq$~i;c>0EbA;?uUi{8%Sul@LK?+(xRpoX zLRBw*v5om5{(a&&BAsXCFV?Wy*&lr`9;(&cyELktcr3nm|E=o&B%pfjNLPSYqsWZ15m7ay{zqk*@;s> zA=8DsPk0KyRU}w>y!y%$$#e;6uNOl#Cyz2SjR&>LEtP8Zl^ZejQN`d&xI+;8CdF=T pr7HA=6p<2jvX`K(2DfY*mpZFl{+BKs)4Rtj=l|8=%$|RZ{|{4{@d*F` From ea24deefcce4d44339105e0fe6d243743b858074 Mon Sep 17 00:00:00 2001 From: Joshna-Medisetty Date: Wed, 19 Aug 2026 14:30:12 -0700 Subject: [PATCH 074/267] [Bugfix][FLUX.2-klein] Fix duplicated image position ids (#6130) Signed-off-by: Joshna Medisetty Signed-off-by: Joshna-Medisetty Co-authored-by: Cursor Co-authored-by: Chendi.Xue --- vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py b/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py index a4350373b..beb0b3787 100644 --- a/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py +++ b/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py @@ -1020,10 +1020,8 @@ class Flux2KleinPipeline( if reference_image_latents is not None: latent_model_input = torch.cat([latents, reference_image_latents], dim=1) - latent_image_ids = latent_ids elif image_latents is not None: latent_model_input = torch.cat([latents, image_latents], dim=1).to(self.transformer.dtype) - latent_image_ids = torch.cat([latent_ids, image_latent_ids], dim=1) positive_kwargs = { "hidden_states": latent_model_input, From efaf4276f93d3e0833198e97ba091ffb530b3dab Mon Sep 17 00:00:00 2001 From: Thanaji Rao Thakkalapelli Date: Wed, 19 Aug 2026 16:30:38 -0500 Subject: [PATCH 075/267] [Bugfix] Record device-agnostic torch.Event on XPU to fix async D2H image corruption [XPU] (#5571) Signed-off-by: Thanaji Rao Thakkalapelli Co-authored-by: Chendi.Xue --- tests/platforms/test_record_device_event.py | 67 +++++++++++++++++++++ vllm_omni/platforms/xpu/platform.py | 14 ++++- 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/tests/platforms/test_record_device_event.py b/tests/platforms/test_record_device_event.py index a979ca940..6a9e0ce06 100644 --- a/tests/platforms/test_record_device_event.py +++ b/tests/platforms/test_record_device_event.py @@ -4,6 +4,8 @@ import pytest +from tests.helpers.mark import hardware_test + pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -38,6 +40,71 @@ class TestCudaOmniPlatformRecordDeviceEvent: assert result is None +class TestXPUOmniPlatformRecordDeviceEvent: + """XPU must record a device-agnostic ``torch.Event``, not ``torch.xpu.Event``. + + The async diffusion output thread gates its D2H copy with + ``torch.Stream.wait_event`` on a generic ``torch.Stream``, and that binding + silently drops the dependency for a ``torch.xpu.Event`` — the copy then races + the still-running forward and the host reads a partially written tensor + (garbage rows at the bottom of the image). + """ + + def test_records_device_agnostic_torch_event(self, mocker): + # vLLM's XPUPlatform imports vllm_xpu_kernels at module scope, and that + # package is absent on non-XPU images. + pytest.importorskip("vllm_xpu_kernels") + + from vllm_omni.platforms.xpu.platform import XPUOmniPlatform + + mock_event = mocker.MagicMock() + mocker.patch("torch.Event", return_value=mock_event) + xpu_event = mocker.patch("torch.xpu.Event") + + result = XPUOmniPlatform.record_device_event() + + assert result is mock_event + mock_event.record.assert_called_once() + xpu_event.assert_not_called() + + @hardware_test(res={"xpu": "B60"}) + def test_recorded_event_orders_side_stream_d2h(self): + """On-device: the event must make a side-stream D2H wait for compute.""" + import torch + + if not torch.xpu.is_available(): + pytest.skip("XPU not available") + + from vllm_omni.platforms.xpu.platform import XPUOmniPlatform + + device = torch.device("xpu", 0) + # Normalized so the matmul chain stays finite; the ``* 0.0`` term keeps + # every element exactly 7.0 while still depending on the whole chain. + base = torch.randn(4096, 4096, device=device) / 64.0 + chained = base.clone() + for _ in range(40): + chained = chained @ base + tensor = torch.full_like(chained, 7.0) + chained * 0.0 + + event = XPUOmniPlatform.record_device_event() + assert event is not None + + # Mirrors WorkerProc._async_output_loop. + d2h_stream = torch.Stream(device=device) + d2h_stream.wait_event(event) + previous = torch.accelerator.current_stream() + torch.accelerator.set_stream(d2h_stream) + try: + host = torch.empty(tensor.shape, dtype=tensor.dtype, pin_memory=True) + host.copy_(tensor, non_blocking=True) + finally: + torch.accelerator.set_stream(previous) + d2h_stream.synchronize() + + copied_early = int((host != 7.0).sum()) + assert copied_early == 0, f"{copied_early} elements copied before compute finished" + + class TestNPUOmniPlatformRecordDeviceEvent: """Test NPUOmniPlatform.record_device_event with mocked torch.npu.""" diff --git a/vllm_omni/platforms/xpu/platform.py b/vllm_omni/platforms/xpu/platform.py index 0d9fe3639..3d2cd2d62 100644 --- a/vllm_omni/platforms/xpu/platform.py +++ b/vllm_omni/platforms/xpu/platform.py @@ -102,8 +102,20 @@ class XPUOmniPlatform(OmniPlatform, XPUPlatform): @classmethod def record_device_event(cls) -> torch.Event | None: + """Record an XPU event on the current stream to mark tensor readiness. + + Deliberately a device-agnostic ``torch.Event`` rather than a + ``torch.xpu.Event``. The consumer (the async diffusion output thread) + waits with ``torch.Stream.wait_event`` on a generic ``torch.Stream``, + and that C-level binding silently no-ops for a ``torch.xpu.Event`` + instead of enqueuing the dependency — the side stream then starts its + D2H copy while the compute stream is still writing the tensor, so the + host reads a partially-written image (garbage rows at the bottom of the + output). ``torch.Event`` dispatches through the accelerator hooks and + the wait is honored, which is the actual fix. + """ try: - event = torch.xpu.Event() + event = torch.Event() event.record() return event except Exception: From 6f55a06fab90509d94b34bfc682312ca6e55fea2 Mon Sep 17 00:00:00 2001 From: Yukim1 <121286183+zwhzzz0821@users.noreply.github.com> Date: Thu, 20 Aug 2026 07:09:12 +0800 Subject: [PATCH 076/267] [Docs] Remove stale stage config references (#6270) Signed-off-by: zwhzzz0821 <2831474076@qq.com> --- .claude/skills/add-tts-model/SKILL.md | 57 ++- .../references/single-stage-ar.md | 56 ++- .claude/skills/review-pr/SKILL.md | 2 +- .../checks/model-addition-checklist.md | 2 +- .../references/modules/configuration.md | 2 +- .../references/process/review-routing.md | 2 +- .claude/skills/vllm-omni-test/SKILL.md | 4 +- .github/CODEOWNERS | 2 - .github/ISSUE_TEMPLATE/400-bug-report.yml | 2 +- benchmarks/tts/README.md | 4 +- docs/configuration/README.md | 5 +- docs/configuration/composable_parallel.md | 4 +- docs/configuration/gpu_memory_utilization.md | 56 ++- docs/configuration/stage_configs.md | 362 +++--------------- docs/contributing/DOCS_GUIDE.md | 2 +- .../test_examples/l4_performance_tests.inc.md | 12 +- docs/contributing/ci/test_writing_guide.md | 12 +- docs/contributing/model/adding_omni_model.md | 39 +- docs/contributing/model/adding_tts_model.md | 195 +++++----- docs/design/architecture_overview.md | 4 +- docs/design/feature/async_chunk.md | 48 ++- .../design/feature/disaggregated_inference.md | 11 +- .../mooncake_store_connector.md | 25 +- .../mooncake_transfer_engine_connector.md | 27 +- .../shared_memory_connector.md | 7 +- .../omni_connectors/yuanrong_connector.md | 19 +- .../yuanrong_transfer_engine_connector.md | 69 ++-- docs/design/feature/prefix_caching.md | 2 +- docs/design/module/model_integration.md | 2 +- docs/design/module/vllm_omni_config.md | 7 +- ...qwen3_omni_tts_performance_optimization.md | 100 +++-- docs/getting_started/quickstart.md | 2 +- docs/serving/speech_api.md | 2 +- docs/serving/video_stream_api.md | 2 +- docs/user_guide/diffusion/execution_modes.md | 12 +- .../diffusion/regional_compilation.md | 2 +- .../examples/offline_inference/mimo_audio.md | 4 +- .../offline_inference/qwen2_5_omni.md | 2 +- .../examples/offline_inference/qwen3_omni.md | 11 +- .../offline_inference/text_to_speech.md | 6 +- .../examples/online_serving/qwen3_omni.md | 2 +- .../examples/online_serving/text_to_speech.md | 2 +- docs/user_guide/quantization/fp8.md | 2 +- docs/user_guide/quantization/msmodelslim.md | 2 +- .../quantization/quantized_kvcache.md | 24 +- .../offline_inference/covo_audio/README.md | 5 +- .../offline_inference/dynin_omni/README.md | 2 +- .../hunyuan_image3/README.md | 21 +- .../offline_inference/mimo_audio/README.md | 4 +- .../ming_flash_omni/README.md | 2 +- examples/offline_inference/minicpmo/README.md | 2 +- .../offline_inference/qwen2_5_omni/README.md | 2 +- .../offline_inference/qwen3_omni/README.md | 11 +- .../qwen3_omni/end2end_async_chunk.py | 4 +- .../offline_inference/step_audio2/README.md | 7 +- .../text_to_speech/README.md | 8 +- .../text_to_speech/ming_tts/README.md | 2 +- .../text_to_speech/voxtral_tts/end2end.py | 2 +- ...letion_client_for_multimodal_generation.py | 2 +- ...letion_client_for_multimodal_generation.py | 2 +- examples/online_serving/personaplex/README.md | 2 +- .../run_curl_multimodal_generation.sh | 2 +- examples/online_serving/qwen3_omni/README.md | 2 +- .../run_curl_multimodal_generation.sh | 2 +- .../replica_data_parallel/bench_replica_dp.py | 2 +- examples/online_serving/step_audio2/README.md | 2 +- .../online_serving/text_to_speech/README.md | 2 +- recipes/Bagel/BAGEL-7B-MoT.md | 6 +- recipes/GLM/GLM-Image.md | 4 +- recipes/Helios/Helios.md | 4 +- recipes/NVIDIA/GR00T-N1.7.md | 2 +- recipes/NVIDIA/Nemotron-Labs-Audex.md | 4 +- vllm_omni/deploy/bagel_single_stage.yaml | 2 +- vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml | 2 +- vllm_omni/deploy/minicpmo_4_5.yaml | 2 +- vllm_omni/deploy/moss_tts_nano.yaml | 4 +- vllm_omni/entrypoints/cli/serve.py | 2 +- .../entrypoints/openai/protocol/audio.py | 2 +- vllm_omni/entrypoints/openai/serving_chat.py | 2 +- .../models/mimo_audio/mimo_audio.py | 4 +- .../models/mimo_audio/mimo_audio_code2wav.py | 2 +- vllm_omni/platforms/npu/worker/base.py | 2 +- vllm_omni/worker/base.py | 2 +- vllm_omni/worker/gpu_generation_worker.py | 4 +- vllm_omni/worker/gpu_memory_utils.py | 4 +- 85 files changed, 566 insertions(+), 794 deletions(-) diff --git a/.claude/skills/add-tts-model/SKILL.md b/.claude/skills/add-tts-model/SKILL.md index ff86ad457..494d6a52b 100644 --- a/.claude/skills/add-tts-model/SKILL.md +++ b/.claude/skills/add-tts-model/SKILL.md @@ -138,9 +138,12 @@ A shared buffer silently corrupts audio across concurrent requests — the sympt - Load codec weights (may need lazy loading from separate checkpoint) - Implement `forward()`: codec codes -> audio waveform - Return `OmniOutput` with `multimodal_outputs` -5. **Create stage config YAML** defining both stages, memory allocation, and model paths -6. **Create stage input processor** for prompt building -7. **Write end2end.py** test script +5. **Define and register the pipeline topology** in + `vllm_omni/model_executor/models//pipeline.py` +6. **Create the deploy YAML** under `vllm_omni/deploy/` for placement, + memory sizing, connectors, and runtime overrides +7. **Create stage input processor** for prompt building and inter-stage handoff +8. **Write end2end.py** test script ### Critical Parameters to Get Right @@ -174,7 +177,7 @@ These bugs appear in almost every new TTS PR. Check all before the first push. S - **All return paths must emit `model_outputs`** — if any early-return branch skips setting `model_outputs`, the serving layer silently drops that step's audio (fish speech: `fix: ensure ALL return paths emit model_outputs`) - **Per-request state isolation** — for batched concurrent requests, key all state by request ID; a shared buffer corrupts audio across requests (fish speech: `fix: per-request vocode + delta emission`) - **Codec tensor device** — move codec codes to the codec decoder's device before calling decode; mismatches cause silent CPU fallback or crashes (fish speech: `fix: use model device for CUDA stream`) -- **AR stage `max_num_seqs`** — set to at least 4 in production configs; for single-stage models this is the only stage. For two-stage models, Stage 0 (AR) needs `max_num_seqs ≥ 4` to pipeline concurrent requests; Stage 1 (codec decoder) typically uses `max_num_seqs: 1` intentionally. Default of 1 everywhere causes audio gaps under concurrency because the codec window round-robins across requests (RFC #2568) +- **AR stage `max_num_seqs`** — set to at least 4 in production deploy configs; for single-stage models this is the only stage. For two-stage models, Stage 0 (AR) needs `max_num_seqs ≥ 4` to pipeline concurrent requests; Stage 1 (codec decoder) is model-specific and may intentionally use `max_num_seqs: 1`. Defaulting the AR stage to 1 causes audio gaps under concurrency because the codec window round-robins across requests (RFC #2568) ### Optional Dependency Handling @@ -190,18 +193,18 @@ pattern, signature constraints, and MOSS-TTS-Nano reference. When the upstream model cannot be cleanly split into an AR stage and a separate decoder, run the full pipeline inside a single AR worker and stream audio through a per-request `inference_stream()` generator keyed by -`_omni_req_id`. Stage config must set `worker_type: ar`, -`engine_output_type: audio`, `final_output: true`, `is_comprehension: true`, -and `async_chunk: false` at the top level. Only extract params from +`_omni_req_id`. Define one `StagePipelineConfig` with +`execution_type=StageExecutionType.LLM_AR`, `engine_output_type="audio"`, +`final_output=True`, and `owns_tokenizer=True`. Set `async_chunk: false` in +the deploy YAML. Only extract params from `additional_information` that you actually forward, or pre-commit fails `ruff F841`. Full walkthrough with the complete `forward()` / `_create_stream_gen()` -skeleton and stage-config fields: +skeleton plus pipeline/deploy definitions: [references/single-stage-ar.md](references/single-stage-ar.md). For an in-tree reference, look for any single-stage AR model under -`vllm_omni/model_executor/models/` — e.g. the MOSS-TTS-Nano integration when -it lands. +`vllm_omni/model_executor/models/`, such as MOSS-TTS-Nano. **VoxCPM2 is a different pattern** and should not reuse this skeleton — it runs the base LM under vLLM PagedAttention with external side-computation. @@ -210,7 +213,8 @@ See `plan/voxcpm2_native_ar_design.md`. ### Deliverables - Model files in `vllm_omni/model_executor/models//` -- Stage config YAML +- Registered `pipeline.py` topology +- Deploy YAML under `vllm_omni/deploy/` - Working `end2end.py` at `examples/offline_inference/text_to_speech//end2end.py` - New section in `examples/offline_inference/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/offline_inference//` dir or a per-model `README.md` inside `text_to_speech//` — the hub README is the documented surface and the mkdocs `generate_examples` hook only descends one level into `examples//`. @@ -400,21 +404,29 @@ See [vllm-omni-test skill](../vllm-omni-test/SKILL.md) § **Runtime send helpers ### Steps -1. **Update stage config YAML**: +1. **Update the pipeline topology** so the producing stage declares its async + handoff processor with `async_chunk_process_next_stage_input_func`. +2. **Update the deploy YAML** to enable async chunk and configure the connector: ```yaml async_chunk: true - codec_chunk_frames: 25 # frames per chunk - codec_left_context_frames: 25 # overlap for smooth boundaries + + connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + codec_streaming: true + codec_chunk_frames: 25 + codec_left_context_frames: 25 ``` -2. **Implement chunk handling in Stage 1**: +3. **Implement chunk handling in Stage 1**: - Accept partial input (chunk of codec codes) - Handle left context for smooth audio boundaries - Return partial audio in `OmniOutput` -3. **Test streaming**: +4. **Test streaming**: - Verify audio quality matches non-streaming output - Check for artifacts at chunk boundaries - Measure TTFA (time to first audio) -4. **Update online serving** to support `stream=true` with PCM output +5. **Update online serving** to support `stream=true` with PCM output ### Streaming Architecture @@ -435,7 +447,8 @@ Stage 0 (AR) Stage 1 (Decoder) ### Deliverables -- Updated stage config with async_chunk enabled +- Updated pipeline async handoff processor and deploy config with + `async_chunk: true` - Smooth streaming audio without boundary artifacts - TTFA metrics @@ -507,14 +520,15 @@ Use this checklist when integrating a new TTS model: - [ ] Config classes created with `model_type` registration - [ ] Stage 0 (AR) implemented and generates correct tokens - [ ] Stage 1 (Decoder) produces correct audio from tokens — dtype float32 for codec decoder -- [ ] Stage 1 `max_num_seqs` ≥ 4 in production config (default 1 causes gaps under concurrency) +- [ ] AR stage `max_num_seqs` ≥ 4 in the production deploy config unless the model has a tested lower limit - [ ] Optional dependency fallbacks handled at `load_weights()` time (torchaudio/soundfile/etc.) - [ ] Streaming: codec codes accumulated across AR steps (not reset per step) - [ ] Streaming: delta audio emitted per chunk, not full re-decoded waveform - [ ] Streaming: all `forward()` return paths emit `model_outputs` - [ ] Streaming: per-request state keyed by request ID (not shared across requests) - [ ] Streaming: codec tensors moved to codec decoder device before decode -- [ ] Stage config YAML created +- [ ] Pipeline topology defined in `pipeline.py` and registered in `OMNI_PIPELINES` +- [ ] Deploy YAML created under `vllm_omni/deploy/` - [ ] `end2end.py` produces audio matching reference quality - [ ] README.md written @@ -531,7 +545,8 @@ Use this checklist when integrating a new TTS model: - [ ] Documentation added (offline + online docs, nav, supported models) ### Phase 4: Async Chunk -- [ ] Stage config updated with `async_chunk: true` +- [ ] Pipeline declares the async handoff processor +- [ ] Deploy config sets `async_chunk: true` and connector chunk parameters - [ ] Stage 1 handles partial chunks correctly - [ ] No audio artifacts at chunk boundaries - [ ] Streaming via API (`stream=true`) works diff --git a/.claude/skills/add-tts-model/references/single-stage-ar.md b/.claude/skills/add-tts-model/references/single-stage-ar.md index ed53d3026..7a48197af 100644 --- a/.claude/skills/add-tts-model/references/single-stage-ar.md +++ b/.claude/skills/add-tts-model/references/single-stage-ar.md @@ -86,12 +86,51 @@ class YourModelForCausalLM(nn.Module): - Handle both `"audio"` (incremental) and `"result"` (final combined) event types from upstream models. -## Stage config +## Pipeline and deploy config -Single stage with `worker_type: ar`, `engine_output_type: audio`, -`final_output: true`, `is_comprehension: true`, and `async_chunk: false` at -the top level. Omitting any of these causes silent misclassification in the -serving layer. +Define the fixed single-stage topology in `pipeline.py`: + +```python +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +YOUR_TTS_PIPELINE = PipelineConfig( + model_type="your_model_name", + default_deploy_config_name="your_model_name.yaml", + model_arch="YourModelForCausalLM", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="your_model_name", + execution_type=StageExecutionType.LLM_AR, + input_sources=(), + owns_tokenizer=True, + engine_output_type="audio", + final_output=True, + final_output_type="audio", + ), + ), +) +``` + +Register it in `vllm_omni/config/pipeline_registry.py`. Keep placement and +runtime sizing in `vllm_omni/deploy/your_model_name.yaml`: + +```yaml +async_chunk: false + +stages: + - stage_id: 0 + devices: "0" + max_num_seqs: 4 + gpu_memory_utilization: 0.8 +``` + +Do not move `execution_type`, output ownership, tokenizer ownership, or model +architecture into the deploy YAML; those are pipeline topology. ## Lint discipline @@ -101,8 +140,5 @@ pre-commit. ## Reference implementation -Look for any single-stage AR model under -`vllm_omni/model_executor/models/` — e.g. `moss_tts_nano/` when its -integration lands. If none is in tree yet, follow the skeleton above and -cross-check against the `is_comprehension: true` / `async_chunk: false` -dispatch in `vllm_omni/entrypoints/openai/serving_speech.py`. +Use `vllm_omni/model_executor/models/moss_tts_nano/pipeline.py` and +`vllm_omni/deploy/moss_tts_nano.yaml` as the in-tree reference. diff --git a/.claude/skills/review-pr/SKILL.md b/.claude/skills/review-pr/SKILL.md index d8b1b43c5..165fa1d5e 100644 --- a/.claude/skills/review-pr/SKILL.md +++ b/.claude/skills/review-pr/SKILL.md @@ -92,7 +92,7 @@ If docs and live code disagree, verify the code/tests and report the drift. | Reference | Read when | | --- | --- | -| [model-addition-checklist.md](references/checks/model-addition-checklist.md) | A model, architecture, loader, processor, registry, or stage config is added. | +| [model-addition-checklist.md](references/checks/model-addition-checklist.md) | A model, architecture, loader, processor, registry, pipeline config, or deploy config is added. | | [perf-verification.md](references/checks/perf-verification.md) | The PR makes a latency, throughput, memory, or quality claim. | | [test-quality-evaluation.md](references/checks/test-quality-evaluation.md) | Tests change, are absent for risky code, or may not exercise production behavior. | | [tests-docs-checklist.md](references/checks/tests-docs-checklist.md) | Coverage, CI markers, examples, user docs, or PR evidence need review. | diff --git a/.claude/skills/review-pr/references/checks/model-addition-checklist.md b/.claude/skills/review-pr/references/checks/model-addition-checklist.md index b60fc936e..bde60d7ab 100644 --- a/.claude/skills/review-pr/references/checks/model-addition-checklist.md +++ b/.claude/skills/review-pr/references/checks/model-addition-checklist.md @@ -1,7 +1,7 @@ # Model Addition Checklist Read this reference when a PR adds or expands a model architecture, pipeline, -loader, processor, registry entry, or stage configuration. +loader, processor, registry entry, pipeline config, or deploy config. Official docs: [model contribution guides](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/model/), [adding an omni model](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/model/adding_omni_model/), diff --git a/.claude/skills/review-pr/references/modules/configuration.md b/.claude/skills/review-pr/references/modules/configuration.md index bdb3a5159..6ecf08e99 100644 --- a/.claude/skills/review-pr/references/modules/configuration.md +++ b/.claude/skills/review-pr/references/modules/configuration.md @@ -3,7 +3,7 @@ Primary design: [vLLM-Omni Configuration](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/vllm_omni_config/). Use for config construction, deploy YAML, registries, schema, defaults, -aliases, CLI projection, stage config, and topology. Inspect the reviewed +aliases, CLI projection, pipeline config, and topology. Inspect the reviewed page's status: while its contract remains deferred or draft, current code and tests—not proposed precedence rules—are authoritative. diff --git a/.claude/skills/review-pr/references/process/review-routing.md b/.claude/skills/review-pr/references/process/review-routing.md index 10768d329..f6455d492 100644 --- a/.claude/skills/review-pr/references/process/review-routing.md +++ b/.claude/skills/review-pr/references/process/review-routing.md @@ -48,7 +48,7 @@ compatibility changes. | Signal | Read | Optional repo-local skill | | --- | --- | --- | -| New or expanded model, loader, processor, registry, or stage config | [model-addition-checklist.md](../checks/model-addition-checklist.md) | [`add-tts-model`](../../../add-tts-model/SKILL.md) or [`add-diffusion-model`](../../../add-diffusion-model/SKILL.md) | +| New or expanded model, loader, processor, registry, pipeline config, or deploy config | [model-addition-checklist.md](../checks/model-addition-checklist.md) | [`add-tts-model`](../../../add-tts-model/SKILL.md) or [`add-diffusion-model`](../../../add-diffusion-model/SKILL.md) | | Latency, throughput, memory, scaling, precision, or quality claim | [perf-verification.md](../checks/perf-verification.md) | [`diffusion-perf-opt`](../../../diffusion-perf-opt/SKILL.md) for diffusion | | Tests changed, absent for risky behavior, or test-only | [test-quality-evaluation.md](../checks/test-quality-evaluation.md) | [`vllm-omni-test`](../../../vllm-omni-test/SKILL.md) | | CI, examples, docs, public behavior, or contributor evidence | [tests-docs-checklist.md](../checks/tests-docs-checklist.md) | None | diff --git a/.claude/skills/vllm-omni-test/SKILL.md b/.claude/skills/vllm-omni-test/SKILL.md index d5102b3f2..3e4d0e489 100644 --- a/.claude/skills/vllm-omni-test/SKILL.md +++ b/.claude/skills/vllm-omni-test/SKILL.md @@ -110,10 +110,10 @@ After choosing offline/online/docs/perf, classify the **product under test** and |-----------|------------------------------|----------------------------|----------------------------------------| | **What it is** | Multimodal LLM pipeline (thinker/talker/stages; text + vision + audio I/O) | Speech synthesis / voice models | Generative diffusion (noise → image, audio, text, or video) | | **Examples** | Qwen2.5-Omni, Qwen3-Omni | Qwen3-TTS, VoxCPM2, Higgs-Audio, Step-Audio2 | Qwen-Image, BAGEL, Wan2.2, HunyuanVideo | -| **Offline runner** | `omni_runner` + `omni_runner_handler`, `generate_multimodal` | `Omni(...).generate` or stage YAML + TTS params | `Omni(...).generate` + `OmniDiffusionSamplingParams` | +| **Offline runner** | `omni_runner` + `omni_runner_handler`, `generate_multimodal` | `Omni(...).generate` with deploy YAML + TTS params | `Omni(...).generate` + `OmniDiffusionSamplingParams` | | **Online client** | `openai_client.send_omni_request` (chat completions, modalities) | `openai_client.send_audio_speech_request` (`/v1/audio/speech`) | `send_diffusion_request` (chat/T2I), `send_video_diffusion_request` (`/v1/videos`, X2V), or `send_images_generations_http_request` / `send_images_edits_http_request` (DALL-E routes) | | **Typical assertions** | Stage outputs, text/audio keywords via `OmniRunnerHandler` / response handler | WAV bytes, stream chunks, speech endpoint contract | Image/video dimensions, `final_output_type`, `assert_diffusion_response` | -| **Stage / deploy YAML** | Per-model omni stage configs (`ci/qwen3_omni_moe.yaml`, …) | `qwen3_tts.yaml`, `voxcpm2.yaml`, … | Often default serve; parallel/offload YAML for heavy DiT | +| **Deploy YAML** | Per-model deploy configs (`ci/qwen3_omni_moe.yaml`, …) | `qwen3_tts.yaml`, `voxcpm2.yaml`, … | Often default serve; parallel/offload YAML for heavy DiT | | **Nightly group** (`test-nightly.yml`) | **Omni Model Test** — `-m "full_model and omni"` | **TTS Model Test** — `-m "full_model and tts"` | **Diffusion X2I(&A&T)** *or* **Diffusion X2V** (see Step 3 table; same `diffusion` marker, different YAML group / file shard) | | **L4 pressure** | Expansion per modality/model as needed | Expansion + accuracy/perf in TTS group | X2I: merge feature combos per [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); X2V: separate nightly group | diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2f7f80673..c92c7b22f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -79,8 +79,6 @@ /vllm_omni/model_executor/models/ @tzhouam @gcanlin @Sy0307 @amy-why-3459 @linyueqian /vllm_omni/model_extras/ @tzhouam @gcanlin @princepride /vllm_omni/plugins/ @gcanlin @tjtanaa @xuechendi -# Config override within model_executor — docs/design/module/vllm_omni_config.md -/vllm_omni/model_executor/stage_configs/ @lishunyang12 @alex-jw-brooks # Execution platforms (hardware backends) — docs/design/module/execution_platforms.md /vllm_omni/platforms/ @gcanlin @tjtanaa @xuechendi diff --git a/.github/ISSUE_TEMPLATE/400-bug-report.yml b/.github/ISSUE_TEMPLATE/400-bug-report.yml index 7cbf0ac0e..6e04ae9e7 100644 --- a/.github/ISSUE_TEMPLATE/400-bug-report.yml +++ b/.github/ISSUE_TEMPLATE/400-bug-report.yml @@ -83,7 +83,7 @@ body: omni = Omni( model="Qwen/Qwen-Image", - stage_configs_path="/path/to/stage_configs.yaml", + deploy_config="/path/to/deploy_config.yaml", ) prompts = [{"prompt": "A scenic watercolor painting of a lighthouse at sunset"}] diff --git a/benchmarks/tts/README.md b/benchmarks/tts/README.md index d3a0f9676..c6b6615d5 100644 --- a/benchmarks/tts/README.md +++ b/benchmarks/tts/README.md @@ -18,8 +18,8 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni --port 8000 ``` The server auto-loads its Deploy YAML from `vllm_omni/deploy/qwen3_tts.yaml` -(Pipeline + Deploy schema introduced in #2383). No `--stage-configs-path` or -`--deploy-config` flag is needed for any registered model. +(Pipeline + Deploy schema introduced in #2383). No explicit `--deploy-config` +flag is needed for a registered model. ### 2. Run the benchmark (`vllm bench serve --omni`) diff --git a/docs/configuration/README.md b/docs/configuration/README.md index b59c29a7f..70a078b97 100644 --- a/docs/configuration/README.md +++ b/docs/configuration/README.md @@ -4,11 +4,12 @@ This section lists the most common options for running vLLM-Omni. For options within a vLLM Engine, please refer to the [vLLM 0.26 configuration guide](https://docs.vllm.ai/en/v0.26.0/configuration/index.html). -Currently, the main options are maintained by stage configs for each model. +Each model defines fixed topology in a registered `PipelineConfig` and runtime +overrides in a deploy YAML. For a specific example, see the [Qwen2.5-Omni deploy config](gh-file:vllm_omni/deploy/qwen2_5_omni.yaml). The matching frozen pipeline topology lives at [vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py](gh-file:vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py). -For introduction, please check [Introduction for stage config](./stage_configs.md) +For an introduction, see [Pipeline and deploy configurations](./stage_configs.md). ## Memory Configuration diff --git a/docs/configuration/composable_parallel.md b/docs/configuration/composable_parallel.md index 473605c69..00af4cad4 100644 --- a/docs/configuration/composable_parallel.md +++ b/docs/configuration/composable_parallel.md @@ -1,6 +1,6 @@ # Composable parallel strategies -In vLLM-Omni, the `composable_parallel` system layers a declarative *parallel strategy* on top of the existing stage configs. A strategy YAML maps each pipeline stage (by its `model_stage` name) to a stack of axis declarations — one per parallelism dimension — that the engine translates into per-stage parallel sizing (`tensor_parallel_size`, `data_parallel_size`, `pipeline_parallel_size`, expert-parallel toggle, replica count) and, when relevant, an orchestrator-wide load-balancer policy. +In vLLM-Omni, the `composable_parallel` system layers a declarative *parallel strategy* on top of the registered pipeline and deploy config. A strategy YAML maps each pipeline stage (by its `model_stage` name) to a stack of axis declarations — one per parallelism dimension — that the engine translates into per-stage parallel sizing (`tensor_parallel_size`, `data_parallel_size`, `pipeline_parallel_size`, expert-parallel toggle, replica count) and, when relevant, an orchestrator-wide load-balancer policy. Strategies are intentionally narrow: each one only writes the axes it declares, leaving every other engine arg untouched. This makes a strategy file a small, reusable overlay you can swap independently of the deploy YAML (see `configuration/stage_configs.md`). @@ -195,7 +195,7 @@ Because `omni_lb_policy` is read once at `AsyncOmniEngine` construction — not Default: no default — required for every entry under `strategies:`. -Strategy keys MUST be `model_stage` *names* (strings), the same labels stage configs use for `engine_args.model_stage` (see [`model_stage` in `stage_configs.md`](stage_configs.md#engine_argsmodel_stage)). Integer `stage_id` values are not accepted; the resolver matches on `model_stage` exactly (`_resolve_stage` in `vllm_omni/config/composable_parallel/apply.py`). +Strategy keys MUST be `model_stage` *names* (strings), the same labels defined by [`StagePipelineConfig.model_stage`](stage_configs.md#pipeline-configuration). Integer `stage_id` values are not accepted; the resolver matches on `model_stage` exactly (`_resolve_stage` in `vllm_omni/config/composable_parallel/apply.py`). ## Common pitfalls diff --git a/docs/configuration/gpu_memory_utilization.md b/docs/configuration/gpu_memory_utilization.md index 741066035..9f0e7439a 100644 --- a/docs/configuration/gpu_memory_utilization.md +++ b/docs/configuration/gpu_memory_utilization.md @@ -80,19 +80,15 @@ When multiple stages share the same GPU, you must ensure the sum of their `gpu_m **Example: Two stages on GPU 0** ```yaml -stage_args: +stages: - stage_id: 0 - runtime: - devices: "0" - engine_args: - gpu_memory_utilization: 0.6 # Uses 60% of GPU 0 + devices: "0" + gpu_memory_utilization: 0.6 # Uses 60% of GPU 0 - stage_id: 1 - runtime: - devices: "0" - engine_args: - gpu_memory_utilization: 0.3 # Uses 30% of GPU 0 - # Total: 90% of GPU 0 (safe, leaves 10% buffer) + devices: "0" + gpu_memory_utilization: 0.3 # Uses 30% of GPU 0 + # Total: 90% of GPU 0 (safe, leaves 10% buffer) ``` **Important:** If stages run on different GPUs, each can use up to 1.0 independently. @@ -103,14 +99,12 @@ When using `tensor_parallel_size > 1`, the model is split across multiple GPUs, **Example: 2-way tensor parallelism** ```yaml -stage_args: +stages: - stage_id: 0 - runtime: - devices: "0,1" # Uses both GPUs - engine_args: - tensor_parallel_size: 2 - gpu_memory_utilization: 0.6 # 60% per GPU - # Model is split, so each GPU uses ~30% of model memory + devices: "0,1" # Uses both GPUs + tensor_parallel_size: 2 + gpu_memory_utilization: 0.6 # 60% per GPU + # Model is split, so each GPU uses ~30% of model memory ``` ## Examples @@ -118,27 +112,25 @@ stage_args: ### Qwen3-Omni-MoE on 2x H100-80GB ```yaml -stage_args: +base_config: /path/to/vllm_omni/deploy/qwen3_omni_moe.yaml + +stages: - stage_id: 0 # Thinker stage with TP=2 - runtime: - devices: "0,1" - engine_args: - tensor_parallel_size: 2 - gpu_memory_utilization: 0.6 # 48GB per GPU + devices: "0,1" + tensor_parallel_size: 2 + gpu_memory_utilization: 0.6 # 48GB per GPU - stage_id: 1 # Talker stage - runtime: - devices: "1" - engine_args: - gpu_memory_utilization: 0.3 # 24GB on GPU 1 + devices: "1" + gpu_memory_utilization: 0.3 # 24GB on GPU 1 - stage_id: 2 # Code2Wav stage - runtime: - devices: "0" - engine_args: - gpu_memory_utilization: 0.1 # 8GB on GPU 0 + devices: "0" + gpu_memory_utilization: 0.1 # 8GB on GPU 0 ``` -**Note:** In this configuration, stages 0 and 2 share GPU 0, but they run at different times in the pipeline, so their memory usage doesn't overlap. +**Note:** Stage 0 uses GPUs 0 and 1, so the combined utilization is `0.7` on +GPU 0 and `0.9` on GPU 1. Keep the sum of all resident stages below `1.0` on +each device. ## Troubleshooting diff --git a/docs/configuration/stage_configs.md b/docs/configuration/stage_configs.md index 38eceb634..cb056f640 100644 --- a/docs/configuration/stage_configs.md +++ b/docs/configuration/stage_configs.md @@ -3,7 +3,46 @@ In vLLM-Omni, a model's `PipelineConfig` defines its fixed stage topology, while a deploy configuration controls how those stages run. !!! note - Default deploy config YAMLs (for example, `vllm_omni/deploy/qwen2_5_omni.yaml`, `vllm_omni/deploy/qwen3_omni_moe.yaml`, and `vllm_omni/deploy/qwen3_tts.yaml`) are bundled and loaded automatically when `--deploy-config` is omitted. The model registry resolves the right pipeline and deploy YAML by `model_type`. + Default deploy config YAMLs (for example, `vllm_omni/deploy/qwen2_5_omni.yaml`, `vllm_omni/deploy/qwen3_omni_moe.yaml`, and `vllm_omni/deploy/qwen3_tts.yaml`) are bundled and loaded automatically when `--deploy-config` is omitted. The resolved pipeline selects its default through `default_deploy_config_name`. + +## Pipeline configuration + +`PipelineConfig` and its `StagePipelineConfig` entries are Python definitions +owned by the model implementation and registered in `OMNI_PIPELINES`. They +describe fixed model topology and are not accepted in deploy YAMLs. + +Common `PipelineConfig` fields include: + +| Field | Description | +|-------|-------------| +| `model_type` | Pipeline identifier used during model and config resolution. | +| `default_deploy_config_name` | Bundled deploy YAML loaded when the user does not pass `deploy_config`. | +| `model_arch` | Default Hugging Face architecture for the pipeline. | +| `hf_architectures` | Architecture names used to identify checkpoints whose `model_type` is shared. | +| `hf_config_predicate` | Optional predicate used to select between pipelines with otherwise identical HF metadata. | +| `diffusers_class_name` | Diffusers `_class_name` used to identify Diffusers-style repositories. | +| `stages` | Ordered tuple of fixed `StagePipelineConfig` definitions. | + +Common `StagePipelineConfig` fields include: + +| Field | Description | +|-------|-------------| +| `stage_id` | Stable stage identifier. | +| `model_stage` | Logical stage name used by runtime and strategy resolution. | +| `execution_type` | `LLM_AR`, `LLM_GENERATION`, or `DIFFUSION`. | +| `input_sources` | Upstream stage IDs that provide this stage's inputs. | +| `final_output` / `final_output_type` | Whether the stage produces a user-visible output and its modality. | +| `owns_tokenizer` | Whether this stage owns the pipeline tokenizer. | +| `model_arch`, `hf_config_name` | Stage-specific model architecture and nested HF config selector. | +| `engine_output_type` | Runtime output representation such as `text`, `latent`, or `audio`. | +| `custom_process_input_func` | Processor applied to this stage's incoming payload. | +| `custom_process_next_stage_input_func` | Processor used for full-payload handoff to the next stage. | +| `async_chunk_process_next_stage_input_func` | Processor used for async chunk handoff. | +| `sampling_constraints` | Model-owned sampling constraints that deploy defaults cannot override. | + +To add or change topology, define and register a new pipeline variant. Use +deploy YAML only for runtime placement, resource sizing, connectors, and other +deployment overrides. ## Deploy configuration schema @@ -13,43 +52,51 @@ The new deploy schema lives under `vllm_omni/deploy/` and is paired with a froze |-------|------|----------|---------|-------------| | `base_config` | str (path) | optional | — | Overlay parent (relative or absolute). `stages:` / `platforms:` deep-merged by stage_id; other scalars overlay-wins. Intended for user-authored overlays; prod yamls stay flat. | | `async_chunk` | bool | optional | `true` | Enable chunked streaming between stages. Pin to `false` if the pipeline runs end-to-end. | +| `session_mode` | str | optional | `"turn"` | Session behavior. Use `"duplex"` only with a pipeline that enables duplex control. | +| `active_stream_window` | int | optional | `0` | Number of active downstream stream slots; `0` preserves all-stream cycling. | +| `duplex_session` | dict | optional | runtime defaults | Full-duplex session lifecycle, buffering, replay, and capacity limits. | | `connectors` | dict | optional | `null` | Named connector specs (`{name, extra}`). Referenced by each stage's `input_connectors` / `output_connectors`. See [Connector schema](#connector-schema). | | `edges` | list | optional | `null` | Explicit edge list for the KV transfer graph. Auto-derived from stage inputs if omitted. | -| `stages` | list | required | — | Per-stage engine args + wiring (see [Stage fields](#stage-fields)). | +| `stages` | list | optional | `[]` | Per-stage runtime overrides matched by `stage_id`. Pipeline stages are still created from `PipelineConfig` when this list is empty. | | `platforms` | dict | optional | `null` | Keyed by `npu` / `rocm` / `xpu`, each contains a `stages:` list with per-platform overrides applied on top of the CUDA defaults. | | `pipeline` | str | optional | `null` | Override the auto-detected pipeline registry key (used for structural variants like `qwen2_5_omni_thinker_only`). | -| `trust_remote_code` | bool | optional | `true` | **Pipeline-wide.** Trust HF remote code on model load; applies to every stage. | +| `trust_remote_code` | bool \| null | optional | `null` | **Pipeline-wide.** Trust HF remote code on model load; applies to every stage when specified. | | `distributed_executor_backend` | str \| null | optional | `null` | **Pipeline-wide.** Distributed executor backend forwarded to vLLM (`"mp"`, `"ray"`, `"external_launcher"`). If omitted, vLLM auto-selects backend from runtime topology. | | `dtype` | str \| null | optional | `null` | **Pipeline-wide.** Model dtype for every stage. | | `quantization` | str \| null | optional | `null` | **Pipeline-wide.** Quantization method for every stage. | -| `enable_prefix_caching` | bool | optional | `false` | **Pipeline-wide.** Prefix cache toggle applied to every stage. | +| `enable_prefix_caching` | bool \| null | optional | `null` | **Pipeline-wide.** Prefix cache toggle applied to every stage when specified. | | `enable_chunked_prefill` | bool \| null | optional | `null` | **Pipeline-wide.** Chunked prefill toggle applied to every stage. | -| `data_parallel_size` | int | optional | `1` | **Pipeline-wide.** DP degree for every stage. | -| `pipeline_parallel_size` | int | optional | `1` | **Pipeline-wide.** PP degree for every stage. | +| `data_parallel_size` | int \| null | optional | `null` | **Pipeline-wide.** DP degree for every stage. | +| `pipeline_parallel_size` | int \| null | optional | `null` | **Pipeline-wide.** PP degree for every stage. | +| `custom_voice_dir` | str \| null | optional | `null` | **Pipeline-wide.** Directory containing custom voice profiles for supported TTS models. | + +For fields whose deploy default is `null`, the deploy layer contributes no +override. The effective value may still come from a platform section, an +explicit CLI or stage override, or the downstream vLLM engine default. Note: for diffusion path, `distributed_executor_backend` currently defaults to `mp`, and `ray` / `external_launcher` are not fully supported yet. ### Stage fields -Each entry under `stages:` accepts any `StageDeployConfig` field directly (no nested `engine_args:`). Only fields whose value legitimately varies across stages live here; pipeline-wide settings (trust_remote_code, distributed_executor_backend, dtype, quantization, prefix/chunked prefill, DP/PP sizes) are declared at the top level and applied to every stage. Unknown keys fall through to `engine_extras:` and are forwarded to the engine. +Each entry under `stages:` accepts any `StageDeployConfig` field directly (no nested `engine_args:`). Only fields whose value legitimately varies across stages live here; pipeline-wide settings (trust_remote_code, distributed_executor_backend, dtype, quantization, prefix/chunked prefill, DP/PP sizes) are declared at the top level and applied to every stage. Unknown keys fall through to `engine_extras:` and are forwarded to the engine. Frequently used fields are listed below; the source-of-truth schema is `StageDeployConfig` in `vllm_omni/config/stage_config.py`. | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| | `stage_id` | int | required | — | Stage identity; matched against `PipelineConfig.stages[*].stage_id`. | -| `max_num_seqs` | int | optional | `64` | Max concurrent sequences per stage. | -| `gpu_memory_utilization` | float | optional | `0.9` | Per-stage total memory target; used for automatic KV-cache sizing. | +| `max_num_seqs` | int \| null | optional | `null` | Max concurrent sequences per stage. | +| `gpu_memory_utilization` | float \| null | optional | `null` | Per-stage total memory target; used for automatic KV-cache sizing. | | `kv_cache_memory_bytes` | int \| null | optional | `null` | Explicit per-rank KV-cache budget in bytes; overrides automatic sizing. | -| `tensor_parallel_size` | int | optional | `1` | TP degree for this stage. | -| `enforce_eager` | bool | optional | `false` | Disable CUDA graphs. | -| `max_num_batched_tokens` | int | optional | `32768` | Per-stage prefill/token budget; also contributes to the native maximum in-flight token limit. | +| `tensor_parallel_size` | int \| null | optional | `null` | TP degree for this stage. | +| `enforce_eager` | bool \| null | optional | `null` | Disable CUDA graphs. | +| `max_num_batched_tokens` | int \| null | optional | `null` | Per-stage prefill/token budget; also contributes to the native maximum in-flight token limit. | | `max_model_len` | int \| null | optional | `null` | Per-sequence context or KV length; `-1` enables native cache-capacity auto-fitting, while values above the HF default auto-set `VLLM_ALLOW_LONG_MAX_MODEL_LEN=1`. | | `async_scheduling` | bool \| null | optional | `null` | Per-stage async scheduling toggle. | -| `devices` | str | optional | `"0"` | `CUDA_VISIBLE_DEVICES`-style device list. | +| `devices` | str \| null | optional | `null` | Device list assigned to this stage. | | `output_connectors` | dict \| null | optional | `null` | Keyed by `to_stage_`; values are names registered under top-level `connectors:`. | | `input_connectors` | dict \| null | optional | `null` | Keyed by `from_stage_`; values are names registered under top-level `connectors:`. | | `default_sampling_params` | dict \| null | optional | `null` | Baseline sampling params. Deep-merged with pipeline `sampling_constraints` (pipeline wins). | -| `engine_extras` | dict | optional | `{}` | Catch-all for keys not listed above; deep-merged across overlays. Also carries per-stage overrides of pipeline-wide settings (e.g. stage-specific `dtype`). | +| `engine_extras` | dict | optional | `{}` | Catch-all for engine fields not listed above; deep-merged across overlays and forwarded to the stage engine. | ### Connector schema @@ -61,7 +108,7 @@ connectors: name: # required — class registered in vllm_omni.distributed extra: # optional — forwarded to the connector's __init__ : - ... + # Additional connector-specific options ``` | Connector class | Use case | `extra` keys | @@ -236,288 +283,3 @@ vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/depl !!! important We are actively iterating on the definition of deployment configurations, and we welcome feedback from users and developers. - -## Legacy stage configuration reference - -The remainder of this page documents the deprecated `stage_args` schema for models that have not migrated to deploy configurations. The following historical Qwen2.5-Omni configuration illustrates that schema; current Qwen2.5-Omni deployments should use `vllm_omni/deploy/qwen2_5_omni.yaml` instead. - -```python -# Historical stage configuration for Qwen2.5-Omni. -stage_args: - - stage_id: 0 # mark the unique id for each stage - runtime: # The disaggregated configuration - process: true # Run this stage in a separate process - devices: "0" # Logical device index for this stage (mapped through CUDA_VISIBLE_DEVICES / ASCEND_RT_VISIBLE_DEVICES if set) - engine_args: # Engine arguments for a certain engine - model_stage: thinker - max_num_seqs: 1 - model_arch: Qwen2_5OmniForConditionalGeneration # The model implementation registered in model_executor/models/registry.py - worker_type: ar # The specific worker used - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler # The specific scehduler used - gpu_memory_utilization: 0.8 # The gpu memory allocation for the stage within a single chip - enforce_eager: true # Now we only support eager mode - trust_remote_code: true # Needed by huggingface config parsing - engine_output_type: latent # It claims that the stage will input latent hiddenstates besides token ids - enable_prefix_caching: false # For request with hiddenstates output, the prefix caching is not supported now - is_comprehension: true # If the stage is a text or multimodal comprehension module. If it is, the AsyncOmni will use its tokenizer as default - final_output: true # If the stage has output as part of final outputs. If it is false, which means that the stage only works as a intermediate role. - final_output_type: text # What is the final output type. It can be text and audio now. - default_sampling_params: # sampling parameters for the stage. Their meaning aligns with vLLM. - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 2048 - seed: 42 - detokenize: True - repetition_penalty: 1.1 - - stage_id: 1 - runtime: - process: true - devices: "1" - engine_args: - model_stage: talker - max_num_seqs: 3 - model_arch: Qwen2_5OmniForConditionalGeneration - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.8 - enforce_eager: true - trust_remote_code: true - enable_prefix_caching: false - engine_output_type: latent - engine_input_source: [0] - default_sampling_params: - temperature: 0.9 - top_p: 0.8 - top_k: 40 - max_tokens: 2048 - seed: 42 - detokenize: True - repetition_penalty: 1.05 - stop_token_ids: [8294] - - stage_id: 2 - runtime: - process: true - devices: "0" # Example: use a different GPU than the previous stage; use "0" if single GPU - engine_args: - model_stage: code2wav - max_num_seqs: 1 - model_arch: Qwen2_5OmniForConditionalGeneration - worker_type: generation - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - gpu_memory_utilization: 0.15 - enforce_eager: true - trust_remote_code: true - enable_prefix_caching: false - engine_output_type: audio - engine_input_source: [1] - final_output: true - final_output_type: audio - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 2048 - seed: 42 - detokenize: True - repetition_penalty: 1.1 - -# Top-level runtime config (concise): default windows and stage edges -runtime: - enabled: true - - edges: - - from: 0 # thinker → talker: trigger only after receiving full input (-1) - to: 1 - - from: 1 # talker → code2wav: trigger only after receiving full input (-1) - to: 2 - -``` - -## Legacy stage configuration arguments - -Each stage in the `stage_args` list contains the following configuration options: - -### `stage_id` - -A unique identifier for each stage in the multi-stage pipeline. Stages are numbered sequentially starting from 0, and this ID is used to reference stages in inter-stage dependencies (e.g., `engine_input_source`). - -### `prompt_expand_func` (Optional) - -A custom Python function hook for the LLM stage (Stage 0) that expands a single incoming prompt object into multiple prompts. This is primarily used for multi-modal Classifier-Free Guidance (CFG), where it generates the necessary companion requests (like a negative text prompt) and tags them with internal roles (e.g., `cfg_text`). This ensures the upstream LLM generates the needed contextual hidden states for both the conditional and unconditional generations simultaneously. - -### `cfg_kv_collect_func` (Optional) - -A custom Python function hook for downstream diffusion stages (Stage 1+) to collect, map, and process the KV caches transferred from the companion requests fired by `prompt_expand_func`. It aggregates the hidden condition states cleanly (e.g., binding them as `cfg_text_past_key_values` and `cfg_text_kv_metadata`), allowing the diffusion runtime to perform CFG smoothly without redundantly evaluating text paths on the DiT workers. - -### `runtime` - -Configuration for disaggregated execution of the stage, controlling how the stage is deployed and executed. - -#### `runtime.process` - -Whether to run this stage in a separate process. When set to `true`, the stage will be executed in an isolated process, enabling better resource isolation and parallel execution across different stages. This is essential for multi-GPU deployments where different stages run on different devices. - -Default: `true` - -#### `runtime.devices` - -Logical device indices for this stage, specified as a string. Values are **logical indices** (`0`, `1`, `2`, ...) — not physical GPU IDs — and are mapped through the platform's visibility env var (`CUDA_VISIBLE_DEVICES` on CUDA, `ASCEND_RT_VISIBLE_DEVICES` on NPU) before being applied via `torch.cuda.set_device()` (or the equivalent). - -Example: if `CUDA_VISIBLE_DEVICES=0,2,4` is set in the environment, then `devices: "0"` selects physical GPU 0 (the first visible), `devices: "1"` selects physical GPU 2, and `devices: "0,1"` makes physical GPUs 0 and 2 available to the stage. If no visibility env var is set, logical and physical IDs coincide. - -Default: `"0"` - -#### `engine_args.max_num_seqs` - -The maximum number of sequences for concurrent processing in this stage. For -LLM stages, this controls the vLLM scheduler's maximum concurrent sequences. -For diffusion stages, this controls scheduler wave capacity in both request -batch mode and step batch mode. `max_num_seqs: 1` is the serial/conservative -path; values above `1` allow compatible requests to batch when the selected -diffusion pipeline supports that batching mode. - -Default: `1` - -#### `engine_args.request_batch_max_wait_ms` - -The maximum time, in milliseconds, that a diffusion request-mode stage may wait -before the first `schedule()` of a new scheduler wave so compatible requests can -accumulate for request-level batching. This only applies to diffusion pipelines -that support request-level batching with `step_execution` disabled. - -Use this together with `max_num_seqs > 1` for bursty serving traffic. `0` -disables admission waiting and preserves the lowest first-request latency. -For diffusion execution and batching tuning, see -[Diffusion Execution Modes](../user_guide/diffusion/execution_modes.md). - -Default: `0.0` - -### `engine_args` - -Engine arguments for configuring the LLM engine, diffusion engine, or other engine types used by this stage. - -#### `engine_args.model_stage` - -The name identifier for this model stage within the multi-stage architecture. This is used internally to distinguish different stages of the same model (e.g., "thinker", "talker", "code2wav" in Qwen2.5-Omni). - -#### `engine_args.model_arch` - -The model architecture class name that is registered in `model_executor/models/registry.py`. This specifies which model implementation to use for this stage. The class must be registered in the model registry for vLLM-Omni to locate and instantiate it. - -#### `engine_args.worker_cls` - -The specific worker class to use for this stage. This determines how the model computations are executed. Examples include `vllm_omni.worker.gpu_ar_worker.GPUARWorker` for autoregressive stages and `vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker` for diffusion-based stages. - -#### `engine_args.scheduler_cls` - -The scheduler class to use for this stage. The scheduler manages request queuing, batching, and execution order. Examples include `vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler` for standard stages and `vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler` for diffusion stages. - -#### `engine_args.gpu_memory_utilization` - -The fraction of GPU memory to allocate for this stage within a single GPU chip. This is a value between 0.0 and 1.0, where 0.8 means 80% of the GPU memory will be used by this stage. This allows fine-grained control over memory allocation when multiple stages share the same GPU or when reserving memory for other operations. - -Default: `0.8` - -!!! tip "Memory Configuration Guide" - For detailed information on how to calculate memory requirements and properly configure `gpu_memory_utilization`, see the [GPU Memory Calculation and Configuration Guide](./gpu_memory_utilization.md). - -#### `engine_args.enforce_eager` - -Whether to enforce eager execution mode. When set to `true`, the engine will run in eager mode without using CUDA graphs or other compilation optimizations. Currently, vLLM-Omni only supports eager mode. - -Default: `true` - -#### `engine_args.trust_remote_code` - -Whether to trust remote code when loading models from Hugging Face. This is required for models that use custom code in their configuration files. Set to `true` when loading models that require custom model implementations. - -Default: `true` - -#### `engine_args.engine_output_type` - -Specifies the type of output produced by this stage's engine. This determines what kind of data flows to downstream stages. Possible values include `latent` (hidden states), `text` (tokenized text), and `audio` (audio waveforms). When set to `latent`, the stage outputs latent hidden states in addition to token IDs, which are consumed by downstream stages. - -Default: `latent` - -#### `engine_args.enable_prefix_caching` - -Whether to enable prefix caching for this stage. Prefix caching can improve performance by caching KV cache for common prompt prefixes. However, for requests that output hidden states (when `engine_output_type` is `latent`), prefix caching is not currently supported and should be set to `false`. - -Default: `false` - -### `is_comprehension` - -Whether this stage is a text or multimodal comprehension module. When set to `true`, the stage acts as a comprehension module that processes input text or multimodal content. If this is the first comprehension stage, `AsyncOmni` will use its tokenizer as the default tokenizer for the entire pipeline. - -Default: `true` - -### `final_output` - -Whether this stage produces output that is part of the final outputs returned to the user. When set to `false`, the stage only works as an intermediate stage, processing data that flows to downstream stages but not contributing directly to the final response. - -Default: `true` - -### `final_output_type` - -The type of final output produced by this stage. This specifies what format the output will be in when returned to the user. Currently supported values are `text` (for text generation) and `audio` (for audio generation). - -Default: `text` - -### `default_sampling_params` - -Default sampling parameters for this stage. These parameters control the generation behavior and align with vLLM's sampling parameter semantics. These defaults are used when no explicit sampling parameters are provided in the request. - -#### `default_sampling_params.temperature` - -Sampling temperature for controlling randomness. Lower values (e.g., 0.0) make the output more deterministic and focused, while higher values increase randomness. - -Default: `0.0` - -#### `default_sampling_params.top_p` - -Nucleus sampling parameter. Only tokens with cumulative probability mass up to `top_p` are considered. This helps filter out low-probability tokens while maintaining diversity. - -Default: `1.0` - -#### `default_sampling_params.top_k` - -Top-k sampling parameter. Only the top `k` most likely tokens are considered. Set to `-1` to disable top-k filtering and consider all tokens. - -Default: `-1` - -#### `default_sampling_params.max_tokens` - -Maximum number of tokens to generate in this stage. This limits the length of the output sequence. - -Default: `2048` - -#### `default_sampling_params.seed` - -Random seed for reproducible generation. When set, the random number generator will be initialized with this seed to ensure consistent outputs across runs. - -Default: `42` - -#### `default_sampling_params.detokenize` - -Whether to detokenize the output tokens into text. When set to `true`, token IDs are converted back to readable text strings. - -Default: `True` - -#### `default_sampling_params.repetition_penalty` - -Penalty applied to tokens that have already appeared in the generated sequence. Values greater than 1.0 discourage repetition, while values less than 1.0 encourage it. A value of 1.0 applies no penalty. - -Default: `1.1` - -### `tts_args` (TTS stages only) - -Configuration for Text-to-Speech specific parameters. This section is only applicable to TTS model stages (e.g., `qwen3_tts`). - -#### `tts_args.max_instructions_length` - -Maximum character length for voice style/emotion instructions. Instructions exceeding this limit will be rejected with a validation error. - -Default: `500` - -This value can be overridden at runtime using the `--tts-max-instructions-length` CLI parameter when starting the server. diff --git a/docs/contributing/DOCS_GUIDE.md b/docs/contributing/DOCS_GUIDE.md index 916f5e60f..57c2fdc54 100644 --- a/docs/contributing/DOCS_GUIDE.md +++ b/docs/contributing/DOCS_GUIDE.md @@ -46,7 +46,7 @@ class Omni: Args: model: Model name or path - stage_configs: Optional stage configurations + deploy_config: Optional path to a deploy configuration **kwargs: Additional arguments passed to the engine Example: diff --git a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md index 8ff27ee08..25d72d9be 100644 --- a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md +++ b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md @@ -31,7 +31,7 @@ Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bul ], "server_params": { "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "stage_config_name": "qwen3_omni.yaml" + "stage_config_name": "qwen3_omni_moe.yaml" }, "benchmark_params": [ { @@ -155,7 +155,7 @@ See also [Markers for Tests](#markers-for-tests) for registered hardware markers | Parameter | Required | Example | Description | | ----------------- | -------- | ---------------------------------- | ----------------------------- | | model | Yes | "Qwen/Qwen3-Omni-30B-A3B-Instruct" | Model name or path | -| stage_config_name | Yes | "qwen3_omni.yaml" | Stage configuration file name | +| stage_config_name | Yes | "qwen3_omni_moe.yaml" | Deploy configuration file name | *Dynamic Configuration (update/delete)* @@ -171,15 +171,15 @@ Supports incremental modifications based on the basic configuration: ``` "update": { "async_chunk": true, // Enable asynchronous chunk processing - "stage_args": { + "stages": { "0": { - "engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk" + "max_num_seqs": 64 } } }, "delete": { - "stage_args": { - "2": ["custom_process_input_func"] // Delete this configuration for stage 2 + "stages": { + "2": ["max_num_batched_tokens"] // Delete this deploy override for stage 2 } } ``` diff --git a/docs/contributing/ci/test_writing_guide.md b/docs/contributing/ci/test_writing_guide.md index ea80ea21e..3b422ec9f 100644 --- a/docs/contributing/ci/test_writing_guide.md +++ b/docs/contributing/ci/test_writing_guide.md @@ -17,7 +17,7 @@ Summary: - **Reliability:** `tests/dfx/reliability/` - Do **not** add new top-level `tests/` directories unrelated to a component (or to `e2e` / `dfx` / `helpers` / `examples` / `buildkite`) -Deploy YAMLs: `vllm_omni/deploy/` via `tests.helpers.stage_config.get_deploy_config_path` (see [stage configs](../../configuration/stage_configs.md)). +Deploy YAMLs: `vllm_omni/deploy/` via `tests.helpers.stage_config.get_deploy_config_path` (see [pipeline and deploy configs](../../configuration/stage_configs.md)). ## Markers for Tests @@ -581,7 +581,7 @@ When you want to add L5-level stability test cases, add or extend the appropriat "test_name": "test_qwen3_omni_stability", "server_params": { "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "stage_config_name": "qwen3_omni.yaml" + "stage_config_name": "qwen3_omni_moe.yaml" }, "benchmark_params": [ { @@ -607,7 +607,7 @@ When you want to add L5-level stability test cases, add or extend the appropriat | Field | Required | Description | | ---------------- | -------- | --------------------------------------------------------------------------- | | test_name | Yes | Unique identifier for the stability test case | -| server_params | Yes | Server-side configuration parameters (model, stage configuration, etc.) | +| server_params | Yes | Server-side configuration parameters (model, deploy configuration, etc.) | | benchmark_params | Yes | Stability benchmark running parameters (supports multiple configurations) | ##### server_params Configuration @@ -617,7 +617,7 @@ When you want to add L5-level stability test cases, add or extend the appropriat | Parameter | Required | Example | Description | | ----------------- | -------- | ---------------------------------- | ----------------------------------- | | model | Yes | "Qwen/Qwen3-Omni-30B-A3B-Instruct" | Model name or path | -| stage_config_name | Yes | "qwen3_omni.yaml" | Stage configuration file name | +| stage_config_name | Yes | "qwen3_omni_moe.yaml" | Deploy configuration file name | ###### Dynamic Configuration (update/delete) @@ -697,8 +697,8 @@ pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow ##### Adding a new model suite 1. Add `test_reliability_.py` under `tests/dfx/reliability/`. -2. Define **`RELIABILITY_SCENARIOS`** and pass them through **`create_reliability_omni_server_params()`** with the correct deploy or e2e stage-config directory (same pattern as existing files). -3. Reuse **`helpers`** for OOM / kill / raw HTTP; prefer **`assert_fault_exception()`** and **`resolve_oom_device_spec()`** from `tests/dfx/conftest.py` for consistent device selection vs stage YAML. +2. Define **`RELIABILITY_SCENARIOS`** and pass them through **`create_reliability_omni_server_params()`** with the correct deploy or e2e deploy-config directory (same pattern as existing files). +3. Reuse **`helpers`** for OOM / kill / raw HTTP; prefer **`assert_fault_exception()`** and **`resolve_oom_device_spec()`** from `tests/dfx/conftest.py` for consistent device selection vs deploy YAML. 4. Register **`slow`** (and **`hardware_test`** if needed); extend **`.buildkite/cuda/test-weekly.yml`** when the suite should run in weekly L5. diff --git a/docs/contributing/model/adding_omni_model.md b/docs/contributing/model/adding_omni_model.md index 4698caf9c..42b9a53d7 100644 --- a/docs/contributing/model/adding_omni_model.md +++ b/docs/contributing/model/adding_omni_model.md @@ -9,7 +9,7 @@ This guide walks through the process of adding a new multi-stage model to vLLM-O 3. [Step-by-Step Implementation](#step-by-step-implementation) 4. [Key Components](#key-components) 5. [Model Registration](#model-registration) -6. [Stage Configuration](#stage-configuration) +6. [Pipeline and Deploy Configuration](#pipeline-and-deploy-configuration) 7. [Stage Input Processors](#stage-input-processors) 8. [Testing](#testing) 9. [Adding a Model Recipe](#adding-a-model-recipe) @@ -316,18 +316,28 @@ _OMNI_MODELS = { The registry uses lazy loading, so the model class is imported only when needed. -## Stage Configuration +## Pipeline and Deploy Configuration -Create a YAML configuration file in `vllm_omni/deploy/`. For a complete example, see the [Qwen3-Omni configuration file](gh-file:vllm_omni/deploy/qwen3_omni_moe.yaml). +Define fixed topology in `vllm_omni/model_executor/models//pipeline.py` +with `PipelineConfig` and `StagePipelineConfig`, then register it in +`vllm_omni/config/pipeline_registry.py`. For a complete example, see the +[Qwen3-Omni pipeline](gh-file:vllm_omni/model_executor/models/qwen3_omni/pipeline.py). -### Key Configuration Fields +### Pipeline topology fields -- **`model_stage`**: Which stage to run ("thinker", "talker", "code2wav", etc.) -- **`model_arch`**: The model architecture name (must match registry) -- **`engine_input_source`**: List of stage IDs that provide input to this stage -- **`custom_process_input_func`**: Function to process inputs from previous stages -- **`final_output`**: Whether this stage produces the final output (True/False) -- **`final_output_type`**: Type of final output ("text", "audio", "image", etc.) +- **`model_stage`**: Logical stage name ("thinker", "talker", "code2wav", etc.) +- **`execution_type`**: Runtime family (`LLM_AR`, `LLM_GENERATION`, or `DIFFUSION`) +- **`model_arch`**: Stage architecture name, when different from the pipeline default +- **`input_sources`**: Upstream stage IDs that provide input to this stage +- **`custom_process_input_func`**: Function that processes inputs from previous stages +- **`final_output`**: Whether this stage produces user-facing output +- **`final_output_type`**: Output modality ("text", "audio", "image", etc.) +- **`owns_tokenizer`**: Whether this stage owns the pipeline tokenizer + +Create the matching runtime defaults in `vllm_omni/deploy/.yaml` and set +`PipelineConfig.default_deploy_config_name` to that file. The deploy YAML owns +GPU placement, memory sizing, parallelism, connectors, and sampling defaults; +it must not redefine topology fields. ## Stage Input Processors @@ -374,7 +384,7 @@ The stage transition process follows these steps: 2. **Transition Detection**: The orchestrator checks if there's a next stage and calls `process_engine_inputs()` on it -3. **Input Processing**: The stage input processor configured in stage YAML (under `vllm_omni/model_executor/stage_input_processors/`) handles the transition: +3. **Input Processing**: The stage input processor registered by the pipeline (implemented under `vllm_omni/model_executor/stage_input_processors/`) handles the transition: ```python def process_engine_inputs( self, stage_list: list[Any], prompt: OmniTokensPrompt | TextPrompt = None @@ -577,7 +587,7 @@ Adding a new model to vLLM-Omni involves: 1. **Create model directory structure** with stage implementations 2. **Implement unified model class** that orchestrates stages 3. **Register model** in `registry.py` -4. **Create stage configuration** YAML file +4. **Define pipeline topology and deploy defaults** in `pipeline.py` and `vllm_omni/deploy/` 5. **Implement stage input processors** for stage transitions 6. **Write tests** to verify functionality 7. **Add model recipe** to the [vllm-project/recipes](https://github.com/vllm-project/recipes) repository (see [Adding a Model Recipe](#adding-a-model-recipe) section) @@ -590,7 +600,8 @@ For a complete reference implementation, see: - **Thinker**: `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py` - **Talker**: `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py` - **Code2Wav**: `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_code2wav.py` -- **Stage config**: `vllm_omni/deploy/qwen3_omni_moe.yaml` +- **Pipeline topology**: `vllm_omni/model_executor/models/qwen3_omni/pipeline.py` +- **Deploy config**: `vllm_omni/deploy/qwen3_omni_moe.yaml` - **Input processors**: `vllm_omni/model_executor/stage_input_processors/qwen3_omni.py` - **Registry**: `vllm_omni/model_executor/models/registry.py` - **Testing**: `vllm_omni/tests/e2e/offline_inference/test_qwen3_omni.py` @@ -598,4 +609,4 @@ For a complete reference implementation, see: For more information, see: - [Architecture Overview](../../design/architecture_overview.md) - [Supported Models](../../models/supported_models.md) -- [Stage Configuration Guide](../../configuration/stage_configs.md) +- [Pipeline and Deploy Configuration Guide](../../configuration/stage_configs.md) diff --git a/docs/contributing/model/adding_tts_model.md b/docs/contributing/model/adding_tts_model.md index 45680ee5d..76aab4291 100644 --- a/docs/contributing/model/adding_tts_model.md +++ b/docs/contributing/model/adding_tts_model.md @@ -20,7 +20,7 @@ pattern, refer to MOSS-TTS-Nano. 4. [Step-by-Step Implementation](#step-by-step-implementation) 5. [Key Components](#key-components) 6. [Model Registration](#model-registration) -7. [Stage Configuration](#stage-configuration) +7. [Pipeline and Deploy Configuration](#pipeline-and-deploy-configuration) 8. [Stage Input Processors](#stage-input-processors) 9. [Online Serving Integration](#online-serving-integration) 10. [Single-Stage Models](#single-stage-models) @@ -427,112 +427,93 @@ _OMNI_MODELS = { The registry uses lazy loading - model classes are only imported when needed. -## Stage Configuration +## Pipeline and Deploy Configuration -Each stage has a `worker_type` that determines how it is scheduled: +Define fixed topology in `models//pipeline.py`. The execution type +selects the scheduler and worker family: -- `worker_type: ar` - autoregressive stage, uses `OmniARScheduler` with PagedAttention -- `worker_type: generation` - non-AR stage (e.g. decoder), uses `OmniGenerationScheduler` +- `StageExecutionType.LLM_AR` uses the autoregressive runtime with PagedAttention. +- `StageExecutionType.LLM_GENERATION` uses the generation runtime for stages such as audio decoders. -Key configuration fields: +Key topology fields belong to `StagePipelineConfig`: | Field | Description | |-------|-------------| -| `model_stage` | Which stage to initialize (`ar_stage`, `decoder`, etc.) | -| `model_arch` | Architecture name, must match `registry.py` | -| `engine_input_source` | List of upstream stage IDs that provide input (e.g. `[0]`) | -| `engine_output_type` | Output type: `latent` for intermediate, `audio` for final | -| `custom_process_next_stage_input_func` | Async chunk processor function path (streaming only) | -| `final_output` | Whether this stage produces the final user-facing output | -| `final_output_type` | Type of final output (`audio`, `text`, etc.) | +| `model_stage` | Logical stage name (`ar_stage`, `decoder`, etc.). | +| `execution_type` | Runtime family for this stage. | +| `input_sources` | Upstream stage IDs that provide input. | +| `model_arch` | Stage-specific architecture name, when it differs from the pipeline default. | +| `engine_output_type` | Output representation such as `latent` or `audio`. | +| `custom_process_next_stage_input_func` | Processor for full-payload stage handoff. | +| `async_chunk_process_next_stage_input_func` | Processor for streaming chunk handoff. | +| `final_output` / `final_output_type` | Whether this stage produces user-facing output and its modality. | +| `owns_tokenizer` | Whether this stage supplies the pipeline tokenizer. | -!!! note - New in-tree models should define frozen topology in `models//pipeline.py`, - register it in `vllm_omni/config/pipeline_registry.py`, and put deployment - knobs in `vllm_omni/deploy/.yaml`. Use `--deploy-config` to load a - custom deployment YAML. +For example, a two-stage TTS topology can be defined as: -### Batch mode +```python +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) -```yaml -# stage_configs/your_model_name.yaml +_PROC = "vllm_omni.model_executor.stage_input_processors.your_model_name" -stage_args: - - stage_id: 0 - stage_type: llm - runtime: - devices: "0" - engine_args: - model_stage: ar_stage - max_num_seqs: 64 - model_arch: YourTTSModelForConditionalGeneration - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - engine_output_type: latent - default_sampling_params: - temperature: 0.9 - top_k: 50 - max_tokens: 2048 - - - stage_id: 1 - stage_type: llm - runtime: - devices: "0" - engine_args: - model_stage: decoder - model_arch: YourTTSModelForConditionalGeneration - worker_type: generation - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - engine_output_type: audio - engine_input_source: [0] - final_output: true - final_output_type: audio +YOUR_TTS_PIPELINE = PipelineConfig( + model_type="your_model_name", + default_deploy_config_name="your_model_name.yaml", + model_arch="YourTTSModelForConditionalGeneration", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="ar_stage", + execution_type=StageExecutionType.LLM_AR, + input_sources=(), + owns_tokenizer=True, + engine_output_type="latent", + custom_process_next_stage_input_func=f"{_PROC}.ar2decoder", + async_chunk_process_next_stage_input_func=f"{_PROC}.ar2decoder_async_chunk", + ), + StagePipelineConfig( + stage_id=1, + model_stage="decoder", + execution_type=StageExecutionType.LLM_GENERATION, + input_sources=(0,), + model_arch="YourTTSDecoder", + engine_output_type="audio", + final_output=True, + final_output_type="audio", + ), + ), +) ``` -### Streaming mode (async_chunk) - -Add `async_chunk: true` at the top level and specify `custom_process_next_stage_input_func` -on Stage 0 to define how intermediate outputs are chunked and forwarded: +Register the pipeline in `vllm_omni/config/pipeline_registry.py`. Runtime +placement and sizing belong in `vllm_omni/deploy/your_model_name.yaml`: ```yaml -# stage_configs/your_model_name_async_chunk.yaml - async_chunk: true -stage_args: +stages: - stage_id: 0 - stage_type: llm - runtime: - devices: "0" - engine_args: - model_stage: ar_stage - max_num_seqs: 64 - model_arch: YourTTSModelForConditionalGeneration - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - engine_output_type: latent - custom_process_next_stage_input_func: > - vllm_omni.model_executor.stage_input_processors.your_model_name.ar2decoder_async_chunk + devices: "0" + max_num_seqs: 64 default_sampling_params: temperature: 0.9 top_k: 50 max_tokens: 2048 - stage_id: 1 - stage_type: llm - runtime: - devices: "0" - engine_args: - model_stage: decoder - model_arch: YourTTSModelForConditionalGeneration - worker_type: generation - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - engine_output_type: audio - engine_input_source: [0] - final_output: true - final_output_type: audio + devices: "0" + max_num_seqs: 1 ``` +Set `async_chunk: false` for end-to-end handoff. The registered pipeline +selects the full-payload or async chunk processor; processor paths and topology +must not be moved into the deploy YAML. Use `--deploy-config` to load a custom +deployment overlay. + ## Stage Input Processors Stage input processors convert Stage 0 outputs into Stage 1 inputs. Create yours in @@ -832,32 +813,40 @@ vllm_omni/deploy/your_model_name.yaml No stage input processor is needed. -### Stage config +### Pipeline and deploy config -Use a single stage with `worker_type: ar`. The `is_comprehension: true` field and the -top-level `async_chunk: false` are required — omitting them causes silent -misclassification in the serving layer. Set `max_num_seqs` to at least 4 for -concurrent production use. +Use a single `LLM_AR` stage that owns the tokenizer and produces final audio: + +```python +YOUR_TTS_PIPELINE = PipelineConfig( + model_type="your_model_name", + default_deploy_config_name="your_model_name.yaml", + model_arch="YourModelForCausalLM", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="your_model_stage_key", + execution_type=StageExecutionType.LLM_AR, + input_sources=(), + owns_tokenizer=True, + engine_output_type="audio", + final_output=True, + final_output_type="audio", + ), + ), +) +``` + +The deploy YAML controls runtime behavior. Set `max_num_seqs` to at least 4 +for concurrent production use: ```yaml -# stage_configs/your_model_name.yaml async_chunk: false -stage_args: +stages: - stage_id: 0 - stage_type: llm - is_comprehension: true # required for serving_speech.py dispatch - runtime: - devices: "0" - engine_args: - model_stage: your_model_stage_key - model_arch: YourModelForCausalLM - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - engine_output_type: audio - max_num_seqs: 4 # min 4 for concurrent requests; default 1 causes gaps - final_output: true - final_output_type: audio + devices: "0" + max_num_seqs: 4 ``` ### Generator-based streaming pattern @@ -1031,4 +1020,4 @@ For more information, see: - [Architecture Overview](../../design/architecture_overview.md) - [Async Chunk Design](../../design/feature/async_chunk.md) -- [Stage Configuration Guide](../../configuration/stage_configs.md) +- [Pipeline and Deploy Configuration Guide](../../configuration/stage_configs.md) diff --git a/docs/design/architecture_overview.md b/docs/design/architecture_overview.md index 4ae1b495e..d7938b62c 100644 --- a/docs/design/architecture_overview.md +++ b/docs/design/architecture_overview.md @@ -335,8 +335,8 @@ The single resolution boundary is implemented by `StageConfigFactory.create_from_model()` and `VllmOmniConfig.from_pipeline_config()` in [`vllm_omni/config`](https://github.com/vllm-project/vllm-omni/tree/main/vllm_omni/config). -The legacy `stage_args` YAML path remains only for models that have not yet -migrated to `PipelineConfig` and `DeployConfig`. +The legacy `stage_args` YAML path has been removed. Model topology now resolves +through `PipelineConfig`, with runtime overrides supplied by `DeployConfig`. In the typed path, each stage configuration derives from `BaseVllmOmniStageConfig` and is specialized as diff --git a/docs/design/feature/async_chunk.md b/docs/design/feature/async_chunk.md index 57b4209b8..068a5ac1b 100644 --- a/docs/design/feature/async_chunk.md +++ b/docs/design/feature/async_chunk.md @@ -211,38 +211,39 @@ In sequential mode, each stage must wait for the previous stage to complete enti ## Configuration -Enable async_chunk in stage configuration YAML: +Enable async chunk mode in the deploy YAML: ```yaml async_chunk: true -stage_args: - - stage_id: 0 - engine_args: - custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk - - stage_id: 1 - engine_args: - custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk ``` -### Stage Configuration +The registered `PipelineConfig` owns the async handoff processor functions. +For Qwen3-Omni these are `thinker2talker_async_chunk` and +`talker2code2wav_async_chunk`; deploy YAML only selects the mode and runtime +sizing. + +### Configuration ownership - `async_chunk: bool`: Enable/disable async chunk mode -- `custom_process_next_stage_input_func: str`: Path to custom chunk processing function; receives `(transfer_manager, pooling_output, request)`. For qwen3-omni: `thinker2talker_async_chunk`, `talker2code2wav_async_chunk` -- `stage_connector_config: dict`: Connector configuration -- `worker_type: str`: Model type, e.g. `"ar"` or `"generation"` (used by OmniChunkTransferAdapter for mode-specific payload handling) +- `async_chunk_process_next_stage_input_func: str`: Pipeline-owned chunk processor path +- `execution_type: StageExecutionType`: Pipeline-owned runtime family +- `connectors: dict`: Deploy-owned connector definitions - `max_num_seqs: int`: Maximum number of sequences for concurrent processing in the stage - ### Connector Configuration ```yaml connectors: - - from_stage: 0 - to_stage: 1 - spec: - name: SharedMemoryConnector - extra: - stage_id: 0 + connector_of_shared_memory: + name: SharedMemoryConnector + +stages: + - stage_id: 0 + output_connectors: + to_stage_1: connector_of_shared_memory + - stage_id: 1 + input_connectors: + from_stage_0: connector_of_shared_memory ``` ### Code2Wav Batch Configuration @@ -250,13 +251,10 @@ connectors: For optimal performance with async_chunk, the code2wav stage should be configured with batching: ```yaml -stage_args: +stages: - stage_id: 2 # code2wav stage - runtime: - devices: "1" - engine_args: - model_stage: code2wav - max_num_seqs: 64 # Enables batched audio generation + devices: "1" + max_num_seqs: 64 # Enables batched audio generation ``` ## Related Files diff --git a/docs/design/feature/disaggregated_inference.md b/docs/design/feature/disaggregated_inference.md index c31086c0d..ba8c8f9fc 100644 --- a/docs/design/feature/disaggregated_inference.md +++ b/docs/design/feature/disaggregated_inference.md @@ -59,19 +59,18 @@ This `metadata` must be passed through the control plane so `get()` can locate t ## Configuration Model -Define connectors in runtime: +Define connectors at the top level of the deploy YAML: ```yaml -runtime: - connectors: - connector_of_shared_memory: - name: SharedMemoryConnector +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector ``` Wire stages to connectors: ```yaml -stage_args: +stages: - stage_id: 0 output_connectors: to_stage_1: connector_of_shared_memory diff --git a/docs/design/feature/omni_connectors/mooncake_store_connector.md b/docs/design/feature/omni_connectors/mooncake_store_connector.md index 2ff116859..ed8e1ef49 100644 --- a/docs/design/feature/omni_connectors/mooncake_store_connector.md +++ b/docs/design/feature/omni_connectors/mooncake_store_connector.md @@ -32,26 +32,25 @@ mooncake_master \ ## Configuration -Define the connector in runtime: +Define the connector at the top level of the deploy YAML: ```yaml -runtime: - connectors: - connector_of_mooncake: - name: MooncakeStoreConnector - extra: - host: "127.0.0.1" - metadata_server: "http://:8080/metadata" - master: ":50051" - segment: 512000000 - localbuf: 64000000 - proto: "tcp" +connectors: + connector_of_mooncake: + name: MooncakeStoreConnector + extra: + host: "127.0.0.1" + metadata_server: "http://:8080/metadata" + master: ":50051" + segment: 512000000 + localbuf: 64000000 + proto: "tcp" ``` Wire stages to the connector: ```yaml -stage_args: +stages: - stage_id: 0 output_connectors: to_stage_1: connector_of_mooncake diff --git a/docs/design/feature/omni_connectors/mooncake_transfer_engine_connector.md b/docs/design/feature/omni_connectors/mooncake_transfer_engine_connector.md index 306a0620b..e23ea897b 100644 --- a/docs/design/feature/omni_connectors/mooncake_transfer_engine_connector.md +++ b/docs/design/feature/omni_connectors/mooncake_transfer_engine_connector.md @@ -21,26 +21,25 @@ InfiniBand/RoCE NICs). ## Configuration -Define the connector in runtime: +Define the connector at the top level of the deploy YAML: ```yaml -runtime: - connectors: - rdma_connector: - name: MooncakeTransferEngineConnector - extra: - host: "auto" # Auto-detect local RDMA IP - zmq_port: 50051 # ZMQ base port (see "Port Offset Scheme" below) - protocol: "rdma" # "rdma" or "tcp" - device_name: "" # RDMA device (e.g., "mlx5_0"), empty for auto-detect - memory_pool_size: 4294967296 # 4 GB (CPU); use 2147483648 (2 GB) for GPU - memory_pool_device: "cpu" # "cpu" for pinned memory (recommended), "cuda" for GPUDirect RDMA +connectors: + rdma_connector: + name: MooncakeTransferEngineConnector + extra: + host: "auto" # Auto-detect local RDMA IP + zmq_port: 50051 # ZMQ base port (see "Port Offset Scheme" below) + protocol: "rdma" # "rdma" or "tcp" + device_name: "" # RDMA device (e.g., "mlx5_0"), empty for auto-detect + memory_pool_size: 4294967296 # 4 GB (CPU); use 2147483648 (2 GB) for GPU + memory_pool_device: "cpu" # "cpu" for pinned memory (recommended), "cuda" for GPUDirect RDMA ``` Wire stages to the connector: ```yaml -stage_args: +stages: - stage_id: 0 output_connectors: to_stage_1: rdma_connector @@ -415,7 +414,7 @@ A receiver connector: - allocates receive buffers from its own pool - requests metadata or transfer service from the sender -The role is not inferred dynamically. It is injected by the stage configuration layer: +The role is not inferred dynamically. It is injected from the deploy connector wiring: - incoming edge for a stage -> `role="receiver"` - outgoing edge for a stage -> `role="sender"` diff --git a/docs/design/feature/omni_connectors/shared_memory_connector.md b/docs/design/feature/omni_connectors/shared_memory_connector.md index 62d77cd6d..1c0272340 100644 --- a/docs/design/feature/omni_connectors/shared_memory_connector.md +++ b/docs/design/feature/omni_connectors/shared_memory_connector.md @@ -13,10 +13,9 @@ segment name is returned in metadata. ## Configuration ```yaml -runtime: - connectors: - connector_of_shared_memory: - name: SharedMemoryConnector +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector ``` ## Notes diff --git a/docs/design/feature/omni_connectors/yuanrong_connector.md b/docs/design/feature/omni_connectors/yuanrong_connector.md index 12127ba3e..828feb99f 100644 --- a/docs/design/feature/omni_connectors/yuanrong_connector.md +++ b/docs/design/feature/omni_connectors/yuanrong_connector.md @@ -65,23 +65,22 @@ dscli stop --worker_address "${WORKER_IP}:31501" ## Configuration -Define the connector in runtime: +Define the connector at the top level of the deploy YAML: ```yaml -runtime: - connectors: - connector_of_yuanrong: - name: YuanrongConnector - extra: - host: "127.0.0.1" - port: 31501 - get_sub_timeout_ms: 1000 +connectors: + connector_of_yuanrong: + name: YuanrongConnector + extra: + host: "127.0.0.1" + port: 31501 + get_sub_timeout_ms: 1000 ``` Wire stages to the connector: ```yaml -stage_args: +stages: - stage_id: 0 output_connectors: to_stage_1: connector_of_yuanrong diff --git a/docs/design/feature/omni_connectors/yuanrong_transfer_engine_connector.md b/docs/design/feature/omni_connectors/yuanrong_transfer_engine_connector.md index 21d5f1841..c98a34896 100644 --- a/docs/design/feature/omni_connectors/yuanrong_transfer_engine_connector.md +++ b/docs/design/feature/omni_connectors/yuanrong_transfer_engine_connector.md @@ -94,26 +94,23 @@ For Ascend P2P, configure the connector with `protocol: "ascend"` and an NPU memory pool: ```yaml -runtime: - enabled: true - defaults: +connectors: + yuanrong_te_connector: + name: YuanrongTransferEngineConnector + extra: + host: "auto" + zmq_port: 50051 + rpc_port: "auto" + protocol: "ascend" + device_name: "auto" + memory_pool_size: 1073741824 + memory_pool_device: "npu" + +edges: + - from: 0 + to: 1 window_size: -1 max_inflight: 1 - connectors: - yuanrong_te_connector: - name: YuanrongTransferEngineConnector - extra: - host: "auto" - zmq_port: 50051 - rpc_port: "auto" - protocol: "ascend" - device_name: "auto" - memory_pool_size: 1073741824 - memory_pool_device: "npu" - edges: - - from: 0 - to: 1 - window_size: -1 ``` Important fields: @@ -141,31 +138,27 @@ For AR 4-way TP sending KV cache to DiT 4-way TP, set the stage TP topology in both stage `omni_kv_config` blocks: ```yaml -stage_args: +stages: - stage_id: 0 - runtime: - devices: "0,1,2,3" - engine_args: - tensor_parallel_size: 4 - omni_kv_config: - need_send_cache: true - rank_mapping: - from_tp: 4 - to_tp: 4 + devices: "0,1,2,3" + tensor_parallel_size: 4 + omni_kv_config: + need_send_cache: true + rank_mapping: + from_tp: 4 + to_tp: 4 output_connectors: to_stage_1: yuanrong_te_connector - stage_id: 1 - runtime: - devices: "4,5,6,7" - engine_args: - parallel_config: - tensor_parallel_size: 4 - omni_kv_config: - need_recv_cache: true - rank_mapping: - from_tp: 4 - to_tp: 4 + devices: "4,5,6,7" + parallel_config: + tensor_parallel_size: 4 + omni_kv_config: + need_recv_cache: true + rank_mapping: + from_tp: 4 + to_tp: 4 input_connectors: from_stage_0: yuanrong_te_connector ``` diff --git a/docs/design/feature/prefix_caching.md b/docs/design/feature/prefix_caching.md index ebad8b691..a8fe4fc50 100644 --- a/docs/design/feature/prefix_caching.md +++ b/docs/design/feature/prefix_caching.md @@ -41,7 +41,7 @@ With this in mind, consider the set of blocks in a 2D layout, where the row repr ### Example !!! note "Note 3" - Prefix caching in vLLM-Omni currently is only supported on AutoRegressive stages with one kv-cache group. It can be enabled/disabled per-stage via the `enable_prefix_caching` parameter in the model's stage config. + Prefix caching in vLLM-Omni currently is only supported on AutoRegressive stages with one kv-cache group. Configure it with the pipeline-wide `enable_prefix_caching` field in the deploy config. The way in which vLLM-Omni ties into vLLM's prefix caching is best understood by example. Say that we have the following: diff --git a/docs/design/module/model_integration.md b/docs/design/module/model_integration.md index 1d8d7ab73..2e848a49c 100644 --- a/docs/design/module/model_integration.md +++ b/docs/design/module/model_integration.md @@ -42,7 +42,7 @@ and execution behavior to stable runtime contracts. ### MODEL-INV-001: Registration is explicit **Rule:** A model integration MUST declare how its model class, loader, input -processor, and stage configuration are selected. +processor, and pipeline configuration are selected. ### MODEL-INV-002: Model code does not route stages diff --git a/docs/design/module/vllm_omni_config.md b/docs/design/module/vllm_omni_config.md index eabb9908f..1ee303b8c 100644 --- a/docs/design/module/vllm_omni_config.md +++ b/docs/design/module/vllm_omni_config.md @@ -10,9 +10,8 @@ owners: primary_code_paths: - vllm_omni/config/** - vllm_omni/deploy/** - - vllm_omni/model_executor/stage_configs/** -related_code_paths: - - vllm_omni/platforms/*/stage_configs/** + - vllm_omni/model_executor/models/*/pipeline.py +related_code_paths: [] depends_on: [] validation_paths: - tests/config/** @@ -95,7 +94,7 @@ parallel input is effective for a diffusion stage. ## Safe-change guide while deferred Changes should trace every affected producer to its runtime consumer and test -the structured, legacy, CLI, and deployment paths that are actually supported. +the structured pipeline, CLI, and deployment paths that are actually supported. For inherited vLLM fields, tests should separately cover constructor behavior and effective `EngineArgs` projection. Do not infer a stable contract from this placeholder page. diff --git a/docs/design/qwen3_omni_tts_performance_optimization.md b/docs/design/qwen3_omni_tts_performance_optimization.md index bc1e29a7f..ff783a02a 100644 --- a/docs/design/qwen3_omni_tts_performance_optimization.md +++ b/docs/design/qwen3_omni_tts_performance_optimization.md @@ -111,7 +111,7 @@ This post walks through each optimization in the same order they are typically e For both Qwen3-Omni and Qwen3-TTS, batching is a pipeline-level optimization: -- Requests are grouped per stage using `runtime.max_batch_size` +- Requests are grouped per stage using `stages[].max_num_seqs` - Each stage executes batch inference with its own scheduler/worker - Stage outputs are routed to downstream stages with per-request mapping preserved @@ -171,7 +171,7 @@ At concurrency 10, batching alone brings Qwen3-TTS RTF from 2.19 (slower than re In decode-heavy serving, repeatedly launching many small kernels from CPU can become a visible overhead. CUDA Graph reduces this overhead by capturing and replaying stable execution graphs. -In stage configs, this is represented by `enforce_eager: false` for stages where graph capture is desired (Thinker/Talker), while Code2Wav keeps eager mode depending on stage behavior. +In deploy configs, this is represented by `enforce_eager: false` for stages where graph capture is desired (Thinker/Talker), while Code2Wav keeps eager mode depending on stage behavior. ### CUDA Graph results on top of batching @@ -320,7 +320,7 @@ In the async chunk pipeline, the standard `codec_chunk_frames` is 25 (each chunk **Dynamic initial chunk sizing (default behavior):** -Rather than using a fixed initial chunk size, vLLM-Omni dynamically selects it based on current server load. The initial chunk size is chosen from power-of-2 steps [2, 4, 8, 16] based on load factor (`active_requests / max_batch_size`): +Rather than using a fixed initial chunk size, vLLM-Omni dynamically selects it based on current server load. The initial chunk size is chosen from power-of-2 steps [2, 4, 8, 16] based on load factor (`active_requests / max_num_seqs`): | Server load | Initial chunk frames | Rationale | | --- | --- | --- | @@ -349,16 +349,15 @@ This overrides the dynamic calculation for that request. **Config (server-side):** ```yaml -runtime: - connectors: - connector_of_shared_memory: - name: SharedMemoryConnector - extra: - codec_streaming: true - codec_chunk_frames: 25 # standard chunk size (~2s of audio) - codec_left_context_frames: 25 - # initial chunk is computed dynamically by default - # set initial_codec_chunk_frames: 2 to force a fixed value +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + codec_streaming: true + codec_chunk_frames: 25 # standard chunk size (~2s of audio) + codec_left_context_frames: 25 + # initial chunk is computed dynamically by default + # set initial_codec_chunk_frames: 2 to force a fixed value ``` The 64 ms TTFP result reported above for Qwen3-TTS at concurrency 1 uses the dynamic initial chunk, which picks `initial_codec_chunk_frames=2` at low load. At higher concurrency the dynamic sizing increases the initial chunk to maintain decode efficiency. @@ -408,7 +407,7 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \ Notes: -- `runtime.max_batch_size` controls stage-level batching. +- `stages[].max_num_seqs` controls stage-level batching. - Thinker/Talker commonly use `enforce_eager: false` for CUDA Graph paths. - Code2Wav often remains eager (`enforce_eager: true`) depending on runtime behavior. - Qwen3-Omni defaults `VLLM_USE_FLASHINFER_MOE_FP16=0`. The Triton has been more stable & faster @@ -426,33 +425,26 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \ ```yaml async_chunk: true -stage_args: +stages: - stage_id: 0 # thinker - runtime: - max_batch_size: 64 - engine_args: - enforce_eager: false - max_num_batched_tokens: 32768 - custom_process_next_stage_input_func: >- - vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk + max_num_seqs: 64 + enforce_eager: false + max_num_batched_tokens: 32768 - stage_id: 1 # talker - runtime: - max_batch_size: 64 - engine_args: - enforce_eager: false - max_num_batched_tokens: 32768 - custom_process_next_stage_input_func: >- - vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk + max_num_seqs: 64 + enforce_eager: false + max_num_batched_tokens: 32768 - stage_id: 2 # code2wav - runtime: - max_batch_size: 64 - engine_args: - enforce_eager: true - max_num_batched_tokens: 51200 + max_num_seqs: 64 + enforce_eager: true + max_num_batched_tokens: 51200 ``` +The async handoff processors are fixed by the registered Qwen3-Omni +`PipelineConfig`; deploy YAML only selects `async_chunk` and runtime sizing. + #### Reproduce Qwen3-Omni benchmarks ```bash @@ -483,7 +475,7 @@ vllm-omni serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \ The default config (`qwen3_tts.yaml`) enables the full optimization stack: -- Batching with `max_batch_size: 10` on the Talker stage +- Batching with `max_num_seqs: 10` on the Talker stage - CUDA Graph on the Talker (`enforce_eager: false`) - Async chunk with streaming transport @@ -500,33 +492,29 @@ vllm-omni serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \ ```yaml async_chunk: true -stage_args: +stages: - stage_id: 0 # Talker (AR decoder) - runtime: - max_batch_size: 10 - engine_args: - enforce_eager: false - max_num_batched_tokens: 512 - custom_process_next_stage_input_func: >- - vllm_omni.model_executor.stage_input_processors.qwen3_tts.talker2code2wav_async_chunk + max_num_seqs: 10 + enforce_eager: false + max_num_batched_tokens: 512 - stage_id: 1 # Code2Wav (vocoder) - runtime: - max_batch_size: 1 - engine_args: - enforce_eager: true - max_num_batched_tokens: 8192 + max_num_seqs: 1 + enforce_eager: true + max_num_batched_tokens: 8192 -runtime: - connectors: - connector_of_shared_memory: - name: SharedMemoryConnector - extra: - codec_streaming: true - codec_chunk_frames: 25 - codec_left_context_frames: 25 +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + codec_streaming: true + codec_chunk_frames: 25 + codec_left_context_frames: 25 ``` +The async Talker-to-Code2Wav processor is defined by the registered Qwen3-TTS +pipeline rather than the deploy YAML. + #### Reproduce Qwen3-TTS benchmarks ```bash diff --git a/docs/getting_started/quickstart.md b/docs/getting_started/quickstart.md index 45ff311cb..a5a02f5a7 100644 --- a/docs/getting_started/quickstart.md +++ b/docs/getting_started/quickstart.md @@ -66,7 +66,7 @@ from vllm_omni.entrypoints.omni import Omni if __name__ == "__main__": omni = Omni( model="Tongyi-MAI/Z-Image-Turbo", - # stage_configs_path="./stage-config.yaml", # See below + # deploy_config="./deploy-config.yaml", # Optional deploy override ) prompts = [ "a cup of coffee on a table", diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index 11c943ca4..e20390385 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -688,7 +688,7 @@ for result in response.json()["results"]: |-----------|--------|---------|-------------| | `tts_batch_max_items` | engine kwarg | 32 | Maximum number of items per batch request | -All items are fanned out to `generate()` concurrently. The engine's stage worker automatically batches them up to the configured `max_batch_size` and queues the rest — no client-side throttling needed. +All items are fanned out to `generate()` concurrently. The engine's stage worker automatically batches them up to the configured `max_num_seqs` and queues the rest — no client-side throttling needed. For best throughput, set both stages' `max_num_seqs` above 1 via `--stage-overrides`. On the current Qwen3-TTS CustomVoice benchmark, stage 1 performed best at `max_num_seqs: 10`: diff --git a/docs/serving/video_stream_api.md b/docs/serving/video_stream_api.md index 88f74affc..688f5fe9c 100644 --- a/docs/serving/video_stream_api.md +++ b/docs/serving/video_stream_api.md @@ -10,7 +10,7 @@ Each server instance runs a single model specified at startup with `vllm serve < ```bash vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \ - --deploy-config vllm_omni/deploy/qwen3_omni.yaml \ + --deploy-config vllm_omni/deploy/qwen3_omni_moe.yaml \ --omni \ --port 8000 \ --trust-remote-code diff --git a/docs/user_guide/diffusion/execution_modes.md b/docs/user_guide/diffusion/execution_modes.md index 9611a8210..ae2f868d8 100644 --- a/docs/user_guide/diffusion/execution_modes.md +++ b/docs/user_guide/diffusion/execution_modes.md @@ -207,18 +207,16 @@ outputs = omni.generate( ) ``` -## Stage Configuration +## Deploy Configuration Use the equivalent engine arguments in a deployment YAML: ```yaml -stage_args: +stages: - stage_id: 0 - stage_type: diffusion - engine_args: - step_execution: false - max_num_seqs: 4 - request_batch_max_wait_ms: 20 + step_execution: false + max_num_seqs: 4 + request_batch_max_wait_ms: 20 ``` For step execution, set `step_execution: true` and remove diff --git a/docs/user_guide/diffusion/regional_compilation.md b/docs/user_guide/diffusion/regional_compilation.md index 469a62b2c..448f19518 100644 --- a/docs/user_guide/diffusion/regional_compilation.md +++ b/docs/user_guide/diffusion/regional_compilation.md @@ -24,7 +24,7 @@ stages: For an experimental whole-transformer compile scope, set `--diffusion-compile-granularity full` or use -`diffusion_compile_granularity: full` in the stage configuration. Full scope may +`diffusion_compile_granularity: full` in the deploy configuration. Full scope may still contain graph breaks; it does not force one graph. It is rejected when HSDP, sequence parallelism, CPU offload, or layerwise offload is enabled. Use regional scope with those features. diff --git a/docs/user_guide/examples/offline_inference/mimo_audio.md b/docs/user_guide/examples/offline_inference/mimo_audio.md index a43d8e408..f1b295bc5 100644 --- a/docs/user_guide/examples/offline_inference/mimo_audio.md +++ b/docs/user_guide/examples/offline_inference/mimo_audio.md @@ -22,7 +22,7 @@ MiMo-Audio provides multiple task variants for audio understanding and generatio ## Setup -Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup. +See the [pipeline and deploy configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation for your hardware. ### Environment Variables @@ -188,7 +188,7 @@ Note: This task uses hardcoded message lists in the script. ### Other -- If the model or stage config fails to load, check [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) for memory and GPU settings. +- If the model or deploy config fails to load, check the [pipeline and deploy configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) for memory and GPU settings. - For errors when reading/writing WAV (e.g. unsupported format), ensure input files are standard WAV/MP3 and that `soundfile` is linked to a working libsndfile (see above). ## Notes diff --git a/docs/user_guide/examples/offline_inference/qwen2_5_omni.md b/docs/user_guide/examples/offline_inference/qwen2_5_omni.md index efd14bc1c..c11cc6189 100644 --- a/docs/user_guide/examples/offline_inference/qwen2_5_omni.md +++ b/docs/user_guide/examples/offline_inference/qwen2_5_omni.md @@ -4,7 +4,7 @@ Source **Note**: The synchronous ``end2end.py`` (using ``Omni``) is still the diff --git a/docs/user_guide/examples/offline_inference/text_to_speech.md b/docs/user_guide/examples/offline_inference/text_to_speech.md index ad7777014..a6544c1e5 100644 --- a/docs/user_guide/examples/offline_inference/text_to_speech.md +++ b/docs/user_guide/examples/offline_inference/text_to_speech.md @@ -69,7 +69,7 @@ python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \ ### Notes - Output: 48 kHz mono WAV. -- Stage config: `vllm_omni/model_executor/stage_configs/voxcpm2.yaml` (default). +- Deploy config: `vllm_omni/deploy/voxcpm2.yaml` (default). --- @@ -152,7 +152,7 @@ python examples/offline_inference/text_to_speech/fish_speech/end2end.py \ --text "Hello, this is a streaming test." \ --streaming ``` -Streaming requires `async_chunk: true` in the stage config. +Streaming requires `async_chunk: true` in the deploy config. ### Notes - Output: 44.1 kHz mono WAV. @@ -274,7 +274,7 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \ --streaming \ --output-dir /tmp/out_stream ``` -Streaming requires `async_chunk: true` in the stage config. +Streaming requires `async_chunk: true` in the deploy config. ### Batched decoding The Code2Wav stage supports batched decoding through the SpeechTokenizer. Configure both stages with `max_num_seqs > 1` via `--stage-overrides` and pass multiple prompts via `--txt-prompts`: diff --git a/docs/user_guide/examples/online_serving/qwen3_omni.md b/docs/user_guide/examples/online_serving/qwen3_omni.md index b4c05bdbe..6ef10b5fd 100644 --- a/docs/user_guide/examples/online_serving/qwen3_omni.md +++ b/docs/user_guide/examples/online_serving/qwen3_omni.md @@ -271,7 +271,7 @@ The script supports the following arguments: vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 ``` -If you have custom stage configs file: +If you have a custom deploy config file: ```bash vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --deploy-config /path/to/deploy_config_file ``` diff --git a/docs/user_guide/examples/online_serving/text_to_speech.md b/docs/user_guide/examples/online_serving/text_to_speech.md index c6a613723..5d4ccbf3b 100644 --- a/docs/user_guide/examples/online_serving/text_to_speech.md +++ b/docs/user_guide/examples/online_serving/text_to_speech.md @@ -419,7 +419,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \ "response_format": "pcm" }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 - ``` -Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming. +Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` in the deploy config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming. ### Streaming WebSocket The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`: diff --git a/docs/user_guide/quantization/fp8.md b/docs/user_guide/quantization/fp8.md index c57642587..010ef4ba7 100644 --- a/docs/user_guide/quantization/fp8.md +++ b/docs/user_guide/quantization/fp8.md @@ -92,7 +92,7 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)]) | FLUX.1 | `black-forest-labs/FLUX.1-dev`, `black-forest-labs/FLUX.1-schnell` | Yes | Yes | All layers | None | | | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | Yes | Not validated | All layers | None | ✅︎ | | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B` | Yes | Yes | All layers | None | | -| HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | All layers; use the Hunyuan stage config for multi-stage runs | None | | +| HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | All layers; use the Hunyuan deploy config for multi-stage runs | None | | | HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None | | | Cosmos3 | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | Yes | Not validated | All layers | None | | | MiniMax-H3 | `MiniMaxAI/MiniMax-H3` (`FL2VA` / `Ref2VA`) | Yes | Not validated | `quantization="fp8"` quantizes eligible DiT and text-encoder linears; mixed-precision input/output heads stay FP32 | None | ✅︎ | diff --git a/docs/user_guide/quantization/msmodelslim.md b/docs/user_guide/quantization/msmodelslim.md index c03628b9d..ea42f6250 100644 --- a/docs/user_guide/quantization/msmodelslim.md +++ b/docs/user_guide/quantization/msmodelslim.md @@ -92,4 +92,4 @@ For HunyuanImage-3.0, use the Hunyuan-specific adaptation linked above. 2. The `ascend` quantization method expects weights produced by the Ascend tooling; it is not a load-time CUDA quantizer. 3. Keep the quantized checkpoint aligned with the same model architecture and - stage config used for BF16 inference. + pipeline and deploy config used for BF16 inference. diff --git a/docs/user_guide/quantization/quantized_kvcache.md b/docs/user_guide/quantization/quantized_kvcache.md index e560032d3..99a90e08a 100644 --- a/docs/user_guide/quantization/quantized_kvcache.md +++ b/docs/user_guide/quantization/quantized_kvcache.md @@ -83,23 +83,23 @@ Online serving: vllm serve --omni --diffusion-kv-cache-dtype fp8 ``` -Stage config: +Deploy config: ```yaml -stage_args: +stages: - stage_id: 0 - stage_type: diffusion - engine_args: - model_stage: dit - diffusion_kv_cache_dtype: "fp8" - diffusion_kv_cache_skip_steps: "0,1" - diffusion_kv_cache_skip_layers: "0-2" + diffusion_kv_cache_dtype: "fp8" + diffusion_kv_cache_skip_steps: "0,1" + diffusion_kv_cache_skip_layers: "0-2" ``` -Legacy YAML keys `kv_cache_dtype`, `kv_cache_skip_steps`, and -`kv_cache_skip_layers` are still accepted when constructing -`OmniDiffusionConfig` (for example via `from_kwargs`); prefer the `diffusion_*` -names for new configs. +The `model_stage` and diffusion execution type belong to the registered +`PipelineConfig`; the deploy YAML only carries runtime overrides. + +The legacy keyword aliases `kv_cache_dtype`, `kv_cache_skip_steps`, and +`kv_cache_skip_layers` remain accepted when constructing +`OmniDiffusionConfig` directly. They are not deploy YAML fields; prefer the +`diffusion_*` names for new code. ## Parameters diff --git a/examples/offline_inference/covo_audio/README.md b/examples/offline_inference/covo_audio/README.md index 80a822fa2..083666ebd 100644 --- a/examples/offline_inference/covo_audio/README.md +++ b/examples/offline_inference/covo_audio/README.md @@ -2,7 +2,8 @@ ## Setup -Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup. +Use `--deploy-config` for deployment overrides such as stage memory allocation. +See the [pipeline and deploy configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/). > **Note** > Covo-Audio code2wav requires `torchdiffeq`. Install it with: `pip install torchdiffeq` @@ -41,7 +42,7 @@ python end2end.py -m /path/to/Covo-Audio-Chat --output-dir ./my_output | `--sampling-rate` | | `16000` | Sampling rate for audio loading (Hz) | | `--output-dir` | | `./output_audio` | Output directory for generated files | | `--num-prompts` | | `1` | Number of prompts to generate | -| `--stage-configs-path` | | (auto) | Path to stage configs YAML file | +| `--deploy-config` | | (auto) | Optional path to a deploy YAML override | | `--log-stats` | | `false` | Enable detailed statistics logging | | `--stage-init-timeout` | | `300` | Stage initialization timeout (seconds) | | `--batch-timeout` | | `5` | Batching timeout (seconds) | diff --git a/examples/offline_inference/dynin_omni/README.md b/examples/offline_inference/dynin_omni/README.md index 19fa8c69e..6cf834142 100644 --- a/examples/offline_inference/dynin_omni/README.md +++ b/examples/offline_inference/dynin_omni/README.md @@ -107,4 +107,4 @@ python /examples/offline_inference/dynin_omni/end2end.py \ - If you want to force local config resolution, pass `--dynin-config-path `. - If you see the warning `max_num_batched_tokens (32768) exceeds max_num_seqs * max_model_len (4096)`, - reduce `max_num_batched_tokens` in stage config (for example, `4096` in CI config). + reduce `max_num_batched_tokens` in the deploy config (for example, `4096` in the CI config). diff --git a/examples/offline_inference/hunyuan_image3/README.md b/examples/offline_inference/hunyuan_image3/README.md index 8b90e6b7f..b4d5af2c7 100644 --- a/examples/offline_inference/hunyuan_image3/README.md +++ b/examples/offline_inference/hunyuan_image3/README.md @@ -7,17 +7,17 @@ YAMLs under `vllm_omni/deploy/`. | File | Topology | Default use | | :--- | :--- | :--- | -| `vllm_omni/deploy/hunyuan_image3.yaml` | AR + DiT | Default for `text2img` and `img2img`. | +| `vllm_omni/deploy/hunyuan_image_3_moe.yaml` | AR + DiT | Default for `text2img` and `img2img`. | | `vllm_omni/deploy/hunyuan_image3_ar.yaml` | AR only | Default for `img2text` and `text2text`. | | `vllm_omni/deploy/hunyuan_image3_dit.yaml` | DiT only | Standalone diffusion stage. Pass it explicitly with `--deploy-config`. | -The example chooses a deploy config automatically when `--deploy-config` and -`--stage-configs-path` are both omitted: +The example chooses a deploy config automatically when `--deploy-config` is +omitted: | `--modality` | `mode` passed to Omni | Default deploy | | :--- | :--- | :--- | -| `text2img` | `text-to-image` | `hunyuan_image3.yaml` | -| `img2img` | `image-editing` | `hunyuan_image3.yaml` | +| `text2img` | `text-to-image` | `hunyuan_image_3_moe.yaml` | +| `img2img` | `image-editing` | `hunyuan_image_3_moe.yaml` | | `img2text` | `image-to-text` | `hunyuan_image3_ar.yaml` | | `text2text` | `text-to-text` | `hunyuan_image3_ar.yaml` | @@ -35,8 +35,8 @@ path; the image endpoints infer the image task from the endpoint and payload. | Online scenario | Server deploy | Request | | :--- | :--- | :--- | -| Text to image | `--deploy-config vllm_omni/deploy/hunyuan_image3.yaml` | `POST /v1/images/generations`, or `POST /v1/chat/completions` with `"modalities": ["image"]`. | -| Image editing | `--deploy-config vllm_omni/deploy/hunyuan_image3.yaml` | `POST /v1/images/edits`. | +| Text to image | `--deploy-config vllm_omni/deploy/hunyuan_image_3_moe.yaml` | `POST /v1/images/generations`, or `POST /v1/chat/completions` with `"modalities": ["image"]`. | +| Image editing | `--deploy-config vllm_omni/deploy/hunyuan_image_3_moe.yaml` | `POST /v1/images/edits`. | | Image/text to text | `--deploy-config vllm_omni/deploy/hunyuan_image3_ar.yaml` | `POST /v1/chat/completions` for text output, for example with `"modalities": ["text"]`. | | DiT-only image generation | `--deploy-config vllm_omni/deploy/hunyuan_image3_dit.yaml` | `POST /v1/images/generations`. | @@ -96,7 +96,7 @@ Override the default full AR + DiT deploy explicitly: python examples/offline_inference/hunyuan_image3/end2end.py \ --model tencent/HunyuanImage-3.0-Instruct \ --modality text2img \ - --deploy-config vllm_omni/deploy/hunyuan_image3.yaml \ + --deploy-config vllm_omni/deploy/hunyuan_image_3_moe.yaml \ --prompts "A cute cat" ``` @@ -117,8 +117,7 @@ python end2end.py --modality text2img \ | Argument | Description | | :--- | :--- | -| `--deploy-config` | Preferred config path for unified deploy YAMLs. | -| `--stage-configs-path` | Legacy stage config path, kept only for compatibility. Prefer `--deploy-config`. | +| `--deploy-config` | Optional path for unified deploy YAML overrides. | | `--additional-config` | JSON object forwarded to diffusion worker `additional_config`. | | `--modality` | Offline-only convenience flag. One of `text2img`, `img2img`, `img2text`, `text2text`. It selects prompt formatting, internal `mode`, and default deploy config for this script. Online serving uses `--deploy-config` plus the endpoint and, for chat completions, request `modalities` instead. | | `--steps` | Number of diffusion inference steps for image generation. | @@ -132,7 +131,7 @@ python end2end.py --modality text2img \ - `hunyuan_image3_ar.yaml` is a 4-card AR-only text/comprehension deploy. - `hunyuan_image3_dit.yaml` is a single-stage DiT deploy with `stage_id: 0`. -- The old HunyuanImage3 YAMLs under `model_executor/stage_configs/` and `platforms/*/stage_configs/` have been folded into the deploy YAMLs. +- Platform-specific overrides are included in the unified deploy YAMLs. ## Prompt Format diff --git a/examples/offline_inference/mimo_audio/README.md b/examples/offline_inference/mimo_audio/README.md index 60f433a4f..126896259 100644 --- a/examples/offline_inference/mimo_audio/README.md +++ b/examples/offline_inference/mimo_audio/README.md @@ -19,7 +19,7 @@ MiMo-Audio provides multiple task variants for audio understanding and generatio ## Setup -Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup. +See the [pipeline and deploy configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation for your hardware. ### Environment Variables @@ -189,7 +189,7 @@ Note: This task uses hardcoded message lists in the script. ### Other -- If the model or stage config fails to load, check [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) for memory and GPU settings. +- If the model or deploy config fails to load, check the [pipeline and deploy configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) for memory and GPU settings. - For errors when reading/writing WAV (e.g. unsupported format), ensure input files are standard WAV/MP3 and that `soundfile` is linked to a working libsndfile (see above). ## Notes diff --git a/examples/offline_inference/ming_flash_omni/README.md b/examples/offline_inference/ming_flash_omni/README.md index ee7e93fd0..29fd59a8d 100644 --- a/examples/offline_inference/ming_flash_omni/README.md +++ b/examples/offline_inference/ming_flash_omni/README.md @@ -14,7 +14,7 @@ For standalone TTS (talker only), see the [Ming-flash-omni-TTS section in the Te ## Setup -Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup. +See the [pipeline and deploy configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation for your hardware. When no `--deploy-config` is passed, the model registry auto-loads the full thinker+talker `vllm_omni/deploy/ming_flash_omni.yaml` (See [Omni-Speech](#omni-speech-thinker--talker)). diff --git a/examples/offline_inference/minicpmo/README.md b/examples/offline_inference/minicpmo/README.md index b80fc0aa6..b827a4e98 100644 --- a/examples/offline_inference/minicpmo/README.md +++ b/examples/offline_inference/minicpmo/README.md @@ -8,7 +8,7 @@ Two-stage pipeline: **thinker** (multimodal understanding) → **talker + Token2 - Install talker deps: `pip install stepaudio2-minicpmo` - `--trust-remote-code` is always passed by `end2end.py` (required for MiniCPMO) -- See [stage configuration docs](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) for memory tuning +- See [pipeline and deploy configuration docs](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) for memory tuning ## Run examples diff --git a/examples/offline_inference/qwen2_5_omni/README.md b/examples/offline_inference/qwen2_5_omni/README.md index 059c10226..490711d34 100644 --- a/examples/offline_inference/qwen2_5_omni/README.md +++ b/examples/offline_inference/qwen2_5_omni/README.md @@ -1,7 +1,7 @@ # Qwen2.5-Omni: Offline inference ## Setup -Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup. +See the [pipeline and deploy configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation for your hardware. ## Run examples diff --git a/examples/offline_inference/qwen3_omni/README.md b/examples/offline_inference/qwen3_omni/README.md index 2f4ee11cc..d216042b4 100644 --- a/examples/offline_inference/qwen3_omni/README.md +++ b/examples/offline_inference/qwen3_omni/README.md @@ -1,7 +1,8 @@ # Qwen3-Omni: Offline inference ## Setup -Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup. +Use `--deploy-config` for deployment overrides such as stage memory allocation. +See the [pipeline and deploy configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/). ## Run examples @@ -70,8 +71,8 @@ For true stage-level concurrency -- where downstream stages (Talker, Code2Wav) start **before** the upstream stage (Thinker) finishes -- use the async_chunk example. This requires: -1. A deploy config YAML with ``async_chunk: true`` (e.g. - ``qwen3_omni_moe.yaml``). +1. A deploy YAML with ``async_chunk: true`` (for example, an overlay based on + ``vllm_omni/deploy/qwen3_omni_moe.yaml``). 2. Hardware that matches the config (e.g. 2x H100 for the default 3-stage config). @@ -97,11 +98,11 @@ bash run_multiple_prompts_async_chunk.sh --max-in-flight 4 python end2end_async_chunk.py --query-type text --modalities text ``` -#### Custom stage config +#### Custom deploy config ```bash python end2end_async_chunk.py \ --query-type use_audio \ - --deploy-config /path/to/your_deploy_config.yaml + --deploy-config /path/to/your_async_chunk.yaml ``` > **Note**: The synchronous ``end2end.py`` (using ``Omni``) is still the diff --git a/examples/offline_inference/qwen3_omni/end2end_async_chunk.py b/examples/offline_inference/qwen3_omni/end2end_async_chunk.py index c224586c4..c4570abae 100644 --- a/examples/offline_inference/qwen3_omni/end2end_async_chunk.py +++ b/examples/offline_inference/qwen3_omni/end2end_async_chunk.py @@ -386,8 +386,8 @@ async def run_all(args): # argparse defaults do not silently override deploy YAML values. async_omni = AsyncOmni.from_cli_args(args, model=args.model) - # Use default sampling params from stage config (they are pre-configured - # in the YAML for each stage). + # Use default sampling params from the resolved pipeline and deploy + # config. # # NOTE: Since we do not set the sampling params directly, .generate in # will automatically set the output kind to delta, since this is what diff --git a/examples/offline_inference/step_audio2/README.md b/examples/offline_inference/step_audio2/README.md index abd21f371..7457863f5 100644 --- a/examples/offline_inference/step_audio2/README.md +++ b/examples/offline_inference/step_audio2/README.md @@ -29,7 +29,7 @@ Step-Audio2 is a two-stage audio model: **Notes:** - Single GPU mode requires high VRAM due to both stages sharing memory - Multi GPU mode separates Stage 0 (Thinker) and Stage 1 (Token2Wav) across GPUs -- VRAM usage can be adjusted via `gpu_memory_utilization` in stage config +- VRAM usage can be adjusted via `gpu_memory_utilization` in the deploy config ## Performance Benchmark @@ -197,11 +197,6 @@ This mode: ### Advanced Options ```bash -# Use custom stage configuration -python end2end.py --query-type audio_to_text \ - --model stepfun-ai/Step-Audio-2-mini \ - --stage-configs-path /path/to/custom_config.yaml - # Use custom deploy configuration python end2end.py --query-type audio_to_text \ --model stepfun-ai/Step-Audio-2-mini \ diff --git a/examples/offline_inference/text_to_speech/README.md b/examples/offline_inference/text_to_speech/README.md index 12f1967ee..dc4f01d57 100644 --- a/examples/offline_inference/text_to_speech/README.md +++ b/examples/offline_inference/text_to_speech/README.md @@ -156,7 +156,7 @@ python examples/offline_inference/text_to_speech/fish_speech/end2end.py \ --text "Hello, this is a streaming test." \ --streaming ``` -Streaming requires `async_chunk: true` in the stage config. +Streaming requires `async_chunk: true` in the deploy config. ### Notes - Output: 44.1 kHz mono WAV. @@ -371,7 +371,7 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \ --streaming \ --output-dir /tmp/out_stream ``` -Streaming requires `async_chunk: true` in the stage config. +Streaming requires `async_chunk: true` in the deploy config. ### Word Timestamps Word-level timestamps are currently a serving-path feature: launch @@ -380,13 +380,13 @@ Word-level timestamps are currently a serving-path feature: launch example is not provided in this release. ### Batched decoding -The Code2Wav stage supports batched decoding through the SpeechTokenizer. Pass multiple prompts via `--txt-prompts` and set `--batch-size` accordingly. To raise `max_num_seqs` on either stage, point `--stage-configs-path` at a stage configs YAML with the desired values (see `vllm_omni/model_executor/stage_configs/` for templates): +The Code2Wav stage supports batched decoding through the SpeechTokenizer. Pass multiple prompts via `--txt-prompts` and set `--batch-size` accordingly. To raise `max_num_seqs` on either stage, create an overlay based on `vllm_omni/deploy/qwen3_tts.yaml` and pass it with `--deploy-config`: ```bash python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \ --query-type CustomVoice \ --txt-prompts examples/offline_inference/text_to_speech/qwen3_tts/benchmark_prompts.txt \ --batch-size 4 \ - --stage-configs-path /path/to/qwen3_tts_batched.yaml + --deploy-config /path/to/qwen3_tts_batched.yaml ``` `--batch-size` must match a CUDA-graph capture size (1, 2, 4, 8, 16…). diff --git a/examples/offline_inference/text_to_speech/ming_tts/README.md b/examples/offline_inference/text_to_speech/ming_tts/README.md index 3fdfaf4d5..87f2df1f5 100644 --- a/examples/offline_inference/text_to_speech/ming_tts/README.md +++ b/examples/offline_inference/text_to_speech/ming_tts/README.md @@ -131,7 +131,7 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \ --enforce-eager ``` -`--streaming` uses `AsyncOmni` and the async_chunk stage config. It currently +`--streaming` uses `AsyncOmni` and the async chunk deploy config. It currently supports one prompt per process invocation; use blocking mode for `--num-prompts > 1`. diff --git a/examples/offline_inference/text_to_speech/voxtral_tts/end2end.py b/examples/offline_inference/text_to_speech/voxtral_tts/end2end.py index c9225c212..e67e9c350 100644 --- a/examples/offline_inference/text_to_speech/voxtral_tts/end2end.py +++ b/examples/offline_inference/text_to_speech/voxtral_tts/end2end.py @@ -304,7 +304,7 @@ def parse_args() -> Namespace: "--cfg-alpha", type=float, default=None, - help="CFG alpha for flow-matching guidance (default: use value from stage config, typically 1.2).", + help="CFG alpha for flow-matching guidance (default: use the deploy config value, typically 1.2).", ) parser.add_argument( "--quantization", diff --git a/examples/online_serving/mimo_audio/openai_chat_completion_client_for_multimodal_generation.py b/examples/online_serving/mimo_audio/openai_chat_completion_client_for_multimodal_generation.py index 3030fa3e3..f2aa159cf 100644 --- a/examples/online_serving/mimo_audio/openai_chat_completion_client_for_multimodal_generation.py +++ b/examples/online_serving/mimo_audio/openai_chat_completion_client_for_multimodal_generation.py @@ -328,7 +328,7 @@ def run_multimodal_generation(args) -> None: extra_body = { "sampling_params_list": sampling_params_list - # Optional, it has a default setting in stage_configs of the corresponding model. + # Optional; defaults come from the resolved pipeline and deploy config. } # Build messages list (filter None so concurrent tasks get valid structure) diff --git a/examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py b/examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py index 63a3f9ec4..fd74ebe71 100644 --- a/examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py +++ b/examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py @@ -415,7 +415,7 @@ def run_multimodal_generation(args, client: OpenAI) -> None: audio_path=audio_path, ) extra_body = { - # Optional, it has default settings in stage configs. you can override them here. + # Optional; defaults come from the resolved pipeline and deploy config. } if args.query_type == "use_audio_in_video": extra_body["mm_processor_kwargs"] = {"use_audio_in_video": True} diff --git a/examples/online_serving/personaplex/README.md b/examples/online_serving/personaplex/README.md index cf105dd6c..3b737d275 100644 --- a/examples/online_serving/personaplex/README.md +++ b/examples/online_serving/personaplex/README.md @@ -38,7 +38,7 @@ full-duplex control plane: CUDA_VISIBLE_DEVICES=0 python -m vllm_omni.entrypoints.cli.main serve \ /path/to/personaplex-7b-v1 \ --omni \ - --stage-configs-path vllm_omni/deploy/personaplex.yaml + --deploy-config vllm_omni/deploy/personaplex.yaml ``` Validate the actual `/v1/realtime?duplex=1` scheduler path with paced 24 kHz diff --git a/examples/online_serving/qwen2_5_omni/run_curl_multimodal_generation.sh b/examples/online_serving/qwen2_5_omni/run_curl_multimodal_generation.sh index ed09bac14..d0ed84b85 100644 --- a/examples/online_serving/qwen2_5_omni/run_curl_multimodal_generation.sh +++ b/examples/online_serving/qwen2_5_omni/run_curl_multimodal_generation.sh @@ -51,7 +51,7 @@ code2wav_sampling_params='{ "detokenize": true, "repetition_penalty": 1.1 }' -# Above is optional, it has a default setting in stage_configs of the corresponding model. +# The block above is optional; defaults come from the resolved pipeline and deploy config. # Define URLs for assets MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg" diff --git a/examples/online_serving/qwen3_omni/README.md b/examples/online_serving/qwen3_omni/README.md index 74a18af32..0403790cc 100644 --- a/examples/online_serving/qwen3_omni/README.md +++ b/examples/online_serving/qwen3_omni/README.md @@ -513,7 +513,7 @@ The script supports the following arguments: vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 ``` -If you have custom stage configs file: +If you have a custom deploy config file: ```bash vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --deploy-config /path/to/deploy_config_file ``` diff --git a/examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh b/examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh index 399c65481..0a059a03e 100644 --- a/examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh +++ b/examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh @@ -50,7 +50,7 @@ code2wav_sampling_params='{ "detokenize": true, "repetition_penalty": 1.1 }' -# Above is optional, it has a default setting in stage_configs of the corresponding model. +# The block above is optional; defaults come from the resolved pipeline and deploy config. # Define URLs for assets MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg" diff --git a/examples/online_serving/replica_data_parallel/bench_replica_dp.py b/examples/online_serving/replica_data_parallel/bench_replica_dp.py index 977f475ac..d17899b78 100755 --- a/examples/online_serving/replica_data_parallel/bench_replica_dp.py +++ b/examples/online_serving/replica_data_parallel/bench_replica_dp.py @@ -15,7 +15,7 @@ so an output is a deterministic function of its own input and nothing else. Measuring replica scaling ------------------------- This client is replica-agnostic. Vary the *server's* replica count -(`runtime.num_replicas` in the stage config; see run_server.sh) and keep the +(`stages[].num_replicas` in the deploy config; see run_server.sh) and keep the client workload fixed, then compare throughput: N=1 -> baseline diff --git a/examples/online_serving/step_audio2/README.md b/examples/online_serving/step_audio2/README.md index 25e2a6785..ba2797dae 100644 --- a/examples/online_serving/step_audio2/README.md +++ b/examples/online_serving/step_audio2/README.md @@ -189,5 +189,5 @@ Async chunk reduces TTFP by **67%** by streaming audio token chunks from Thinker - Check server logs for errors ### Out of memory -- Reduce `gpu_memory_utilization` in stage configs +- Reduce `gpu_memory_utilization` in the deploy config - Use a smaller batch size diff --git a/examples/online_serving/text_to_speech/README.md b/examples/online_serving/text_to_speech/README.md index 1ea191a37..4127eb6ef 100644 --- a/examples/online_serving/text_to_speech/README.md +++ b/examples/online_serving/text_to_speech/README.md @@ -570,7 +570,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \ "response_format": "pcm" }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 - ``` -Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming. +Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` in the deploy config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming. ### Streaming WebSocket The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`: diff --git a/recipes/Bagel/BAGEL-7B-MoT.md b/recipes/Bagel/BAGEL-7B-MoT.md index 4c2d75aca..3b4b6c594 100644 --- a/recipes/Bagel/BAGEL-7B-MoT.md +++ b/recipes/Bagel/BAGEL-7B-MoT.md @@ -57,14 +57,14 @@ move the diffusion stage to a second GPU in a custom deploy config. #### Offline Commands Run text-to-image with the shared offline example from the repository root. -Pick the topology with `--stage-configs-path` and forward BAGEL-specific +Pick the topology with `--deploy-config` and forward BAGEL-specific generation parameters as a JSON object through `--extra-body`: ```bash # Two-stage (Thinker + DiT), shares one GPU by default python examples/offline_inference/text_to_image/text_to_image.py \ --model ByteDance-Seed/BAGEL-7B-MoT \ - --stage-configs-path vllm_omni/deploy/bagel.yaml \ + --deploy-config vllm_omni/deploy/bagel.yaml \ --prompt "A beautiful sunset over mountains" \ --height 1024 \ --width 1024 \ @@ -79,7 +79,7 @@ python examples/offline_inference/text_to_image/text_to_image.py \ # Single-stage (DiT only, with internal LLM/ViT/VAE) python examples/offline_inference/text_to_image/text_to_image.py \ --model ByteDance-Seed/BAGEL-7B-MoT \ - --stage-configs-path vllm_omni/deploy/bagel_single_stage.yaml \ + --deploy-config vllm_omni/deploy/bagel_single_stage.yaml \ --prompt "A beautiful sunset over mountains" \ --height 1024 \ --width 1024 \ diff --git a/recipes/GLM/GLM-Image.md b/recipes/GLM/GLM-Image.md index bd11558c9..df6b512cd 100644 --- a/recipes/GLM/GLM-Image.md +++ b/recipes/GLM/GLM-Image.md @@ -52,7 +52,7 @@ Start the server from the repository root: vllm serve zai-org/GLM-Image --omni --port 8091 ``` -To use the bundled stage config explicitly (same default as above): +To use the bundled deploy config explicitly (same default as above): ```bash vllm serve zai-org/GLM-Image \ @@ -106,7 +106,7 @@ Overall summary from the run’s metrics. Rough wall-time split: **Stage 0 (AR)* #### Notes - Memory usage: Roughly **~38 GiB + KV** on Stage 0 (AR) and **~20 GiB** on Stage 1 (DiT+VAE) per the user guide; two 80 GB cards match the default split. -- Key flags: `--omni` is required; `--stage-configs-path` is optional unless you use a custom YAML (for example single-GPU). +- Key flags: `--omni` is required; `--deploy-config` is optional unless you use a custom YAML (for example single-GPU). - Keep **Transformers ≥ 5.5.1** (this recipe used **5.5.4**) so `glm_image` configs resolve; otherwise Stage 0 can fail at `ModelConfig` validation. - Known limitations: This starter recipe follows the dual-GPU online path documented under `examples/online_serving/glm_image`. The first request may be slower due to warmup. - Generation time: about **61 s** wall time end-to-end for the sample above (50 inference steps, 1024×1024). diff --git a/recipes/Helios/Helios.md b/recipes/Helios/Helios.md index fdc0660bb..f1c11ed92 100644 --- a/recipes/Helios/Helios.md +++ b/recipes/Helios/Helios.md @@ -79,7 +79,7 @@ Total generation time: seconds ( ms) Saved generated video to helios_t2v_base.mp4 ``` -#### Important flags and stage configs +#### Important flags and deploy config - `--model BestWishYsh/Helios-Base` selects the base Helios checkpoint used by this recipe. @@ -88,7 +88,7 @@ Saved generated video to helios_t2v_base.mp4 - `--cache-backend cache_dit` enables the cache-dit acceleration path. - `--enable-cache-dit-summary` prints cache-dit summary information after diffusion forward passes. -- No separate stage config is required for this offline recipe; the shared +- No separate deploy config is required for this offline recipe; the shared `text_to_video.py` example configures the pipeline through its arguments. - Helios-specific knobs (declared in `vllm_omni/model_extras/helios.py`) are passed via the generic `--extra-body` JSON flag: diff --git a/recipes/NVIDIA/GR00T-N1.7.md b/recipes/NVIDIA/GR00T-N1.7.md index 0a5270222..826309835 100644 --- a/recipes/NVIDIA/GR00T-N1.7.md +++ b/recipes/NVIDIA/GR00T-N1.7.md @@ -44,7 +44,7 @@ vllm serve nvidia/GR00T-N1.7-3B \ --host 127.0.0.1 \ --port 8000 \ --served-model-name gr00t-n1d7 \ - --stage-configs-path vllm_omni/deploy/Gr00tN1d7.yaml + --deploy-config vllm_omni/deploy/Gr00tN1d7.yaml ``` Notes: diff --git a/recipes/NVIDIA/Nemotron-Labs-Audex.md b/recipes/NVIDIA/Nemotron-Labs-Audex.md index 315d31804..13be59f4c 100644 --- a/recipes/NVIDIA/Nemotron-Labs-Audex.md +++ b/recipes/NVIDIA/Nemotron-Labs-Audex.md @@ -54,7 +54,7 @@ NemotronH (~3B active parameters). Pick the pipeline by task: vllm serve nvidia/Nemotron-Labs-Audex-2B \ --host 0.0.0.0 --port 8097 \ --trust-remote-code --omni -# other modes: add --stage-configs-path vllm_omni/deploy/audex_{tta,thinker_only,s2s}.yaml +# other modes: add --deploy-config vllm_omni/deploy/audex_{tta,thinker_only,s2s}.yaml # or use the launcher: MODE=s2s PORT=8098 examples/online_serving/audex/run_server.sh ``` @@ -112,7 +112,7 @@ auto-resolves to the 2B-tuned config otherwise): vllm serve nvidia/Nemotron-Labs-Audex-30B-A3B \ --host 0.0.0.0 --port 8097 \ --trust-remote-code --omni \ - --stage-configs-path vllm_omni/deploy/audex_tts_30b.yaml + --deploy-config vllm_omni/deploy/audex_tts_30b.yaml # or: SIZE=30b MODE=tts examples/online_serving/audex/run_server.sh ``` diff --git a/vllm_omni/deploy/bagel_single_stage.yaml b/vllm_omni/deploy/bagel_single_stage.yaml index bcfbad253..fb0872af8 100644 --- a/vllm_omni/deploy/bagel_single_stage.yaml +++ b/vllm_omni/deploy/bagel_single_stage.yaml @@ -9,7 +9,7 @@ # --deploy-config vllm_omni/deploy/bagel_single_stage.yaml # # Or programmatically: -# Omni(model="...", deploy_config_path="vllm_omni/deploy/bagel_single_stage.yaml") +# Omni(model="...", deploy_config="vllm_omni/deploy/bagel_single_stage.yaml") pipeline: bagel_single_stage async_chunk: false diff --git a/vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml b/vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml index 360b72974..b6e5bc7fe 100644 --- a/vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml +++ b/vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml @@ -11,7 +11,7 @@ # # Or programmatically: # Omni(model="fishaudio/s2-pro", -# deploy_config_path="vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml") +# deploy_config="vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml") async_chunk: true enable_chunked_prefill: false diff --git a/vllm_omni/deploy/minicpmo_4_5.yaml b/vllm_omni/deploy/minicpmo_4_5.yaml index 04ddad608..532de5e79 100644 --- a/vllm_omni/deploy/minicpmo_4_5.yaml +++ b/vllm_omni/deploy/minicpmo_4_5.yaml @@ -32,7 +32,7 @@ stages: max_num_batched_tokens: 16384 devices: "0" # Required for MiniCPM interleaved image/audio Daily-Omni packs (~78% recipe). - # Keep these in YAML so they survive both --deploy-config and --stage-configs-path. + # Keep these in YAML so they are applied through --deploy-config. interleave_mm_strings: true media_io_kwargs: video: diff --git a/vllm_omni/deploy/moss_tts_nano.yaml b/vllm_omni/deploy/moss_tts_nano.yaml index 2c8fc54c0..e9eab184b 100644 --- a/vllm_omni/deploy/moss_tts_nano.yaml +++ b/vllm_omni/deploy/moss_tts_nano.yaml @@ -6,8 +6,8 @@ # * enforce_eager=true — the model uses upstream inference_stream() with # trust_remote_code; CUDA graph capture is not wired up. # * gpu_memory_utilization=0.3 — the AR LM + codec fit in ~2 GiB. -# * max_num_seqs=4 — mirrors the legacy stage_configs contract; per-request -# streaming generators are stored in self._stream_gens. +# * max_num_seqs=4 — supports per-request streaming generators stored in +# self._stream_gens. # # MOSS-TTS-Nano has no multimodal input preprocessing, so skip_mm_profiling # bypasses the dummy-MM pass at profile time. diff --git a/vllm_omni/entrypoints/cli/serve.py b/vllm_omni/entrypoints/cli/serve.py index 62ee10d94..cf80a2afa 100644 --- a/vllm_omni/entrypoints/cli/serve.py +++ b/vllm_omni/entrypoints/cli/serve.py @@ -829,7 +829,7 @@ class OmniServeCommand(CLISubcommand): "--tts-max-instructions-length", type=int, default=None, - help="Maximum length for TTS voice style instructions (overrides stage config, default: 500).", + help="Maximum length for TTS voice style instructions (overrides the pipeline default, default: 500).", ) # Disable safety guardrails for this server (currently only applicable for Cosmos3) diff --git a/vllm_omni/entrypoints/openai/protocol/audio.py b/vllm_omni/entrypoints/openai/protocol/audio.py index e309f1cb7..da529ab2a 100644 --- a/vllm_omni/entrypoints/openai/protocol/audio.py +++ b/vllm_omni/entrypoints/openai/protocol/audio.py @@ -521,7 +521,7 @@ class StreamingSpeechSessionConfig(BaseModel): initial_codec_chunk_frames: int | None = Field( default=None, ge=0, - description="Initial chunk size for reduced TTFA. Overrides stage config for this session.", + description="Initial chunk size for reduced TTFA. Overrides the deploy config for this session.", ) non_streaming_mode: bool | None = Field( default=None, diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index d408e11a9..752d93577 100755 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -1272,7 +1272,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): max_tokens is NOT computed dynamically — it uses the deploy YAML default. Args: - default_params: Default SamplingParams from stage config YAML. + default_params: Default SamplingParams from the resolved pipeline and deploy configuration. request: The chat completion request containing user-provided values. Returns: diff --git a/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py b/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py index 2af6abd42..4309d6177 100644 --- a/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py +++ b/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py @@ -284,14 +284,14 @@ class MiMoAudioDataParser(MultiModalDataParser): if not self.audio_tokenizer_path: raise ValueError( "Audio tokenizer path is not set. Provide " - "`model_config.audio_tokenizer_path` in the stage config " + "`model_config.audio_tokenizer_path` in the model configuration " "or export MIMO_AUDIO_TOKENIZER_PATH." ) if not os.path.exists(self.audio_tokenizer_path): raise ValueError( "Audio tokenizer not exists. Provide " - "`model_config.audio_tokenizer_path` in the stage config " + "`model_config.audio_tokenizer_path` in the model configuration " "or export MIMO_AUDIO_TOKENIZER_PATH." ) diff --git a/vllm_omni/model_executor/models/mimo_audio/mimo_audio_code2wav.py b/vllm_omni/model_executor/models/mimo_audio/mimo_audio_code2wav.py index 26482c6bc..019670cf9 100644 --- a/vllm_omni/model_executor/models/mimo_audio/mimo_audio_code2wav.py +++ b/vllm_omni/model_executor/models/mimo_audio/mimo_audio_code2wav.py @@ -459,7 +459,7 @@ class MiMoAudioToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP): if not self.audio_tokenizer_path: raise ValueError( "Audio tokenizer path is not set. Provide " - "`model_config.audio_tokenizer_path` in the stage config " + "`model_config.audio_tokenizer_path` in the model configuration " "or export MIMO_AUDIO_TOKENIZER_PATH." ) diff --git a/vllm_omni/platforms/npu/worker/base.py b/vllm_omni/platforms/npu/worker/base.py index ff1a860b0..5dd929ea1 100644 --- a/vllm_omni/platforms/npu/worker/base.py +++ b/vllm_omni/platforms/npu/worker/base.py @@ -45,7 +45,7 @@ class OmniNPUWorkerBase(NPUWorker): if self.profiler is None: raise RuntimeError( "Profiling is not enabled. For diffusion models, set --profiler-config via CLI. " - "For omni models, add profiler_config to your stage config file." + "For omni models, add profiler_config to the relevant deploy config stage." ) if is_start: from vllm_omni.profiler import OmniTorchProfilerWrapper diff --git a/vllm_omni/worker/base.py b/vllm_omni/worker/base.py index c1e229118..320ac2cae 100644 --- a/vllm_omni/worker/base.py +++ b/vllm_omni/worker/base.py @@ -74,7 +74,7 @@ class OmniGPUWorkerBase(GPUWorker): if self.profiler is None: raise RuntimeError( "Profiling is not enabled. For diffusion models, set --profiler-config via CLI. " - "For omni models, add profiler_config to your stage config file." + "For omni models, add profiler_config to the relevant deploy config stage." ) if is_start: from vllm_omni.profiler import OmniTorchProfilerWrapper diff --git a/vllm_omni/worker/gpu_generation_worker.py b/vllm_omni/worker/gpu_generation_worker.py index 38a88c84c..4a7aaf711 100644 --- a/vllm_omni/worker/gpu_generation_worker.py +++ b/vllm_omni/worker/gpu_generation_worker.py @@ -23,8 +23,8 @@ logger = init_logger(__name__) class GPUGenerationWorker(OmniWorkerMixin, OmniGPUWorkerBase): """GPU Worker for Generation model (non-autoregressive waveform generation). - Usage in stage config: - worker_cls: "vllm_omni.worker.gpu_generation_model_runner.GPUGenerationModelRunner" + Selected for stages whose pipeline topology uses + ``execution_type=StageExecutionType.LLM_GENERATION``. """ @instrument(span_name="Init device") diff --git a/vllm_omni/worker/gpu_memory_utils.py b/vllm_omni/worker/gpu_memory_utils.py index d005c5e58..1a8072c83 100644 --- a/vllm_omni/worker/gpu_memory_utils.py +++ b/vllm_omni/worker/gpu_memory_utils.py @@ -96,7 +96,7 @@ def get_process_gpu_memory(local_rank: int) -> int | None: except Exception as e: raise RuntimeError( f"Failed to get NVML handle for device '{device_id}' (local_rank={local_rank}). " - f"Check CUDA_VISIBLE_DEVICES or stage config 'devices' setting." + f"Check CUDA_VISIBLE_DEVICES or the deploy config `devices` setting." ) from e else: # No CUDA_VISIBLE_DEVICES or local_rank out of range: use index directly @@ -104,7 +104,7 @@ def get_process_gpu_memory(local_rank: int) -> int | None: if local_rank >= device_count: raise RuntimeError( f"Invalid GPU device {local_rank}. Only {device_count} GPU(s) available. " - f"Check CUDA_VISIBLE_DEVICES or stage config 'devices' setting." + f"Check CUDA_VISIBLE_DEVICES or the deploy config `devices` setting." ) handle = nvmlDeviceGetHandleByIndex(local_rank) From bd17618c11cf9362043b9d43dc744f99dc52dfb6 Mon Sep 17 00:00:00 2001 From: Joshna-Medisetty Date: Wed, 19 Aug 2026 18:06:33 -0700 Subject: [PATCH 077/267] [XPU][Bugfix] Make Omni AR async output path device-agnostic (#5569) Signed-off-by: Joshna Medisetty Co-authored-by: Cursor --- vllm_omni/platforms/xpu/utils.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/vllm_omni/platforms/xpu/utils.py b/vllm_omni/platforms/xpu/utils.py index 18437cae3..6eb314745 100644 --- a/vllm_omni/platforms/xpu/utils.py +++ b/vllm_omni/platforms/xpu/utils.py @@ -1,16 +1,27 @@ from contextlib import contextmanager +from functools import partial import torch +from vllm.utils.torch_utils import supports_xpu_graph @contextmanager def torch_cuda_wrapper(): - try: - # replace cuda APIs with xpu APIs, this should work by default - torch.cuda.Stream = torch.xpu.Stream - torch.cuda.default_stream = torch.xpu.current_stream - torch.cuda.current_stream = torch.xpu.current_stream - torch.cuda.stream = torch.xpu.stream - yield - finally: - pass + # Use partial() so callables differ from torch.xpu.* (Dynamo registration). + torch.cuda.Stream = torch.xpu.Stream + torch.cuda.default_stream = partial(torch.xpu.current_stream) + torch.cuda.current_stream = partial(torch.xpu.current_stream) + torch.cuda.stream = partial(torch.xpu.stream) + torch.cuda.set_stream = partial(torch.xpu.set_stream) + torch.cuda.set_device = partial(torch.xpu.set_device) + + def _xpu_event(*args, blocking=None, **kwargs): + return torch.xpu.Event(*args, **kwargs) + + torch.cuda.Event = _xpu_event + if supports_xpu_graph(): + torch.cuda.graph = partial(torch.xpu.graph) + torch.cuda.CUDAGraph = torch.xpu.XPUGraph + torch.cuda.graph_pool_handle = partial(torch.xpu.graph_pool_handle) + torch.cuda.is_current_stream_capturing = torch.xpu.is_current_stream_capturing + yield From 47b18936766ea8ca97e030dce9c70f2e79315fa1 Mon Sep 17 00:00:00 2001 From: Shenglei Fu <117230642+ShengleiFu@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:49:44 -0700 Subject: [PATCH 078/267] [Perf][CI] Reuse the Whisper judge worker across a test module (#6208) Signed-off-by: Shenglei Fu Co-authored-by: Hongsheng Liu --- tests/conftest.py | 1 + tests/helpers/fixtures/media.py | 24 +++ tests/helpers/media.py | 175 ++++++++++++---- tests/helpers/runtime.py | 21 +- tests/helpers/tests/test_media.py | 193 ++++++++++++++++-- .../tests/test_runtime_transcriber_release.py | 74 +++++++ 6 files changed, 428 insertions(+), 60 deletions(-) create mode 100644 tests/helpers/fixtures/media.py create mode 100644 tests/helpers/tests/test_runtime_transcriber_release.py diff --git a/tests/conftest.py b/tests/conftest.py index b453bbcbf..5c1276ec2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,6 +21,7 @@ pytest_plugins = ( "tests.helpers.fixtures.config", "tests.helpers.fixtures.env", "tests.helpers.fixtures.log", + "tests.helpers.fixtures.media", "tests.helpers.fixtures.run_args", "tests.helpers.fixtures.runtime", "tests.helpers.fixtures.speaker_cache", diff --git a/tests/helpers/fixtures/media.py b/tests/helpers/fixtures/media.py new file mode 100644 index 000000000..c34de093e --- /dev/null +++ b/tests/helpers/fixtures/media.py @@ -0,0 +1,24 @@ +"""Fixtures for the reused Whisper transcription worker.""" + +import sys + +import pytest + + +@pytest.fixture(autouse=True, scope="module") +def release_audio_transcriber_after_module(): + """Free the transcription worker's device memory once a test module is done. + + A backstop. The server and runner fixtures already release the worker when + each of their instances tears down, which is what keeps Whisper off the + device while the next parametrized server initializes. This covers the + remaining case: a module that transcribes without using those fixtures. + + Reached through ``sys.modules`` because ``tests/conftest.py`` deliberately + keeps ``tests.helpers.media`` lazy: a module that never transcribed should + not pay to import it. + """ + yield + media = sys.modules.get("tests.helpers.media") + if media is not None: + media.release_audio_transcriber() diff --git a/tests/helpers/media.py b/tests/helpers/media.py index 8eb82fc1f..ed8ef035e 100644 --- a/tests/helpers/media.py +++ b/tests/helpers/media.py @@ -3,7 +3,6 @@ import atexit import base64 import concurrent.futures -import gc import hashlib import io import logging @@ -14,7 +13,9 @@ import random import re import subprocess import tempfile +import threading import time +from concurrent.futures.process import BrokenProcessPool from contextlib import contextmanager from pathlib import Path from typing import Any @@ -670,59 +671,152 @@ def _serialize_whisper_model_download(model_size: str = "small"): f.close() -def _whisper_transcribe_in_current_process( - output_path: str, model_size: str = "small", language: str | None = None -) -> str: - import whisper - +def _select_whisper_device() -> str: device_index = None from vllm_omni.platforms import current_omni_platform if current_omni_platform.is_available(): n = current_omni_platform.get_device_count() # Single-GPU runners (e.g. the L4 nightly): the model server already - # occupies device 0. Loading Whisper there, once per concurrent - # request, competes for VRAM and OOMs. Only borrow an accelerator - # when a spare device exists; otherwise validate on CPU. + # occupies device 0, and a Whisper model resident there competes with + # it for VRAM and OOMs. Only borrow an accelerator when a spare device + # exists; otherwise validate on CPU. if n > 1: device_index = n - 1 - if device_index is not None: - torch_device = current_omni_platform.get_torch_device(device_index) - current_omni_platform.set_device(torch_device) - device = str(torch_device) - use_accelerator = True - else: - use_accelerator = False - device = "cpu" + if device_index is None: + return "cpu" - with _serialize_whisper_model_download(model_size): - model = whisper.load_model(model_size, device=device) - try: - text = model.transcribe( - output_path, - temperature=0.0, - word_timestamps=True, - condition_on_previous_text=False, - # None keeps whisper's auto-detection. Do not default this to a - # language: callers include non-English audio tests. - language=language, - )["text"] - finally: - del model - gc.collect() - if use_accelerator: - current_omni_platform.synchronize() - current_omni_platform.empty_cache() + torch_device = current_omni_platform.get_torch_device(device_index) + current_omni_platform.set_device(torch_device) + return str(torch_device) + + +# Populated in the transcription worker, not in the pytest process. +_WHISPER_MODELS: dict[str, Any] = {} + + +def _get_whisper_model(model_size: str) -> Any: + model = _WHISPER_MODELS.get(model_size) + if model is None: + import whisper + + # The device is picked on first load and the model stays on it for the + # worker's lifetime: the current server or runner fixture instance, or + # the test module for callers that transcribe without those fixtures. + device = _select_whisper_device() + with _serialize_whisper_model_download(model_size): + model = whisper.load_model(model_size, device=device) + _WHISPER_MODELS[model_size] = model + return model + + +def _whisper_transcribe_in_current_process( + output_path: str, model_size: str = "small", language: str | None = None +) -> str: + model = _get_whisper_model(model_size) + text = model.transcribe( + output_path, + temperature=0.0, + word_timestamps=True, + condition_on_previous_text=False, + # None keeps whisper's auto-detection. Do not default this to a + # language: callers include non-English audio tests. + language=language, + )["text"] return text or "" +# Serializes a whole submit->result->cleanup on the parent side, so at most one +# call is ever in flight on the single-worker pool. The child already runs +# transcriptions serially, so this costs no throughput; it lets a failed call +# discard the worker without disrupting another caller's in-flight future. +_TRANSCRIBER_CALL_LOCK = threading.Lock() +# Guards the _TRANSCRIBER pointer itself. +_TRANSCRIBER_LOCK = threading.Lock() +_TRANSCRIBER: concurrent.futures.ProcessPoolExecutor | None = None + + +def _get_transcriber() -> concurrent.futures.ProcessPoolExecutor: + global _TRANSCRIBER + with _TRANSCRIBER_LOCK: + if _TRANSCRIBER is None: + ctx = multiprocessing.get_context("spawn") + _TRANSCRIBER = concurrent.futures.ProcessPoolExecutor(max_workers=1, mp_context=ctx) + return _TRANSCRIBER + + +def _discard_transcriber(executor: concurrent.futures.ProcessPoolExecutor) -> None: + """Drop ``executor``, but only while it is still the current one. + + Identity-checked so a stale reference can never shut down a newer worker that + was installed after ``executor`` was replaced. + """ + global _TRANSCRIBER + with _TRANSCRIBER_LOCK: + if _TRANSCRIBER is not executor: + return + _TRANSCRIBER = None + # Joining the worker can block; do it outside the lock. + executor.shutdown(wait=True) + + +def release_audio_transcriber() -> None: + """Shut the transcription worker down, freeing the device memory its models hold. + + Called when a server or runner fixture instance tears down, so that the next + one -- including the next parametrization inside the same test module -- does + not initialize its model while Whisper still occupies the device. A module + teardown fixture repeats it for tests that transcribe without those fixtures. + + Takes the call lock, so it waits for any in-flight transcription to finish + rather than shutting the worker down underneath it. + """ + global _TRANSCRIBER + with _TRANSCRIBER_CALL_LOCK: + with _TRANSCRIBER_LOCK: + executor, _TRANSCRIBER = _TRANSCRIBER, None + if executor is not None: + executor.shutdown(wait=True) + + def convert_audio_file_to_text(output_path: str, model_size: str = "small", language: str | None = None) -> str: - """Convert an audio file to text in an isolated subprocess.""" - ctx = multiprocessing.get_context("spawn") - with concurrent.futures.ProcessPoolExecutor(max_workers=1, mp_context=ctx) as executor: - future = executor.submit(_whisper_transcribe_in_current_process, output_path, model_size, language) - return future.result() + """Convert an audio file to text in a reused, isolated subprocess. + + The worker outlives the call so its Whisper model is loaded once rather than + once per transcription. The call lock serializes callers onto the single + worker (the child already transcribes serially, so this costs no throughput), + which lets a failed call tear the worker down without racing another caller. + The worker caches one model per size it is asked for, so a run that escalates + to a stronger ASR keeps both models resident until release. + + Any failure discards the worker, restoring the failure isolation of the old + one-process-per-call design: a failure (a ``torch`` OOM, or a + ``KeyboardInterrupt``/``SystemExit`` transported through the future or + raised while ``result()`` blocks) propagates unchanged after the worker -- + and its resident model -- is torn down, and a dead worker + (``BrokenProcessPool``) is additionally retried once. + """ + with _TRANSCRIBER_CALL_LOCK: + for attempt in range(2): + executor = _get_transcriber() + try: + return executor.submit( + _whisper_transcribe_in_current_process, output_path, model_size, language + ).result() + except BrokenProcessPool: + _discard_transcriber(executor) + if attempt == 1: + raise + except BaseException: + # Any other failure -- a task exception (a torch OOM included), or + # a KeyboardInterrupt/SystemExit transported through the future or + # interrupting result() -- leaves the worker and its model + # resident; drop it so it cannot contaminate later calls, matching + # the old per-call teardown. Do not retry. + _discard_transcriber(executor) + raise + raise AssertionError("unreachable") def convert_audio_bytes_to_text(raw_bytes: bytes, model_size: str = "small", language: str | None = None) -> str: @@ -749,4 +843,5 @@ __all__ = [ "get_asset_path", "load_test_audio_data_url", "preprocess_text", + "release_audio_transcriber", ] diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index 4d06eb86a..56f7aaccc 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -16,6 +16,7 @@ import tempfile import threading import time from collections.abc import Generator +from contextlib import contextmanager from dataclasses import asdict, dataclass from io import BytesIO from pathlib import Path @@ -50,6 +51,7 @@ from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup from tests.helpers.media import ( _merge_base64_audio_to_segment, decode_b64_image, + release_audio_transcriber, ) from tests.model_tests.diffusion.utils import resolve_tiny_model_path from vllm_omni.config.stage_config import resolve_deploy_yaml @@ -3143,6 +3145,21 @@ class OmniRunnerHandler: # --------------------------------------------------------------------------- +@contextmanager +def _whisper_device_free_around(): + """Keep the Whisper judge off the accelerator around a server/runner lifecycle. + + Released on entry so the next instance -- e.g. the next parametrization in + the same module -- initializes on a clean device, and again on exit + (including when a test raises) so it does not linger for the instance after. + """ + release_audio_transcriber() + try: + yield + finally: + release_audio_transcriber() + + def iter_omni_server( request: Any, run_level: str, @@ -3152,7 +3169,7 @@ def iter_omni_server( """Start/stop an Omni HTTP server; used by ``omni_server`` / ``omni_server_function`` fixtures.""" from tests.helpers.stage_config import stage_config_path_for_run_level - with omni_fixture_lock: + with omni_fixture_lock, _whisper_device_free_around(): params: OmniServerParams = request.param # For now, when a tiny model is substituted, we preserve the original model # name via --served-model-name (so that the server still accepts requests with @@ -3240,7 +3257,7 @@ def iter_omni_runner( """Yield an :class:`OmniRunner`; used by ``omni_runner`` / ``omni_runner_function`` fixtures.""" from tests.helpers.stage_config import stage_config_path_for_run_level - with omni_fixture_lock: + with omni_fixture_lock, _whisper_device_free_around(): param = request.param if not isinstance(param, (tuple, list)) or len(param) not in (2, 3): raise ValueError( diff --git a/tests/helpers/tests/test_media.py b/tests/helpers/tests/test_media.py index 4a8e5393d..8045f7a69 100644 --- a/tests/helpers/tests/test_media.py +++ b/tests/helpers/tests/test_media.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import sys +from concurrent.futures.process import BrokenProcessPool +from contextlib import nullcontext from types import SimpleNamespace import numpy as np @@ -22,7 +24,11 @@ def _stub_transcribe(monkeypatch) -> dict: captured.update(kwargs) return {"text": "London"} - monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace(load_model=lambda size, device=None: FakeModel())) + monkeypatch.setitem( + sys.modules, + "whisper", + SimpleNamespace(load_model=lambda size, device=None: FakeModel()), + ) # Keep the unit test off the accelerator probe so it stays hermetic. monkeypatch.setitem( sys.modules, @@ -50,32 +56,183 @@ def test_transcribe_defaults_to_auto_language(monkeypatch): assert captured.get("language") is None +class _FakeExecutor: + """Stand-in for ProcessPoolExecutor recording what was submitted and shut down.""" + + def __init__(self, outcomes=()): + self._outcomes = list(outcomes) + self.submitted: list[tuple] = [] + self.shutdown_calls = 0 + + def submit(self, fn, *args): + self.submitted.append((fn, args)) + outcome = self._outcomes.pop(0) if self._outcomes else "London" + + def _result(): + if isinstance(outcome, BaseException): + raise outcome + return outcome + + return SimpleNamespace(result=_result) + + def shutdown(self, wait=True): + self.shutdown_calls += 1 + + +def _patch_executors(monkeypatch, outcomes_per_executor=()) -> list[_FakeExecutor]: + """Replace the pool constructor and hand back the fakes it produces, in creation order.""" + created: list[_FakeExecutor] = [] + + def factory(*args, **kwargs): + index = len(created) + outcomes = outcomes_per_executor[index] if index < len(outcomes_per_executor) else () + executor = _FakeExecutor(outcomes) + executor.init_kwargs = kwargs + created.append(executor) + return executor + + monkeypatch.setattr(media.concurrent.futures, "ProcessPoolExecutor", factory) + return created + + +@pytest.fixture(autouse=True) +def _reset_transcriber_singletons(): + """The worker and its model cache are module-level state; do not leak them across tests.""" + media.release_audio_transcriber() + media._WHISPER_MODELS.clear() + yield + media.release_audio_transcriber() + media._WHISPER_MODELS.clear() + + def test_bytes_entrypoint_forwards_language_to_subprocess(monkeypatch, tmp_path): """Cover the two hops the tests above skip: bytes -> file -> executor.submit. The real call crosses a spawn ProcessPoolExecutor, so capture what gets submitted rather than what the worker eventually does. """ - submitted: dict = {} - - class FakeExecutor: - def __enter__(self): - return self - - def __exit__(self, *exc): - return False - - def submit(self, fn, *args): - submitted["fn"] = fn - submitted["args"] = args - return SimpleNamespace(result=lambda: "London") - - monkeypatch.setattr(media.concurrent.futures, "ProcessPoolExecutor", lambda *a, **kw: FakeExecutor()) + created = _patch_executors(monkeypatch) wav = tmp_path / "clip.wav" sf.write(wav, np.zeros(2400, dtype=np.float32), 24000) assert media.convert_audio_bytes_to_text(wav.read_bytes(), "small", "en") == "London" - assert submitted["fn"] is media._whisper_transcribe_in_current_process - assert submitted["args"][1:] == ("small", "en") + fn, args = created[0].submitted[0] + assert fn is media._whisper_transcribe_in_current_process + assert args[1:] == ("small", "en") + + +def test_parent_reuses_one_spawned_worker_across_calls(monkeypatch): + """Two calls share one spawned worker. + + Pins the parent-side contract: the pool is constructed once, with + ``spawn`` (CUDA is not fork-safe) and ``max_workers=1`` (one resident + model). Reuse of the worker's *loaded model* is pinned by the worker-side + cache test below, which is where the startup cost actually lives. + """ + created = _patch_executors(monkeypatch) + + media.convert_audio_file_to_text("/tmp/a.wav") + media.convert_audio_file_to_text("/tmp/b.wav") + + assert len(created) == 1 + assert len(created[0].submitted) == 2 + assert created[0].init_kwargs["max_workers"] == 1 + assert created[0].init_kwargs["mp_context"].get_start_method() == "spawn" + + +def test_release_forces_a_new_process_pool(monkeypatch): + created = _patch_executors(monkeypatch) + + media.convert_audio_file_to_text("/tmp/a.wav") + media.release_audio_transcriber() + media.convert_audio_file_to_text("/tmp/b.wav") + + assert len(created) == 2 + assert created[0].shutdown_calls == 1 + + +def test_worker_loads_each_model_size_once(monkeypatch): + """The worker-side cache is what removes the repeated whisper.load_model cost.""" + loaded: list[str] = [] + + class FakeModel: + def transcribe(self, path, **kwargs): + return {"text": "London"} + + def load_model(size, device=None): + loaded.append(size) + return FakeModel() + + monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace(load_model=load_model)) + monkeypatch.setitem( + sys.modules, + "vllm_omni.platforms", + SimpleNamespace(current_omni_platform=SimpleNamespace(is_available=lambda: False)), + ) + monkeypatch.setattr(media, "_serialize_whisper_model_download", lambda model_size: nullcontext()) + + media._whisper_transcribe_in_current_process("/tmp/a.wav", "small") + media._whisper_transcribe_in_current_process("/tmp/b.wav", "small") + + assert loaded == ["small"] + + # The ASR escalation path asks the same worker for a stronger model. + media._whisper_transcribe_in_current_process("/tmp/c.wav", "large-v3") + + assert loaded == ["small", "large-v3"] + + +@pytest.mark.parametrize( + ("second_outcome", "recovers"), + [(["London"], True), ([BrokenProcessPool("second")], False)], +) +def test_broken_pool_is_discarded_and_retried_once(monkeypatch, second_outcome, recovers): + """A dead worker is discarded and the call retried exactly once. + + The retry either succeeds, or -- if the replacement also dies -- fails with + both pools discarded and no executor left installed. + """ + created = _patch_executors( + monkeypatch, + outcomes_per_executor=([BrokenProcessPool("first")], second_outcome), + ) + + if recovers: + assert media.convert_audio_file_to_text("/tmp/a.wav") == "London" + else: + with pytest.raises(BrokenProcessPool): + media.convert_audio_file_to_text("/tmp/a.wav") + assert created[1].shutdown_calls == 1 + assert media._TRANSCRIBER is None + + assert len(created) == 2 # discarded the first, retried on a fresh one + assert created[0].shutdown_calls == 1 + + +@pytest.mark.parametrize( + "error", + [RuntimeError("CUDA out of memory"), KeyboardInterrupt()], + ids=["Exception", "BaseException"], +) +def test_transcription_failure_propagates_and_discards_the_worker(monkeypatch, error): + """Any failure -- an Exception or a BaseException like KeyboardInterrupt -- + propagates AND tears the worker down. + + It arrives as the task's own exception, not ``BrokenProcessPool``, but the + worker -- and its resident model -- is still discarded, restoring the old + one-process-per-call isolation. It is not retried; the next call builds a + fresh worker. + """ + created = _patch_executors(monkeypatch, outcomes_per_executor=([error],)) + + with pytest.raises(type(error)): + media.convert_audio_file_to_text("/tmp/a.wav") + + assert len(created) == 1 # not retried + assert created[0].shutdown_calls == 1 + assert media._TRANSCRIBER is None + + assert media.convert_audio_file_to_text("/tmp/b.wav") == "London" + assert len(created) == 2 # a fresh worker for the next call diff --git a/tests/helpers/tests/test_runtime_transcriber_release.py b/tests/helpers/tests/test_runtime_transcriber_release.py new file mode 100644 index 000000000..6bc7d4221 --- /dev/null +++ b/tests/helpers/tests/test_runtime_transcriber_release.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""The server/runner fixtures free the Whisper judge around each instance. + +iter_omni_server and iter_omni_runner both wrap their body in the same +_whisper_device_free_around() context manager, so one representative path +exercises its enter/exit and exception behavior. +""" + +import threading +from types import SimpleNamespace + +import pytest + +from tests.helpers import runtime +from tests.helpers.runtime import OmniServerParams + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +@pytest.fixture +def events(monkeypatch) -> list[str]: + """Record construction and release in order.""" + calls: list[str] = [] + + class FakeServer: + def __init__(self, *args, **kwargs): + calls.append("construct") + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + monkeypatch.setattr(runtime, "OmniServer", FakeServer) + monkeypatch.setattr(runtime, "release_audio_transcriber", lambda: calls.append("release")) + monkeypatch.setattr( + "tests.helpers.stage_config.stage_config_path_for_run_level", + lambda path, run_level: None, + ) + return calls + + +def _server_generator(): + request = SimpleNamespace( + param=OmniServerParams(model="fake-model"), + node=SimpleNamespace(get_closest_marker=lambda name: None), + ) + return runtime.iter_omni_server(request, "core_model", "", threading.Lock()) + + +def test_releases_before_construction_and_after_teardown(events): + generator = _server_generator() + generator.send(None) + + assert events == ["release", "construct"], "the device must be clear before the model loads" + + with pytest.raises(StopIteration): + generator.send(None) + + assert events == ["release", "construct", "release"] + + +def test_failing_test_still_releases(events): + """A test that raises must not strand Whisper on the device.""" + generator = _server_generator() + generator.send(None) + + with pytest.raises(RuntimeError, match="assertion blew up"): + generator.throw(RuntimeError("assertion blew up")) + + assert events[-1] == "release" From 9c61d5b67b2c57536b1b6246ef036d4e29bdb3d1 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Thu, 20 Aug 2026 11:50:07 +0800 Subject: [PATCH 079/267] =?UTF-8?q?[Bugfix][MiniCPM-o]=20Stop=20Talker=20m?= =?UTF-8?q?id-utterance=20truncation=20and=20load=20cod=E2=80=A6=20(#6346)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: amy-why-3459 Co-authored-by: Cursor --- .buildkite/npu/test-npu-nightly.yml | 49 ++ tests/config/test_config_factory.py | 23 +- tests/dfx/perf/tests/test_minicpmo_4_5.json | 84 +++ .../test_minicpmo_4_5_duplex_seed_tts.json | 18 + .../perf/tests/test_minicpmo_4_5_ready.json | 1 + tests/e2e/online_serving/test_minicpmo_4_5.py | 20 +- .../test_minicpmo_4_5_expansion.py | 49 +- .../engine/test_stage_request_token_budget.py | 74 +++ .../minicpmo_4_5/test_code2wav_batching.py | 39 ++ .../models/minicpmo_4_5/test_pipeline.py | 4 + .../minicpmo_4_5/test_talker_batching.py | 523 ++++++++---------- tests/worker/test_native_duplex_hooks.py | 126 +++++ vllm_omni/config/stage_config.py | 8 +- vllm_omni/deploy/minicpmo_4_5.yaml | 14 +- vllm_omni/deploy/minicpmo_4_5_2gpu.yaml | 13 +- vllm_omni/deploy/minicpmo_4_5_3gpu.yaml | 12 +- vllm_omni/deploy/minicpmo_4_5_8x4090.yaml | 12 +- vllm_omni/deploy/minicpmo_4_5_duplex.yaml | 9 +- vllm_omni/engine/orchestrator.py | 12 +- .../experimental/fullduplex/model_executor.py | 60 +- .../models/minicpmo_4_5/batched_token2wav.py | 151 ++++- .../minicpmo_4_5/minicpmo_4_5_omni_llm.py | 4 + .../minicpmo_4_5/minicpmo_4_5_omni_tts.py | 424 +++++--------- .../models/minicpmo_4_5/pipeline.py | 7 +- .../minicpmo_4_5_omni.py | 3 + .../npu/worker/npu_ar_model_runner.py | 20 +- vllm_omni/worker/gpu_ar_model_runner.py | 37 +- 27 files changed, 1118 insertions(+), 678 deletions(-) create mode 100644 tests/engine/test_stage_request_token_budget.py diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index b89f61a39..e049783ce 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -81,6 +81,55 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json" exit $$EXIT + - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test" + key: nightly-omni-performance-minicpmo-4-5-npu + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_2 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT + - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test (A2)" + key: nightly-omni-performance-minicpmo-4-5-npu-a2 + timeout_in_minutes: 180 + mirror_hardwares: a2b3_npu_1 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + HF_HUB_OFFLINE: "1" + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A2" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT + - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test" + key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts-npu + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_2 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT - group: ":card_index_dividers: TTS Model Test" key: nightly-tts-test-group depends_on: upload-nightly-pipeline diff --git a/tests/config/test_config_factory.py b/tests/config/test_config_factory.py index 41fc15df5..cacc8142f 100644 --- a/tests/config/test_config_factory.py +++ b/tests/config/test_config_factory.py @@ -1127,9 +1127,11 @@ class TestDeployConfigLoading: assert all("Async" not in (stage.scheduler_cls or "") for stage in stages) assert [stage.devices for stage in deploy.stages] == ["0", "0", "0"] assert deploy.stages[1].enforce_eager is False - assert stages[1].yaml_extras["default_sampling_params"]["max_tokens"] == 2048 + assert stages[1].yaml_extras["default_sampling_params"]["max_tokens"] == 4096 assert stages[1].yaml_extras["default_sampling_params"]["min_tokens"] == 0 - assert stages[1].yaml_extras["default_sampling_params"]["stop_token_ids"] == [1] + assert stages[1].yaml_extras["default_sampling_params"]["temperature"] == 0.8 + assert stages[1].yaml_extras["default_sampling_params"]["stop_token_ids"] == [6561] + assert "codec_sampling_params" not in stages[1].yaml_engine_args @pytest.mark.parametrize( ("filename", "stage0_devices", "stage1_devices", "stage2_devices", "stage1_replicas"), @@ -2111,6 +2113,23 @@ class TestMingFlashOmniPipeline: class TestBaseConfigInheritance: """Test deploy YAML base_config inheritance.""" + def test_minicpmo_overlays_inherit_talker_sampling_params(self): + """Overlays must keep the base Talker codec Sampler knobs.""" + for filename in ( + "minicpmo_4_5_duplex.yaml", + "minicpmo_4_5_3gpu_stage1_replicas.yaml", + "minicpmo_4_5_4gpu_stage1_replicas.yaml", + "minicpmo_4_5_8x4090_stage1_replicas.yaml", + ): + path = Path(get_deploy_config_path(filename)) + if not path.exists(): + pytest.skip(f"{filename} not found") + deploy = load_deploy_config(path) + sampling = deploy.stages[1].default_sampling_params + assert sampling is not None, f"{filename} stage 1 lost default_sampling_params" + assert sampling["temperature"] == 0.8, filename + assert sampling["top_k"] == 100, filename + def test_ci_inherits_from_main(self): ci_path = Path(get_deploy_config_path("ci/qwen3_omni_moe.yaml")) if not ci_path.exists(): diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5.json b/tests/dfx/perf/tests/test_minicpmo_4_5.json index cfb57169b..9296cf5b3 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5.json @@ -39,6 +39,7 @@ 8 ], "no_oversample": true, + "disable_shuffle": true, "trust_remote_code": true, "seed_tts_locale": "en", "extra_body": { @@ -79,9 +80,92 @@ 0.8568, 1.5425 ] + }, + "A3": { + "request_throughput": [ + 0.5383, + 0.6042, + 0.7547 + ], + "mean_ttft_ms": [ + 333.2633, + 533.8656, + 547.3388 + ], + "mean_e2el_ms": [ + 1857.2154, + 6600.5101, + 10450.6811 + ], + "mean_audio_ttfp_ms": [ + 986.4666, + 3411.0754, + 3352.7515 + ], + "mean_audio_rtf": [ + 0.4423, + 1.5734, + 2.3024 + ] } } } ] + }, + { + "test_name": "test_minicpmo_4_5_a2", + "mark": [ + { + "hardware_marks": { + "res": { + "npu": "A2" + }, + "num_cards": 1 + } + }, + "full_model", + "omni" + ], + "server_params": { + "model": "openbmb/MiniCPM-o-4_5", + "extra_cli_args": [ + "--deploy-config", + "vllm_omni/deploy/minicpmo_4_5.yaml", + "--trust-remote-code" + ] + }, + "benchmark_params": [ + { + "dataset_name": "seed-tts", + "dataset_path": "zhaochenyang20/seed-tts-eval", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8 + ], + "no_oversample": true, + "disable_shuffle": true, + "trust_remote_code": true, + "seed_tts_locale": "en", + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, + "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration" + } + ] } ] diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json b/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json index cfbcc3abb..6ef27680c 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json @@ -35,6 +35,7 @@ 1 ], "no_oversample": true, + "disable_shuffle": true, "trust_remote_code": true, "seed_tts_locale": "en", "seed_tts_turns_per_session": 4, @@ -60,6 +61,23 @@ "mean_audio_rtf": [ 0.46163 ] + }, + "A3": { + "request_throughput": [ + 0.1594 + ], + "mean_ttft_ms": [ + 274.164 + ], + "mean_e2el_ms": [ + 6250.7696 + ], + "mean_audio_ttfp_ms": [ + 994.422 + ], + "mean_audio_rtf": [ + 0.46163 + ] } } } diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json b/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json index e327be510..15d52e6ad 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json @@ -37,6 +37,7 @@ 4 ], "no_oversample": true, + "disable_shuffle": true, "trust_remote_code": true, "seed_tts_locale": "en", "extra_body": { diff --git a/tests/e2e/online_serving/test_minicpmo_4_5.py b/tests/e2e/online_serving/test_minicpmo_4_5.py index 6953ef291..9c8bcd908 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5.py @@ -65,6 +65,15 @@ def get_max_batch_size(size_type="few"): return batch_sizes.get(size_type, 5) +# Close and emit <|tts_bos|> so Talker speaks the answer, not reasoning. +_TTS_EXTRA_BODY = { + "chat_template_kwargs": { + "use_tts_template": True, + "enable_thinking": False, + } +} + + @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni @@ -111,6 +120,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None: "messages": messages, "stream": True, "key_words": {"audio": ["Beijing"]}, + "extra_body": _TTS_EXTRA_BODY, } openai_client.send_omni_request(request_config) @@ -141,12 +151,7 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None: "stream": True, "key_words": {"text": ["Beijing"]}, "modalities": ["text", "audio"], - "extra_body": { - "chat_template_kwargs": { - "use_tts_template": True, - "enable_thinking": False, - } - }, + "extra_body": _TTS_EXTRA_BODY, } openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) @@ -175,6 +180,7 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None: "model": omni_server.model, "messages": messages, "stream": True, + "extra_body": _TTS_EXTRA_BODY, } openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) @@ -203,6 +209,7 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None: "model": omni_server.model, "messages": messages, "stream": True, + "extra_body": _TTS_EXTRA_BODY, } openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) @@ -236,6 +243,7 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None: "model": omni_server.model, "messages": messages, "stream": True, + "extra_body": _TTS_EXTRA_BODY, } openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) diff --git a/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py b/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py index e84985e3e..5a21e8692 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py @@ -1,7 +1,7 @@ """E2E online expansion tests for MiniCPM-o 4.5. These cover modality combinations, separate Code2Wav behavior, sequential -request isolation, longer audio output, and Chinese speech. +request isolation, longer audio output, long-form generation, and Chinese speech. Like Qwen3-Omni expansion, ``default`` exercises ``--no-async-chunk`` and ``async_chunk`` exercises ``--async-chunk``. @@ -75,6 +75,15 @@ def get_max_batch_size(size_type="few"): return batch_sizes.get(size_type, 5) +# Close and emit <|tts_bos|> so Talker speaks the answer, not reasoning. +_TTS_EXTRA_BODY = { + "chat_template_kwargs": { + "use_tts_template": True, + "enable_thinking": False, + } +} + + @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_video_to_text_001(omni_server, openai_client) -> None: @@ -127,6 +136,7 @@ def test_sequential_requests_independent(omni_server, openai_client) -> None: "model": omni_server.model, "messages": messages_1, "stream": True, + "extra_body": _TTS_EXTRA_BODY, }, request_num=1, ) @@ -138,6 +148,7 @@ def test_sequential_requests_independent(omni_server, openai_client) -> None: "messages": messages_2, "stream": True, "key_words": {"text": ["Beijing"]}, + "extra_body": _TTS_EXTRA_BODY, }, request_num=1, ) @@ -164,20 +175,39 @@ def test_text_to_audio_long_output_001(omni_server, openai_client) -> None: "messages": messages, "modalities": ["text", "audio"], "stream": True, - # Delimit the assistant answer with the TTS template so the Talker - # does not spend its codec-token budget speaking hidden reasoning. - "extra_body": { - "chat_template_kwargs": { - "use_tts_template": True, - "enable_thinking": False, - } - }, + "extra_body": _TTS_EXTRA_BODY, "key_words": {"audio": ["Beijing"]}, } openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_text_to_audio_long_form_001(omni_server, openai_client) -> None: + """ + Input Modal: text only (long-form generation prompt). + Output Modal: text, audio (default ``modalities``); + Input Setting: stream=True + Datasets: single request + """ + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + content_text="帮我讲一个300字的故事.", + ) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + "extra_body": _TTS_EXTRA_BODY, + } + responses = openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) + text = responses[0].text_content if responses else "" + word_count = len(text.split()) + assert word_count >= 200, f"Expected at least 200 words in long output, got {word_count}" + + @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_chinese_text_to_audio(omni_server, openai_client) -> None: @@ -193,5 +223,6 @@ def test_chinese_text_to_audio(omni_server, openai_client) -> None: "messages": messages, "stream": True, "key_words": {"text": ["北京"]}, + "extra_body": _TTS_EXTRA_BODY, } openai_client.send_omni_request(request_config) diff --git a/tests/engine/test_stage_request_token_budget.py b/tests/engine/test_stage_request_token_budget.py new file mode 100644 index 000000000..045dfd31f --- /dev/null +++ b/tests/engine/test_stage_request_token_budget.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Token-budget clamping when the orchestrator builds a stage request.""" + +from __future__ import annotations + +import pytest +from vllm.sampling_params import SamplingParams + +from vllm_omni.engine.orchestrator import build_engine_core_request_from_tokens + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class _StageModelConfig: + """Only the field ``build_engine_core_request_from_tokens`` reads.""" + + max_model_len = 4096 + + +class TestStageRequestTokenBudget: + def test_min_tokens_is_clamped_to_the_remaining_context(self): + # A min_tokens wider than the leftover window makes ``check_stop`` + # return early forever: the request can neither stop nor be granted + # more tokens, so it stalls instead of being length-capped. + params = SamplingParams(max_tokens=4096, min_tokens=50) + + request = build_engine_core_request_from_tokens( + request_id="req-clamped", + prompt={"prompt_token_ids": [0] * 4050}, + params=params, + model_config=_StageModelConfig(), + ) + + assert request.sampling_params.min_tokens == 46 + assert params.min_tokens == 50 + + def test_min_tokens_survives_when_the_prompt_leaves_room(self): + params = SamplingParams(max_tokens=4096, min_tokens=50) + + request = build_engine_core_request_from_tokens( + request_id="req-untouched", + prompt={"prompt_token_ids": [0] * 68}, + params=params, + model_config=_StageModelConfig(), + ) + + assert request.sampling_params.min_tokens == 50 + assert request.sampling_params.max_tokens == 4096 + + def test_min_tokens_floors_at_zero_when_the_prompt_fills_the_context(self): + params = SamplingParams(max_tokens=16, min_tokens=8) + + request = build_engine_core_request_from_tokens( + request_id="req-full", + prompt={"prompt_token_ids": [0] * 4096}, + params=params, + model_config=_StageModelConfig(), + ) + + assert request.sampling_params.min_tokens == 0 + + def test_absent_max_tokens_still_defaults_to_the_remaining_context(self): + params = SamplingParams(min_tokens=0) + params.max_tokens = None + + request = build_engine_core_request_from_tokens( + request_id="req-default", + prompt={"prompt_token_ids": [0] * 96}, + params=params, + model_config=_StageModelConfig(), + ) + + assert request.sampling_params.max_tokens == 4000 diff --git a/tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py b/tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py index ba3c7292b..a1ef3b2ec 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py +++ b/tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py @@ -7,6 +7,8 @@ import torch.nn as nn from vllm_omni.model_executor.models.minicpmo_4_5.batched_token2wav import ( BatchedToken2Wav, + plan_token2wav_encode_slices, + relpos_encode_token_budget, ) from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_code2wav import ( MiniCPMO45Code2Wav, @@ -217,6 +219,43 @@ def test_adapter_runs_true_batch_cfg_and_splits_request_caches(): assert cache1[0, 0, 0, 0, 0].item() == 20 +def test_relpos_encode_token_budget_leaves_room_for_upsample_and_cache(): + # The NPU crash was 6968 vs 985: 3333 tokens * 2 + cache overflowed max_pos=5000. + assert relpos_encode_token_budget(max_pos=5000, stride=2, cache_offset=150, lookahead=3) <= 1024 + assert relpos_encode_token_budget(max_pos=5000, stride=2, cache_offset=150, lookahead=3, cap=3000) == 2346 + assert plan_token2wav_encode_slices(3333, max_frames=1024, min_nonfinal=4, last_chunk=True) == [ + (0, 1024), + (1024, 2048), + (2048, 3072), + (3072, 3333), + ] + assert plan_token2wav_encode_slices(50, max_frames=40, min_nonfinal=20, last_chunk=False) == [ + (0, 30), + (30, 50), + ] + + +def test_decode_batch_splits_overlong_prefill_inside_relpos_budget(): + token2wav = _FakeToken2Wav() + adapter = BatchedToken2Wav(token2wav) + prompt = adapter.prepare_prompt("shared", "/fake/prompt.wav") + states = adapter.setup_batch(prompt, 1) + setup_encodes = len(token2wav.flow.encoder.calls) + adapter._max_encode_token_frames = lambda _states: 4 + + audios, _ = adapter.decode_batch( + torch.arange(10, dtype=torch.long).reshape(1, -1), + prompt, + states, + last_chunk=True, + ) + + assert token2wav.flow.encoder.calls[setup_encodes:] == [1, 1, 1] + assert token2wav.flow.encoder.last_chunk_calls[setup_encodes:] == [False, False, True] + assert len(audios) == 1 + assert audios[0].numel() > 0 + + def test_fade_in_out_limits_overlap_to_available_previous_audio(): speech = torch.arange(6, dtype=torch.float32).reshape(1, -1) previous = torch.full((1, 3), 2.0) diff --git a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py index bbc011284..42bc364cb 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py +++ b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py @@ -105,6 +105,10 @@ class TestPipelineTopology: assert talker.engine_output_type == "latent" # scope KV cache / mrope sizing to talker sub-config assert talker.hf_config_name == "tts_config" + assert talker.sampling_constraints == { + "detokenize": False, + "stop_token_ids": [6561], + } assert talker.custom_process_next_stage_input_func == ( "vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.tts2code2wav_full_payload" ) diff --git a/tests/model_executor/models/minicpmo_4_5/test_talker_batching.py b/tests/model_executor/models/minicpmo_4_5/test_talker_batching.py index 8a7593623..4115c1a29 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_talker_batching.py +++ b/tests/model_executor/models/minicpmo_4_5/test_talker_batching.py @@ -4,6 +4,8 @@ from __future__ import annotations +from dataclasses import dataclass + import pytest import torch import torch.nn as nn @@ -11,18 +13,30 @@ import torch.nn as nn from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( MiniCPMO45OmniForConditionalGeneration, ) +from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import ( + ConditionalChatTTSConfig, +) from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( + _DUPLEX_CODEC_TOKENS_PER_CHUNK, _REPETITION_PENALTY_CHUNK_SIZE, MiniCPMO45OmniTTSForConditionalGeneration, _apply_batched_repetition_penalty, - _max_audio_tokens, + _native_duplex_chunk_budget, _restore_weight_norm_weight, + blank_scheduler_prompt_for_penalties, ) from vllm_omni.utils.mm_outputs import to_payload_element pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +@dataclass +class _PenaltyMetadata: + """Stand-in for the penalty-relevant slice of vLLM's SamplingMetadata.""" + + prompt_token_ids: torch.Tensor | None + + class _FakeNativeTalker(nn.Module): has_preprocess = True @@ -35,6 +49,12 @@ class _FakeNativeTalker(nn.Module): return torch.ones(2, 4) +def test_tts_config_exposes_codec_vocab_to_vllm() -> None: + cfg = ConditionalChatTTSConfig(num_audio_tokens=6562) + assert cfg.vocab_size == 6562 + assert cfg.eos_token_id == 6561 + + def test_wrapper_always_delegates_talker_to_native_ar_path() -> None: model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration) nn.Module.__init__(model) @@ -55,12 +75,14 @@ def _make_talker() -> MiniCPMO45OmniTTSForConditionalGeneration: talker = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration) nn.Module.__init__(talker) talker._num_audio_tokens = 8 - talker._batch_stop_logits = None - talker._request_generators = {} + talker._codec_eos_id = 7 + talker._force_eos_rows = None + talker._mask_eos_rows = None talker._request_audio_states = {} talker._deferred_cleanup_ids = set() - talker._codec_min_tokens = 50 - talker._codec_seed = 42 + talker.head_code = nn.ModuleList([nn.Linear(2, 8, bias=False)]) + with torch.no_grad(): + talker.head_code[0].weight.copy_(torch.eye(8, 2)) return talker @@ -90,46 +112,6 @@ def _reference_repetition_penalty( return torch.where(logits < 0, logits * alpha, logits / alpha) -def _make_sampling_talker() -> MiniCPMO45OmniTTSForConditionalGeneration: - talker = _make_talker() - talker._codec_temperature = 0.8 - talker._codec_top_k = 5 - talker._codec_top_p = 0.85 - talker._codec_repetition_penalty = 1.05 - talker._request_audio_states = { - "req-a": {"min_tokens": 2}, - "req-b": {"min_tokens": 0}, - } - talker.head_code = nn.ModuleList([nn.Linear(2, 8, bias=False)]) - with torch.no_grad(): - talker.head_code[0].weight.copy_( - torch.tensor( - [ - [0.1, -0.3], - [0.2, 0.4], - [-0.5, 0.2], - [0.7, -0.1], - [-0.2, 0.8], - [0.6, 0.3], - [-0.4, -0.7], - [0.3, 0.5], - ] - ) - ) - return talker - - -@pytest.mark.parametrize( - ("condition_tokens", "expected"), - [(3, 64), (100, 1000), (1000, 2048)], -) -def test_audio_token_limit_scales_with_condition_length( - condition_tokens: int, - expected: int, -) -> None: - assert _max_audio_tokens(condition_tokens) == expected - - def test_weight_norm_restore_matches_checkpoint_parametrization_in_bfloat16() -> None: generator = torch.Generator().manual_seed(42) weight_v = torch.randn(8, 16, generator=generator, dtype=torch.bfloat16) @@ -220,133 +202,212 @@ def test_batched_repetition_penalty_matches_rows_across_chunks(mocker) -> None: ] -def test_batched_codec_sampling_is_request_order_independent() -> None: - talker = _make_sampling_talker() - histories = [torch.tensor([1, 1, 2]), torch.tensor([3, 4, 4])] - hidden = torch.tensor([[0.25, -0.75], [-0.5, 0.5]]) +def test_scheduler_prompt_is_fully_blanked_for_penalties() -> None: + # The handoff path fills the prompt with 0s and the non-handoff path passes + # thinker token ids; neither is codec history, so every position is padded. + prompt = torch.tensor([[0, 0, 0], [12, 0, 340]]) - forward_order = talker._sample_audio_codes( - hidden, - histories, - ["req-a", "req-b"], - [0, 0], - ) - talker._request_generators = {} - reverse_order = talker._sample_audio_codes( - hidden.flip(0), - list(reversed(histories)), - ["req-b", "req-a"], - [0, 0], + blanked = blank_scheduler_prompt_for_penalties(prompt, vocab_size=8) + + assert blanked.tolist() == [[8, 8, 8], [8, 8, 8]] + assert prompt.tolist() == [[0, 0, 0], [12, 0, 340]] + + +def test_blanked_prompt_leaves_every_codec_logit_unpenalized() -> None: + from vllm.model_executor.layers.utils import apply_penalties + + vocab_size = 8 + logits = torch.arange(1.0, vocab_size + 1.0).unsqueeze(0) + prompt = blank_scheduler_prompt_for_penalties(torch.tensor([[0, 0, 3]]), vocab_size) + + penalized = apply_penalties( + logits.clone(), + prompt, + torch.full((1, 1), vocab_size, dtype=torch.long), + torch.zeros(1), + torch.zeros(1), + torch.full((1,), 1.02), ) - assert torch.equal(forward_order[0], reverse_order[1]) - assert torch.equal(forward_order[1], reverse_order[0]) + torch.testing.assert_close(penalized, logits) -def test_batched_codec_sampling_preserves_request_rng_after_compaction() -> None: - talker = _make_sampling_talker() - history_a = torch.tensor([1, 1, 2]) - history_b = torch.tensor([3, 4, 4]) - hidden_a = torch.tensor([[0.25, -0.75]]) - hidden_b = torch.tensor([[-0.5, 0.5]]) - - first_batch = talker._sample_audio_codes( - torch.cat([hidden_a, hidden_b]), - [history_a, history_b], - ["req-a", "req-b"], - [0, 0], - ) - compacted_b = talker._sample_audio_codes( - hidden_b, - [history_b], - ["req-b"], - [1], - ) - - talker._request_generators = {} - standalone_b_first = talker._sample_audio_codes( - hidden_b, - [history_b], - ["req-b"], - [0], - )[0] - standalone_b_second = talker._sample_audio_codes( - hidden_b, - [history_b], - ["req-b"], - [1], - )[0] - - assert torch.equal(first_batch[1], standalone_b_first) - assert torch.equal(compacted_b[0], standalone_b_second) - - -def test_talker_emits_request_aligned_codec_deltas_after_compaction(mocker) -> None: +def test_talker_sample_blanks_penalty_prompt_without_touching_runner_state(mocker) -> None: talker = _make_talker() - seen: list[tuple[str, list[float], list[int]]] = [] + metadata = _PenaltyMetadata(prompt_token_ids=torch.tensor([[0, 2, 0]])) + captured: dict[str, torch.Tensor] = {} - def sample(hidden, histories, request_ids, steps): - assert steps == [0, 0] - seen.extend( - (request_id, hidden[row].reshape(-1).tolist(), histories[row].tolist()) - for row, request_id in enumerate(request_ids) - ) - return torch.tensor([2, 3]) + class _Recorder: + def __call__(self, logits, sampling_metadata): + captured["prompt"] = sampling_metadata.prompt_token_ids + return None - batched_sample = mocker.patch.object(talker, "_sample_audio_codes", side_effect=sample) + mocker.patch( + "vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts.Sampler", + return_value=_Recorder(), + ) + talker.sample(torch.zeros(1, 8), metadata) + + assert captured["prompt"].tolist() == [[8, 8, 8]] + assert metadata.prompt_token_ids.tolist() == [[0, 2, 0]] + + +def test_compute_logits_returns_codec_vocab() -> None: + talker = _make_talker() + hidden = torch.tensor([[1.0, 0.0], [0.0, 1.0]]) + + logits = talker.compute_logits(hidden) + + assert logits.shape == (2, 8) + assert torch.allclose(logits[0], talker.head_code[0](hidden[0])) + + +def test_compute_logits_forces_eos_for_empty_speech() -> None: + talker = _make_talker() + talker._request_audio_states["req-empty"] = {"finished": True} + + output = talker.make_omni_output( + torch.ones(1, 2), + model_intermediate_buffer=[{"request_id": "req-empty", "codes": {"audio": torch.empty(0)}}], + request_token_spans=[(0, 1)], + ) + logits = talker.compute_logits(output.text_hidden_states) + + assert output.multimodal_outputs["codes"]["audio"][0].numel() == 0 + assert output.multimodal_outputs["meta"]["finished"][0].item() is True + assert int(logits.argmax(dim=-1)) == 7 + + +def test_native_duplex_chunk_budget_masks_eos_until_generate_chunk_is_full() -> None: + assert _native_duplex_chunk_budget({}) == (_DUPLEX_CODEC_TOKENS_PER_CHUNK, _DUPLEX_CODEC_TOKENS_PER_CHUNK) + assert _native_duplex_chunk_budget({"turn_start": True}) == (_DUPLEX_CODEC_TOKENS_PER_CHUNK, 0) + assert _native_duplex_chunk_budget({"turn_end": True}) == (_DUPLEX_CODEC_TOKENS_PER_CHUNK, 0) + + talker = _make_talker() + talker._request_audio_states["req-chunk"] = { + "finished": False, + "step": 0, + "max_tokens": _DUPLEX_CODEC_TOKENS_PER_CHUNK, + "min_tokens": _DUPLEX_CODEC_TOKENS_PER_CHUNK, + } + + output = talker.make_omni_output( + torch.ones(1, 2), + model_intermediate_buffer=[{"request_id": "req-chunk", "codes": {"audio": torch.empty(0)}}], + request_token_spans=[(0, 1)], + ) + logits = talker.compute_logits(output.text_hidden_states) + + assert logits[0, 7].item() == float("-inf") + assert torch.isfinite(logits[0, 0]) + assert output.multimodal_outputs["meta"]["finished"][0].item() is False + + +def test_native_duplex_chunk_budget_forces_eos_after_twenty_five_frames() -> None: + talker = _make_talker() + talker._request_audio_states["req-full"] = { + "finished": False, + "step": 24, + "max_tokens": _DUPLEX_CODEC_TOKENS_PER_CHUNK, + "min_tokens": _DUPLEX_CODEC_TOKENS_PER_CHUNK, + } + + output = talker.make_omni_output( + torch.ones(1, 2), + model_intermediate_buffer=[{"request_id": "req-full", "codes": {"audio": torch.tensor([[3]])}}], + request_token_spans=[(0, 1)], + ) + logits = talker.compute_logits(output.text_hidden_states) + + assert talker._request_audio_states["req-full"]["step"] == 25 + assert talker._request_audio_states["req-full"]["finished"] is True + assert int(logits.argmax(dim=-1)) == 7 + assert output.multimodal_outputs["meta"]["finished"][0].item() is True + + +def test_simplex_talker_does_not_force_eos_after_a_duplex_sized_chunk() -> None: + talker = _make_talker() + talker._request_audio_states["req-simplex"] = {"finished": False, "step": 0} + + output = talker.make_omni_output( + torch.tensor([[1.0, 0.0]]), + model_intermediate_buffer=[{"request_id": "req-simplex", "codes": {"audio": torch.tensor([[3]])}}], + request_token_spans=[(0, 1)], + ) + logits = talker.compute_logits(output.text_hidden_states) + + assert talker._request_audio_states["req-simplex"]["step"] == 0 + assert torch.allclose(logits[0], talker.head_code[0](output.text_hidden_states[0]).float()) + + +def test_native_duplex_prefill_records_generate_chunk_budget() -> None: + talker = _make_talker() + talker.emb_text = nn.Embedding(1, 2) + talker.emb_code = nn.ModuleList([nn.Embedding(8, 2)]) + talker._text_eos_id = 0 + talker._tts_bos_id = 0 + + _, _, mid_turn = talker.preprocess( + torch.zeros(2, dtype=torch.long), + None, + _omni_is_prefill=True, + request_id="req-mid", + native_duplex=True, + tts_token_ids=torch.empty(0, dtype=torch.long), + tts_hidden_states=torch.empty(0, 2), + ) + _, _, boundary = talker.preprocess( + torch.zeros(2, dtype=torch.long), + None, + _omni_is_prefill=True, + request_id="req-end", + native_duplex=True, + meta={"turn_end": True}, + tts_token_ids=torch.empty(0, dtype=torch.long), + tts_hidden_states=torch.empty(0, 2), + ) + + assert mid_turn["audio_state"]["max_tokens"] == _DUPLEX_CODEC_TOKENS_PER_CHUNK + assert mid_turn["audio_state"]["min_tokens"] == _DUPLEX_CODEC_TOKENS_PER_CHUNK + assert boundary["audio_state"]["min_tokens"] == 0 + + +def test_make_omni_output_packs_the_previous_codec_id() -> None: + talker = _make_talker() infos = [ - {"request_id": "req-a", "audio_codes": {"accumulated": torch.tensor([1])}}, - {"request_id": "req-b", "audio_codes": {"accumulated": torch.empty(0, dtype=torch.long)}}, + {"request_id": "req-a", "codes": {"audio": torch.tensor([[2]])}}, + {"request_id": "req-b", "codes": {"audio": torch.tensor([[3]])}}, ] output = talker.make_omni_output( - torch.tensor([[1.0, 0.0], [2.0, 0.0], [3.0, 0.0]]), + torch.ones(2, 2), model_intermediate_buffer=infos, - request_token_spans=[(0, 2), (2, 3)], + request_token_spans=[(0, 1), (1, 2)], ) - assert seen == [ - ("req-a", [2.0, 0.0], [1]), - ("req-b", [3.0, 0.0], []), - ] - batched_sample.assert_called_once() - assert infos[0]["audio_codes"]["accumulated"].tolist() == [1, 2] - assert infos[1]["audio_codes"]["accumulated"].tolist() == [3] - assert set(output.multimodal_outputs) == {"codes", "meta"} - assert "model_outputs" not in output.multimodal_outputs - assert "sr" not in output.multimodal_outputs assert _routed(output, 0)["codes"]["audio"].tolist() == [[2]] assert _routed(output, 1)["codes"]["audio"].tolist() == [[3]] assert _routed(output, 0)["meta"]["finished"].item() is False - assert set(output.multimodal_outputs["meta"]) == {"finished"} - assert talker.compute_logits(output.text_hidden_states).argmax(dim=-1).tolist() == [0, 0] -def test_talker_projects_request_aligned_duplex_metadata(mocker) -> None: +def test_talker_projects_request_aligned_duplex_metadata() -> None: talker = _make_talker() - mocker.patch.object(talker, "_sample_audio_codes", return_value=torch.tensor([2, 2])) infos = [ { "request_id": "req-a", "native_duplex": True, "duplex": {"epoch": 3, "turn_id": 7}, "ids": {"tts": [41]}, - "meta": { - "native_duplex_segment_text": "first", - "turn_eos_token_id": 99, - }, - "audio_codes": {"accumulated": torch.empty(0, dtype=torch.long)}, + "meta": {"native_duplex_segment_text": "first", "turn_eos_token_id": 99}, + "codes": {"audio": torch.empty(0)}, }, { "request_id": "req-b", "native_duplex": True, "duplex": {"epoch": 4, "turn_id": 8}, "ids": {"tts": [42, 99]}, - "meta": { - "native_duplex_segment_text": "second", - "turn_eos_token_id": 99, - }, - "audio_codes": {"accumulated": torch.empty(0, dtype=torch.long)}, + "meta": {"native_duplex_segment_text": "second", "turn_eos_token_id": 99}, + "codes": {"audio": torch.empty(0)}, }, ] @@ -360,7 +421,6 @@ def test_talker_projects_request_aligned_duplex_metadata(mocker) -> None: assert [value.item() for value in meta["native_duplex"]] == [True, True] assert [value.item() for value in meta["duplex_epoch"]] == [3, 4] assert [value.item() for value in meta["duplex_turn_id"]] == [7, 8] - assert "native_duplex_segment_text" not in meta assert [bytes(value.tolist()).decode("utf-8") for value in meta["llm_output_text_utf8"]] == [ "first", "second", @@ -368,14 +428,9 @@ def test_talker_projects_request_aligned_duplex_metadata(mocker) -> None: assert [value.item() for value in meta["turn_end"]] == [False, True] -def test_talker_rejects_native_duplex_without_fence_identity(mocker) -> None: +def test_talker_rejects_native_duplex_without_fence_identity() -> None: talker = _make_talker() - mocker.patch.object(talker, "_sample_audio_codes", return_value=torch.tensor([2])) - info = { - "request_id": "req-missing-fence", - "native_duplex": True, - "audio_codes": {"accumulated": torch.empty(0, dtype=torch.long)}, - } + info = {"request_id": "req-missing-fence", "native_duplex": True, "codes": {"audio": torch.empty(0)}} with pytest.raises(RuntimeError, match="requires non-negative integer epoch and turn_id"): talker.make_omni_output( @@ -385,113 +440,35 @@ def test_talker_rejects_native_duplex_without_fence_identity(mocker) -> None: ) -def test_incomplete_prefill_emits_no_code_and_does_not_advance_state(mocker) -> None: +def test_decode_preprocess_embeds_the_sampled_codec_id() -> None: talker = _make_talker() - sample = mocker.patch.object(talker, "_sample_audio_codes", return_value=torch.tensor([2])) - infos = [ - { - "request_id": "req-prefill", - "audio_state": {"step": 0}, - "audio_codes": {"accumulated": torch.empty(0, dtype=torch.long)}, - }, - { - "request_id": "req-decode", - "audio_state": {"step": 4}, - "audio_codes": {"accumulated": torch.tensor([1])}, - }, - ] + talker.emb_code = nn.ModuleList([nn.Embedding(8, 4)]) + with torch.no_grad(): + talker.emb_code[0].weight.copy_(torch.arange(32, dtype=torch.float32).reshape(8, 4)) - output = talker.make_omni_output( - torch.tensor([[1.0, 0.0], [2.0, 0.0], [3.0, 0.0]]), - model_intermediate_buffer=infos, - request_token_spans=[(0, 2), (2, 3)], - request_sample_eligible=[False, True], + _, embeds, updates = talker.preprocess( + torch.tensor([3]), + None, + request_id="req-decode", + audio_state={"finished": False}, ) - sample.assert_called_once() - assert sample.call_args.args[2] == ["req-decode"] - assert infos[0]["audio_state"]["step"] == 0 - assert infos[0]["audio_codes"]["accumulated"].numel() == 0 - assert infos[1]["audio_state"]["step"] == 5 - assert _routed(output, 0)["codes"]["audio"].shape == (0, 1) - assert _routed(output, 1)["codes"]["audio"].tolist() == [[2]] + assert torch.equal(embeds, talker.emb_code[0](torch.tensor([3]))) + assert updates["codes"]["audio"].tolist() == [[3]] -def test_eos_is_terminal_once_and_never_enters_codec_history(mocker) -> None: +def test_decode_preprocess_drops_codec_eos() -> None: talker = _make_talker() - sample = mocker.patch.object(talker, "_sample_audio_codes", return_value=torch.tensor([7])) - info = { - "request_id": "req-stop", - "audio_state": {"step": 3}, - "audio_codes": {"accumulated": torch.tensor([4, 5])}, - } + talker.emb_code = nn.ModuleList([nn.Embedding(8, 4)]) - first = talker.make_omni_output( - torch.ones(1, 2), - model_intermediate_buffer=[info], - request_token_spans=[(0, 1)], - ) - first_logits = talker.compute_logits(first.text_hidden_states) - second = talker.make_omni_output( - torch.ones(1, 2), - model_intermediate_buffer=[info], - request_token_spans=[(0, 1)], + _, _, updates = talker.preprocess( + torch.tensor([7]), + None, + request_id="req-eos", + audio_state={"finished": False}, ) - sample.assert_called_once() - assert info["audio_codes"]["accumulated"].tolist() == [4, 5] - assert first.multimodal_outputs["codes"]["audio"][0].shape == (0, 1) - assert first.multimodal_outputs["meta"]["finished"][0].item() is True - assert second.multimodal_outputs["meta"]["finished"][0].item() is False - assert first_logits.argmax(dim=-1).tolist() == [1] - assert talker.compute_logits(second.text_hidden_states).argmax(dim=-1).tolist() == [1] - - -def test_max_token_terminal_drops_unconsumed_codec_delta(mocker) -> None: - talker = _make_talker() - mocker.patch.object(talker, "_sample_audio_codes", return_value=torch.tensor([3])) - info = { - "request_id": "req-limit", - "audio_state": {"step": 1, "max_tokens": 2}, - "audio_codes": {"accumulated": torch.tensor([4, 5])}, - } - - output = talker.make_omni_output( - torch.ones(1, 2), - model_intermediate_buffer=[info], - request_token_spans=[(0, 1)], - ) - - # MiniCPMTTS.generate_chunk samples once at the max-token boundary to - # advance RNG state, but the sampled code is not fed into KV or returned. - assert info["audio_codes"]["accumulated"].tolist() == [4, 5] - assert output.multimodal_outputs["codes"]["audio"][0].shape == (0, 1) - assert output.multimodal_outputs["meta"]["finished"][0].item() is True - assert talker.compute_logits(output.text_hidden_states).argmax(dim=-1).tolist() == [1] - - -def test_request_local_state_survives_missing_runner_buffer_update(mocker) -> None: - talker = _make_talker() - mocker.patch.object(talker, "_sample_audio_codes", return_value=torch.tensor([3])) - first_info = { - "request_id": "req-local-state", - "audio_state": {"step": 1, "max_tokens": 3}, - "audio_codes": {"accumulated": torch.tensor([4])}, - } - - talker.make_omni_output( - torch.ones(1, 2), - model_intermediate_buffer=[first_info], - request_token_spans=[(0, 1)], - ) - second = talker.make_omni_output( - torch.ones(1, 2), - model_intermediate_buffer=[{"request_id": "req-local-state"}], - request_token_spans=[(0, 1)], - ) - - assert second.multimodal_outputs["meta"]["finished"][0].item() is True - assert talker._request_audio_states["req-local-state"]["step"] == 3 + assert updates["codes"]["audio"].numel() == 0 def test_missing_conditioning_fails_clearly() -> None: @@ -524,18 +501,19 @@ def test_empty_speech_segment_finishes_without_sampling_codes() -> None: assert torch.equal(embeds, talker.emb_text(torch.tensor([5, 6]))) assert updates["audio_state"]["finished"] is True + assert updates["codes"]["audio"].numel() == 0 - # Stage 1's sampling min_tokens keeps scheduling decode steps until the stop - # token becomes eligible, and those steps have no previous code to embed. - _, decode_embeds, _ = talker.preprocess( + # min_tokens can keep scheduling leftover decode rows after an empty + # speech segment. Those rows have no previous codec id to embed. + _, decode_embeds, decode_updates = talker.preprocess( torch.zeros(1, dtype=torch.long), None, request_id="req-empty", audio_state=updates["audio_state"], - audio_codes=updates["audio_codes"], ) assert decode_embeds.shape == (1, 4) + assert decode_updates["codes"]["audio"].numel() == 0 def test_chunked_prefill_tail_aligns_condition_with_prompt_length(mocker) -> None: @@ -544,7 +522,7 @@ def test_chunked_prefill_tail_aligns_condition_with_prompt_length(mocker) -> Non condition = torch.arange(18, dtype=torch.float32).reshape(9, 2) mocker.patch.object(talker, "_build_condition_embeddings", return_value=condition) - _, embeds, _ = talker.preprocess( + _, embeds, updates = talker.preprocess( torch.zeros(9, dtype=torch.long), None, _omni_is_prefill=True, @@ -556,46 +534,9 @@ def test_chunked_prefill_tail_aligns_condition_with_prompt_length(mocker) -> Non ) assert torch.equal(embeds, condition) + assert updates["codes"]["audio"].numel() == 0 state = talker._request_audio_states["req-chunked-prefill"] - assert state["min_tokens"] == 50 - assert state["max_tokens"] == 64 - - -@pytest.mark.parametrize( - ("meta", "expected_min_tokens"), - [ - ({"turn_start": True}, 0), - ({}, 26), - ({"turn_end": True}, 0), - ], -) -def test_native_duplex_prefill_uses_official_chunk_limits( - mocker, - meta, - expected_min_tokens, -) -> None: - talker = _make_talker() - talker.emb_text = nn.Embedding(1, 2) - mocker.patch.object( - talker, - "_build_condition_embeddings", - return_value=torch.ones(3, 2), - ) - - talker.preprocess( - torch.zeros(3, dtype=torch.long), - None, - _omni_is_prefill=True, - request_id="req-duplex-chunk", - native_duplex=True, - meta=meta, - tts_token_ids=torch.tensor([1]), - tts_hidden_states=torch.ones(1, 2), - ) - - state = talker._request_audio_states["req-duplex-chunk"] - assert state["min_tokens"] == expected_min_tokens - assert state["max_tokens"] == 26 + assert state["finished"] is False def test_native_duplex_condition_matches_official_text_plus_audio_bos() -> None: @@ -626,13 +567,11 @@ def test_native_duplex_condition_matches_official_text_plus_audio_bos() -> None: assert condition.shape[0] == token_ids.shape[0] + 1 -def test_request_cleanup_evicts_ar_rng_and_decode_state() -> None: +def test_request_cleanup_evicts_decode_state() -> None: talker = _make_talker() - talker._request_generators["req-done"] = torch.Generator() - talker._request_audio_states["req-done"] = {"step": 1} + talker._request_audio_states["req-done"] = {"finished": False} talker.on_requests_finished(["req-done"]) talker._flush_deferred_cleanup() - assert "req-done" not in talker._request_generators assert "req-done" not in talker._request_audio_states diff --git a/tests/worker/test_native_duplex_hooks.py b/tests/worker/test_native_duplex_hooks.py index 383379202..ba50be72c 100644 --- a/tests/worker/test_native_duplex_hooks.py +++ b/tests/worker/test_native_duplex_hooks.py @@ -1,5 +1,6 @@ from __future__ import annotations +import ast import inspect from types import SimpleNamespace @@ -2466,3 +2467,128 @@ def test_minicpmo_stage0_session_context_includes_resolved_ref_audio(): assert state.context_token_ids == [1, 2, 3, 151683, 151683, 4, 5] assert len(state.context_embeds) == 6 + + +# --------------------------------------------------------------------------- +# Cross-platform wiring: every AR model runner must reach the duplex hook. +# +# GPUARModelRunner and NPUARModelRunner are siblings -- neither inherits the +# other's _sample -- so the NPU runner once copied GPU _sample without the +# prepare_duplex_sampling call. Parse source instead of importing NPUARModelRunner: +# that class needs vllm_ascend, which CPU test images do not have. +# --------------------------------------------------------------------------- + +_REQUIRED_DUPLEX_CALLS = { + "__init__": "_init_duplex_sampling_state", + "load_model": "_resolve_duplex_sampling_hook", + "_update_states": "_update_duplex_sampling_states", + "_sample": "_apply_duplex_sampling", +} + +_MODEL_SAMPLER_CALL = "model_sample" + + +def _repo_root(): + from pathlib import Path + + return Path(__file__).parents[2] + + +def _ar_runner_sources(): + root = _repo_root() + paths = [root / "vllm_omni/worker/gpu_ar_model_runner.py"] + paths.extend(sorted(root.glob("vllm_omni/platforms/*/worker/*_ar_model_runner.py"))) + return paths + + +def _ar_runner_classdefs(): + root = _repo_root() + found = {} + for path in _ar_runner_sources(): + module = ast.parse(path.read_text()) + for node in module.body: + if isinstance(node, ast.ClassDef) and node.name.endswith("ARModelRunner"): + found[node.name] = (node, str(path.relative_to(root))) + assert found, "no AR model runners discovered; the glob in _ar_runner_sources() is stale" + return found + + +def _method_calls(method): + calls = {} + for node in ast.walk(method): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute): + calls.setdefault(node.func.attr, node.lineno) + return calls + + +def _wires_duplex_sampling_directly(classdef) -> bool: + bases = {base.id for base in classdef.bases if isinstance(base, ast.Name)} + if "DuplexSamplingRunnerMixin" not in bases: + return False + methods = {node.name: node for node in classdef.body if isinstance(node, ast.FunctionDef)} + return all( + method_name in methods and required_call in _method_calls(methods[method_name]) + for method_name, required_call in _REQUIRED_DUPLEX_CALLS.items() + ) + + +def test_every_ar_runner_reaches_the_duplex_sampling_hook(): + classdefs = _ar_runner_classdefs() + wired = {name for name, (classdef, _path) in classdefs.items() if _wires_duplex_sampling_directly(classdef)} + + for class_name, (classdef, path) in sorted(classdefs.items()): + if class_name in wired: + continue + base_names = {base.id for base in classdef.bases if isinstance(base, ast.Name)} + assert base_names & wired, ( + f"{path}::{class_name} neither wires the duplex sampling hook nor derives from a " + f"runner that does (wired runners: {sorted(wired)}). Inherit DuplexSamplingRunnerMixin " + f"and call {sorted(_REQUIRED_DUPLEX_CALLS.values())} from " + f"{sorted(_REQUIRED_DUPLEX_CALLS)} respectively." + ) + + +@pytest.mark.parametrize("class_name", ["GPUARModelRunner", "NPUARModelRunner"]) +def test_ar_runner_calls_every_duplex_sampling_hook_site(class_name: str): + classdefs = _ar_runner_classdefs() + assert class_name in classdefs, f"{class_name} not found by _ar_runner_sources()" + classdef, path = classdefs[class_name] + + bases = {base.id for base in classdef.bases if isinstance(base, ast.Name)} + assert "DuplexSamplingRunnerMixin" in bases, ( + f"{path}::{class_name} must inherit DuplexSamplingRunnerMixin so the duplex " + f"sampling hook is wired the same way on every platform" + ) + + methods = {node.name: node for node in classdef.body if isinstance(node, ast.FunctionDef)} + for method_name, required_call in _REQUIRED_DUPLEX_CALLS.items(): + method = methods.get(method_name) + assert method is not None, f"{path}::{class_name}.{method_name} is missing; it must call {required_call}" + calls = _method_calls(method) + assert required_call in calls, f"{path}::{class_name}.{method_name} must call self.{required_call}" + + +@pytest.mark.parametrize("class_name", ["GPUARModelRunner", "NPUARModelRunner"]) +def test_ar_runner_applies_duplex_sampling_before_the_model_sampler(class_name: str): + classdef, path = _ar_runner_classdefs()[class_name] + sample = next(node for node in classdef.body if isinstance(node, ast.FunctionDef) and node.name == "_sample") + + apply_lineno = _method_calls(sample).get("_apply_duplex_sampling") + sampler_lineno = next( + ( + node.lineno + for node in ast.walk(sample) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == _MODEL_SAMPLER_CALL + ), + None, + ) + assert apply_lineno is not None, f"{path}::{class_name}._sample must call self._apply_duplex_sampling" + assert sampler_lineno is not None, ( + f"{path}::{class_name}._sample no longer calls {_MODEL_SAMPLER_CALL}(); update _MODEL_SAMPLER_CALL " + f"so the ordering check keeps testing something" + ) + assert apply_lineno < sampler_lineno, ( + f"{path}::{class_name}._sample calls self._apply_duplex_sampling at line {apply_lineno}, after " + f"{_MODEL_SAMPLER_CALL}() at line {sampler_lineno}. The hook masks the logits and publishes the " + f"row -> session map the sampler reads, so running it afterwards is a silent no-op." + ) diff --git a/vllm_omni/config/stage_config.py b/vllm_omni/config/stage_config.py index cdfd35ee2..0e4457733 100644 --- a/vllm_omni/config/stage_config.py +++ b/vllm_omni/config/stage_config.py @@ -532,7 +532,13 @@ def _parse_stage_deploy(stage_data: dict[str, Any]) -> StageDeployConfig: _DEEP_MERGE_KEYS = frozenset( - {"default_sampling_params", "default_pooling_params", "subtalker_sampling_params", "engine_extras", "engine_args"} + { + "default_sampling_params", + "default_pooling_params", + "subtalker_sampling_params", + "engine_extras", + "engine_args", + } ) diff --git a/vllm_omni/deploy/minicpmo_4_5.yaml b/vllm_omni/deploy/minicpmo_4_5.yaml index 532de5e79..cd7705f52 100644 --- a/vllm_omni/deploy/minicpmo_4_5.yaml +++ b/vllm_omni/deploy/minicpmo_4_5.yaml @@ -64,15 +64,15 @@ stages: devices: "0" output_connectors: to_stage_2: connector_of_shared_memory - # Codec sampling comes from the checkpoint's tts_config. These parameters - # govern only vLLM's binary continue/stop token. + # Talker is a single-vocab codec LM. These knobs go to vLLM's Sampler + # (head_code logits). EOS is tts_config.eos_token_id (num_audio_tokens-1). default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 + temperature: 0.8 + top_p: 0.8 + top_k: 100 + repetition_penalty: 1.02 min_tokens: 50 - max_tokens: 2048 - stop_token_ids: [1] + max_tokens: 4096 seed: 42 detokenize: false diff --git a/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml b/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml index 5b43206b8..fd56b376f 100644 --- a/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml +++ b/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml @@ -52,16 +52,13 @@ stages: devices: "1" output_connectors: to_stage_2: connector_of_shared_memory - # Codec sampling is model-owned by the checkpoint's tts_config (defaults: - # T=0.8, top-k=25, top-p=0.85, repetition penalty=1.05, seed=42). - # These parameters only govern vLLM's binary continue/stop token. default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 + temperature: 0.8 + top_p: 0.8 + top_k: 100 + repetition_penalty: 1.02 min_tokens: 50 - max_tokens: 2048 - stop_token_ids: [1] + max_tokens: 4096 seed: 42 detokenize: false diff --git a/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml b/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml index edbde0c5d..0bb21b177 100644 --- a/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml +++ b/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml @@ -57,15 +57,13 @@ stages: devices: "1" output_connectors: to_stage_2: connector_of_shared_memory - # vLLM samples the binary continue/stop row; codec sampling comes from - # the checkpoint's tts_config and currently uses a deterministic seed. default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 + temperature: 0.8 + top_p: 0.8 + top_k: 100 + repetition_penalty: 1.02 min_tokens: 50 - max_tokens: 2048 - stop_token_ids: [1] + max_tokens: 4096 seed: 42 detokenize: false diff --git a/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml b/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml index 813a2e2eb..0734e2d96 100644 --- a/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml +++ b/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml @@ -69,15 +69,13 @@ stages: devices: "4" output_connectors: to_stage_2: connector_of_shared_memory - # vLLM samples the binary continue/stop row; codec sampling comes from - # the checkpoint's tts_config and currently uses a deterministic seed. default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 + temperature: 0.8 + top_p: 0.8 + top_k: 100 + repetition_penalty: 1.02 min_tokens: 50 - max_tokens: 2048 - stop_token_ids: [1] + max_tokens: 4096 seed: 42 detokenize: false diff --git a/vllm_omni/deploy/minicpmo_4_5_duplex.yaml b/vllm_omni/deploy/minicpmo_4_5_duplex.yaml index df0e40be9..b51c46a9d 100644 --- a/vllm_omni/deploy/minicpmo_4_5_duplex.yaml +++ b/vllm_omni/deploy/minicpmo_4_5_duplex.yaml @@ -29,12 +29,13 @@ stages: async_scheduling: false # Talker context is 4096 (tts_config.max_position_embeddings). max_model_len: 4096 - # The split Talker exposes a binary continue/stop vocabulary; native - # segment and turn boundaries travel as explicit output metadata. + # Overlay only the knobs duplex changes. Codec temperature / top_k stay + # on the base YAML. Native duplex still ends each Talker request after + # one generate_chunk (26 samples, last is codec EOS) inside the model; + # YAML max_tokens is only the hard ceiling. default_sampling_params: - max_tokens: 2048 + max_tokens: 4096 min_tokens: 0 - stop_token_ids: [1] - stage_id: 2 max_num_seqs: 2 diff --git a/vllm_omni/engine/orchestrator.py b/vllm_omni/engine/orchestrator.py index f112983e1..0fb3fb14c 100644 --- a/vllm_omni/engine/orchestrator.py +++ b/vllm_omni/engine/orchestrator.py @@ -127,8 +127,16 @@ def build_engine_core_request_from_tokens( pooling_params = None if isinstance(params, SamplingParams): sampling_params = params.clone() - if sampling_params.max_tokens is None and model_config is not None: - sampling_params.max_tokens = model_config.max_model_len - len(prompt_token_ids) + if model_config is not None: + remaining = model_config.max_model_len - len(prompt_token_ids) + if sampling_params.max_tokens is None: + sampling_params.max_tokens = remaining + # ``check_stop`` returns early while ``min_tokens`` is unmet, so a + # min_tokens wider than the leftover context window never reaches + # the length cap: the request stalls once the scheduler can grant + # no further tokens. Stage prompts are built here, so clamp now. + if sampling_params.min_tokens > remaining: + sampling_params.min_tokens = max(0, remaining) else: pooling_params = params.clone() diff --git a/vllm_omni/experimental/fullduplex/model_executor.py b/vllm_omni/experimental/fullduplex/model_executor.py index eebe01876..58ba597df 100644 --- a/vllm_omni/experimental/fullduplex/model_executor.py +++ b/vllm_omni/experimental/fullduplex/model_executor.py @@ -4,7 +4,12 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + import torch + from vllm.v1.core.sched.output import SchedulerOutput + from vllm.v1.sample.metadata import SamplingMetadata @dataclass(frozen=True, slots=True) @@ -103,4 +108,55 @@ class DuplexSamplingHelper: self.hook_active = False -__all__ = ["DuplexSamplingHelper", "DuplexSamplingRow"] +class DuplexSamplingRunnerMixin: + """Runner-side wiring for the experimental duplex sampling hook. + + GPU and NPU AR runners are siblings, not parent and child, so neither + inherits the other's ``_sample``. Keep the hook here and have each runner + opt in, instead of copying the GPU ``_sample`` path and dropping the + ``prepare_duplex_sampling`` call. + """ + + def _init_duplex_sampling_state(self) -> None: + """Reset hook resolution. Call from ``__init__``, before ``load_model``.""" + self._duplex_sampling_hook = None + self._duplex_sampling_hook_resolved = False + + def _resolve_duplex_sampling_hook(self, *, force: bool = False): + """Bind ``model.prepare_duplex_sampling``. Call with ``force`` after load.""" + if not force and getattr(self, "_duplex_sampling_hook_resolved", False): + return self._duplex_sampling_hook + candidate = getattr(getattr(self, "model", None), "prepare_duplex_sampling", None) + self._duplex_sampling_hook = candidate if callable(candidate) else None + self._duplex_sampling_hook_resolved = True + if self._duplex_sampling_hook is not None and not hasattr(self, "_duplex_sampling_helper"): + self._duplex_sampling_helper = DuplexSamplingHelper() + return self._duplex_sampling_hook + + def _update_duplex_sampling_states(self, scheduler_output: SchedulerOutput) -> None: + """Refresh which batch rows are duplex rows. Call from ``_update_states``.""" + if self._resolve_duplex_sampling_hook() is None: + return + helper = getattr(self, "_duplex_sampling_helper", None) + if helper is not None: + helper.update_states(self, scheduler_output) + + def _apply_duplex_sampling(self, logits: torch.Tensor, prepared_sampling_metadata: SamplingMetadata) -> None: + """Hand the model its duplex rows. Call from ``_sample``, before the sampler.""" + prepare_duplex_sampling = self._resolve_duplex_sampling_hook() + if prepare_duplex_sampling is None: + return + helper = getattr(self, "_duplex_sampling_helper", None) + rows = helper.rows(self) if helper is not None and helper.active_request_ids else () + if rows or (helper is not None and helper.hook_active): + prepare_duplex_sampling(logits, prepared_sampling_metadata, rows) + if helper is not None: + helper.hook_active = bool(rows) + + def _clear_duplex_sampling(self) -> None: + helper = getattr(self, "_duplex_sampling_helper", None) + if helper is not None: + helper.clear() + + +__all__ = ["DuplexSamplingHelper", "DuplexSamplingRow", "DuplexSamplingRunnerMixin"] diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py b/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py index fe5c8a57e..1553a5921 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py @@ -19,6 +19,65 @@ from .cuda_graph_wrapper import CFMGraphWrapper, HiFTGraphWrapper logger = init_logger(__name__) _SILENCE_TOKEN = 4218 +# CosyVoice2 RelPos PE is built for max_len=5000 and `forward_chunk` never +# calls ``extend_pe``. After the 2x upsample a single Code2Wav prefill longer +# than this many codec tokens overflows the PE slice (matrix_ac vs matrix_bd). +_DEFAULT_RELPOS_MAX_POS = 5000 +_MAX_ENCODE_TOKEN_CAP = 1024 + + +def relpos_encode_token_budget( + *, + max_pos: int, + stride: int, + cache_offset: int, + lookahead: int, + cap: int = _MAX_ENCODE_TOKEN_CAP, +) -> int: + """How many codec tokens ``forward_chunk`` can take before RelPos PE wraps. + + ``position_encoding(size)`` needs ``size <= max_pos``. After upsample, + ``size = cache_offset * stride + ~stride * token_frames`` (plus last-chunk + lookahead pad). + """ + stride = max(1, int(stride)) + lookahead = max(0, int(lookahead)) + room = int(max_pos) // stride - max(0, int(cache_offset)) - lookahead - 1 + return max(lookahead + 1, min(int(cap), room)) + + +def plan_token2wav_encode_slices( + num_frames: int, + *, + max_frames: int, + min_nonfinal: int, + last_chunk: bool, +) -> list[tuple[int, int]]: + """Split a Code2Wav token span so every non-final piece has a lookahead window.""" + if num_frames <= 0: + return [] + min_nonfinal = max(1, int(min_nonfinal)) + max_frames = max(int(max_frames), min_nonfinal) + last_min = 1 if last_chunk else min_nonfinal + if num_frames <= max_frames: + return [(0, num_frames)] + + slices: list[tuple[int, int]] = [] + start = 0 + while start < num_frames: + remaining = num_frames - start + if remaining <= max_frames: + slices.append((start, num_frames)) + break + take = max_frames + tail = remaining - take + if tail <= max_frames and tail < last_min: + take = remaining - last_min + if take < min_nonfinal: + take = remaining + slices.append((start, start + take)) + start += take + return slices def _autocast_disabled(device: torch.device): @@ -237,6 +296,41 @@ class BatchedToken2Wav(nn.Module): width = getattr(layer, "pre_lookahead_len", None) return int(width) if width is not None else None + def _relpos_max_pos(self) -> int: + embed = getattr(self.flow.encoder, "embed", None) + pe = getattr(embed, "pe", None) + if isinstance(pe, torch.Tensor) and pe.numel() > 0: + return max(1, int(pe.size(1) // 2)) + return _DEFAULT_RELPOS_MAX_POS + + def _upsample_stride(self) -> int: + stride = getattr(getattr(self.flow.encoder, "up_layer", None), "stride", None) + return max(1, int(stride)) if stride is not None else 2 + + def _max_encode_token_frames(self, states: list[BatchedToken2WavState]) -> int: + att = None + if states: + att = self._stack_flow_cache(states).get("conformer_att_cache") + offset1 = int(att.shape[3] // 2) if att is not None else 0 + lookahead = self._pre_lookahead_len() or 0 + return relpos_encode_token_budget( + max_pos=self._relpos_max_pos(), + stride=self._upsample_stride(), + cache_offset=offset1, + lookahead=lookahead, + ) + + def _ensure_relpos_pe(self, tokens: torch.Tensor, att_cache: torch.Tensor | None) -> None: + """Grow CosyVoice RelPos PE before ``forward_chunk``, which never calls extend_pe.""" + embed = getattr(self.flow.encoder, "embed", None) + extend_pe = getattr(embed, "extend_pe", None) + if not callable(extend_pe): + return + offset1 = int(att_cache.shape[3] // 2) if att_cache is not None else 0 + lookahead = self._pre_lookahead_len() or 0 + needed = offset1 * self._upsample_stride() + self._upsample_stride() * (int(tokens.shape[1]) + lookahead + 1) + extend_pe(tokens.new_zeros((1, max(needed, 1)))) + def _encode_chunk( self, tokens: torch.Tensor, @@ -245,6 +339,7 @@ class BatchedToken2Wav(nn.Module): cnn_cache: torch.Tensor | None, att_cache: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + self._ensure_relpos_pe(tokens, att_cache) embedded = self.flow.input_embedding(tokens) hidden, new_cnn, new_att = self.flow.encoder.forward_chunk( xs=embedded, @@ -508,14 +603,68 @@ class BatchedToken2Wav(nn.Module): # must carry at least one full kernel. Only the final chunk is allowed # to be shorter: ``forward_chunk`` zero-pads it by the lookahead width. lookahead = self._pre_lookahead_len() + num_frames = int(tokens.shape[1]) if lookahead is not None and not last_chunk: - num_frames = int(tokens.shape[1]) if num_frames <= lookahead: raise RuntimeError( "MiniCPMO45Code2WavBatchError " f'{{"reason":"chunk_below_lookahead_window","frames":{num_frames},' f'"minimum":{lookahead + 1}}}' ) + # A non-async Talker dump can land thousands of codec tokens in one + # Code2Wav prefill. CosyVoice RelPos PE (max_len=5000) plus 2x upsample + # cannot score that in one ``forward_chunk`` (6968 vs 985 on NPU). + max_frames = self._max_encode_token_frames(states) + slices = plan_token2wav_encode_slices( + num_frames, + max_frames=max_frames, + min_nonfinal=(lookahead + 1) if lookahead is not None else 1, + last_chunk=last_chunk, + ) + if len(slices) > 1: + logger.info( + "MiniCPM-o Code2Wav splitting %d codec tokens into %d encoder windows " + "(max_frames=%d) to stay inside RelPos PE.", + num_frames, + len(slices), + max_frames, + ) + parts: list[list[torch.Tensor]] = [[] for _ in range(batch_size)] + current = states + for index, (start, end) in enumerate(slices): + is_last_piece = index == len(slices) - 1 + audios, current = self._decode_batch_once( + tokens[:, start:end], + features, + current, + last_chunk=last_chunk and is_last_piece, + flush_encoder=flush_encoder and is_last_piece, + ) + for row, audio in enumerate(audios): + parts[row].append(audio) + merged = [ + torch.cat(row_parts) if row_parts else tokens.new_zeros((0,), dtype=torch.float32) + for row_parts in parts + ] + return merged, current + return self._decode_batch_once( + tokens, + features, + states, + last_chunk=last_chunk, + flush_encoder=flush_encoder, + ) + + def _decode_batch_once( + self, + tokens: torch.Tensor, + features: PromptFeatures, + states: list[BatchedToken2WavState], + *, + last_chunk: bool, + flush_encoder: bool = False, + ) -> tuple[list[torch.Tensor], list[BatchedToken2WavState]]: + batch_size = int(tokens.shape[0]) flow_cache = self._stack_flow_cache(states) speakers = features.speaker_embedding.expand(batch_size, -1) with self._autocast(tokens.device): diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py index 490d8e0cf..af9401872 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py @@ -301,6 +301,10 @@ class ConditionalChatTTSConfig(PretrainedConfig): self.top_p = top_p self.top_k = top_k self.repetition_penalty = repetition_penalty + # Talker stage is a single-vocab codec LM: vLLM's Sampler and + # embed_tokens both address this table, not the Thinker text vocab. + self.vocab_size = int(num_audio_tokens) + self.eos_token_id = int(num_audio_tokens) - 1 class MiniCPMOConfig(Qwen2Config): diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py index 9e9dfe952..0480176b7 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py @@ -8,11 +8,12 @@ Pipeline: 1. Receive thinker hidden_states + full token IDs via additional_information 2. Extract tts_bos..tts_eos region 3. Build condition: emb_text(tokens) + projector_semantic(hidden) (hidden_text_merge) - 4. Continuously generate request-aligned discrete audio-code deltas + 4. Project last hidden through head_code; vLLM Sampler picks the codec id + 5. Next decode embeds that id with emb_code and emits it to Code2Wav """ -from collections.abc import Iterable, Sequence -from dataclasses import dataclass +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import replace from typing import Any import torch @@ -32,45 +33,34 @@ from vllm_omni.platforms import current_omni_platform logger = init_logger(__name__) -_REPETITION_WINDOW = 16 _REPETITION_PENALTY_CHUNK_SIZE = 16 -_MIN_AUDIO_TOKENS = 64 -_MAX_AUDIO_TOKENS = 2048 -_AUDIO_TOKENS_PER_TEXT_TOKEN = 10 -# Codec-token sampling happens inside the model; vLLM sampling parameters -# only choose the Talker's binary continue/stop row. -_CODEC_SEED = 42 -_CODEC_TEMPERATURE = 0.8 -_CODEC_TOP_K = 25 -_CODEC_TOP_P = 0.85 -_CODEC_REPETITION_PENALTY = 1.05 -_CODEC_MIN_TOKENS = 50 +# Native duplex Talker must finish after one MiniCPMTTS.generate_chunk: +# 25 codec frames (``codec_chunk_frames``) plus the terminating sample. +# Without this, the single-vocab Sampler keeps the stage-1 request alive +# until codec EOS / 4096 and Thinker never starts the next model turn. _DUPLEX_CODEC_TOKENS_PER_CHUNK = 26 -@dataclass(slots=True) -class _PendingCodecSample: - output_index: int - hidden_row: torch.Tensor - codes: torch.Tensor - request_id: str - step: int - state: dict[str, Any] - info: dict[str, Any] +def _native_duplex_chunk_budget(meta: Mapping[str, Any] | None) -> tuple[int, int]: + """Return ``(max_tokens, min_tokens)`` for one native-duplex Talker request.""" + boundary = isinstance(meta, Mapping) and (bool(meta.get("turn_start")) or bool(meta.get("turn_end"))) + ceiling = _DUPLEX_CODEC_TOKENS_PER_CHUNK + return ceiling, 0 if boundary else ceiling -def _max_audio_tokens(condition_tokens: int) -> int: - """Bound codec generation with a conservative text-length estimate. +def blank_scheduler_prompt_for_penalties( + prompt_token_ids: torch.Tensor, + vocab_size: int, +) -> torch.Tensor: + """Return a penalty prompt whose every position is the pad id (``vocab_size``). - EOS is masked for the first 50 steps, so a direct ``text_tokens * 10`` - limit can terminate short responses before EOS is eligible. The 2048 - ceiling matches the checkpoint's native generation default and keeps the - sequence within the Talker's 4096-position context. + No Talker prompt position is codec history: prefill conditioning arrives as + embeddings from ``preprocess`` and decode embeds sampled ids with + ``emb_code``. The scheduler ids are placeholders (``llm2tts`` fills them + with ``0``, or with thinker token ids on the non-handoff path), so scoring + them would tax unrelated codec tokens. """ - return max( - _MIN_AUDIO_TOKENS, - min(_MAX_AUDIO_TOKENS, condition_tokens * _AUDIO_TOKENS_PER_TEXT_TOKEN), - ) + return torch.full_like(prompt_token_ids, int(vocab_size)) def _restore_weight_norm_weight(weight_g: torch.Tensor, weight_v: torch.Tensor) -> torch.Tensor: @@ -122,32 +112,6 @@ def _apply_batched_repetition_penalty( return penalized -def _apply_top_k_top_p( - logits: torch.Tensor, - *, - top_k: int | None, - top_p: float | None, - min_tokens_to_keep: int = 3, -) -> torch.Tensor: - """Apply the same candidate floors as the upstream Transformers warpers.""" - filtered = logits.clone() - vocab_size = filtered.shape[-1] - # MiniCPM-o's gen_logits() appends TopPLogitsWarper before - # TopKLogitsWarper. The order is observable for fixed-seed sampling. - if top_p is not None and 0.0 < top_p < 1.0: - sorted_logits, sorted_indices = torch.sort(filtered, descending=False, dim=-1) - cumulative_probs = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1) - remove = cumulative_probs <= (1.0 - float(top_p)) - remove[..., -min_tokens_to_keep:] = False - remove = remove.scatter(-1, sorted_indices, remove) - filtered.masked_fill_(remove, float("-inf")) - if top_k is not None and top_k > 0: - keep = min(vocab_size, max(int(top_k), min_tokens_to_keep)) - threshold = torch.topk(filtered, keep, dim=-1).values[..., -1, None] - filtered.masked_fill_(filtered < threshold, float("-inf")) - return filtered - - class _MiniCPMTTSProjector(nn.Module): """Checkpoint-compatible hidden-state projector used by MiniCPMTTS.""" @@ -173,8 +137,8 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): config: MiniCPMOConfig = vllm_config.model_config.hf_config self.config = config self.vllm_config = vllm_config - self._batch_stop_logits: torch.Tensor | None = None - self._request_generators: dict[str, torch.Generator] = {} + self._force_eos_rows: list[bool] | None = None + self._mask_eos_rows: list[bool] | None = None self._request_audio_states: dict[str, dict[str, Any]] = {} self._deferred_cleanup_ids: set[str] = set() @@ -186,23 +150,18 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): self._tts_bos_id = getattr(tts_config, "audio_bos_token_id", 151687) self._text_eos_id = getattr(tts_config, "text_eos_token_id", 151692) self._num_audio_tokens = getattr(tts_config, "num_audio_tokens", 6562) + self._codec_eos_id = int(getattr(tts_config, "eos_token_id", self._num_audio_tokens - 1)) self._hidden_size = getattr(tts_config, "hidden_size", 768) self._normalize = getattr(tts_config, "normalize_projected_hidden", True) - self._codec_seed = int(getattr(tts_config, "seed", _CODEC_SEED)) - self._codec_temperature = float(getattr(tts_config, "temperature", _CODEC_TEMPERATURE)) - self._codec_top_k = int(getattr(tts_config, "top_k", _CODEC_TOP_K)) - self._codec_top_p = float(getattr(tts_config, "top_p", _CODEC_TOP_P)) - self._codec_repetition_penalty = float(getattr(tts_config, "repetition_penalty", _CODEC_REPETITION_PENALTY)) - self._codec_min_tokens = int(getattr(tts_config, "min_new_tokens", _CODEC_MIN_TOKENS)) else: self._tts_config = None + self._codec_eos_id = 0 self.has_preprocess = True self.has_postprocess = False - self.gpu_resident_buffer_keys: set[tuple[str, str]] = { - ("audio_codes", "current"), - ("audio_codes", "accumulated"), - } + # Same-step codes travel through make_omni_output from the previous + # sampled id (decode preprocess embeds that id via emb_code). + self.gpu_resident_buffer_keys: set[tuple[str, str]] = {("codes", "audio")} self._init_native_talker(prefix) def _init_native_talker(self, prefix: str) -> None: @@ -326,6 +285,7 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): info_dict.get("request_id"), ) native_duplex = bool(info_dict.get("native_duplex", False)) + meta = info_dict.get("meta") full_embeds = self._build_condition_embeddings( token_ids, hidden_states, @@ -333,7 +293,6 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): ) offset = int(info_dict.get("_omni_num_computed_tokens", 0)) request_id = str(info_dict.get("request_id", "0")) - meta = info_dict.get("meta") # The handoff rebuilds only the tail-aligned Talker condition. # Materialize zero-token embeddings for any scheduler prompt # prefix so chunked prefill can slice from a non-zero offset. @@ -356,21 +315,11 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): f"tts_ids={token_ids.shape[0]} tts_hidden={hidden_states.shape[0]} " f"prompt_len={info_dict.get('_omni_prompt_len')}" ) - duplex_boundary = isinstance(meta, dict) and ( - bool(meta.get("turn_start", False)) or bool(meta.get("turn_end", False)) - ) + state: dict[str, Any] = {"finished": empty_condition, "step": 0} if native_duplex: - max_tokens = _DUPLEX_CODEC_TOKENS_PER_CHUNK - min_tokens = 0 if duplex_boundary else _DUPLEX_CODEC_TOKENS_PER_CHUNK - else: - max_tokens = _max_audio_tokens(int(token_ids.numel())) - min_tokens = self._codec_min_tokens - state = { - "step": 0, - "max_tokens": max_tokens, - "min_tokens": min_tokens, - "finished": empty_condition, - } + max_tokens, min_tokens = _native_duplex_chunk_budget(meta if isinstance(meta, Mapping) else None) + state["max_tokens"] = max_tokens + state["min_tokens"] = min_tokens request_states = getattr(self, "_request_audio_states", None) if request_states is None: request_states = {} @@ -382,98 +331,38 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): embeds, { "audio_state": state, - "audio_codes": { - "current": empty_codes, - "accumulated": empty_codes, - }, + # Prefill has no previous codec id. vLLM samples the first + # code after this forward; the next decode emits it. + "codes": {"audio": empty_codes}, }, ) - current = (info_dict.get("audio_codes", {}) or {}).get("current") - if not isinstance(current, torch.Tensor) or current.numel() != 1: - if state.get("finished"): - # A request that finished before sampling any code can still be - # scheduled for decode steps while sampling min_tokens masks the - # stop token. make_omni_output ignores its hidden states, so any - # shape-correct embedding will do. - weight = self.emb_code[0].weight - return input_ids, weight.new_zeros((span_len, weight.shape[1])), {} - raise RuntimeError("MiniCPM-o Talker decode is missing the previous request-local audio code") - code = current.to(device=self.emb_code[0].weight.device, dtype=torch.long).reshape(1) + request_id = str(info_dict.get("request_id", "0")) + stored = self._request_audio_states.get(request_id) + if isinstance(stored, dict): + state = stored + if isinstance(state, dict) and state.get("finished"): + # An empty speech segment can still be scheduled until EOS is + # eligible. The sampler is forced to EOS; any shape-correct + # embedding is enough for these leftover decode rows. + weight = self.emb_code[0].weight + empty_codes = torch.empty(0, dtype=torch.long, device=weight.device) + return input_ids, weight.new_zeros((span_len, weight.shape[1])), {"codes": {"audio": empty_codes}} + + # Decode: vLLM's previous sampled codec id is this step's input. + # Embed it with the codec table (not the 32k Llama embed_tokens) and + # hand the same id to make_omni_output so Code2Wav sees it this step. + code = input_ids.to(device=self.emb_code[0].weight.device, dtype=torch.long).reshape(-1)[-1:] embeds = self.emb_code[0](code) - return input_ids, embeds, {} - - def _request_generator(self, request_id: str, device: torch.device) -> torch.Generator: - generator = self._request_generators.get(request_id) - if generator is None: - generator = torch.Generator(device=device) - generator.manual_seed(self._codec_seed) - self._request_generators[request_id] = generator - return generator - - def _sample_audio_codes( - self, - hidden_states: torch.Tensor, - histories: Sequence[torch.Tensor], - request_ids: Sequence[str], - steps: Sequence[int], - ) -> torch.Tensor: - batch_size = int(hidden_states.shape[0]) - if not (len(histories) == len(request_ids) == len(steps) == batch_size): - raise ValueError( - "MiniCPM-o batched codec sampling requires one history, request id, " - f"and step per hidden row, got batch={batch_size}, histories={len(histories)}, " - f"request_ids={len(request_ids)}, steps={len(steps)}" - ) - if batch_size == 0: - return torch.empty(0, dtype=torch.long, device=hidden_states.device) - - logits = self.head_code[0](hidden_states).float() / self._codec_temperature - eos_id = self._num_audio_tokens - 1 - logits = _apply_batched_repetition_penalty( - logits, - histories, - penalty=self._codec_repetition_penalty, - window_size=_REPETITION_WINDOW, - ) - request_states = getattr(self, "_request_audio_states", {}) - mask_eos_values: list[bool] = [] - for request_id, step in zip(request_ids, steps, strict=True): - state = request_states.get(request_id) - min_tokens = ( - int(state.get("min_tokens", self._codec_min_tokens)) - if isinstance(state, dict) - else self._codec_min_tokens - ) - mask_eos_values.append(step < min_tokens) - mask_eos = torch.tensor( - mask_eos_values, - dtype=torch.bool, - device=logits.device, - ) - logits[:, eos_id].masked_fill_(mask_eos, float("-inf")) - logits = _apply_top_k_top_p( - logits, - top_k=self._codec_top_k, - top_p=self._codec_top_p, - min_tokens_to_keep=3, - ) - probabilities = torch.softmax(logits, dim=-1) - # torch.multinomial accepts one generator for the entire call. Keep it - # per row so compaction, request ordering, or another request finishing - # cannot perturb request-local RNG streams; the expensive projection - # and logit transforms above remain fully batched. - return torch.cat( - [ - torch.multinomial( - probabilities[row : row + 1], - num_samples=1, - generator=self._request_generator(request_id, probabilities.device), - ) - for row, request_id in enumerate(request_ids) - ], - dim=0, - ).reshape(-1) + if int(code.item()) == int(self._codec_eos_id): + if isinstance(state, dict): + state["finished"] = True + elif stored is None: + self._request_audio_states[request_id] = {"finished": True, "step": 0} + delta = torch.empty(0, dtype=torch.long, device=code.device) + else: + delta = code.reshape(1, 1) + return input_ids, embeds, {"codes": {"audio": delta}} def make_omni_output( self, @@ -487,18 +376,8 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): spans = kwargs.get("request_token_spans") if spans is None or len(spans) != len(infos): raise RuntimeError("MiniCPM-o continuous Talker requires one request_token_span per request") - sample_eligible = kwargs.get("request_sample_eligible") - if sample_eligible is None: - sample_eligible = [True] * len(infos) - if len(sample_eligible) != len(infos): - raise RuntimeError( - f"MiniCPM-o continuous Talker received {len(sample_eligible)} sampling flags for {len(infos)} requests" - ) emit_duplex_metadata = any(isinstance(info, dict) and info.get("native_duplex") is True for info in infos) - # Rows default to continue. Only previously finished or newly terminal - # requests stop; prefill/ineligible rows stay aligned as False. - stop_flags = [False] * len(infos) native_duplex_flags: list[torch.Tensor] = [] duplex_epochs: list[torch.Tensor] = [] duplex_turn_ids: list[torch.Tensor] = [] @@ -507,7 +386,8 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): empty_delta = hidden.new_empty((0, 1), dtype=torch.long) codec_deltas = [empty_delta for _ in infos] terminal_flags = [torch.tensor(False, dtype=torch.bool) for _ in infos] - pending_samples: list[_PendingCodecSample] = [] + force_eos_rows = [False] * len(infos) + mask_eos_rows = [False] * len(infos) for index, info in enumerate(infos): info_dict = info if isinstance(info, dict) else {} native_duplex = info_dict.get("native_duplex") is True @@ -554,100 +434,39 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): if not isinstance(info, dict): continue - start, end = spans[index] - end = min(int(end), int(hidden.shape[0])) - if int(start) >= end: - continue request_id = str(info.get("request_id", index)) - request_states = getattr(self, "_request_audio_states", None) - if request_states is None: - request_states = {} - self._request_audio_states = request_states - state = request_states.get(request_id) + state = self._request_audio_states.get(request_id) if not isinstance(state, dict): state = dict(info.get("audio_state", {}) or {}) - request_states[request_id] = state - if state.get("finished"): - stop_flags[index] = True - continue - if not sample_eligible[index]: - # vLLM computes a logit row for incomplete chunked prefills but - # discards its sampled token. Advancing codec/RNG state here - # would make output depend on prefill chunking and compaction. - continue - codes = state.get("codes") - if not isinstance(codes, torch.Tensor): - codes = (info.get("audio_codes", {}) or {}).get("accumulated") - if not isinstance(codes, torch.Tensor): - codes = torch.empty(0, dtype=torch.long, device=hidden.device) - else: - codes = codes.to(device=hidden.device, dtype=torch.long).reshape(-1) + self._request_audio_states[request_id] = state + empty_speech = bool(state.get("finished")) + codes = info.get("codes", {}) + audio = codes.get("audio") if isinstance(codes, Mapping) else None + if isinstance(audio, torch.Tensor) and audio.numel() > 0: + codec_deltas[index] = audio.to(device=hidden.device, dtype=torch.long).reshape(-1, 1) + if state.get("max_tokens") is not None: + state["step"] = int(state.get("step", 0)) + 1 + max_tokens = state.get("max_tokens") + min_tokens = state.get("min_tokens") step = int(state.get("step", 0)) - pending_samples.append( - _PendingCodecSample( - output_index=index, - hidden_row=hidden[end - 1 : end], - codes=codes, - request_id=request_id, - step=step, - state=state, - info=info, - ) - ) + # 26 samples include the terminating EOS, so force it after 25 + # forwarded frames — the same cadence as generate_chunk. + hit_chunk_limit = max_tokens is not None and step >= int(max_tokens) - 1 + chunk_done = empty_speech or hit_chunk_limit + if hit_chunk_limit: + # The next sampled id is forced EOS and will finish the + # request; mark the chunk done on this step so the data + # plane does not wait for a follow-up empty decode. + state["finished"] = True + force_eos_rows[index] = chunk_done + mask_eos_rows[index] = not force_eos_rows[index] and min_tokens is not None and step < int(min_tokens) + terminal_flags[index] = torch.tensor(chunk_done, dtype=torch.bool) - if pending_samples: - active_hidden_rows = [pending.hidden_row for pending in pending_samples] - active_hidden = ( - active_hidden_rows[0] if len(active_hidden_rows) == 1 else torch.cat(active_hidden_rows, dim=0) - ) - sampled_batch = self._sample_audio_codes( - active_hidden, - [pending.codes for pending in pending_samples], - [pending.request_id for pending in pending_samples], - [pending.step for pending in pending_samples], - ) - # One batched device-to-host synchronization replaces one .item() - # synchronization per request. - sampled_ids = sampled_batch.detach().to(device="cpu").tolist() - else: - sampled_batch = hidden.new_empty((0,), dtype=torch.long) - sampled_ids = [] - - for row, pending in enumerate(pending_samples): - sampled = sampled_batch[row].reshape(()) - sampled_id = int(sampled_ids[row]) - codes = pending.codes - state = pending.state - info = pending.info - is_eos = sampled_id == self._num_audio_tokens - 1 - state["step"] = int(state.get("step", 0)) + 1 - reached_limit = int(state["step"]) >= int(state.get("max_tokens", 2048)) - finished = is_eos or reached_limit - state["finished"] = finished - # MiniCPMTTS.generate_chunk consumes the boundary sample but - # returns only codes that were fed into the retained KV state. - if not is_eos and not reached_limit: - codes = torch.cat([codes[-(_REPETITION_WINDOW - 1) :], sampled.reshape(1)]) - delta = sampled.reshape(1, 1) - else: - delta = empty_delta - state["codes"] = codes - info["audio_state"] = state - info["audio_codes"] = { - "current": sampled.reshape(1), - "accumulated": codes, - } - codec_deltas[pending.output_index] = delta - terminal_flags[pending.output_index] = torch.tensor(finished, dtype=torch.bool) - stop_flags[pending.output_index] = finished - - self._batch_stop_logits = ( - hidden.new_tensor([[float("-inf"), 0.0] if finished else [0.0, float("-inf")] for finished in stop_flags]) - if stop_flags - else hidden.new_empty((0, 2)) - ) - # Lists are deliberate: the runner routes element i to request i, - # preserving compaction alignment while emitting only this step's code. + # Empty-speech and finished duplex chunks must sample codec EOS so + # the scheduler releases the request; mid-chunk rows mask EOS until + # the 26-token generate_chunk budget is met. + self._force_eos_rows = force_eos_rows + self._mask_eos_rows = mask_eos_rows meta_outputs = {"finished": terminal_flags} if emit_duplex_metadata: meta_outputs.update( @@ -659,13 +478,12 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): "turn_end": turn_end_flags, } ) - multimodal_outputs: dict[str, Any] = { - "codes": {"audio": codec_deltas}, - "meta": meta_outputs, - } return OmniOutput( text_hidden_states=hidden, - multimodal_outputs=multimodal_outputs, + multimodal_outputs={ + "codes": {"audio": codec_deltas}, + "meta": meta_outputs, + }, ) def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None: @@ -674,7 +492,6 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): def _flush_deferred_cleanup(self) -> None: request_audio_states = getattr(self, "_request_audio_states", {}) for request_id in self._deferred_cleanup_ids: - self._request_generators.pop(request_id, None) request_audio_states.pop(request_id, None) self._deferred_cleanup_ids.clear() @@ -723,18 +540,38 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): def compute_logits(self, hidden_states, *args, **kwargs): if not isinstance(hidden_states, torch.Tensor): return None - if self._batch_stop_logits is None: - return torch.zeros( - hidden_states.shape[0], - 2, - device=hidden_states.device, - dtype=torch.float32, - ) - logits = self._batch_stop_logits - self._batch_stop_logits = None + if hidden_states.numel() == 0: + return hidden_states.new_empty((0, int(self._num_audio_tokens))) + logits = self.head_code[0](hidden_states).float() + force_eos = self._force_eos_rows + mask_eos = self._mask_eos_rows + self._force_eos_rows = None + self._mask_eos_rows = None + need_force = bool(force_eos and len(force_eos) == logits.shape[0] and any(force_eos)) + need_mask = bool(mask_eos and len(mask_eos) == logits.shape[0] and any(mask_eos)) + if need_force or need_mask: + logits = logits.clone() + eos_id = int(self._codec_eos_id) + if need_force: + assert force_eos is not None + forced = torch.tensor(force_eos, dtype=torch.bool, device=logits.device) + logits[forced] = float("-inf") + logits[forced, eos_id] = 0.0 + if need_mask: + assert mask_eos is not None + masked = torch.tensor(mask_eos, dtype=torch.bool, device=logits.device) + logits[masked, eos_id] = float("-inf") return logits def sample(self, logits, sampling_metadata): + prompt_ids = getattr(sampling_metadata, "prompt_token_ids", None) + if isinstance(logits, torch.Tensor) and isinstance(prompt_ids, torch.Tensor): + # Copy rather than mutate: the runner may hand us the input batch's + # own persistent SamplingMetadata. + sampling_metadata = replace( + sampling_metadata, + prompt_token_ids=blank_scheduler_prompt_for_penalties(prompt_ids, logits.shape[-1]), + ) return Sampler()(logits, sampling_metadata) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): @@ -782,9 +619,10 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): return loaded def get_input_embeddings(self, input_ids, multimodal_embeddings=None, **kwargs): - if hasattr(self, "emb_text") and self.emb_text is not None: - return self.emb_text(input_ids) - return torch.zeros(input_ids.shape[0], 1) + del multimodal_embeddings + # Decode tokens live in the codec table. Prefill overwrites these + # embeddings in preprocess with emb_text + projected thinker hidden. + return self.emb_code[0](input_ids) def embed_input_ids(self, input_ids, **kwargs): return self.get_input_embeddings(input_ids, **kwargs) diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py index b8fed05f1..f4e8e4c32 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py @@ -62,7 +62,12 @@ MINICPMO_4_5_PIPELINE = PipelineConfig( custom_process_input_func=f"{_PROC}.llm2tts", custom_process_next_stage_input_func=f"{_PROC}.tts2code2wav_full_payload", async_chunk_process_next_stage_input_func=f"{_PROC}.tts2code2wav_async_chunk", - sampling_constraints={"detokenize": False}, + sampling_constraints={ + "detokenize": False, + # MiniCPM-o 4.5 codec EOS is tts_config.num_audio_tokens - 1. + # Same pattern as Qwen3 talker's stop_token_ids: [2150]. + "stop_token_ids": [6561], + }, ), StagePipelineConfig( stage_id=2, diff --git a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py index d12a3b382..cd5860786 100644 --- a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py +++ b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py @@ -945,6 +945,9 @@ def llm2tts( if handoff_ids is not None and handoff_hidden is not None: condition_suffix_length = 1 if is_native_duplex_handoff else 2 condition_length = max(len(handoff_ids), len(handoff_hidden)) + condition_suffix_length + # Dummy ids only reserve scheduler slots; prefill overwrites the + # embeddings. Talker.sample() blanks the whole prompt out of the + # repetition penalty, so codec id 0 is not taxed from step one. scheduler_prompt_token_ids = [0] * condition_length handoff_meta = model_intermediate_buffer.setdefault("meta", {}) handoff_meta["next_stage_prompt_len"] = condition_length diff --git a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py index 63f3912f7..1829fafa2 100644 --- a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py @@ -48,6 +48,7 @@ from vllm_omni.distributed.omni_connectors.utils.config import ( get_stage_connector_role, stage_sends_async_output, ) +from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRunnerMixin from vllm_omni.outputs import OmniModelRunnerOutput from vllm_omni.platforms.npu.worker.npu_model_runner import OmniNPUModelRunner from vllm_omni.utils.mm_outputs import build_mm_cpu, partition_payload_list, to_payload_element @@ -106,7 +107,7 @@ class ExecuteModelState(NamedTuple): batch_desc: BatchDescriptor multimodal_outputs: Any # Omni-Specific -class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): +class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin, DuplexSamplingRunnerMixin): """Autoregressive NPU model runner that returns hidden states per request.""" def __init__(self, *args, **kwargs): @@ -140,8 +141,18 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): kv_transfer_manager=self.kv_transfer_manager, ) self._downstream_payload_cache: dict[str, bool] = {} + self._init_duplex_sampling_state() # -------------------------------------- Omni-new ------------------------------------------------- + def load_model(self, *args, **kwargs) -> None: + super().load_model(*args, **kwargs) + self._resolve_duplex_sampling_hook(force=True) + + def _update_states(self, scheduler_output: SchedulerOutput): + deferred_state_corrections_fn = super()._update_states(scheduler_output) + self._update_duplex_sampling_states(scheduler_output) + return deferred_state_corrections_fn + def _make_buffer(self, *size, dtype, numpy=True): # Prevent ray from pinning the buffer due to large size from vllm_omni.distributed.ray_utils.utils import ( @@ -899,10 +910,9 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): self.input_batch.idx_mapping_np, self.input_batch.positions[self.input_batch.logits_indices], ) - sampler_output = model_sample( - logits, - self._sampling_metadata_for_model_sampler(sampling_metadata), - ) + prepared_sampling_metadata = self._sampling_metadata_for_model_sampler(sampling_metadata) + self._apply_duplex_sampling(logits, prepared_sampling_metadata) + sampler_output = model_sample(logits, prepared_sampling_metadata) if sampler_output is not None: return sampler_output return self.sampler( diff --git a/vllm_omni/worker/gpu_ar_model_runner.py b/vllm_omni/worker/gpu_ar_model_runner.py index b3c388e3a..e66f759f8 100644 --- a/vllm_omni/worker/gpu_ar_model_runner.py +++ b/vllm_omni/worker/gpu_ar_model_runner.py @@ -45,6 +45,7 @@ from vllm_omni.distributed.omni_connectors.utils.config import ( get_stage_connector_role, stage_sends_async_output, ) +from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRunnerMixin from vllm_omni.outputs import OmniModelRunnerOutput from vllm_omni.utils.mm_outputs import ( build_mm_cpu, @@ -294,7 +295,7 @@ class ExecuteModelState(NamedTuple): slot_mappings: dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None = None -class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin): +class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin, DuplexSamplingRunnerMixin): """Autoregressive GPU model runner that returns hidden states per request. Follows the v0.12 two-phase execute/sample flow from GPUModelRunner, and @@ -362,25 +363,12 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin): kv_transfer_manager=self.kv_transfer_manager, ) self._downstream_payload_cache: dict[str, bool] = {} - self._duplex_sampling_hook = None - self._duplex_sampling_hook_resolved = False + self._init_duplex_sampling_state() def load_model(self, *args, **kwargs) -> None: super().load_model(*args, **kwargs) self._resolve_duplex_sampling_hook(force=True) - def _resolve_duplex_sampling_hook(self, *, force: bool = False): - if not force and getattr(self, "_duplex_sampling_hook_resolved", False): - return self._duplex_sampling_hook - candidate = getattr(getattr(self, "model", None), "prepare_duplex_sampling", None) - self._duplex_sampling_hook = candidate if callable(candidate) else None - self._duplex_sampling_hook_resolved = True - if self._duplex_sampling_hook is not None and not hasattr(self, "_duplex_sampling_helper"): - from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingHelper - - self._duplex_sampling_helper = DuplexSamplingHelper() - return self._duplex_sampling_hook - def _make_buffer(self, *size, dtype, numpy=True): # Prevent ray from pinning the buffer due to large size from vllm_omni.distributed.ray_utils.utils import ( @@ -449,11 +437,7 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin): def _update_states(self, scheduler_output: SchedulerOutput) -> Callable | None: deferred_state_corrections_fn = super()._update_states(scheduler_output) - if self._resolve_duplex_sampling_hook() is None: - return deferred_state_corrections_fn - helper = getattr(self, "_duplex_sampling_helper", None) - if helper is not None: - helper.update_states(self, scheduler_output) + self._update_duplex_sampling_states(scheduler_output) return deferred_state_corrections_fn def _request_final_stage_id(self, req_id: str) -> int | None: @@ -581,9 +565,7 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin): self._downstream_payload_cache.clear() if hasattr(self, "model_intermediate_buffer"): self.model_intermediate_buffer.clear() - duplex_helper = getattr(self, "_duplex_sampling_helper", None) - if duplex_helper is not None: - duplex_helper.clear() + self._clear_duplex_sampling() # 5. Release all CUDA graphs unconditionally (upstream only does this # on ROCm; on CUDA the graphs are only freed by Python GC during @@ -1501,14 +1483,7 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin): self.input_batch.positions[self.input_batch.logits_indices], ) prepared_sampling_metadata = self._sampling_metadata_for_model_sampler(sampling_metadata) - prepare_duplex_sampling = self._resolve_duplex_sampling_hook() - if prepare_duplex_sampling is not None: - helper = getattr(self, "_duplex_sampling_helper", None) - rows = helper.rows(self) if helper is not None and helper.active_request_ids else () - if rows or (helper is not None and helper.hook_active): - prepare_duplex_sampling(logits, prepared_sampling_metadata, rows) - if helper is not None: - helper.hook_active = bool(rows) + self._apply_duplex_sampling(logits, prepared_sampling_metadata) sampler_output = model_sample(logits, prepared_sampling_metadata) if sampler_output is not None: return sampler_output From 38cf9adc380f8bc7c73a6ca4463e625a1bad57d0 Mon Sep 17 00:00:00 2001 From: linzhenpl07 <60651557+linzhenpl07@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:42:14 +0800 Subject: [PATCH 080/267] [BugFix][Diffusion] Fix MiniMax-H3 VAE hang when decoder tiles are fewer than ranks (#6345) Signed-off-by: linzhenpl07 Co-authored-by: Claude Opus 5 --- .../minimax_h3/test_minimax_h3_vae_tiling.py | 214 ++++++++++++++++++ vllm_omni/diffusion/models/minimax_h3/vae.py | 84 ++++++- 2 files changed, 293 insertions(+), 5 deletions(-) create mode 100644 tests/diffusion/models/minimax_h3/test_minimax_h3_vae_tiling.py diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_vae_tiling.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_vae_tiling.py new file mode 100644 index 000000000..ef04019f6 --- /dev/null +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_vae_tiling.py @@ -0,0 +1,214 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""CPU tests for the decoder tile-shortage guard in the MiniMax-H3 video VAE.""" + +import sys +import types + +import pytest +import torch + +from vllm_omni.diffusion.models.minimax_h3.vae import MiniMaxH3VideoVAE + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + +RATIO = 16 +# The checkpoint keeps two independent grids -- ``split_tiles`` selects +# ``decoder_tile_size``/``decoder_tile_overlap_min`` when ``is_decoder`` is set +# and ``tile_size``/``tile_overlap_min`` otherwise. The shipped H3 config leaves +# both at 256/64, so the two happen to coincide there, but that is a config +# coincidence. The double deliberately gives them different values so that +# passing the wrong flag changes the tile count and fails a test. +TILE_SIZE, OVERLAP = 256, 64 # decoder, matching the shipped config +ENCODER_TILE_SIZE, ENCODER_OVERLAP = 128, 32 + + +class _FakeCheckpointModel: + """The parts of the checkpoint's AutoencoderKL the guard touches.""" + + def __init__(self): + self.vae_ratio = RATIO + self.parallel_tiling = True + self.decoded_with = [] + + def split_tiles(self, input_len, is_decoder=False): + """Same grid arithmetic as the checkpoint, on pixel dimensions.""" + tile = TILE_SIZE if is_decoder else ENCODER_TILE_SIZE + overlap = OVERLAP if is_decoder else ENCODER_OVERLAP + if tile >= input_len: + return [0], [input_len], [] + n = -(-input_len // tile) + while tile * n - overlap * (n - 1) - input_len < 0: + n += 1 + return list(range(n)), [tile] * n, [overlap] * (n - 1) + + +def _vae(parallel_size, state): + """A stand-in instance: the guard only reads .model/.remote/.parallel_size.""" + module = types.ModuleType("fake_ckpt.parallel") + module.get_parallel_state = lambda: state + sys.modules.setdefault("fake_ckpt", types.ModuleType("fake_ckpt")) + sys.modules["fake_ckpt.parallel"] = module + + remote = type("Remote", (), {"__module__": "fake_ckpt.klvae"})() + + vae = object.__new__(MiniMaxH3VideoVAE) + vae.model = _FakeCheckpointModel() + vae.remote = remote + vae.parallel_size = parallel_size + return vae + + +def _latent(h, w): + return torch.zeros(1, 24, 9, h, w) + + +@pytest.mark.parametrize( + ("h", "w", "expected"), + [ + (16, 16, 1), # 256x256 px -> single tile + (24, 16, 2), # 384x256 px + (24, 24, 4), # 384x384 px + (48, 84, 28), # 768x1344 px, the shipped H3 request + (96, 168, 112), # 1536x2688 px, a larger canvas the request path also allows + ], +) +def test_tile_count_matches_the_checkpoint_grid(h, w, expected): + assert MiniMaxH3VideoVAE._decoder_tile_count(_vae(1, {}), _latent(h, w)) == expected + + +@pytest.mark.parametrize(("h", "w", "decoder_tiles", "encoder_tiles"), [(16, 16, 1, 9), (24, 24, 4, 16)]) +def test_tile_count_uses_the_decoder_grid_not_the_encoder_one(h, w, decoder_tiles, encoder_tiles): + """Passing ``is_decoder=False`` would silently change the count, so pin it.""" + vae = _vae(1, {}) + assert MiniMaxH3VideoVAE._decoder_tile_count(vae, _latent(h, w)) == decoder_tiles + + px_h, px_w = h * RATIO, w * RATIO + as_encoder = len(vae.model.split_tiles(px_h, False)[0]) * len(vae.model.split_tiles(px_w, False)[0]) + assert as_encoder == encoder_tiles != decoder_tiles + + +def test_rank_local_tiling_restores_the_group_state(): + state = {"sp_size": 4, "sp_rank": 2, "sp_enabled": True, "sp_process_group": "pg"} + vae = _vae(4, state) + + with vae._rank_local_tiling(): + assert state["sp_size"] == 1 + assert state["sp_rank"] == 0 + assert state["sp_enabled"] is False + assert state["sp_process_group"] is None + assert vae.model.parallel_tiling is False + + assert state == {"sp_size": 4, "sp_rank": 2, "sp_enabled": True, "sp_process_group": "pg"} + assert vae.model.parallel_tiling is True + + +def test_rank_local_tiling_restores_after_an_exception(): + state = {"sp_size": 4, "sp_rank": 1, "sp_enabled": True, "sp_process_group": "pg"} + vae = _vae(4, state) + + with pytest.raises(RuntimeError, match="decode failed"): + with vae._rank_local_tiling(): + raise RuntimeError("decode failed") + + assert state["sp_size"] == 4 + assert vae.model.parallel_tiling is True + + +# --------------------------------------------------------------------------- +# Dispatch: every one of these calls ``decode_latent``, so deleting the guard +# turns the whole matrix red rather than leaving it green. +# --------------------------------------------------------------------------- + + +class _RecordingProcessor: + @staticmethod + def revert_tensor(decoded): + return decoded + + +def _dispatch_vae(parallel_size): + """A VAE whose ``decode_base`` records the parallel state it ran under.""" + state = { + "group_size": parallel_size, + "group_rank": 0, + "local_process_group": object(), + "sp_size": parallel_size, + "sp_rank": 0, + "sp_enabled": True, + "sp_process_group": object(), + "tp_size": 1, + "tp_rank": 0, + } + vae = _vae(parallel_size, state) + vae.config_dict = { + "latent_channels": 24, + "latents_mean": [0.0] * 24, + "latents_std": [1.0] * 24, + } + vae.model.processor = _RecordingProcessor() + + seen = {} + + def decode_base(sample): + seen["sp_size"] = state["sp_size"] + seen["sp_enabled"] = state["sp_enabled"] + seen["sp_process_group"] = state["sp_process_group"] + seen["parallel_tiling"] = vae.model.parallel_tiling + # the guard never inspects the decoded shape, so keep it small + return torch.zeros(1, 3, 2, 4, 4) + + vae.model.decode_base = decode_base + return vae, state, seen + + +@pytest.mark.parametrize( + ("parallel_size", "h", "w", "falls_back"), + [ + (4, 24, 16, True), # 2 tiles, ranks 2 and 3 would get none + (4, 16, 16, True), # 1 tile + (4, 24, 24, False), # 4 tiles, exactly enough + (4, 48, 84, False), # 28 tiles, the shipped 768x1344 request + (4, 96, 168, False), # 112 tiles, 1536x2688 + (2, 24, 16, False), # 2 tiles across 2 ranks + (1, 16, 16, False), # never parallel, nothing to guard + ], +) +def test_decode_latent_falls_back_exactly_when_tiles_are_short(parallel_size, h, w, falls_back): + """Fewer tiles than ranks is the hang condition; equal or more is fine.""" + vae, _, seen = _dispatch_vae(parallel_size) + + MiniMaxH3VideoVAE.decode_latent(vae, _latent(h, w)) + + # only the guard clears ``parallel_tiling``, so it is the signal that it fired + assert (seen["parallel_tiling"] is False) is falls_back + assert seen["sp_size"] == (1 if falls_back else parallel_size) + + +def test_decode_latent_isolates_and_restores_the_whole_group_state(): + """1 tile, 4 ranks: ``decode_base`` must observe rank-local tiling.""" + vae, state, seen = _dispatch_vae(4) + + frames = MiniMaxH3VideoVAE.decode_latent(vae, _latent(16, 16)) + + assert seen["sp_size"] == 1 + assert seen["sp_enabled"] is False + assert seen["sp_process_group"] is None + assert seen["parallel_tiling"] is False + assert frames.ndim == 5 and frames.dtype is torch.float32 + # the group state is restored once decode returns + assert state["sp_size"] == 4 and state["sp_enabled"] is True + assert vae.model.parallel_tiling is True + + +def test_decode_latent_leaves_the_group_state_alone_when_the_grid_is_large_enough(): + """The shipped 768x1344 request is 28 tiles: the guard must not fire.""" + vae, state, seen = _dispatch_vae(4) + + MiniMaxH3VideoVAE.decode_latent(vae, _latent(48, 84)) + + assert seen["sp_size"] == 4 + assert seen["sp_enabled"] is True + assert seen["sp_process_group"] is state["sp_process_group"] + assert seen["parallel_tiling"] is True diff --git a/vllm_omni/diffusion/models/minimax_h3/vae.py b/vllm_omni/diffusion/models/minimax_h3/vae.py index bf75ab7f6..ffde5ee7c 100644 --- a/vllm_omni/diffusion/models/minimax_h3/vae.py +++ b/vllm_omni/diffusion/models/minimax_h3/vae.py @@ -5,7 +5,8 @@ from __future__ import annotations import importlib import json -from contextlib import AbstractContextManager +from collections.abc import Iterator +from contextlib import AbstractContextManager, contextmanager, nullcontext from pathlib import Path from typing import Any @@ -14,6 +15,7 @@ import torch.distributed as dist import torch.nn as nn from PIL import Image from transformers.dynamic_module_utils import get_class_from_dynamic_module +from vllm.logger import init_logger from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import ( DistributedVaeMixin, @@ -28,6 +30,9 @@ MINIMAX_H3_AUDIO_SAMPLE_RATE = 32000 MINIMAX_H3_AUDIO_CHANNELS = 2 +logger = init_logger(__name__) + + def _load_component_config(component_path: str) -> dict[str, Any]: config_path = Path(component_path) / "config.json" config = json.loads(config_path.read_text(encoding="utf-8")) @@ -170,9 +175,7 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin): self.parallel_size = parallel_size enabled = parallel_size > 1 - package = self.remote.__class__.__module__.rsplit(".", 1)[0] - parallel_module = importlib.import_module(f"{package}.parallel") - state = parallel_module.get_parallel_state() + state = self._native_parallel_state() state.clear() state.update( group_size=parallel_size, @@ -187,6 +190,57 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin): ) self.model.parallel_tiling = enabled + def _native_parallel_state(self) -> dict[str, Any]: + """Return the checkpoint's own mutable parallel-state dict.""" + + package = self.remote.__class__.__module__.rsplit(".", 1)[0] + parallel_module = importlib.import_module(f"{package}.parallel") + return parallel_module.get_parallel_state() + + def _decoder_tile_count(self, latent: torch.Tensor) -> int: + """Number of decoder tiles the checkpoint will split ``latent`` into. + + Mirrors the checkpoint's ``decode_tiled``: the grid is computed from the + pixel-space dimensions, so it is a pure function of the latent shape and + resolves identically on every rank. + """ + + ratio = int(self.model.vae_ratio) + rows, _, _ = self.model.split_tiles(int(latent.shape[-2]) * ratio, True) + cols, _, _ = self.model.split_tiles(int(latent.shape[-1]) * ratio, True) + return len(rows) * len(cols) + + @contextmanager + def _rank_local_tiling(self) -> Iterator[None]: + """Run one decode with tiling kept on this rank, then restore the group. + + Used only when there are fewer tiles than ranks. Every rank then decodes + every tile, which is slower than sharing the work but is correct and + involves no collective. + """ + + state = self._native_parallel_state() + saved_state = dict(state) + saved_tiling = self.model.parallel_tiling + state.update( + group_size=1, + group_rank=0, + local_process_group=None, + sp_size=1, + sp_rank=0, + sp_enabled=False, + sp_process_group=None, + tp_size=1, + tp_rank=0, + ) + self.model.parallel_tiling = False + try: + yield + finally: + state.clear() + state.update(saved_state) + self.model.parallel_tiling = saved_tiling + def is_distributed_enabled(self) -> bool: return self.parallel_size > 1 and dist.is_initialized() @@ -293,7 +347,27 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin): device=latent.device, dtype=latent.dtype, ).view(1, channels, 1, 1, 1) - decoded = self.model.decode_base(latent * std + mean) + # The checkpoint hands rank r the tiles ``range(r, num_tiles, sp_size)`` + # and then rejects an empty share inside the gather. A rank with no + # tiles raises and leaves the collective while the others block in it + # forever, so too few tiles hangs the whole stage rather than failing + # it. Tile count depends only on the latent shape, so every rank takes + # this branch together. + num_tiles = self._decoder_tile_count(latent) + if self.parallel_size > 1 and num_tiles < self.parallel_size: + logger.warning_once( + "MiniMax-H3 VAE decode splits into %d tile(s) but the tile group has " + "%d ranks; decoding rank-locally for this shape instead, which is " + "slower but avoids ranks without tiles hanging the collective.", + num_tiles, + self.parallel_size, + ) + tiling_context: AbstractContextManager = self._rank_local_tiling() + else: + tiling_context = nullcontext() + + with tiling_context: + decoded = self.model.decode_base(latent * std + mean) frames = self.model.processor.revert_tensor(decoded) if frames.ndim == 4: frames = frames.unsqueeze(0).transpose(1, 2) From b58feb1420f347e5a5cdec98d2e6ea54b9be66d1 Mon Sep 17 00:00:00 2001 From: R0CKSTAR Date: Thu, 20 Aug 2026 14:19:14 +0800 Subject: [PATCH 081/267] [MUSA] Fix SwiGLU to use fused op (#6364) Signed-off-by: Xiaodong Ye --- vllm_omni/diffusion/layers/activation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm_omni/diffusion/layers/activation.py b/vllm_omni/diffusion/layers/activation.py index c9658cf6e..9d51b03d2 100644 --- a/vllm_omni/diffusion/layers/activation.py +++ b/vllm_omni/diffusion/layers/activation.py @@ -22,7 +22,7 @@ class SiluAndMul(CustomOp): return F.silu(x[..., :d]) * x[..., d:] def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: - if x.device.type != "cuda": + if x.device.type == "cpu": return self.forward_native(x) d = x.shape[-1] // 2 output_shape = x.shape[:-1] + (d,) From 480e10ba9c1cae673086da493e60c9d5edc92c22 Mon Sep 17 00:00:00 2001 From: Sy03 <1370724210@qq.com> Date: Thu, 20 Aug 2026 00:25:49 -0700 Subject: [PATCH 082/267] [CI/Build] Match perf warmups to benchmark concurrency (#6356) Signed-off-by: Sy03 <24841857+Sy0307@users.noreply.github.com> Co-authored-by: Sy03 <24841857+Sy0307@users.noreply.github.com> --- tests/dfx/conftest.py | 3 ++- tests/dfx/perf/scripts/run_benchmark.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/dfx/conftest.py b/tests/dfx/conftest.py index 7e3ab77c7..2e22da531 100644 --- a/tests/dfx/conftest.py +++ b/tests/dfx/conftest.py @@ -653,6 +653,7 @@ def run_benchmark( random_input_len: Any | None = None, random_output_len: Any | None = None, resource_label: str | None = None, + num_warmups: int = 2, ) -> dict[str, Any]: """Run one ``vllm bench serve --omni`` iteration and return parsed metrics. @@ -677,7 +678,7 @@ def run_benchmark( + args + [ "--num-warmups", - "2", + str(num_warmups), "--save-result", "--result-dir", os.environ.get("BENCHMARK_DIR", "tests"), diff --git a/tests/dfx/perf/scripts/run_benchmark.py b/tests/dfx/perf/scripts/run_benchmark.py index 03dfac1a0..1eee88ac7 100644 --- a/tests/dfx/perf/scripts/run_benchmark.py +++ b/tests/dfx/perf/scripts/run_benchmark.py @@ -279,5 +279,6 @@ def test_performance_benchmark(omni_server, benchmark_params): random_input_len=params.get("random_input_len"), random_output_len=params.get("random_output_len"), resource_label=resource_label, + num_warmups=max(2, int(concurrency)), ) assert_result(result, params, num_prompt) From b57dc70ec4b0dab61c69974a87d47fbcaa56564e Mon Sep 17 00:00:00 2001 From: NATURE Date: Thu, 20 Aug 2026 15:59:39 +0800 Subject: [PATCH 083/267] [Bugfix] Reclaim resumable async-chunk requests on finish (#6360) Signed-off-by: natureofnature Co-authored-by: Ruirui Yang | Rein <73573651+R2-Y@users.noreply.github.com> Co-authored-by: Hongsheng Liu --- ...mni_generation_scheduler_update_session.py | 3 +- ...st_omni_scheduler_finish_requests_purge.py | 68 +++++++++ .../test_chunk_transfer_adapter.py | 136 +++++++++++++++++- vllm_omni/core/sched/omni_scheduler_mixin.py | 100 ++++++++++--- .../chunk_transfer_adapter.py | 28 ++-- 5 files changed, 301 insertions(+), 34 deletions(-) diff --git a/tests/core/sched/test_omni_generation_scheduler_update_session.py b/tests/core/sched/test_omni_generation_scheduler_update_session.py index 0f654715a..247f0ce84 100644 --- a/tests/core/sched/test_omni_generation_scheduler_update_session.py +++ b/tests/core/sched/test_omni_generation_scheduler_update_session.py @@ -410,12 +410,13 @@ class TestRealignRequestStatusToQueues: assert stale.status == RequestStatus.RUNNING assert clean.status == RequestStatus.RUNNING - def test_finished_request_is_skipped(self) -> None: + def test_non_resumable_finished_request_is_skipped(self) -> None: """Already-finished requests must not be touched -- they may have legitimate finished statuses (FINISHED_STOPPED etc.) that a status flip would corrupt. """ req = _make_request(request_id="req-finished") + req.resumable = False req.status = RequestStatus.FINISHED_STOPPED sched = _RealignSchedulerStub( diff --git a/tests/core/sched/test_omni_scheduler_finish_requests_purge.py b/tests/core/sched/test_omni_scheduler_finish_requests_purge.py index fbfeb384f..770c85578 100644 --- a/tests/core/sched/test_omni_scheduler_finish_requests_purge.py +++ b/tests/core/sched/test_omni_scheduler_finish_requests_purge.py @@ -157,3 +157,71 @@ def test_finish_requests_leaves_healthy_running_intact( ) assert scheduler.running == [alive] + + +@pytest.mark.parametrize(("scheduler_cls", "scheduler_mod"), _SCHEDULER_PARAMS) +def test_finish_requests_preserves_unrelated_resumable_segment( + monkeypatch: pytest.MonkeyPatch, + scheduler_cls, + scheduler_mod, +) -> None: + paused = _StubRequest("req-paused", RequestStatus.FINISHED_STOPPED) + paused.resumable = True + leaving = _StubRequest("req-leaving", RequestStatus.FINISHED_STOPPED) + scheduler = _make_scheduler( + scheduler_cls, + requests={paused.request_id: paused}, + running=[paused], + waiting=[], + ) + + def fake_finish_requests(self, request_ids, finished_status): + return [leaving] + + monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests) + + scheduler_cls.finish_requests(scheduler, [leaving.request_id], RequestStatus.FINISHED_ABORTED) + + assert scheduler.running == [paused] + assert scheduler.requests == {paused.request_id: paused} + + +@pytest.mark.parametrize(("scheduler_cls", "scheduler_mod"), _SCHEDULER_PARAMS) +def test_finish_requests_materializes_single_pass_request_ids( + monkeypatch: pytest.MonkeyPatch, + scheduler_cls, + scheduler_mod, +) -> None: + request = _StubRequest("req-generator", RequestStatus.RUNNING) + scheduler = _make_scheduler( + scheduler_cls, + requests={request.request_id: request}, + running=[request], + waiting=[], + ) + + def fake_finish_requests(self, request_ids, finished_status): + assert tuple(request_ids) == (request.request_id,) + return [request] + + monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests) + + result = scheduler_cls.finish_requests( + scheduler, + (request_id for request_id in [request.request_id]), + RequestStatus.FINISHED_ABORTED, + ) + + assert result == [request] + + +@pytest.mark.parametrize("scheduler_cls", [OmniARScheduler, OmniGenerationScheduler]) +def test_finish_requests_does_not_reopen_off_queue_deferred_free_terminal(scheduler_cls) -> None: + terminal = _StubRequest("req-deferred-free", RequestStatus.FINISHED_STOPPED) + terminal.resumable = True + scheduler = _make_scheduler(scheduler_cls, requests={terminal.request_id: terminal}, running=[], waiting=[]) + scheduler._free_request = lambda *args, **kwargs: pytest.fail("off-queue request was freed") + + assert scheduler_cls.finish_requests(scheduler, [terminal.request_id], RequestStatus.FINISHED_ABORTED) == [] + assert scheduler.requests == {terminal.request_id: terminal} + assert terminal.status == RequestStatus.FINISHED_STOPPED diff --git a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py index 063751b01..fcfd284bf 100644 --- a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py +++ b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py @@ -3,8 +3,8 @@ import threading from collections import deque -from types import SimpleNamespace -from unittest.mock import patch +from types import MethodType, SimpleNamespace +from unittest.mock import Mock, patch import pytest import torch @@ -13,6 +13,7 @@ from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler from vllm.v1.metrics.stats import PrefillStats, PromptTokenStats from vllm.v1.request import RequestStatus +from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler from vllm_omni.data_entry_keys import CodesStruct, MetaStruct, OmniPayload, OmniPayloadStruct from vllm_omni.distributed.omni_connectors.adapter import construct_next_stage_streaming_input_prompt @@ -32,9 +33,14 @@ class DummyWaitingQueue(list): def add_request(self, request): self.append(request) + def remove_requests(self, requests): + remove = set(requests) + self[:] = [request for request in self if request not in remove] + def _req(req_id: str, status: RequestStatus, external_req_id: str | None = None): - return SimpleNamespace( + request = Mock( + client_index=0, request_id=req_id, external_req_id=external_req_id or req_id, status=status, @@ -44,8 +50,10 @@ def _req(req_id: str, status: RequestStatus, external_req_id: str | None = None) num_output_placeholders=0, prefill_stats=None, additional_information=None, - is_finished=lambda: status == RequestStatus.FINISHED_STOPPED, + resumable=False, ) + request.is_finished = lambda: RequestStatus.is_finished(request.status) + return request def test_streaming_payload_can_replace_placeholder_prompt(mocker: MockerFixture) -> None: @@ -1006,12 +1014,30 @@ def test_finish_requests_restores_status(build_adapter): prior = RequestStatus.RUNNING request = _req(req_id, RequestStatus.WAITING_FOR_CHUNK) adapter.requests_origin_status[req_id] = prior + adapter.waiting_for_chunk_running_requests.append(request) requests_map = {req_id: request} adapter.finish_requests([req_id], RequestStatus.FINISHED_ABORTED, requests_map) assert request.status == prior assert req_id not in adapter.requests_origin_status + assert not adapter.waiting_for_chunk_running_requests + + +def test_finish_requests_does_not_restore_stale_status_without_connector_ownership(build_adapter): + adapter, _ = build_adapter(stage_id=1) + request = _req("req-restored", RequestStatus.WAITING_FOR_STREAMING_REQ) + request.resumable = True + adapter.requests_origin_status[request.request_id] = RequestStatus.RUNNING + + adapter.finish_requests( + [request.request_id], + RequestStatus.FINISHED_ABORTED, + {request.request_id: request}, + ) + + assert request.status == RequestStatus.WAITING_FOR_STREAMING_REQ + assert request.request_id not in adapter.requests_origin_status def test_finish_requests_removes_zombies_from_chunk_waiting_deques(build_adapter): @@ -1061,6 +1087,108 @@ def test_finish_requests_releases_active_stream_slot(build_adapter): assert waiting.status == RequestStatus.WAITING_FOR_CHUNK +@pytest.mark.parametrize("scheduler_cls", [OmniGenerationScheduler, OmniARScheduler]) +@pytest.mark.parametrize( + ("placement", "origin_status", "initial_status", "streaming_counter_owned"), + [ + ("hidden", RequestStatus.RUNNING, RequestStatus.FINISHED_STOPPED, False), + ("running", RequestStatus.WAITING, RequestStatus.FINISHED_STOPPED, False), + ("waiting", RequestStatus.WAITING, RequestStatus.FINISHED_STOPPED, False), + ("skipped", RequestStatus.WAITING, RequestStatus.FINISHED_STOPPED, True), + ("skipped_streaming", RequestStatus.RUNNING, RequestStatus.WAITING_FOR_STREAMING_REQ, True), + ("skipped_hidden", RequestStatus.RUNNING, RequestStatus.FINISHED_STOPPED, True), + ("skipped_non_streaming", RequestStatus.WAITING, RequestStatus.WAITING, False), + ], +) +def test_finish_requests_reclaims_resumable_segment_and_reuses_capacity( + build_adapter, + scheduler_cls, + placement, + origin_status, + initial_status, + streaming_counter_owned, +): + adapter, _ = build_adapter(stage_id=1, active_stream_window=2) + request = _req( + "req-segment-stop", + initial_status, + external_req_id="ext-segment-stop", + ) + request.resumable = True + adapter.requests_origin_status[request.request_id] = origin_status + if placement in {"hidden", "skipped_hidden"}: + adapter.waiting_for_chunk_running_requests.append(request) + adapter._active_streams[request.request_id] = request + + scheduler = scheduler_cls.__new__(scheduler_cls) + scheduler.chunk_transfer_adapter = adapter + scheduler.input_coordinator = None + scheduler.requests = {request.request_id: request} + scheduler.running = [request] if placement == "running" else [] + scheduler.waiting = DummyWaitingQueue([request] if placement == "waiting" else []) + scheduler.skipped_waiting = DummyWaitingQueue([request] if placement.startswith("skipped") else []) + scheduler.num_waiting_for_streaming_input = int(streaming_counter_owned) + + freed = [] + + def fake_free_request(self, live, delay_free_blocks=False): + del delay_free_blocks + freed.append(live.request_id) + self.requests.pop(live.request_id) + return None, None + + scheduler._free_request = MethodType(fake_free_request, scheduler) + + first = scheduler_cls.finish_requests(scheduler, [request.request_id], RequestStatus.FINISHED_ABORTED) + second = scheduler_cls.finish_requests(scheduler, [request.request_id], RequestStatus.FINISHED_ABORTED) + + assert len(first) == 1 + assert second == [] + assert freed == [request.request_id] + assert request.request_id not in scheduler.requests + assert request.request_id not in adapter._active_streams + assert not adapter.waiting_for_chunk_running_requests + assert scheduler.running == [] + assert list(scheduler.waiting) == [] + assert list(scheduler.skipped_waiting) == [] + assert scheduler.num_waiting_for_streaming_input == 0 + + fresh_a = _req("req-fresh-a", RequestStatus.WAITING) + fresh_b = _req("req-fresh-b", RequestStatus.WAITING) + assert adapter._ensure_active_stream(fresh_a) + assert adapter._ensure_active_stream(fresh_b) + assert set(adapter._active_streams) == {fresh_a.request_id, fresh_b.request_id} + + +@pytest.mark.parametrize("scheduler_cls", [OmniGenerationScheduler, OmniARScheduler]) +def test_finish_requests_does_not_reopen_off_queue_resumable_segment(build_adapter, scheduler_cls): + adapter, _ = build_adapter(stage_id=1, active_stream_window=2) + request = _req("req-off-queue", RequestStatus.FINISHED_STOPPED) + request.resumable = True + adapter.requests_origin_status[request.request_id] = RequestStatus.RUNNING + + scheduler = scheduler_cls.__new__(scheduler_cls) + scheduler.chunk_transfer_adapter = adapter + scheduler.input_coordinator = None + scheduler.requests = {request.request_id: request} + scheduler.running = [] + scheduler.waiting = DummyWaitingQueue() + scheduler.skipped_waiting = DummyWaitingQueue() + scheduler._free_request = lambda *args, **kwargs: pytest.fail("off-queue request was freed") + + assert ( + scheduler_cls.finish_requests( + scheduler, + [request.request_id], + RequestStatus.FINISHED_ABORTED, + ) + == [] + ) + assert scheduler.requests == {request.request_id: request} + assert request.status == RequestStatus.FINISHED_STOPPED + assert request.request_id not in adapter.requests_origin_status + + def test_restore_queues_skips_requests_missing_from_scheduler_requests(build_adapter): adapter, _ = build_adapter(stage_id=1) zombie = _req("req-zombie", RequestStatus.WAITING_FOR_CHUNK) diff --git a/vllm_omni/core/sched/omni_scheduler_mixin.py b/vllm_omni/core/sched/omni_scheduler_mixin.py index 7ce5f4470..05ab26e06 100644 --- a/vllm_omni/core/sched/omni_scheduler_mixin.py +++ b/vllm_omni/core/sched/omni_scheduler_mixin.py @@ -3,7 +3,7 @@ from __future__ import annotations import math import os import time -from collections.abc import Iterable +from collections.abc import Iterable, Iterator from typing import Any import torch @@ -571,12 +571,35 @@ class OmniSchedulerMixin: finished_status: RequestStatus, ) -> list[Request]: """Finish requests and clean all Omni-owned queue/coordinator state.""" + if isinstance(request_ids, str): + target_request_ids = {request_ids} + elif request_ids is None: + target_request_ids = set(self.requests) + else: + if isinstance(request_ids, Iterator): + request_ids = tuple(request_ids) + target_request_ids = set(request_ids) + + skipped_waiting_ids = {r.request_id for r in getattr(self, "skipped_waiting", ())} + pre_adapter_streaming_wait_ids = { + rid + for rid in target_request_ids & skipped_waiting_ids + if (req := self.requests.get(rid)) is not None + and ( + req.status == RequestStatus.WAITING_FOR_STREAMING_REQ + or (getattr(req, "resumable", False) and req.status == RequestStatus.FINISHED_STOPPED) + ) + } + if self.chunk_transfer_adapter: self.chunk_transfer_adapter.finish_requests(request_ids, finished_status, self.requests) - self._realign_request_status_to_queues(request_ids) + self._realign_request_status_to_queues( + request_ids, + pre_adapter_streaming_wait_ids=pre_adapter_streaming_wait_ids, + ) finished = super().finish_requests(request_ids, finished_status) - self._purge_finished_from_running() + self._purge_finished_from_running(target_request_ids) for request in finished: self._free_input_coordinator_request(request.request_id) @@ -592,6 +615,8 @@ class OmniSchedulerMixin: def _realign_request_status_to_queues( self, request_ids: str | Iterable[str] | None, + *, + pre_adapter_streaming_wait_ids: set[str] | None = None, ) -> None: """Realign ``request.status`` to actual queue membership. @@ -618,7 +643,13 @@ class OmniSchedulerMixin: Realign here: if a request lives in ``self.running`` but its status is not ``RUNNING``, set it to ``RUNNING``; symmetrically flip ``RUNNING → WAITING`` when the request is actually in - ``self.waiting``. This is a localized safety net for + ``self.waiting``. A resumable segment stop still held in + ``self.skipped_waiting`` is restored to + ``WAITING_FOR_STREAMING_REQ`` so upstream also balances its + paused-session counter. Because adapter cleanup may restore a + connector-owned request's prior status first, ``finish_requests`` + snapshots these counter-bearing rows before invoking the adapter. + This is a localized safety net for ``requests_origin_status`` staleness on the admit transition; it does not touch the adapter's invariants and is complementary to the chunk-transfer-adapter deque purge that already runs @@ -651,17 +682,36 @@ class OmniSchedulerMixin: running_ids = {r.request_id for r in self.running} waiting_ids = {r.request_id for r in self.waiting} + skipped_waiting_ids = {r.request_id for r in getattr(self, "skipped_waiting", ())} for rid in ids_to_align: req = self.requests.get(rid) - if req is None or req.is_finished(): + if req is None: continue - if rid in running_ids and req.status != RequestStatus.RUNNING: + # A persistent Session may be closed after its current segment + # reached FINISHED_STOPPED but while it is still owned by a live + # scheduler/connector queue. vLLM skips already-finished requests + # in finish_requests(), leaking the KV and worker slot. Only + # recover requests with positive queue ownership; an off-queue + # terminal can legitimately be waiting for deferred block free. + resumable_segment_stop = bool( + getattr(req, "resumable", False) and req.status == RequestStatus.FINISHED_STOPPED + ) + streaming_wait = ( + rid in pre_adapter_streaming_wait_ids + if pre_adapter_streaming_wait_ids is not None + else resumable_segment_stop + ) + if req.is_finished() and not (resumable_segment_stop or streaming_wait): + continue + if rid in skipped_waiting_ids and streaming_wait: + req.status = RequestStatus.WAITING_FOR_STREAMING_REQ + elif rid in running_ids and req.status != RequestStatus.RUNNING: req.status = RequestStatus.RUNNING - elif rid in waiting_ids and req.status == RequestStatus.RUNNING: + elif rid in waiting_ids and (req.status == RequestStatus.RUNNING or resumable_segment_stop): req.status = RequestStatus.WAITING - def _purge_finished_from_running(self) -> None: + def _purge_finished_from_running(self, target_request_ids: set[str] | None = None) -> None: """Defensive post-finish sweep of ``self.running``. Belt-and-suspenders to ``_realign_request_status_to_queues``: @@ -678,18 +728,14 @@ class OmniSchedulerMixin: residue after ``super().finish_requests`` so any stale entries are reclaimed). - Scope of the predicate. ``is_finished()`` covers entries the - upstream ``finish_requests`` already drained from ``self.requests`` - but failed to remove from ``self.running``; the - ``request_id not in self.requests`` arm catches the same surface - from a different angle and is the post-cleanup mirror of the - deque purge ``_purge_untracked_chunk_requests`` already runs at - the chunk-transfer-adapter layer. It does **not** by itself make - arbitrary direct deletions of ``self.requests`` safe -- callers - that pop ``self.requests`` outside the standard finish path - still have to go through ``_free_request`` (or equivalent) for - block / connector / coordinator cleanup. This sweep only - reclaims the ``self.running`` slot reference. + A resumable ``FINISHED_STOPPED`` request may legitimately remain + in ``self.running`` between realtime segments. Preserve such a + request unless it belongs to this finish call. Other finished or + untracked entries are stale and can be swept defensively. + + When ``target_request_ids`` is ``None`` or empty, every resumable + ``FINISHED_STOPPED`` entry is preserved. Production + ``finish_requests`` always passes its resolved finish set. In-place via ``self.running[:] = ...`` for minor consistency with idiomatic vLLM scheduler mutation; upstream @@ -706,4 +752,16 @@ class OmniSchedulerMixin: """ if not self.running: return - self.running[:] = [req for req in self.running if not req.is_finished() and req.request_id in self.requests] + target_request_ids = target_request_ids or set() + + def keep_running(req: Request) -> bool: + if req.request_id not in self.requests: + return False + if not req.is_finished(): + return True + resumable_segment_stop = bool( + getattr(req, "resumable", False) and req.status == RequestStatus.FINISHED_STOPPED + ) + return resumable_segment_stop and req.request_id not in target_request_ids + + self.running[:] = [req for req in self.running if keep_running(req)] diff --git a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py index ae1cb74b9..4a5abcbe5 100644 --- a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py +++ b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py @@ -869,13 +869,30 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): else: request_ids = requests.keys() + connector_owned_ids = { + request.request_id + for queue in ( + self.waiting_for_chunk_waiting_requests, + self.waiting_for_chunk_running_requests, + self._held_non_active, + ) + for request in queue + } + # First pass: collect requests to remove from queues for req_id in request_ids: request = requests.get(req_id) if requests else None - if request is None or request.is_finished(): + if request is None: # Invalid request ID. continue - if req_id in self.requests_origin_status: + resumable_segment_stop = bool( + getattr(request, "resumable", False) and request.status == RequestStatus.FINISHED_STOPPED + ) + if request.is_finished() and not resumable_segment_stop: + continue + # Once restored to a scheduler queue, the saved origin is stale and + # must not overwrite statuses such as WAITING_FOR_STREAMING_REQ. + if req_id in self.requests_origin_status and req_id in connector_owned_ids: request.status = self.requests_origin_status.pop(req_id) request_ids = set(request_ids) @@ -891,11 +908,6 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): ) for req_id in request_ids: - self._active_streams.pop(req_id, None) - self.requests_with_ready_chunks.discard(req_id) - self.upstream_exhausted_requests.discard(req_id) - self._finished_load_reqs.discard(req_id) - self._cancelled_load_reqs.add(req_id) - self._waiting_since.pop(req_id, None) + self.cleanup_receiver(req_id) return [] From 53df9fe62b3e05b388916351ecd758cb770e5cb4 Mon Sep 17 00:00:00 2001 From: WeiQing Chen <40507679+david6666666@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:13:34 +0800 Subject: [PATCH 084/267] [Bugfix] Use dedicated WORLD group for distributed VAE communication (#6401) Signed-off-by: david6666666 <530634352@qq.com> --- .../test_distributed_vae_executor.py | 21 +++++++++++++++++++ .../autoencoders/distributed_vae_executor.py | 10 +++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/tests/diffusion/distributed/test_distributed_vae_executor.py b/tests/diffusion/distributed/test_distributed_vae_executor.py index c122e9105..dcaa8e01a 100644 --- a/tests/diffusion/distributed/test_distributed_vae_executor.py +++ b/tests/diffusion/distributed/test_distributed_vae_executor.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from types import SimpleNamespace import pytest @@ -58,6 +59,11 @@ class E2EOperator: return torch.cat(tiles, dim=0) +@dataclass +class FakeWorldGroup: + device_group: object + + class DummyMixin(DistributedVaeMixin): def __init__(self): self.use_tiling = True @@ -74,6 +80,16 @@ def mock_dist(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(dist, "broadcast", lambda *args, **kwargs: None) +@pytest.fixture(autouse=True) +def mock_world_group(monkeypatch: pytest.MonkeyPatch): + group = object() + monkeypatch.setattr( + "vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor.get_world_group", + lambda: FakeWorldGroup(device_group=group), + ) + return group + + @pytest.fixture(autouse=True) def mock_dist_vae_executor(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(DistributedVaeExecutor, "gather_tensors", lambda self, x: [x]) @@ -85,6 +101,11 @@ def mock_dist_vae_executor(monkeypatch: pytest.MonkeyPatch): # ============================ +def test_uses_dedicated_world_device_group(mock_world_group): + executor = DistributedVaeExecutor() + assert executor.group is mock_world_group + + def test_balance_tasks(): executor = DistributedVaeExecutor() workloads = [2, 5, 13, 8, 2, 4] # each is 17 diff --git a/vllm_omni/diffusion/distributed/autoencoders/distributed_vae_executor.py b/vllm_omni/diffusion/distributed/autoencoders/distributed_vae_executor.py index 9d84cdf53..e8ae6bf32 100644 --- a/vllm_omni/diffusion/distributed/autoencoders/distributed_vae_executor.py +++ b/vllm_omni/diffusion/distributed/autoencoders/distributed_vae_executor.py @@ -9,6 +9,8 @@ import torch import torch.distributed as dist from vllm.logger import init_logger +from vllm_omni.diffusion.distributed.parallel_state import get_world_group + logger = init_logger(__name__) @@ -43,10 +45,10 @@ class DistributedVaeExecutor: """ def __init__(self): - # VAE tiles are scheduled across the complete worker WORLD. - self.group = None - self.world_size = dist.get_world_size() - self.rank = dist.get_rank() + # Use a dedicated process group spanning the complete worker WORLD. + self.group = get_world_group().device_group + self.world_size = dist.get_world_size(self.group) + self.rank = dist.get_rank(self.group) self.parallel_size = 1 self.parallel_mode = "tile" From 48e4f6a38504bc236a28d06e7c2837594c64e46f Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Thu, 20 Aug 2026 16:24:08 +0800 Subject: [PATCH 085/267] =?UTF-8?q?[Bugfix][MiniCPM-o][NPU]=20Skip=20Code2?= =?UTF-8?q?Wav=20dynamo=20unwrap=20when=20flow.encode=E2=80=A6=20(#6397)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: amy-why-3459 --- .../model_executor/models/minicpmo_4_5/batched_token2wav.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py b/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py index 1553a5921..f8692f349 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py @@ -160,7 +160,9 @@ class BatchedToken2Wav(nn.Module): self._trt_stepper = trt_stepper self.flow = token2wav.flow self.hift = token2wav.hift - _undecorate_dynamo(self.flow.encoder, "forward_chunk") + encoder = getattr(self.flow, "encoder", None) + if encoder is not None: + _undecorate_dynamo(encoder, "forward_chunk") hift_parameter = next(self.hift.parameters(), None) if hift_parameter is not None and hift_parameter.device.type == "cuda": # Prime the CUDA state used by HiFT during backend construction. From 10906e3a6db947290581b88dc4b6f35bd2a26b41 Mon Sep 17 00:00:00 2001 From: Zhou Taichang Date: Thu, 20 Aug 2026 16:47:50 +0800 Subject: [PATCH 086/267] [Refactor] Worker/ModelRunner runner correctness fixes (G2/N) (#5452) Signed-off-by: tzhouam Co-authored-by: Claude Fable 5 --- .../models/test_gepard_wiring.py | 11 +- .../voxcpm2/test_talker_output_marker.py | 73 ++ .../npu/test_minicpmo_code2wav_npugraph.py | 1 + tests/worker/test_capture_replay_contract.py | 28 +- tests/worker/test_gpu_ar_model_runner.py | 656 +++++++++++++++++- .../test_gpu_generation_model_runner.py | 121 +++- .../models/voxcpm2/voxcpm2_talker.py | 9 + vllm_omni/worker/gpu_ar_model_runner.py | 218 ++---- .../worker/gpu_generation_model_runner.py | 43 +- vllm_omni/worker/output/__init__.py | 21 + vllm_omni/worker/output/payload_build.py | 162 +++++ vllm_omni/worker/sampling_utils.py | 16 +- vllm_omni/worker/sparse_audio.py | 232 +++++++ 13 files changed, 1412 insertions(+), 179 deletions(-) create mode 100644 tests/model_executor/models/voxcpm2/test_talker_output_marker.py create mode 100644 vllm_omni/worker/output/__init__.py create mode 100644 vllm_omni/worker/output/payload_build.py create mode 100644 vllm_omni/worker/sparse_audio.py diff --git a/tests/model_executor/models/test_gepard_wiring.py b/tests/model_executor/models/test_gepard_wiring.py index b38b65639..9a08ddf0a 100644 --- a/tests/model_executor/models/test_gepard_wiring.py +++ b/tests/model_executor/models/test_gepard_wiring.py @@ -10,9 +10,10 @@ survives the runner's sparse-audio routing. These drive the real ``preprocess`` -> ``forward`` -> ``make_omni_output`` sequence through a stand-in for ``GPUARModelRunner.execute_model``, and use the -runner's own ``_resolve_sparse_mm_routing`` to decide what actually reaches the -caller. Only the LM backbone and the frame sampler are stubbed out; every -method under test is the real one, so a test here goes red when the engine +shared ``sparse_audio.resolve_sparse_mm_routing`` helper used by the runner to +decide what actually reaches the caller. Only the LM backbone and the frame +sampler are stubbed out; every method under test is the real one, so a test +here goes red when the engine path breaks — which a test that calls the flush by hand cannot do. """ @@ -33,7 +34,7 @@ from vllm_omni.model_executor.models.gepard.gepard_talker import ( ) from vllm_omni.model_executor.models.gepard.nanocodec import NanoCodec from vllm_omni.model_executor.models.gepard.prompt import build_gepard_prompt_ids -from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner +from vllm_omni.worker import sparse_audio pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -219,7 +220,7 @@ class _MiniEngine: self.committed_frames[req_id] = state.frame_count # --- runner: sparse-audio routing decides what reaches the caller --- - downstream, sparse_index, is_sparse = GPUARModelRunner._resolve_sparse_mm_routing( + downstream, sparse_index, is_sparse = sparse_audio.resolve_sparse_mm_routing( engine_output_type="audio", req_ids_output_copy=list(live), downstream_req_ids=list(live), diff --git a/tests/model_executor/models/voxcpm2/test_talker_output_marker.py b/tests/model_executor/models/voxcpm2/test_talker_output_marker.py new file mode 100644 index 000000000..1a76525fe --- /dev/null +++ b/tests/model_executor/models/voxcpm2/test_talker_output_marker.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""VoxCPM2 dense-mode audio outputs must carry the sparse-alignment marker. + +Dense mode (`_uses_sparse_audio_outputs()` False) still yields a strict +SUBSET of the batch whenever some requests emit no audio in a step (e.g. +prefill phase). The runner routes per-request `model_outputs` lists by batch +position unless `meta.sparse_audio` declares `meta.req_id` alignment — so a +subset without the marker misroutes audio across requests (RFC #5450 C3 +review follow-up). +""" + +from __future__ import annotations + +import functools + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +torch = pytest.importorskip("torch") + + +@functools.lru_cache(maxsize=1) +def _talker_cls(): + """Defer talker import (pulls vLLM model_executor) until first use.""" + from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import ( + VoxCPM2TalkerForConditionalGeneration, + ) + + return VoxCPM2TalkerForConditionalGeneration + + +def _make_dense_talker(): + cls = _talker_cls() + talker = cls.__new__(cls) + # Dense mode: emit every step, no delayed copy, no batched VAE decode. + talker._audio_emit_every = 1 + talker._vae_decode_every = 1 + talker._enable_delayed_audio_copy = False + talker._coalesce_audio_d2h = False + talker._sample_rate = 16000 + talker._audio_queue = [] + return talker + + +def test_dense_subset_audio_carries_sparse_alignment_marker(): + talker = _make_dense_talker() + # Batch had [r0, r1, r2]; only r1/r2 produced audio this step. + talker._audio_queue = [("r1", torch.zeros(4)), ("r2", torch.ones(4))] + + out = talker.make_omni_output(torch.zeros(1)) + + mm = out.multimodal_outputs + assert mm["meta"] == {"req_id": ["r1", "r2"], "sparse_audio": ["1"]} + assert len(mm["model_outputs"]) == 2 + assert len(mm["sr"]) == 2 + # (The coalesce-D2H sibling branch carries the same marker; it needs + # device-resident chunks, so its marker contract is pinned here via the + # shared dense-subset shape rather than a GPU-only test.) + + +def test_dense_full_batch_audio_still_carries_marker_and_order(): + # Full coverage is the degenerate subset: the marker stays truthful and + # `meta.req_id` order matches the emitted list order. + talker = _make_dense_talker() + talker._audio_queue = [("r0", torch.zeros(2)), ("r1", torch.ones(2))] + + out = talker.make_omni_output(torch.zeros(1)) + + mm = out.multimodal_outputs + assert mm["meta"]["req_id"] == ["r0", "r1"] + assert mm["meta"]["sparse_audio"] == ["1"] diff --git a/tests/platforms/npu/test_minicpmo_code2wav_npugraph.py b/tests/platforms/npu/test_minicpmo_code2wav_npugraph.py index 1c35c37aa..9a069fb8f 100644 --- a/tests/platforms/npu/test_minicpmo_code2wav_npugraph.py +++ b/tests/platforms/npu/test_minicpmo_code2wav_npugraph.py @@ -32,6 +32,7 @@ pytestmark = [ class _Flow(nn.Module): def __init__(self, estimator: nn.Module): super().__init__() + self.encoder = nn.Identity() self.decoder = SimpleNamespace(estimator=estimator) diff --git a/tests/worker/test_capture_replay_contract.py b/tests/worker/test_capture_replay_contract.py index 504aaa7b0..a0ca095f3 100644 --- a/tests/worker/test_capture_replay_contract.py +++ b/tests/worker/test_capture_replay_contract.py @@ -14,9 +14,9 @@ cannot be masked by an adjacent ``else``/``elif`` that happens to read it. It matches by attribute chain (``self.input_ids.gpu``), so local variable renames do not break it. -Scope: the BASE runner (``OmniGPUModelRunner``) only. The generation runner is -KNOWN to lack the ``has_preprocess`` branch; extend this -check to the generation file when it adds the branch. +Scope: the BASE runner (``OmniGPUModelRunner``) and the generation runner's +``_dummy_run`` override (which gained the branch in RFC #5450 C6; its +``_preprocess`` is inherited from the base). """ import ast @@ -25,6 +25,7 @@ import textwrap import pytest +from vllm_omni.worker.gpu_generation_model_runner import GPUGenerationModelRunner from vllm_omni.worker.gpu_model_runner import OmniGPUModelRunner pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -69,11 +70,24 @@ def _has_preprocess_branch_reads_both_buffers(method) -> bool: return any(_REQUIRED_BUFFERS <= _attr_chains_in(node.body) for node in guarded) -@pytest.mark.parametrize("method_name", ["_dummy_run", "_preprocess"]) -def test_has_preprocess_branch_reads_shared_buffers(method_name): - method = getattr(OmniGPUModelRunner, method_name) +@pytest.mark.parametrize( + ("runner_cls", "method_name"), + [ + (OmniGPUModelRunner, "_dummy_run"), + (OmniGPUModelRunner, "_preprocess"), + (GPUGenerationModelRunner, "_dummy_run"), + ], +) +def test_has_preprocess_branch_reads_shared_buffers(runner_cls, method_name): + method = getattr(runner_cls, method_name) assert _has_preprocess_branch_reads_both_buffers(method), ( - f"{OmniGPUModelRunner.__name__}.{method_name}: the has_preprocess branch no " + f"{runner_cls.__name__}.{method_name}: the has_preprocess branch no " "longer reads BOTH self.input_ids.gpu and self.inputs_embeds.gpu — capture " "and replay would diverge." ) + + +def test_generation_dummy_run_overrides_with_branch(): + # Guard against the override being deleted (which would silently fall back + # to the base implementation and vacuously pass the parametrized check). + assert "_dummy_run" in GPUGenerationModelRunner.__dict__ diff --git a/tests/worker/test_gpu_ar_model_runner.py b/tests/worker/test_gpu_ar_model_runner.py index 967407169..99f609ea5 100644 --- a/tests/worker/test_gpu_ar_model_runner.py +++ b/tests/worker/test_gpu_ar_model_runner.py @@ -1,8 +1,12 @@ import asyncio import copy +import re from collections.abc import Sequence +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier from types import SimpleNamespace from typing import Any +from unittest.mock import patch import numpy as np import pytest @@ -15,12 +19,15 @@ from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechReques from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech from vllm_omni.entrypoints.openai.tts_adapters.base import PreparedRequest from vllm_omni.outputs import OmniModelRunnerOutput +from vllm_omni.worker import sparse_audio from vllm_omni.worker.gpu_ar_model_runner import ( ExecuteModelState, GPUARModelRunner, OmniAsyncGPUModelRunnerOutput, ) +from vllm_omni.worker.output import payload_build from vllm_omni.worker.runner_assisted_metadata import RunnerAssistedFullAttentionMetadataRequest +from vllm_omni.worker.sampling_utils import clamp_prompt_ids_to_penalty_padding pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -301,11 +308,13 @@ def test_resolve_pooler_payload_req_ids_downstream_stage_uses_filtered_requests( def test_sparse_mm_req_ids_requires_sparse_audio_marker(): - assert GPUARModelRunner._sparse_mm_req_ids({"meta": {"req_id": ["r1"]}}) is None - assert GPUARModelRunner._sparse_mm_req_ids({"meta.req_id": ["r1"]}) is None + assert sparse_audio.resolve_sparse_mm_req_ids({"meta": {"req_id": ["r1"]}}) is None + assert sparse_audio.resolve_sparse_mm_req_ids({"meta.req_id": ["r1"]}) is None - assert GPUARModelRunner._sparse_mm_req_ids({"meta": {"req_id": ["r1"], "sparse_audio": ["1"]}}) == ["r1"] - assert GPUARModelRunner._sparse_mm_req_ids({"meta.req_id": ["r1"], "meta.sparse_audio": ["1"]}) == ["r1"] + assert sparse_audio.resolve_sparse_mm_req_ids({"meta": {"req_id": ["r1"], "sparse_audio": ["1"]}}) == ["r1"] + assert sparse_audio.resolve_sparse_mm_req_ids({"meta.req_id": ["r1"], "meta.sparse_audio": ["1"]}) == ["r1"] + assert sparse_audio.resolve_sparse_mm_req_ids({"meta": {"req_id": ["r1"]}, "meta.sparse_audio": ["1"]}) == ["r1"] + assert sparse_audio.resolve_sparse_mm_req_ids({"meta": {"sparse_audio": ["1"]}, "meta.req_id": ["r1"]}) == ["r1"] def test_runner_assisted_full_attention_metadata_request_is_opt_in(): @@ -1198,3 +1207,642 @@ def test_build_omni_output_uses_combined_prefix_cache_mm_payload_for_partial_dow assert torch.equal(output.inter_stage_outputs[0]["codes.ref"], ref_codes[0]) assert torch.equal(output.inter_stage_outputs[2]["codes.ref"], ref_codes[2]) + + +class TestPromptIdsClampPaddingConvention: + """Pin `sampling_utils.clamp_prompt_ids_to_penalty_padding` (the clamp + `sample_tokens` applies before penalties). + + The audit (RFC #5450 C2) originally flagged the clamp as an off-by-one + (OOB index). It is not: upstream penalties allocate `vocab_size + 1` bins + and drop the last column, so `vocab_size` is the designed padding value + (vllm/model_executor/layers/utils.py::get_token_bin_counts_and_mask). + These tests call the runner's own helper, so a future "fix" to the clamp + fails here. + """ + + LOGITS_VOCAB = 8 + BATCH_VOCAB = 10 # wider input_batch vocab; its pad value is BATCH_VOCAB + + def _padded_prompt(self): + # Two real tokens + two padding slots padded with the batch vocab size. + return torch.tensor([[1, 2, self.BATCH_VOCAB, self.BATCH_VOCAB]]) + + def test_clamp_to_logits_vocab_keeps_padding_inert(self): + from vllm.model_executor.layers.utils import get_token_bin_counts_and_mask + + # The runner's own helper, not a re-implementation. + clamped = clamp_prompt_ids_to_penalty_padding(self._padded_prompt(), self.LOGITS_VOCAB) + + _, mask = get_token_bin_counts_and_mask(clamped, self.LOGITS_VOCAB, 1) + + expected = torch.zeros(1, self.LOGITS_VOCAB, dtype=torch.bool) + expected[0, 1] = True + expected[0, 2] = True + # Pad ids landed in the dropped (vocab_size-th) bin: no penalty effect. + assert torch.equal(mask, expected) + + def test_clamp_one_lower_would_corrupt_last_token_penalties(self): + from vllm.model_executor.layers.utils import get_token_bin_counts_and_mask + + # The "fix" the audit first proposed - shown here to be the actual bug: + # padding would count as occurrences of the last real vocab token. + clamped = self._padded_prompt().clamp(max=self.LOGITS_VOCAB - 1) + + _, mask = get_token_bin_counts_and_mask(clamped, self.LOGITS_VOCAB, 1) + + assert bool(mask[0, self.LOGITS_VOCAB - 1]) + + def test_unclamped_padding_overflows_penalty_bins(self): + from vllm.model_executor.layers.utils import get_token_bin_counts_and_mask + + # Why the clamp exists: the batch-level pad value (BATCH_VOCAB) is out + # of range even for the vocab_size+1 penalty bins of a narrower head. + with pytest.raises(RuntimeError): + get_token_bin_counts_and_mask(self._padded_prompt(), self.LOGITS_VOCAB, 1) + + +class TestSparseAudioMarkerRobustness: + """RFC #5450 C3: `sparse_audio.is_sparse_audio_marker` must not crash on tensor/array + markers, and the combined prefix-cache payload builder must never ship one + request's data as another's.""" + + def test_marker_list_and_str_forms(self): + marker = sparse_audio.is_sparse_audio_marker + assert marker(["1"]) is True + assert marker(["0"]) is False + assert marker("true") is True + assert marker("off") is False + + def test_marker_enforced_canonical_forms_no_error(self): + # The designed carrier (list-of-str; bare str accepted) and marker + # absence (None) are the ONLY silent forms. + marker = sparse_audio.is_sparse_audio_marker + with patch.object(sparse_audio.logger, "error") as err: + assert marker(["1"]) is True + assert marker(["0"]) is False + assert marker(["0", "1"]) is True + assert marker("true") is True + assert marker("off") is False + assert marker(None) is False + err.assert_not_called() + + def test_marker_illegal_carriers_raise_one_error_per_type(self): + # Enforcement, not normalization: any carrier outside the design + # contract is a producer bug - it raises InvalidSparseDeclarationError + # (caught inside the module by resolve_sparse_mm_routing, which + # fails closed) + # and logs at error level once per offending type. Interpreting + # deviants would need a second truth predicate, and predicate + # divergence is exactly how the np.array(["0"])-routes-as-sparse bug + # happened. + marker = sparse_audio.is_sparse_audio_marker + for value in (torch.tensor(1), torch.tensor([1, 1]), np.array(["1"]), 1, 1.0, True, ["1", 1]): + with ( + patch.object(sparse_audio, "_logged_offenders", set()), + patch.object(sparse_audio.logger, "error") as err, + ): + with pytest.raises(sparse_audio.InvalidSparseDeclarationError): + marker(value) + assert err.call_count == 1, f"expected one error for {value!r}" + with pytest.raises(sparse_audio.InvalidSparseDeclarationError): + marker(value) + assert err.call_count == 1 # deduped per type, not per step + + def test_marker_device_tensor_rejected_without_read(self): + # Deviant carriers are never read element-wise, so a device tensor + # can never introduce a D2H sync here. Meta tensors hold no data - + # any read (`.item()`/`bool()`) raises a RuntimeError, so seeing OUR + # exception type (not RuntimeError) proves the data is never touched. + marker = sparse_audio.is_sparse_audio_marker + with patch.object(sparse_audio, "_logged_offenders", set()): + with pytest.raises(sparse_audio.InvalidSparseDeclarationError): + marker(torch.ones((), device="meta")) + with pytest.raises(sparse_audio.InvalidSparseDeclarationError): + marker(torch.ones(3, device="meta")) + + def test_invalid_marker_fails_closed_at_routing(self): + # A present-but-invalid sparse declaration most likely belongs to a + # producer that INTENDED sparse output; falling back to dense would + # batch-index-assign its payloads to the WRONG requests. Routing + # must fail closed: no payload routed for the step (empty sparse + # set), never dense misrouting. + with patch.object(sparse_audio, "_logged_offenders", set()): + downstream, index, is_sparse = sparse_audio.resolve_sparse_mm_routing( + engine_output_type="audio", + req_ids_output_copy=["r0", "r1"], + downstream_req_ids=["r0", "r1"], + multimodal_outputs={ + "meta": {"req_id": ["r0"], "sparse_audio": torch.tensor([1, 1])}, + "model_outputs": [torch.zeros(1)], + }, + ) + assert downstream == [] + assert index == {} + assert is_sparse is True + + def test_fail_closed_routing_error_remains_observable(self): + def route_invalid_marker(): + return sparse_audio.resolve_sparse_mm_routing( + engine_output_type="audio", + req_ids_output_copy=["r0"], + downstream_req_ids=["r0"], + multimodal_outputs={ + "meta": {"req_id": ["r0"], "sparse_audio": torch.tensor([1, 1])}, + "model_outputs": [torch.zeros(1)], + }, + ) + + with ( + patch.object(sparse_audio, "_logged_offenders", set()), + patch.object(sparse_audio, "_fail_closed_last_log_at", None), + patch.object(sparse_audio, "_fail_closed_suppressed", 0), + patch.object(sparse_audio.time, "monotonic", side_effect=[100.0, 120.0, 161.0]), + patch.object(sparse_audio.logger, "error") as err, + ): + for _ in range(3): + assert route_invalid_marker() == ([], {}, True) + + routing_errors = [call for call in err.call_args_list if "no multimodal payload" in call.args[0]] + assert len(routing_errors) == 2 + assert [call.args[1] for call in routing_errors] == [0, 1] + + def test_fail_closed_logging_is_thread_safe(self): + workers = 8 + ready = Barrier(workers) + + def route_invalid_marker(): + return sparse_audio.resolve_sparse_mm_routing( + engine_output_type="audio", + req_ids_output_copy=["r0"], + downstream_req_ids=["r0"], + multimodal_outputs={ + "meta": {"req_id": ["r0"], "sparse_audio": torch.tensor([1, 1])}, + }, + ) + + def route_concurrently(_): + ready.wait(timeout=10) + return route_invalid_marker() + + with ( + patch.object(sparse_audio, "_logged_offenders", set()), + patch.object(sparse_audio, "_fail_closed_last_log_at", None), + patch.object(sparse_audio, "_fail_closed_suppressed", 0), + patch.object(sparse_audio.time, "monotonic", return_value=100.0) as clock, + patch.object(sparse_audio.logger, "error") as err, + ): + with ThreadPoolExecutor(max_workers=workers) as executor: + results = list(executor.map(route_concurrently, range(workers))) + assert results == [([], {}, True)] * workers + + clock.return_value = 161.0 + assert route_invalid_marker() == ([], {}, True) + + routing_errors = [call for call in err.call_args_list if "no multimodal payload" in call.args[0]] + assert len(routing_errors) == 2 + assert [call.args[1] for call in routing_errors] == [0, workers - 1] + classifier_errors = [call for call in err.call_args_list if "Illegal sparse-audio marker" in call.args[0]] + assert len(classifier_errors) == 1 + + def test_duplicate_sparse_metadata_must_be_consistent(self): + equal = { + "meta": {"req_id": ["r1"], "sparse_audio": ["1"]}, + "meta.req_id": ["r1"], + "meta.sparse_audio": ["true"], + } + assert sparse_audio.resolve_sparse_mm_req_ids(equal) == ["r1"] + + invalid_declarations = ( + { + "meta": {"req_id": ["r1"], "sparse_audio": ["1"]}, + "meta.req_id": ["r2"], + }, + { + "meta": {"req_id": ["r1"], "sparse_audio": ["1"]}, + "meta.req_id": "r1", + }, + { + "meta": {"req_id": ["r1"], "sparse_audio": ["0"]}, + "meta.sparse_audio": ["1"], + }, + {"meta": {"req_id": ["r1", "r1"], "sparse_audio": ["1"]}}, + ) + with patch.object(sparse_audio, "_logged_offenders", set()): + for declaration in invalid_declarations: + with pytest.raises(sparse_audio.InvalidSparseDeclarationError): + sparse_audio.resolve_sparse_mm_req_ids(declaration) + + def test_conflicting_sparse_metadata_fails_closed_at_public_boundary(self): + downstream, index, is_sparse = sparse_audio.resolve_sparse_mm_routing( + engine_output_type="audio", + req_ids_output_copy=["r1", "r2"], + downstream_req_ids=["r1", "r2"], + multimodal_outputs={ + "meta": {"req_id": ["r1"], "sparse_audio": ["1"]}, + "meta.req_id": ["r2"], + }, + ) + assert downstream == [] + assert index == {} + assert is_sparse is True + + def test_flat_marker_with_nested_req_ids_is_not_erased(self): + downstream, index, is_sparse = sparse_audio.resolve_sparse_mm_routing( + engine_output_type="audio", + req_ids_output_copy=["r0", "r1"], + downstream_req_ids=["r0", "r1"], + multimodal_outputs={"meta": {"req_id": ["r1"]}, "meta.sparse_audio": ["1"]}, + ) + assert downstream == ["r1"] + assert index == {"r1": 0} + assert is_sparse is True + + def test_sparse_marker_with_unusable_req_ids_fails_closed(self): + # The marker says "sparse" but the alignment list is unusable - the + # declaration as a whole is out of contract; same fail-closed shape. + with patch.object(sparse_audio, "_logged_offenders", set()): + downstream, index, is_sparse = sparse_audio.resolve_sparse_mm_routing( + engine_output_type="audio", + req_ids_output_copy=["r0"], + downstream_req_ids=["r0"], + multimodal_outputs={"meta": {"req_id": "r0", "sparse_audio": ["1"]}}, + ) + assert downstream == [] + assert is_sparse is True + + def test_nested_marker_without_req_id_fails_closed(self): + # {"meta": {"sparse_audio": ["1"]}} - marker set, req_id missing. + # The flattened-encoding fallback must not ERASE the nested marker + # (which used to silently demote the declaration to dense); this is + # an unusable-req_id invalid declaration -> fail closed. + with patch.object(sparse_audio, "_logged_offenders", set()): + downstream, index, is_sparse = sparse_audio.resolve_sparse_mm_routing( + engine_output_type="audio", + req_ids_output_copy=["r0"], + downstream_req_ids=["r0"], + multimodal_outputs={"meta": {"sparse_audio": ["1"]}}, + ) + assert downstream == [] + assert is_sparse is True + + def test_invalid_marker_on_non_audio_pipeline_keeps_dense_routing(self): + # Non-audio pipelines never consult sparse routing; the carrier + # error is logged by the classifier but routing is unaffected. + with patch.object(sparse_audio, "_logged_offenders", set()): + downstream, index, is_sparse = sparse_audio.resolve_sparse_mm_routing( + engine_output_type="latent", + req_ids_output_copy=["r0", "r1"], + downstream_req_ids=["r0", "r1"], + multimodal_outputs={"meta": {"req_id": ["r0"], "sparse_audio": torch.tensor([1, 1])}}, + ) + assert downstream == ["r0", "r1"] + assert is_sparse is False + + def test_legal_marker_still_routes_sparse(self): + downstream, index, is_sparse = sparse_audio.resolve_sparse_mm_routing( + engine_output_type="audio", + req_ids_output_copy=["r0", "r1", "r2"], + downstream_req_ids=["r0", "r1", "r2"], + multimodal_outputs={"meta": {"req_id": ["r1", "r2"], "sparse_audio": ["1"]}}, + ) + assert downstream == ["r1", "r2"] + assert index == {"r1": 0, "r2": 1} + assert is_sparse is True + + def test_combined_payload_unwraps_aligned_list(self): + combined = {"codes.audio": {"req-1": [torch.zeros(1), torch.ones(1)]}} + payload = payload_build.build_combined_prefix_cache_mm_payload(combined, rid="req-1", list_idx=1) + assert torch.equal(payload["codes.audio"], torch.ones(1)) + + def test_combined_payload_broadcasts_qwen_request_invariant_singletons(self): + req_ids = ("r0", "r1") + tts_embeddings = { + "embed.tts_bos": torch.full((1, 1, 2), 1.0), + "embed.tts_eos": torch.full((1, 1, 2), 2.0), + "embed.tts_pad": torch.full((1, 1, 2), 3.0), + } + combined = {key: {rid: [value.clone()] for rid in req_ids} for key, value in tts_embeddings.items()} + + for idx, rid in enumerate(req_ids): + payload = payload_build.build_omni_mm_payload( + combined_multimodal_outputs=combined, + mm_cpu=None, + rid=rid, + idx=idx, + start=idx, + end=idx + 1, + audio_sparse_output=False, + sparse_mm_index={}, + hidden_seq_len=len(req_ids), + scheduled_seq_len=len(req_ids), + ) + assert payload.keys() == tts_embeddings.keys() + for key, expected in tts_embeddings.items(): + assert torch.equal(payload[key], expected) + + def test_combined_payload_drops_sparse_protocol_metadata(self): + combined = { + "meta.sparse_audio": {"req-1": ["1"]}, + "meta.req_id": {"req-1": ["req-1"]}, + "sr": {"req-1": 22050}, + } + payload = payload_build.build_combined_prefix_cache_mm_payload(combined, rid="req-1", list_idx=2) + assert payload == {"sr": 22050} + + def test_combined_payload_misalignment_drops_key_not_request_zero(self): + # A short multi-element list used to silently return `v[0]` - request + # 0's payload shipped under request `rid`. Never substitute another + # request's data; the key is dropped with an error log (same policy + # as the non-combined sparse mismatch) rather than raised - v1 + # runners don't catch per-request exceptions, so a raise would turn a + # per-request protocol break into a whole-stage outage. + combined = { + "codes.audio": {"req-3": [torch.zeros(1), torch.ones(1)]}, + "sr": {"req-3": 22050}, + } + payload = payload_build.build_combined_prefix_cache_mm_payload(combined, rid="req-3", list_idx=2) + assert "codes.audio" not in payload + assert payload["sr"] == 22050 + + def test_direct_path_sparse_mismatch_drops_key_and_keeps_payload_shape(self): + # Mirror of the combined-path policy on the non-prefix-cache path: + # a sparse index beyond the per-request list drops that key (error + # log) and the rest of the request's payload survives. + payload = payload_build.build_omni_mm_payload( + combined_multimodal_outputs=None, + mm_cpu={"codes.audio": [torch.zeros(1)]}, + rid="r2", + idx=1, + start=0, + end=1, + audio_sparse_output=True, + sparse_mm_index={"r2": 1}, + hidden_seq_len=1, + scheduled_seq_len=1, + ) + assert payload == {} + + def test_combined_payload_uses_sparse_index_under_sparse_routing(self): + # Batch [r0, r1, r2], sparse outputs only for [r1, r2]: producers + # align per-request lists to the SPARSE order, so r1 (batch idx 1, + # sparse idx 0) must get element 0 and r2 (batch idx 2, sparse idx 1) + # element 1. Indexing by batch idx shipped r2's payload to r1 and + # went out of range for r2. The sparse->list_idx resolution lives in + # `_build_omni_mm_payload` (the combined helper itself is + # sparse-agnostic), so this exercises the caller. + sparse_mm_index = {"r1": 0, "r2": 1} + combined = { + "codes.audio": { + "r1": [torch.zeros(1), torch.ones(1)], + "r2": [torch.zeros(1), torch.ones(1)], + } + } + payloads = {} + for rid, idx in (("r1", 1), ("r2", 2)): + payloads[rid] = payload_build.build_omni_mm_payload( + combined_multimodal_outputs=combined, + mm_cpu=None, + rid=rid, + idx=idx, + start=0, + end=1, + audio_sparse_output=True, + sparse_mm_index=sparse_mm_index, + hidden_seq_len=1, + scheduled_seq_len=1, + ) + assert torch.equal(payloads["r1"]["codes.audio"], torch.zeros(1)) + assert torch.equal(payloads["r2"]["codes.audio"], torch.ones(1)) + + +class TestMergeModelKvTransferMetadata: + """RFC #5450 C4: model KV-transfer metadata must merge into a COPY of the + scheduler's finished-request data - `scheduler_output` can be shared with + the engine-core process and must never be mutated by the runner.""" + + def _runner_with_model(self, model): + runner = object.__new__(GPUARModelRunner) + runner.model = model + return runner + + def test_merge_does_not_mutate_scheduler_data(self): + class _Model: + def get_kv_transfer_metadata(self, req_id, num_computed_tokens): + return {"talker_codes": [7], "seen_tokens": num_computed_tokens} + + original_meta = {"a": 1} + original = {"r1": {"seq_len": 3, "block_ids": [4], "custom_metadata": original_meta}} + runner = self._runner_with_model(_Model()) + + merged = runner._merge_model_kv_transfer_metadata(original) + + # The manager sees the merged view... + assert merged["r1"]["custom_metadata"] == {"a": 1, "talker_codes": [7], "seen_tokens": 3} + assert merged["r1"]["seq_len"] == 3 + # ...while the engine-shared originals are untouched. + assert original["r1"]["custom_metadata"] is original_meta + assert original_meta == {"a": 1} + assert "talker_codes" not in original["r1"]["custom_metadata"] + + def test_merge_without_model_meta_passes_entry_through(self): + class _Model: + def get_kv_transfer_metadata(self, req_id, num_computed_tokens): + return None + + data = {"seq_len": 2, "block_ids": [1]} + runner = self._runner_with_model(_Model()) + + merged = runner._merge_model_kv_transfer_metadata({"r1": data}) + + assert merged["r1"] is data + + def test_merge_failure_raises_instead_of_shipping_partial_transfer(self): + # RFC #5450 C7: the bare `except Exception -> logger.warning` used to + # drop the model's custom metadata and let the KV transfer proceed + # incomplete; the failure then surfaced much later on the consumer + # stage. It must fail at the defect instead. + class _Model: + def get_kv_transfer_metadata(self, req_id, num_computed_tokens): + raise RuntimeError("boom") + + data = {"seq_len": 2, "block_ids": [1], "custom_metadata": {"a": 1}} + runner = self._runner_with_model(_Model()) + + with pytest.raises(RuntimeError, match="boom"): + runner._merge_model_kv_transfer_metadata({"r1": data}) + assert data["custom_metadata"] == {"a": 1} + + +class TestDownstreamPayloadMemoization: + """RFC #5450 C7: `_request_needs_downstream_stage_payload` must not + memoize before the deciding marker exists - the final-stage id arrives via + `model_intermediate_buffer`, which can be unpopulated on the first call.""" + + def _runner(self, stages): + runner = object.__new__(GPUARModelRunner) + runner._downstream_payload_cache = {} + runner._request_final_stage_id = lambda rid: stages[rid] + return runner + + def test_missing_marker_defaults_true_without_memoizing(self): + stages = {"r1": None} + runner = self._runner(stages) + + assert runner._request_needs_downstream_stage_payload("r1") is True + # Not cached: the answer must refresh once the marker arrives. + assert "r1" not in runner._downstream_payload_cache + + stages["r1"] = 0 # marker lands: this stage IS the final stage + assert runner._request_needs_downstream_stage_payload("r1") is False + assert runner._downstream_payload_cache["r1"] is False + + def test_memoizes_once_marker_known(self): + stages = {"r1": 2} + runner = self._runner(stages) + + assert runner._request_needs_downstream_stage_payload("r1") is True + assert runner._downstream_payload_cache["r1"] is True + # Cached value now authoritative for the request's lifetime. + stages["r1"] = 0 + assert runner._request_needs_downstream_stage_payload("r1") is True + + +# Matches a real ASSIGNMENT of the capability (optionally `self.`-qualified +# and/or annotated), not a bare mention: a comment, a docstring, or a read +# such as `getattr(self.model, "prefer_model_sampler", False)` must not count +# as declaring it. `=(?!=)` keeps `==` comparisons out. +_PREFER_MODEL_SAMPLER_ASSIGNMENT = re.compile( + r"^[ \t]*(?:self\.)?prefer_model_sampler[ \t]*(?::[^=\n]*)?=(?!=)[ \t]*(?P[^\n]*)", + re.MULTILINE, +) + + +def _declares_prefer_model_sampler(source: str) -> bool: + """Whether `source` opts INTO the model-sampler contract. + + An explicit `prefer_model_sampler = False` is an opt-OUT and does not + count; any other right-hand side does, including runtime-conditional + forms (minicpmo_4_5 assigns `self.model_stage in {...}`), which can + evaluate True. + """ + for match in _PREFER_MODEL_SAMPLER_ASSIGNMENT.finditer(source): + if match.group("rhs").split("#")[0].strip() != "False": + return True + return False + + +class TestPreferModelSamplerNoneFallback: + """RFC #5450 C7: a `prefer_model_sampler` model returning None from + `sample()` DECLINES the step and the runner falls back to the default + sampler. The fallback is load-bearing (cosyvoice3/glm_tts decline for + empty-logits / inapplicable-stage steps), so it must keep working - now + as a documented contract with a one-time visibility log.""" + + def _runner(self, model_sample_result, default_result): + runner = object.__new__(GPUARModelRunner) + runner.input_batch = SimpleNamespace( + sampling_metadata="smd", + update_async_output_token_ids=lambda: None, + ) + runner.model = SimpleNamespace( + prefer_model_sampler=True, + sample=lambda logits, metadata: model_sample_result, + ) + runner._sampling_metadata_for_model_sampler = lambda smd: smd + runner._resolve_duplex_sampling_hook = lambda: None + runner.sampler = lambda logits, sampling_metadata: default_result + return runner + + def test_none_falls_back_to_default_sampler_and_warns_once(self): + import logging + + from vllm_omni.worker import gpu_ar_model_runner as ar_module + + class _UniqueDecliningSampler: + prefer_model_sampler = True + + @staticmethod + def sample(logits, metadata): + return None + + sentinel = object() + runner = self._runner(model_sample_result=None, default_result=sentinel) + runner.model = _UniqueDecliningSampler() + + records: list[logging.LogRecord] = [] + + class _Capture(logging.Handler): + def emit(self, record): + records.append(record) + + handler = _Capture(level=logging.WARNING) + ar_module.logger.addHandler(handler) + try: + for _ in range(3): + out = GPUARModelRunner._sample(runner, torch.zeros(1, 4), None) + assert out is sentinel + finally: + ar_module.logger.removeHandler(handler) + + fallback_records = [r for r in records if "falling back to the default sampler" in r.getMessage()] + assert len(fallback_records) == 1 + assert "_UniqueDecliningSampler" in fallback_records[0].getMessage() + + def test_model_sampler_output_short_circuits_default(self): + model_out = object() + runner = self._runner(model_sample_result=model_out, default_result=object()) + + out = GPUARModelRunner._sample(runner, torch.zeros(1, 4), None) + + assert out is model_out + + def test_declaration_matcher_ignores_mentions_and_opt_outs(self): + # Guards the guard: the inventory below is only meaningful if + # "declares" means an actual opt-in assignment. + assert _declares_prefer_model_sampler(" prefer_model_sampler = True") + assert _declares_prefer_model_sampler(" prefer_model_sampler: bool = True") + assert _declares_prefer_model_sampler(' self.prefer_model_sampler = stage in {"llm"}') + assert not _declares_prefer_model_sampler(" prefer_model_sampler = False") + assert not _declares_prefer_model_sampler(" prefer_model_sampler = False # opted out") + assert not _declares_prefer_model_sampler(" # prefer_model_sampler = True (not anymore)") + assert not _declares_prefer_model_sampler(' """Does not use prefer_model_sampler."""') + assert not _declares_prefer_model_sampler(' x = getattr(model, "prefer_model_sampler", False)') + assert not _declares_prefer_model_sampler(" if prefer_model_sampler == True:") + + def test_in_tree_declarer_inventory_is_exact(self): + # The contract's audience: an exact inventory, so ADDING a declarer + # fails this test and the new model owner consciously signs up for the + # documented None-fallback semantics (update the set when they do). + import pathlib + + import vllm_omni + + models_dir = pathlib.Path(vllm_omni.__file__).parent / "model_executor" / "models" + declarers = { + p.parent.name + for p in models_dir.rglob("*.py") + if _declares_prefer_model_sampler(p.read_text(encoding="utf-8", errors="ignore")) + } + expected = { + "cosyvoice3", + "glm_tts", + "higgs_audio_v2", + "higgs_audio_v3", + "hunyuan_image3", + "minicpmo_4_5", + "minimax_music3", + } + assert declarers == expected, ( + "The set of models declaring `prefer_model_sampler` changed:\n" + f" added: {sorted(declarers - expected) or 'none'}\n" + f" removed: {sorted(expected - declarers) or 'none'}\n\n" + "This is a deliberate consent gate, not a broken test. Declaring " + "`prefer_model_sampler` opts a model into the runner's documented " + "None-fallback contract: `model.sample()` may return None to decline " + "a step, and the runner then falls back to the DEFAULT sampler " + "(see `GPUARModelRunner._sample`; RFC #5450 C7.3).\n\n" + "If you ADDED a declarer: confirm your model relies on -- or at " + "least tolerates -- that fallback, then add its directory name to " + "`expected` above. If you REMOVED one, drop its name." + ) diff --git a/tests/worker/test_gpu_generation_model_runner.py b/tests/worker/test_gpu_generation_model_runner.py index a6df4732f..b97168161 100644 --- a/tests/worker/test_gpu_generation_model_runner.py +++ b/tests/worker/test_gpu_generation_model_runner.py @@ -16,14 +16,14 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] class _DummyInputBatch: - def __init__(self): - self.req_ids = ["req-1"] - self.req_id_to_index = {"req-1": 0} - self.num_reqs = 1 + def __init__(self, num_reqs: int = 1): + self.req_ids = [f"req-{i + 1}" for i in range(num_reqs)] + self.req_id_to_index = {rid: i for i, rid in enumerate(self.req_ids)} + self.num_reqs = num_reqs self.vocab_size = 10 -def _make_runner(multimodal_outputs): +def _make_runner(multimodal_outputs, num_reqs: int = 1): runner = object.__new__(GPUGenerationModelRunner) runner.execute_model_state = ExecuteModelState( None, @@ -40,7 +40,7 @@ def _make_runner(multimodal_outputs): None, ) runner.kv_connector_output = None - runner.input_batch = _DummyInputBatch() + runner.input_batch = _DummyInputBatch(num_reqs) runner.use_async_scheduling = False runner.device = torch.device("cpu") runner.supports_mm_inputs = False @@ -111,6 +111,12 @@ def _make_guard_runner(): runner.cache_config = CacheConfig() runner.input_batch = _DummyInputBatch() runner.model = object() + runner.parallel_config = SimpleNamespace( + distributed_executor_backend=None, + data_parallel_size=1, + ) + runner._dummy_run_calls = [] + runner._dummy_run = lambda n: runner._dummy_run_calls.append(n) runner._update_states = lambda scheduler_output: None runner.synchronize_input_prep = contextlib.nullcontext runner.attach_omni_connector_output = lambda result: result @@ -121,9 +127,112 @@ def _make_guard_runner(): def test_execute_model_skips_non_positive_scheduled_span(monkeypatch, total): """#5196: a negative span is truthy, so it used to reach `_prepare_inputs`, whose `assert total_num_scheduled_tokens > 0` killed the stage EngineCore.""" + monkeypatch.setattr(gen_runner_module, "has_ec_transfer", lambda: False) monkeypatch.setattr(gen_runner_module, "has_kv_transfer_group", lambda: False) runner = _make_guard_runner() output = GPUGenerationModelRunner.execute_model(runner, _StubSchedulerOutput(total)) assert output is EMPTY_MODEL_RUNNER_OUTPUT + assert runner._dummy_run_calls == [] + + +@pytest.mark.parametrize("total", [-1, 0]) +def test_execute_model_zero_tokens_runs_dp_dummy_run(monkeypatch, total): + """A 0-token step under external_launcher + DP>1 must `_dummy_run(1)` so + `coordinate_batch_across_dp` stays in sync (it used to be shadowed by an + unconditional early return and never ran -> DP hang risk).""" + monkeypatch.setattr(gen_runner_module, "has_ec_transfer", lambda: False) + monkeypatch.setattr(gen_runner_module, "has_kv_transfer_group", lambda: False) + runner = _make_guard_runner() + runner.parallel_config = SimpleNamespace( + distributed_executor_backend="external_launcher", + data_parallel_size=2, + ) + + output = GPUGenerationModelRunner.execute_model(runner, _StubSchedulerOutput(total)) + + assert output is EMPTY_MODEL_RUNNER_OUTPUT + assert runner._dummy_run_calls == [1] + + +def test_execute_model_zero_tokens_kv_connector_no_forward(monkeypatch): + """With a KV-transfer group, a 0-token step must go through + `kv_connector_no_forward` (also shadowed by the old early return).""" + monkeypatch.setattr(gen_runner_module, "has_ec_transfer", lambda: False) + monkeypatch.setattr(gen_runner_module, "has_kv_transfer_group", lambda: True) + runner = _make_guard_runner() + sentinel = object() + calls = [] + runner.kv_connector_no_forward = lambda scheduler_output, vllm_config: ( + calls.append(scheduler_output), + sentinel, + )[1] + runner.vllm_config = object() + + output = GPUGenerationModelRunner.execute_model(runner, _StubSchedulerOutput(0)) + + assert output is sentinel + assert len(calls) == 1 + + +def test_execute_model_zero_tokens_ec_producer_takes_encoder_path(monkeypatch): + """AR lock-step: the EC-producer block sits ABOVE the zero-token block + (matching GPUARModelRunner), so a 0-token step on an EC producer returns + the empty ENCODER output. Before the C1 reorder, the unconditional early + return won and produced EMPTY_MODEL_RUNNER_OUTPUT instead — this is the + one cell whose behaviour the reorder changed.""" + monkeypatch.setattr(gen_runner_module, "has_ec_transfer", lambda: True) + monkeypatch.setattr(gen_runner_module, "get_ec_transfer", lambda: SimpleNamespace(is_consumer=False)) + monkeypatch.setattr(gen_runner_module, "has_kv_transfer_group", lambda: False) + sentinel = object() + monkeypatch.setattr(gen_runner_module, "make_empty_encoder_model_runner_output", lambda so: sentinel) + runner = _make_guard_runner() + runner.encoder_cache = {} + runner.maybe_get_ec_connector_output = lambda scheduler_output, encoder_cache: contextlib.nullcontext() + encoder_calls = [] + runner._execute_mm_encoder = lambda so: encoder_calls.append(so) + + output = GPUGenerationModelRunner.execute_model(runner, _StubSchedulerOutput(0)) + + assert output is sentinel + assert len(encoder_calls) == 1 + assert runner._dummy_run_calls == [] + + +class TestSampleTokensBatched: + """RFC #5450 C5: `sample_tokens` must slice per request for num_reqs > 1. + The tensor branch's asserts used to force num_reqs == 1 and the list + branch built a length-1 payload misaligned with `req_ids`.""" + + def test_tensor_output_sliced_per_request(self): + multimodal_outputs = torch.arange(12, dtype=torch.float32).reshape(2, 2, 3) + runner = _make_runner(multimodal_outputs, num_reqs=2) + + output = GPUGenerationModelRunner.sample_tokens(runner) + + assert len(output.multimodal_outputs) == 2 + assert torch.equal(output.multimodal_outputs[0]["model_outputs"], multimodal_outputs[0]) + assert torch.equal(output.multimodal_outputs[1]["model_outputs"], multimodal_outputs[1]) + + def test_list_output_one_entry_per_request(self): + entries = [torch.zeros(2, 1), torch.ones(2, 1)] + runner = _make_runner(list(entries), num_reqs=2) + + output = GPUGenerationModelRunner.sample_tokens(runner) + + assert len(output.multimodal_outputs) == 2 + assert torch.equal(output.multimodal_outputs[0]["model_outputs"], entries[0]) + assert torch.equal(output.multimodal_outputs[1]["model_outputs"], entries[1]) + + def test_tensor_output_batch_mismatch_raises(self): + runner = _make_runner(torch.randn(1, 2, 3), num_reqs=2) + + with pytest.raises(ValueError, match="leading dim 1 .* 2 requests"): + GPUGenerationModelRunner.sample_tokens(runner) + + def test_list_output_batch_mismatch_raises(self): + runner = _make_runner([torch.randn(2, 1)], num_reqs=2) + + with pytest.raises(ValueError, match="length 1 .* 2 requests"): + GPUGenerationModelRunner.sample_tokens(runner) diff --git a/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py b/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py index 3a1677d49..7dda87e96 100644 --- a/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py +++ b/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py @@ -3013,9 +3013,18 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): merged_cpu = merged.detach().cpu().float() mm["model_outputs"] = list(merged_cpu.split(sizes)) mm["sr"] = [sr for _ in ready_req_ids] + # Dense mode still yields a strict SUBSET of the batch when + # some requests emit no audio this step (e.g. prefill + # phase). Without the marker the runner indexes these + # per-request lists by batch position and misroutes audio + # across requests; the marker declares meta.req_id + # alignment. + mm["meta"] = {"req_id": ready_req_ids, "sparse_audio": ["1"]} else: mm["model_outputs"] = list(audio_by_req.values()) mm["sr"] = [sr for _ in audio_by_req] + # Same subset hazard as the coalesce branch above. + mm["meta"] = {"req_id": list(audio_by_req), "sparse_audio": ["1"]} elif self._uses_sparse_audio_outputs(): mm["model_outputs"] = [] mm["sr"] = [] diff --git a/vllm_omni/worker/gpu_ar_model_runner.py b/vllm_omni/worker/gpu_ar_model_runner.py index e66f759f8..c87d17cc2 100644 --- a/vllm_omni/worker/gpu_ar_model_runner.py +++ b/vllm_omni/worker/gpu_ar_model_runner.py @@ -51,12 +51,13 @@ from vllm_omni.utils.mm_outputs import ( build_mm_cpu, partition_payload_list, snapshot_mm_payload, - to_payload_element, ) from vllm_omni.worker.gpu_model_runner import OmniGPUModelRunner from vllm_omni.worker.omni_connector_model_runner_mixin import OmniConnectorModelRunnerMixin +from vllm_omni.worker.output.payload_build import build_omni_mm_payload from vllm_omni.worker.runner_assisted_metadata import RunnerAssistedFullAttentionMetadataRequest -from vllm_omni.worker.sampling_utils import sanitize_min_tokens_stop_ids +from vllm_omni.worker.sampling_utils import clamp_prompt_ids_to_penalty_padding, sanitize_min_tokens_stop_ids +from vllm_omni.worker.sparse_audio import resolve_sparse_mm_routing logger = init_logger(__name__) @@ -457,9 +458,15 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin, Duplex cached = self._downstream_payload_cache.get(req_id) if cached is not None: return cached - # Conservative default: keep payload if marker is missing. final_stage_id = self._request_final_stage_id(req_id) - needs_payload = final_stage_id is None or final_stage_id > 0 + if final_stage_id is None: + # Conservative default while the marker is missing: keep the + # payload, but do NOT memoize - the marker arrives via + # `model_intermediate_buffer`, which may be unpopulated on the + # first call (memoizing here pinned the request to True forever, + # never refreshing once the marker landed). + return True + needs_payload = final_stage_id > 0 self._downstream_payload_cache[req_id] = needs_payload return needs_payload @@ -472,50 +479,6 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin, Duplex downstream_req_ids = req_ids_output_copy return engine_output_type, downstream_req_ids - @staticmethod - def _sparse_mm_req_ids(multimodal_outputs: Any) -> list[str] | None: - if not isinstance(multimodal_outputs, dict): - return None - meta = multimodal_outputs.get("meta") - req_ids = None - sparse_audio = False - if isinstance(meta, dict): - req_ids = meta.get("req_id") - sparse_audio = GPUARModelRunner._is_sparse_audio_marker(meta.get("sparse_audio")) - if req_ids is None: - req_ids = multimodal_outputs.get("meta.req_id") - sparse_audio = GPUARModelRunner._is_sparse_audio_marker(multimodal_outputs.get("meta.sparse_audio")) - if not sparse_audio: - return None - if not isinstance(req_ids, list): - return None - return [rid for rid in req_ids if isinstance(rid, str)] - - @staticmethod - def _resolve_sparse_mm_routing( - *, - engine_output_type: str, - req_ids_output_copy: list[str], - downstream_req_ids: list[str], - multimodal_outputs: Any, - ) -> tuple[list[str], dict[str, int], bool]: - sparse_mm_req_ids = GPUARModelRunner._sparse_mm_req_ids(multimodal_outputs) - sparse_mm_index = {rid: i for i, rid in enumerate(sparse_mm_req_ids or [])} - if engine_output_type != "audio" or sparse_mm_req_ids is None: - return downstream_req_ids, sparse_mm_index, False - - sparse_req_id_set = set(sparse_mm_req_ids) - sparse_downstream_req_ids = [rid for rid in req_ids_output_copy if rid in sparse_req_id_set] - return sparse_downstream_req_ids, sparse_mm_index, True - - @staticmethod - def _is_sparse_audio_marker(value: Any) -> bool: - if isinstance(value, list): - return any(str(item).lower() in ("1", "true", "yes", "on") for item in value) - if isinstance(value, str): - return value.lower() in ("1", "true", "yes", "on") - return bool(value) - def capture_model(self) -> int: result = super().capture_model() self._capture_talker_mtp_graphs() @@ -862,79 +825,6 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin, Duplex ) return hidden_states_cpu, combined_hidden_states, combined_multimodal_outputs - @staticmethod - def _build_combined_prefix_cache_mm_payload( - combined_multimodal_outputs: dict, - *, - rid: str, - idx: int, - ) -> dict[str, object]: - def _unwrap_lists(v): - if isinstance(v, list): - return v[idx] if idx < len(v) else v[0] - if isinstance(v, dict): - return {k: _unwrap_lists(sv) for k, sv in v.items()} - return v - - return { - mm_key: _unwrap_lists(combined_multimodal_outputs[mm_key][rid]) - for mm_key in combined_multimodal_outputs.keys() - } - - def _build_omni_mm_payload( - self, - *, - combined_multimodal_outputs: dict | None, - mm_cpu: dict[str, object] | None, - rid: str, - idx: int, - start: int, - end: int, - audio_sparse_output: bool, - sparse_mm_index: dict[str, int], - hidden_seq_len: int, - scheduled_seq_len: int, - ) -> dict[str, object]: - if combined_multimodal_outputs: - return self._build_combined_prefix_cache_mm_payload( - combined_multimodal_outputs, - rid=rid, - idx=idx, - ) - - mm_payload: dict[str, object] = {} - if not mm_cpu: - return mm_payload - - for mm_key, mm_val in mm_cpu.items(): - if mm_key in {"meta.req_id", "meta.sparse_audio"}: - continue - if audio_sparse_output and isinstance(mm_val, list): - sparse_idx = sparse_mm_index.get(rid) - if sparse_idx is None: - continue - if sparse_idx >= len(mm_val): - logger.warning( - "Sparse multimodal payload mismatch for request %s: index %d >= %d.", - rid, - sparse_idx, - len(mm_val), - ) - continue - sparse_val = mm_val[sparse_idx] - mm_payload[mm_key] = sparse_val.clone() if isinstance(sparse_val, torch.Tensor) else sparse_val - continue - mm_payload[mm_key] = to_payload_element( - element=mm_val, - idx=idx, - start=start, - end=end, - pass_lists_through=False, - seq_len=hidden_seq_len, - scheduled_seq_len=scheduled_seq_len, - ) - return mm_payload - def _build_omni_pooler_payload( self, *, @@ -967,7 +857,7 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin, Duplex if req_hidden_states is not None: payload["hidden"] = req_hidden_states - mm_payload = self._build_omni_mm_payload( + mm_payload = build_omni_mm_payload( combined_multimodal_outputs=combined_multimodal_outputs, mm_cpu=mm_cpu, rid=rid, @@ -982,6 +872,47 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin, Duplex payload.update(mm_payload) return payload + def _merge_model_kv_transfer_metadata( + self, + finished_reqs: dict[str, dict[str, Any]], + ) -> dict[str, dict[str, Any]]: + """Merge model-provided KV-transfer metadata copy-on-write. + + The outer mapping is always new; a per-request dict is shallow-copied + (with a fresh merged ``custom_metadata``) only when the model supplies + metadata for it, and passes through unchanged otherwise. + `scheduler_output` (and its per-request dicts) can be shared with the + engine-core process, so nothing here mutates them — the ngram block + in `execute_model` `replace()`-copies for exactly the same reason. The + merged mapping is only consumed by + `kv_transfer_manager.handle_finished_requests_kv_transfer`. + """ + merged: dict[str, dict[str, Any]] = {} + for req_id, data in finished_reqs.items(): + try: + # NOTE: seq_len is the same as num_computed_tokens_cpu in current + # async scheduling, since both exclude async placeholders. We use + # seq_len since we control it, just in case upstream async scheduler + # semantics change in the future. + model_meta = self.model.get_kv_transfer_metadata( + req_id, + num_computed_tokens=data.get("seq_len"), + ) + except Exception: + # A swallowed failure here used to ship the KV transfer WITHOUT + # the model's custom metadata and surface much later as a + # corrupt/incomplete transfer on the consumer stage. Fail at + # the point of the defect instead. + logger.exception("Failed to get custom KV-transfer metadata from model for %s", req_id) + raise + if model_meta: + data = { + **data, + "custom_metadata": {**(data.get("custom_metadata") or {}), **model_meta}, + } + merged[req_id] = data + return merged + @torch.inference_mode() def execute_model( self, @@ -1012,24 +943,7 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin, Duplex # [Omni] Handle KV transfer BEFORE updating states (which removes finished requests) finished_reqs = getattr(scheduler_output, "finished_requests_needing_kv_transfer", {}) if finished_reqs and hasattr(self.model, "get_kv_transfer_metadata"): - for req_id, data in finished_reqs.items(): - try: - # NOTE: seq_len is the same as num_computed_tokens_cpu in current - # async scheduling, since both exclude async placeholders. We use - # seq_len since we control it, just in case upstream async scheduler - # semantics change in the future. - num_computed = data.get("seq_len") - - model_meta = self.model.get_kv_transfer_metadata( - req_id, - num_computed_tokens=num_computed, - ) - if model_meta: - existing = data.get("custom_metadata") or {} - existing.update(model_meta) - data["custom_metadata"] = existing - except Exception as e: - logger.warning(f"Failed to get custom metadata from model for {req_id}: {e}") + finished_reqs = self._merge_model_kv_transfer_metadata(finished_reqs) self.kv_extracted_req_ids = self.kv_transfer_manager.handle_finished_requests_kv_transfer( finished_reqs=finished_reqs, kv_caches=self.kv_caches, @@ -1467,6 +1381,16 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin, Duplex logits: torch.Tensor | None, spec_decode_metadata: Any, ): + """Sample tokens, preferring the model's own sampler when declared. + + Model-interface contract for `prefer_model_sampler` models: a custom + `model.sample(logits, sampling_metadata)` may return ``None`` to + DECLINE the step, and the runner then falls back to the default + sampler. Declarers rely on this deliberately for empty-logits or + inapplicable-stage steps (e.g. cosyvoice3, glm_tts), so the + fallthrough is load-bearing, not an error path; it is logged once per + model for visibility. + """ sampling_metadata = self.input_batch.sampling_metadata if spec_decode_metadata is None: model_sample = getattr(self.model, "sample", None) @@ -1487,6 +1411,14 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin, Duplex sampler_output = model_sample(logits, prepared_sampling_metadata) if sampler_output is not None: return sampler_output + # Contract: None => fall back to the default sampler (see + # docstring). If a custom sampler returns None by accident, + # its tokens silently come from the default sampler - the + # one-time log is the visibility hook for that case. + logger.warning_once( + "prefer_model_sampler model %s returned None from sample(); falling back to the default sampler.", + type(self.model).__name__, + ) return self.sampler( logits=logits, sampling_metadata=sampling_metadata, @@ -1889,7 +1821,7 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin, Duplex combined_multimodal_outputs = None engine_output_type, downstream_req_ids = self._resolve_pooler_payload_req_ids(req_ids_output_copy) - downstream_req_ids, sparse_mm_index, audio_sparse_output = self._resolve_sparse_mm_routing( + downstream_req_ids, sparse_mm_index, audio_sparse_output = resolve_sparse_mm_routing( engine_output_type=engine_output_type, req_ids_output_copy=req_ids_output_copy, downstream_req_ids=downstream_req_ids, @@ -2089,13 +2021,13 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin, Duplex if grammar_output is not None: apply_grammar_bitmask(scheduler_output, grammar_output, self.input_batch, logits) - # Correct padding values of prompt_token_ids to match the logits vocabulary size + # Correct padding values of prompt_token_ids to match the logits vocabulary size. if logits is not None and not self.input_batch.sampling_metadata.no_penalties: smd = self.input_batch.sampling_metadata if smd.prompt_token_ids is not None: logits_vocab = logits.shape[-1] if self.input_batch.vocab_size > logits_vocab: - smd.prompt_token_ids = smd.prompt_token_ids.clamp(max=logits_vocab) + smd.prompt_token_ids = clamp_prompt_ids_to_penalty_padding(smd.prompt_token_ids, logits_vocab) # Drop min-tokens stop ids the head cannot emit (e.g. the text # tokenizer EOS folded into all_stop_token_ids on a narrow codec diff --git a/vllm_omni/worker/gpu_generation_model_runner.py b/vllm_omni/worker/gpu_generation_model_runner.py index 7cf26d005..a397c2e9d 100644 --- a/vllm_omni/worker/gpu_generation_model_runner.py +++ b/vllm_omni/worker/gpu_generation_model_runner.py @@ -167,10 +167,6 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"): self.model.on_requests_finished(scheduler_output.finished_req_ids) - # `<= 0`: upstream can schedule a negative span, which is truthy (#5196). - if scheduler_output.total_num_scheduled_tokens <= 0: - return self.attach_omni_connector_output(EMPTY_MODEL_RUNNER_OUTPUT) - if has_ec_transfer() and not get_ec_transfer().is_consumer: with self.maybe_get_ec_connector_output( scheduler_output, @@ -179,7 +175,10 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin self._execute_mm_encoder(scheduler_output) return self.attach_omni_connector_output(make_empty_encoder_model_runner_output(scheduler_output)) - if not num_scheduled_tokens: + # OMNI: keep this block in lock-step with the same block in + # GPUARModelRunner.execute_model. + # `<= 0`: upstream can schedule a negative span, which is truthy (#5196). + if num_scheduled_tokens <= 0: if ( self.parallel_config.distributed_executor_backend == "external_launcher" and self.parallel_config.data_parallel_size > 1 @@ -439,16 +438,29 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin # pooler_output is no longer used for multimodal data. per_req_payloads: list[dict[str, object]] = [] if isinstance(multimodal_outputs_raw, torch.Tensor): - assert multimodal_outputs_raw.shape[0] == 1, ( - "model should return a single tensor, to return multiple tensors, use a dict" - ) - assert multimodal_outputs_raw.shape[0] == self.input_batch.num_reqs - for i in range(self.input_batch.num_reqs): + # One row per request. The old asserts (`shape[0] == 1` AND + # `shape[0] == num_reqs`) jointly forced num_reqs == 1, silently + # rejecting batched steps the generation scheduler does admit. + num_reqs = self.input_batch.num_reqs + if multimodal_outputs_raw.shape[0] != num_reqs: + raise ValueError( + f"Multimodal output tensor has leading dim {multimodal_outputs_raw.shape[0]} " + f"but the batch has {num_reqs} requests (one row per request; " + "to return multiple tensors per request, use a dict)." + ) + for i in range(num_reqs): per_req_payloads.append({"model_outputs": multimodal_outputs_raw[i].detach().to("cpu").contiguous()}) elif isinstance(multimodal_outputs_raw, list): - assert len(multimodal_outputs_raw) == 1, ( - "model should return a single list, to return multiple lists, use a dict" - ) + # One entry per request. The old `len == 1` assert did not check + # num_reqs, so a batched step built a length-1 payload list that + # misaligned with `req_ids` downstream. + num_reqs = self.input_batch.num_reqs + if len(multimodal_outputs_raw) != num_reqs: + raise ValueError( + f"Multimodal output list has length {len(multimodal_outputs_raw)} " + f"but the batch has {num_reqs} requests (one entry per request; " + "to return multiple lists per request, use a dict)." + ) for out in multimodal_outputs_raw: per_req_payloads.append( {"model_outputs": out.detach().to("cpu").contiguous() if out is not None else None} @@ -794,6 +806,11 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin input_ids = None inputs_embeds = self.inputs_embeds.gpu[:num_tokens_padded] model_kwargs = self._init_model_kwargs() + elif getattr(getattr(self, "model", None), "has_preprocess", False): + # Capture CUDA graph with inputs_embeds path so replay reads + # from the same buffer that _preprocess writes into. + input_ids = self.input_ids.gpu[:num_tokens_padded] + inputs_embeds = self.inputs_embeds.gpu[:num_tokens_padded] else: input_ids = self.input_ids.gpu[:num_tokens_padded] inputs_embeds = None diff --git a/vllm_omni/worker/output/__init__.py b/vllm_omni/worker/output/__init__.py new file mode 100644 index 000000000..33ea418a3 --- /dev/null +++ b/vllm_omni/worker/output/__init__.py @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Worker output helpers. + +Package seeded ahead of the async-output refactor (RFC #5450 series, G5-C2), +which relocates the remaining payload-copy helpers, snapshot types, and +`ExecuteModelState` here. Currently hosts the per-request multimodal payload +builders. +""" + +from vllm_omni.worker.output.payload_build import ( + build_combined_prefix_cache_mm_payload, + build_omni_mm_payload, + unwrap_combined_payload_value, +) + +__all__ = [ + "build_combined_prefix_cache_mm_payload", + "build_omni_mm_payload", + "unwrap_combined_payload_value", +] diff --git a/vllm_omni/worker/output/payload_build.py b/vllm_omni/worker/output/payload_build.py new file mode 100644 index 000000000..0f65ac293 --- /dev/null +++ b/vllm_omni/worker/output/payload_build.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Per-request multimodal payload builders. + +State-free extraction helpers that turn a step's `multimodal_outputs` +(combined prefix-cache-merged form, or the direct `mm_cpu` form) into one +request's payload dict. Sparse-routing context (`audio_sparse_output`, +`sparse_mm_index`) is resolved upstream by +`vllm_omni.worker.sparse_audio.resolve_sparse_mm_routing` and arrives here +as plain arguments. +""" + +from typing import Any + +import torch +from vllm.logger import init_logger + +from vllm_omni.utils.mm_outputs import to_payload_element + +logger = init_logger(__name__) + +_SPARSE_AUDIO_PROTOCOL_KEYS = frozenset({"meta.req_id", "meta.sparse_audio"}) + + +def unwrap_combined_payload_value( + value: Any, + *, + rid: str, + list_idx: int, + mm_key: str, +) -> tuple[bool, Any]: + """Unwrap one value from the prefix-cache-merged payload. + + Returns ``(keep, unwrapped)``: ``keep`` is False when a non-singleton + per-request list is misaligned and must be dropped. A singleton list is + request-invariant passthrough data and is shared across the batch. + """ + if isinstance(value, list): + if len(value) == 1: + # Prefix-cache passthrough intentionally preserves list + # containers and copies the whole list under every request id. + # Qwen3-Omni uses this shape for its request-invariant TTS + # BOS/EOS/PAD embeddings, so every batch index unwraps the same + # singleton. Sparse one-request lists are safe here too: routing + # limits them to the declared request at sparse index zero. + return True, value[0] + if list_idx < len(value): + return True, value[list_idx] + # A non-singleton list shorter than the request's index is per-request + # data that lost alignment. Never substitute another request's + # element (`v[0]` used to ship request 0's payload as `rid`'s); drop + # the key loudly instead — the SAME protocol-break policy as the + # sparse mismatch in + # `build_omni_mm_payload`. Not a raise: v1 runners do not catch + # per-request exceptions, so raising would turn a per-request + # protocol break into a whole-stage outage. + logger.error( + "Combined prefix-cache mm payload misaligned for request %s: index %d out of " + "range for per-request list of length %d; dropping key %s.", + rid, + list_idx, + len(value), + mm_key, + ) + return False, None + if isinstance(value, dict): + nested = {} + for k, sv in value.items(): + keep, unwrapped = unwrap_combined_payload_value(sv, rid=rid, list_idx=list_idx, mm_key=mm_key) + if keep: + nested[k] = unwrapped + return True, nested + return True, value + + +def build_combined_prefix_cache_mm_payload( + combined_multimodal_outputs: dict, + *, + rid: str, + list_idx: int, +) -> dict[str, object]: + """Unwrap one request's entry from the prefix-cache-merged payload. + + Sparse-agnostic by design: ``list_idx`` arrives pre-resolved by the + caller (`build_omni_mm_payload`, which owns the sparse-routing context) + — batch index normally, sparse index under sparse routing. + """ + payload: dict[str, object] = {} + for mm_key in combined_multimodal_outputs.keys(): + if mm_key in _SPARSE_AUDIO_PROTOCOL_KEYS: + continue + keep, unwrapped = unwrap_combined_payload_value( + combined_multimodal_outputs[mm_key][rid], + rid=rid, + list_idx=list_idx, + mm_key=mm_key, + ) + if keep: + payload[mm_key] = unwrapped + return payload + + +def build_omni_mm_payload( + *, + combined_multimodal_outputs: dict | None, + mm_cpu: dict[str, object] | None, + rid: str, + idx: int, + start: int, + end: int, + audio_sparse_output: bool, + sparse_mm_index: dict[str, int], + hidden_seq_len: int, + scheduled_seq_len: int, +) -> dict[str, object]: + """Build one request's multimodal payload for the step.""" + if combined_multimodal_outputs: + # Sparse-audio producers emit per-request lists aligned to the + # sparse request order (`meta.req_id`), not the batch order; under + # sparse routing the request's sparse index selects its element. + list_idx = sparse_mm_index.get(rid, idx) if audio_sparse_output else idx + return build_combined_prefix_cache_mm_payload( + combined_multimodal_outputs, + rid=rid, + list_idx=list_idx, + ) + + mm_payload: dict[str, object] = {} + if not mm_cpu: + return mm_payload + + for mm_key, mm_val in mm_cpu.items(): + if mm_key in _SPARSE_AUDIO_PROTOCOL_KEYS: + continue + if audio_sparse_output and isinstance(mm_val, list): + sparse_idx = sparse_mm_index.get(rid) + if sparse_idx is None: + continue + if sparse_idx >= len(mm_val): + # Misalignment means the sparse-marker protocol broke; the + # request's output for this key is dropped, so be loud. + logger.error( + "Sparse multimodal payload mismatch for request %s: index %d >= %d; dropping key %s.", + rid, + sparse_idx, + len(mm_val), + mm_key, + ) + continue + sparse_val = mm_val[sparse_idx] + mm_payload[mm_key] = sparse_val.clone() if isinstance(sparse_val, torch.Tensor) else sparse_val + continue + mm_payload[mm_key] = to_payload_element( + element=mm_val, + idx=idx, + start=start, + end=end, + pass_lists_through=False, + seq_len=hidden_seq_len, + scheduled_seq_len=scheduled_seq_len, + ) + return mm_payload diff --git a/vllm_omni/worker/sampling_utils.py b/vllm_omni/worker/sampling_utils.py index c38a9bbdc..845881138 100644 --- a/vllm_omni/worker/sampling_utils.py +++ b/vllm_omni/worker/sampling_utils.py @@ -8,7 +8,21 @@ from vllm.v1.sample.logits_processor import LogitsProcessors, MinTokensLogitsPro logger = init_logger(__name__) -__all__ = ["sanitize_min_tokens_stop_ids"] +__all__ = ["clamp_prompt_ids_to_penalty_padding", "sanitize_min_tokens_stop_ids"] + + +def clamp_prompt_ids_to_penalty_padding(prompt_token_ids: torch.Tensor, logits_vocab: int) -> torch.Tensor: + """Clamp batch-level pad ids down to ``logits_vocab`` — upstream's + designed penalty padding value. + + ``max=logits_vocab`` (NOT ``logits_vocab - 1``) is deliberate: upstream + penalty computation allocates ``vocab_size + 1`` bins and drops the last + column, so ``vocab_size`` is the padding value that never affects + penalties (vllm/model_executor/layers/utils.py:: + get_token_bin_counts_and_mask). Clamping one lower would count padding + as real occurrences of the last vocab token. + """ + return prompt_token_ids.clamp(max=logits_vocab) def sanitize_min_tokens_stop_ids(logitsprocs: LogitsProcessors, logits_vocab: int) -> None: diff --git a/vllm_omni/worker/sparse_audio.py b/vllm_omni/worker/sparse_audio.py new file mode 100644 index 000000000..f82c7a59e --- /dev/null +++ b/vllm_omni/worker/sparse_audio.py @@ -0,0 +1,232 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Sparse-audio output protocol: marker classification and payload routing. + +A step's per-request `multimodal_outputs` lists may cover only a SUBSET of +the batch (requests that produced audio this step). The producer declares +this with two metadata keys (nested ``meta`` dict or flattened ``meta.*``): + +* ``meta.req_id`` — the request ids that DO have output, in list order; +* ``meta.sparse_audio`` — the marker saying "per-request lists are aligned + to ``meta.req_id`` order, not batch order". + +The marker's carrier form is ENFORCED, not normalized. The design contract +is a list of strings (e.g. ``["1"]`` — what every in-tree producer emits); a +bare string is also accepted. Interpreting deviant carriers would need a +second truth predicate, and predicate divergence is exactly how +``np.array(["0"])`` came to route as sparse while ``["0"]`` routed as dense; +per the refactor series' Policy A the runner<->model interface owes no +tolerance shim. + +A present-but-invalid declaration (illegal marker carrier, or marker set +with an unusable ``meta.req_id``) FAILS CLOSED: it most likely belongs to a +producer that INTENDED sparse output, and falling back to dense would +batch-index-assign its payloads to the WRONG requests. `InvalidSparseDeclarationError` +is raised by the classifiers and caught INSIDE this module by +`resolve_sparse_mm_routing`, which routes no payload for the affected step — +the exception never crosses the module boundary, so it can never escape a +v1 runner (which does not catch per-request exceptions). + +This module is the single home of the contract until the model-state +refactor series (G9-C2.6) turns the marker into a typed bool resolved at +one adapter site and unifies the nested/flattened encodings. The NPU runner +still carries a tolerant pre-enforcement copy — NPU-track handoff item. +""" + +import threading +import time +from typing import Any + +from vllm.logger import init_logger + +logger = init_logger(__name__) + +# The DESIGNED marker carrier is a list of strings (bare str also accepted); +# truthiness of a string item: +_SPARSE_MARKER_TRUTHY = ("1", "true", "yes", "on") + +# Per-process dedup so a broken producer logs once per offending shape, not +# once per step: +_logged_offenders: set[str] = set() +_error_state_lock = threading.Lock() + +# A persistent invalid producer must remain observable without flooding the +# per-step output path. Classifier errors above stay deduplicated per carrier +# type; the routing consequence is reported at most once per interval, with a +# count of repeats suppressed since the previous report. +_FAIL_CLOSED_LOG_INTERVAL_S = 60.0 +_fail_closed_last_log_at: float | None = None +_fail_closed_suppressed = 0 + + +class InvalidSparseDeclarationError(ValueError): + """Sparse-audio metadata is PRESENT but out of contract. + + Internal control signal: raised by `is_sparse_audio_marker` / + `resolve_sparse_mm_req_ids` and caught by `resolve_sparse_mm_routing` + in this same module, which fails closed. Callers outside this module + only ever see the routing result, never the exception. + """ + + +def _error_once(key: str, msg: str, *args: Any) -> None: + with _error_state_lock: + if key in _logged_offenders: + return + _logged_offenders.add(key) + logger.error(msg, *args) + + +def _error_fail_closed_rate_limited(cause: str) -> None: + global _fail_closed_last_log_at, _fail_closed_suppressed + + now = time.monotonic() + with _error_state_lock: + if _fail_closed_last_log_at is not None and now - _fail_closed_last_log_at < _FAIL_CLOSED_LOG_INTERVAL_S: + _fail_closed_suppressed += 1 + return + suppressed = _fail_closed_suppressed + _fail_closed_last_log_at = now + _fail_closed_suppressed = 0 + + logger.error( + "Invalid sparse-audio declaration: failing closed - no multimodal payload is routed " + "for the affected step(s). Fix the emitting model. Suppressed repeats before this " + "report: %d. Cause: %s", + suppressed, + cause, + ) + + +def is_sparse_audio_marker(value: Any) -> bool: + """Classify a ``meta.sparse_audio`` marker. + + Legal carriers (list of strings, bare string) evaluate against the + literal truthy set; ``None`` means "marker absent" and returns ``False``. + An ILLEGAL carrier (tensors, arrays, numbers, lists with non-string + items) is a producer bug: logged at error level once per offending type, + then raised as `InvalidSparseDeclarationError`. Deviant carriers are never + read element-wise, so a device tensor can never introduce a D2H sync + here. + """ + if value is None: + return False + if isinstance(value, str): + return value.lower() in _SPARSE_MARKER_TRUTHY + if isinstance(value, list) and all(isinstance(item, str) for item in value): + return any(item.lower() in _SPARSE_MARKER_TRUTHY for item in value) + vtype = type(value).__name__ + _error_once( + vtype, + 'Illegal sparse-audio marker of type %s: the marker MUST be a list of strings (e.g. ["1"]). ' + "The emitting model is out of contract; routing fails closed for the affected step(s).", + vtype, + ) + raise InvalidSparseDeclarationError(f"illegal sparse-audio marker carrier: {vtype}") + + +def resolve_sparse_mm_req_ids(multimodal_outputs: Any) -> list[str] | None: + """Resolve the sparse request-id list. + + Returns the request-id list when a legal sparse declaration is present, + and ``None`` when there is no sparse declaration (dense). Raises + `InvalidSparseDeclarationError` when the declaration is present but out of + contract (illegal marker carrier, or marker set with an unusable + ``meta.req_id``). + """ + if not isinstance(multimodal_outputs, dict): + return None + meta = multimodal_outputs.get("meta") + nested_req_ids = meta.get("req_id") if isinstance(meta, dict) else None + flat_req_ids = multimodal_outputs.get("meta.req_id") + + # Resolve both marker encodings independently, even when the request ids + # came from the other encoding. Previously a nested `req_id` prevented the + # flattened marker from being read, while the mirror combination worked. + # Evaluating both also preserves the strict contract: an illegal carrier + # on either side wins over an otherwise-valid declaration and fails closed. + nested_marker_value = meta.get("sparse_audio") if isinstance(meta, dict) else None + flat_marker_value = multimodal_outputs.get("meta.sparse_audio") + nested_marker = is_sparse_audio_marker(nested_marker_value) + flat_marker = is_sparse_audio_marker(flat_marker_value) + if nested_marker_value is not None and flat_marker_value is not None and nested_marker != flat_marker: + _error_once( + "marker:conflict", + "Nested and flattened sparse-audio markers disagree. Failing closed.", + ) + raise InvalidSparseDeclarationError("nested and flattened sparse markers disagree") + sparse_audio = nested_marker or flat_marker + if not sparse_audio: + return None + + req_id_carriers = [ + (name, value) for name, value in (("nested", nested_req_ids), ("flattened", flat_req_ids)) if value is not None + ] + if not req_id_carriers: + req_id_carriers = [("missing", None)] + + validated_req_ids: list[list[str]] = [] + for carrier_name, req_ids in req_id_carriers: + if isinstance(req_ids, list) and all(isinstance(rid, str) for rid in req_ids): + validated_req_ids.append(req_ids) + continue + rtype = type(req_ids).__name__ + _error_once( + f"req_id:{carrier_name}:{rtype}", + "Sparse-audio marker is set but the %s meta.req_id carrier is unusable (%s; " + "must be a list of request-id strings). Failing closed.", + carrier_name, + rtype, + ) + raise InvalidSparseDeclarationError(f"sparse marker set but {carrier_name} meta.req_id is unusable: {rtype}") + + req_ids = validated_req_ids[0] + if len(validated_req_ids) == 2 and validated_req_ids[1] != req_ids: + _error_once( + "req_id:conflict", + "Nested and flattened sparse-audio request-id lists disagree. Failing closed.", + ) + raise InvalidSparseDeclarationError("nested and flattened sparse request-id lists disagree") + if len(req_ids) != len(set(req_ids)): + _error_once( + "req_id:duplicate", + "Sparse-audio request-id list contains duplicates. Failing closed.", + ) + raise InvalidSparseDeclarationError("sparse request-id list contains duplicates") + return list(req_ids) + + +def resolve_sparse_mm_routing( + *, + engine_output_type: str, + req_ids_output_copy: list[str], + downstream_req_ids: list[str], + multimodal_outputs: Any, +) -> tuple[list[str], dict[str, int], bool]: + """Resolve `(downstream_req_ids, sparse_mm_index, audio_sparse_output)`. + + The only entry point runners use; all protocol errors are absorbed here. + """ + try: + sparse_mm_req_ids = resolve_sparse_mm_req_ids(multimodal_outputs) + except InvalidSparseDeclarationError as exc: + if engine_output_type != "audio": + # Non-audio pipelines never consult sparse routing; the carrier + # error is already logged by the classifier. + return downstream_req_ids, {}, False + # FAIL CLOSED: a present-but-invalid sparse declaration most likely + # belongs to a producer that INTENDED sparse output; routing it as + # dense would batch-index-assign its payloads to the WRONG requests. + # Route no payload for this step instead - a visible, bounded gap + # rather than silent cross-request corruption. (Empty-sparse is an + # already-exercised path: the zero-audio sparse step ships + # req_id=[] through the same shape.) + _error_fail_closed_rate_limited(str(exc)) + return [], {}, True + sparse_mm_index = {rid: i for i, rid in enumerate(sparse_mm_req_ids or [])} + if engine_output_type != "audio" or sparse_mm_req_ids is None: + return downstream_req_ids, sparse_mm_index, False + + sparse_req_id_set = set(sparse_mm_req_ids) + sparse_downstream_req_ids = [rid for rid in req_ids_output_copy if rid in sparse_req_id_set] + return sparse_downstream_req_ids, sparse_mm_index, True From 273ffce130ec648fb0b600d7a6895cfb9c91bd52 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Thu, 20 Aug 2026 21:07:04 +0800 Subject: [PATCH 087/267] [CI/Build] Strengthen pre-commit with markdownlint, SPDX, and policy hooks (#6273) Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: [Your Name] Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com> --- .claude/skills/add-diffusion-model/SKILL.md | 18 + .claude/skills/add-tts-model/SKILL.md | 2 +- .../add-tts-model/references/precommit-dco.md | 12 + .claude/skills/precheck-pr/SKILL.md | 2 + .../precheck-pr/references/checklists.md | 1 + .claude/skills/review-pr/SKILL.md | 5 +- .../references/process/review-execution.md | 11 + .claude/skills/vllm-omni-test/SKILL.md | 1 + .github/mergify.yml | 2 +- .github/workflows/pre-commit.yml | 8 +- .gitignore | 3 + .markdownlint.yaml | 17 + .pre-commit-config.yaml | 100 ++++- CONTRIBUTING.md | 8 +- docs/contributing/README.md | 145 +++++++ docs/contributing/ci/failures.md | 9 +- docs/contributing/ci/test_writing_guide.md | 300 ++++++-------- docs/contributing/model/adding_tts_model.md | 35 +- pyproject.toml | 23 +- tests/diffusion/test_diffusion_ipc.py | 50 +-- tools/pre_commit/check_buildkite.py | 118 ++++++ tools/pre_commit/check_forbidden_imports.py | 368 ++++++++++++++++++ tools/pre_commit/check_pickle_imports.py | 87 ----- tools/pre_commit/check_spdx_header.py | 206 ++++++++++ tools/pre_commit/check_torch_cuda.py | 109 ++++++ tools/pre_commit/mypy.py | 97 +++++ tools/pre_commit/shellcheck.sh | 79 ++++ .../attention/ops/minimax_h3_modulation.py | 4 +- .../models/indextts2/gpt/conformer_encoder.py | 4 +- .../models/minimax_music3/prompt.py | 5 +- 30 files changed, 1497 insertions(+), 332 deletions(-) create mode 100644 .markdownlint.yaml create mode 100644 tools/pre_commit/check_buildkite.py create mode 100644 tools/pre_commit/check_forbidden_imports.py delete mode 100644 tools/pre_commit/check_pickle_imports.py create mode 100644 tools/pre_commit/check_spdx_header.py create mode 100644 tools/pre_commit/check_torch_cuda.py create mode 100644 tools/pre_commit/mypy.py create mode 100755 tools/pre_commit/shellcheck.sh diff --git a/.claude/skills/add-diffusion-model/SKILL.md b/.claude/skills/add-diffusion-model/SKILL.md index bae8cd1b5..111f8e9bf 100644 --- a/.claude/skills/add-diffusion-model/SKILL.md +++ b/.claude/skills/add-diffusion-model/SKILL.md @@ -610,6 +610,24 @@ See the [Profiling Single-Stage Diffusion](../../../docs/contributing/profiling. --- +## Pre-commit conventions + +New library files must pass the local gates in +[docs/contributing/README.md](../../../docs/contributing/README.md#linting). +That page is the full hook list (SPDX, forbidden imports including Hugging Face +Hub / Triton / pickle, `torch.cuda`, mypy, test marks, markdownlint, Buildkite, +shellcheck). In particular: + +- SPDX copyright is `vLLM-Omni project` (stale `vLLM project` is rewritten). +- Use `import regex as re` and `pybase64` in `vllm_omni/`; do not import stdlib + `re` or `base64`. Hugging Face Hub downloads go through + `vllm.transformers_utils.repo_utils`. +- Do not add `torch.cuda.*` call sites; use `current_omni_platform`. +- New `tests/**/test_*.py` files need a CI level mark and a hardware mark. +- Do not expand `CHECK_IMPORTS[*].allowed_files` or `ALLOWED_FILES` without review. +- GitHub Actions skips SPDX/shellcheck/mypy-3.10/test-marks/markdownlint; run + `pre-commit` locally. + ## Iterative Development Tips 1. **Start minimal**: Basic generation first, no parallelism/caching diff --git a/.claude/skills/add-tts-model/SKILL.md b/.claude/skills/add-tts-model/SKILL.md index 494d6a52b..034ac7cd4 100644 --- a/.claude/skills/add-tts-model/SKILL.md +++ b/.claude/skills/add-tts-model/SKILL.md @@ -563,7 +563,7 @@ Use this checklist when integrating a new TTS model: - [ ] `pre-commit run --files ` passes before every push - [ ] Every commit has `Signed-off-by` matching the author email (`git commit -s`) - [ ] `git config user.email` matches the email registered on your GitHub account -- [ ] Details and failure-recovery commands: [references/precommit-dco.md](references/precommit-dco.md) +- [ ] Hook list, failure table, and DCO recovery: [references/precommit-dco.md](references/precommit-dco.md) ## References diff --git a/.claude/skills/add-tts-model/references/precommit-dco.md b/.claude/skills/add-tts-model/references/precommit-dco.md index 86a1f42ce..160ce71b0 100644 --- a/.claude/skills/add-tts-model/references/precommit-dco.md +++ b/.claude/skills/add-tts-model/references/precommit-dco.md @@ -30,6 +30,18 @@ but the changes are correct — stage the modified files and re-commit. | `ruff F841` | Variable extracted but never forwarded to model call | Remove the extraction or wire it through | | `ruff E402` | Import added below function definitions | Move to top-level import block | | `ruff format` | Line length, spacing, quote style | Accept auto-fix, stage, re-commit | +| `check-spdx-header` | Missing header, or copyright still says `vLLM project` | Two-line Omni header (`vLLM-Omni project`); restage the rewrite | +| `check-forbidden-imports` | Stdlib `re`/`base64`, pickle, Hugging Face Hub API, or direct Triton/TileLang | `import regex as re` and `pybase64`; use `vllm.transformers_utils.repo_utils`. Do **not** add the file to `allowed_files` without review | +| `check-torch-cuda-call` | New `torch.cuda.*` call site | Use `current_omni_platform` / `OmniPlatform`; do not grow `ALLOWED_FILES` without review | +| `check-tts-adapter-migration` | New `self._tts_model_type` branch in `serving_speech.py` | Put per-model logic in `tts_adapters/`. Lower `MAX_MODEL_TYPE_BRANCHES` when removing branches; do not raise it without review | +| `check-test-ci-coverage` | New `tests/**/test_*.py` missing level or hardware mark | Add `core_model`/`advanced_model`/… plus `cpu`/`cuda`/`hardware_test(` | +| `mypy-3.10` | Type error on changed `vllm_omni/` files | Fix the types; model trees are excluded. Extra versions: `pre-commit run --hook-stage manual mypy-3.12` | +| `markdownlint-cli2` | Docs/README markdown lint | Accept auto-fix under `docs/` / `recipes/` / root README | +| `check-buildkite` | Invalid `.buildkite/*.yml` | Fix the YAML; do not grow `SKIP_FILES` without review | +| `shellcheck` | No `shellcheck` on PATH, or script warning | `apt-get`/`dnf`/`brew install shellcheck`, or `shellcheck.exe` on Windows PATH | + +Canonical policy, CI `SKIP` list, and allowlist locations: +[docs/contributing/README.md](../../../docs/contributing/README.md#linting). ## DCO sign-off diff --git a/.claude/skills/precheck-pr/SKILL.md b/.claude/skills/precheck-pr/SKILL.md index 4c8086029..ea3a4ed34 100644 --- a/.claude/skills/precheck-pr/SKILL.md +++ b/.claude/skills/precheck-pr/SKILL.md @@ -67,6 +67,8 @@ Ask: "Quick mode or full mode?" Then walk the checklist for the detected PR type **Also run the Examples-Policy check on every PR** using [references/examples-policy.md](references/examples-policy.md). Inspect only Python paths introduced by the diff. Treat a new model-, checkpoint-, vendor-, or family-specific Python example as blocking; do not report pre-existing example debt. +**Also confirm local pre-commit gates** from [docs/contributing/README.md](../../../docs/contributing/README.md#linting). GitHub Actions `SKIP` does not prove they passed. Full new-hook list: Omni SPDX (`vLLM-Omni project`); forbidden imports (stdlib `re`/`base64`, pickle, Hugging Face Hub API, Triton/TileLang); no new `torch.cuda` call sites; shellcheck (macOS/Windows must install the binary); mypy-3.10; test files have CI level + hardware marks; TTS `_tts_model_type` branches in `serving_speech.py` must not increase; markdownlint on `docs/`; Buildkite YAML schema. ✗ for growing `allowed_files` / `ALLOWED_FILES` or raising `MAX_MODEL_TYPE_BRANCHES` without review. + ### Step 5: Print Report ``` diff --git a/.claude/skills/precheck-pr/references/checklists.md b/.claude/skills/precheck-pr/references/checklists.md index b65a66713..dc7f85ea9 100644 --- a/.claude/skills/precheck-pr/references/checklists.md +++ b/.claude/skills/precheck-pr/references/checklists.md @@ -164,6 +164,7 @@ Diffusion models live under `vllm_omni/diffusion/`. In addition to the New Model - [ ] **PR body matches diff:** description doesn't claim changes not in the diff - [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts - [ ] **CI gates passing:** DCO, pre-commit, build — check `gh pr view --json statusCheckRollup` if a PR already exists +- [ ] **Local pre-commit gates:** full list in [Linting](../../../../docs/contributing/README.md#linting). Blocking when the diff introduces: stale `vLLM project` SPDX; stdlib `re`/`base64` or pickle in library code; new `torch.cuda` call sites; `tests/**/test_*.py` without level+hardware marks; new `_tts_model_type` branches in `serving_speech.py`; invalid `.buildkite` YAML. macOS/Windows need a native `shellcheck`. ✗ for growing `CHECK_IMPORTS[*].allowed_files`, `ALLOWED_FILES`, or `MAX_MODEL_TYPE_BRANCHES` without review. ### Full (adds) diff --git a/.claude/skills/review-pr/SKILL.md b/.claude/skills/review-pr/SKILL.md index 165fa1d5e..63046190d 100644 --- a/.claude/skills/review-pr/SKILL.md +++ b/.claude/skills/review-pr/SKILL.md @@ -21,7 +21,10 @@ Make every finding: - **Calibrated:** match severity to user and maintainer impact. Do not report unrelated backlog, style already enforced by pre-commit, or a -missing test that would not protect changed behavior. +missing test that would not protect changed behavior. Do report new allowlist +or budget entries in `check_forbidden_imports.py` / `check_torch_cuda.py` / +`check_tts_adapter.py` / `check_buildkite.py` unless the PR justifies them: +those are policy changes, not lint noise. ## Select the input and depth diff --git a/.claude/skills/review-pr/references/process/review-execution.md b/.claude/skills/review-pr/references/process/review-execution.md index f06f27933..7c0305254 100644 --- a/.claude/skills/review-pr/references/process/review-execution.md +++ b/.claude/skills/review-pr/references/process/review-execution.md @@ -111,6 +111,17 @@ Mark early findings as preliminary and continue. This is not a GitHub comment. but do not publish review events without separate authorization. - Record DCO, pre-commit, required CI, and mergeability. Pending or unknown gates do not block source review. +- GitHub Actions `SKIP` omits SPDX, shellcheck, markdownlint, mypy-3.10, and + test-mark coverage. A green GHA pre-commit job does not prove those local + gates passed. New files still need the full Linting list: Omni SPDX + (`vLLM-Omni project`), no stdlib `re`/`base64` in `vllm_omni/`, no new + pickle / Hugging Face Hub API / `torch.cuda` call sites, test marks, TTS + adapter ratchet, Buildkite schema, and native shellcheck on macOS/Windows. + See [Linting](https://vllm-omni.readthedocs.io/en/latest/contributing/#linting). +- Expanding `CHECK_IMPORTS[*].allowed_files`, `ALLOWED_FILES`, + `MAX_MODEL_TYPE_BRANCHES`, or Buildkite `SKIP_FILES` is a policy change. + Do not rubber-stamp allowlist/budget growth; require justification and + prefer fixing the call site. - Treat a failed gate as its own evidence. Do not restate its formatting/lint output as a new code-review finding. - Open CI logs only when the first failing step overlaps the frozen diff or diff --git a/.claude/skills/vllm-omni-test/SKILL.md b/.claude/skills/vllm-omni-test/SKILL.md index 3e4d0e489..1ccdb3324 100644 --- a/.claude/skills/vllm-omni-test/SKILL.md +++ b/.claude/skills/vllm-omni-test/SKILL.md @@ -274,6 +274,7 @@ def test_text_to_video_001(omni_server, openai_client) -> None: **3. Cross-cutting rules** +- New `.py` files need the Omni SPDX header (`Copyright contributors to the vLLM-Omni project`, not `vLLM project`). `tests/` may keep stdlib `re`/`base64`; pickle is still banned unless the file is already on the pickle allowlist. Every collected `tests/**/test_*.py` needs a CI **level** mark and a **hardware** mark/helper (`check-test-ci-coverage` runs locally; GHA skips it). Do not grow `allowed_files` to land a test. Policy: [docs/contributing/README.md](../../../docs/contributing/README.md#linting). - Reuse existing fixtures for the chosen scenario; do not mix “online client” assumptions into offline `OmniRunner` tests without a clear reason. - Avoid external network dependency in assertions unless the scenario is explicitly “online serving” or doc examples that require a model hub (then align with CI secrets/cache). - Keep **one test function = one intent** (one modality combo, one endpoint contract, or one acceleration combo). diff --git a/.github/mergify.yml b/.github/mergify.yml index 4e4c68425..fdba32b17 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -45,7 +45,7 @@ pull_request_rules: > # For mypy (substitute "3.10" with the failing version if needed) > pre-commit run --hook-stage manual mypy-3.10 > # For markdownlint - > pre-commit run --hook-stage manual markdownlint + > pre-commit run markdownlint-cli2 > ``` > diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 2bd0cc133..a08b385f3 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -24,9 +24,7 @@ jobs: python-version: "3.12" cache: 'pip' - uses: pre-commit/action@v3.0.1 - # Don't run the pytest mark check against everything in the CI - # for now since a lot of files are still missing them. This check - # is used to help contributors remember to add the marks when they - # commit. + # Don't run historical-debt hooks against the whole tree in CI yet. + # They still run locally on changed files. env: - SKIP: check-test-ci-coverage + SKIP: check-test-ci-coverage,markdownlint-cli2,shellcheck,check-spdx-header,mypy-3.10 diff --git a/.gitignore b/.gitignore index 7502df8a0..9155a85cc 100644 --- a/.gitignore +++ b/.gitignore @@ -278,3 +278,6 @@ tests/.ci_generated/ # Local Lance precision-matching dump scripts (not part of the repo) lance_compare/ + +# Linting binaries downloaded by local pre-commit hooks +shellcheck*/ diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 000000000..329597772 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,17 @@ +# Keep these rules aligned with vLLM so MkDocs Material syntax is tolerated. +MD007: + indent: 4 +MD013: false +# GitHub READMEs start with a centered logo, not an H1. +MD041: false +MD024: + siblings_only: true +MD025: + # Allow front matter title to be different from the first heading in the document. + front_matter_title: "" +MD031: + list_items: false +MD033: false +MD046: false +MD052: false +MD059: false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 535bc1452..65b4fd27b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: - id: ruff-format - repo: https://github.com/crate-ci/typos - rev: typos-dict-v0.13.13 + rev: v1.49.0 hooks: - id: typos # only for staged files @@ -38,7 +38,19 @@ repos: - id: actionlint files: ^\.github/workflows/.*\.ya?ml$ + - repo: https://github.com/DavidAnson/markdownlint-cli2 + rev: v0.23.2 + hooks: + - id: markdownlint-cli2 + language_version: lts + args: [--fix] + files: (^docs/|^recipes/|(^|/)README\.md$|^CONTRIBUTING\.md$) + # Agent skill docs and CLAUDE.md are not user-facing docs/READMEs. + exclude: (^|/)CLAUDE\.md$|^\.claude/|^\.cursor/ + # Keep `local` last so `suggestion` stays the final hook. Autofix repos + # (ruff, typos, markdownlint, trailing-whitespace, etc.) must run first; + # otherwise their "files were modified" output displaces the SKIP tip. - repo: local hooks: - id: signoff-commit @@ -54,6 +66,45 @@ repos: verbose: true stages: [commit-msg] + # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup + # is less awkward. Local wrapper groups files so follow_imports=skip on + # tests does not leak into library code. + # mypy-3.10 runs on git commit (changed files). CI SKIPs it because + # --all-files is not clean yet. Extra versions are manual-stage only + # (not in default `pre-commit run --all-files`): + # pre-commit run --hook-stage manual mypy-3.12 + - id: mypy-3.10 + name: Run mypy for Python 3.10 + entry: python tools/pre_commit/mypy.py "3.10" + <<: &mypy_common + language: python + types_or: [python, pyi] + require_serial: true + additional_dependencies: + - mypy==1.11.1 + - regex + - types-cachetools + - types-setuptools + - types-PyYAML + - types-requests + - types-torch + - pydantic + - id: mypy-3.11 + name: Run mypy for Python 3.11 + entry: python tools/pre_commit/mypy.py "3.11" + <<: *mypy_common + stages: [manual] + - id: mypy-3.12 + name: Run mypy for Python 3.12 + entry: python tools/pre_commit/mypy.py "3.12" + <<: *mypy_common + stages: [manual] + - id: mypy-3.13 + name: Run mypy for Python 3.13 + entry: python tools/pre_commit/mypy.py "3.13" + <<: *mypy_common + stages: [manual] + - id: check-test-ci-coverage name: Ensure test files have pytestmarks entry: python tools/pre_commit/check_test_marks.py @@ -71,7 +122,45 @@ repos: # pre-commit hides output from passing hooks unless the hook is verbose. verbose: true - # Keep `suggestion` last + - id: shellcheck + name: Lint shell scripts + # language: script breaks on Windows (shebang path loses slashes). + # Invoke the vLLM-style wrapper with bash so Linux CI and Win/Git Bash work. + entry: bash + args: [tools/pre_commit/shellcheck.sh] + language: system + types: [shell] + + - id: check-spdx-header + name: Check SPDX headers + entry: python tools/pre_commit/check_spdx_header.py + language: python + types_or: [python, pyi, rust, proto, shell] + files: \.(py|pyi|rs|proto|sh)$ + + - id: check-forbidden-imports + name: Check forbidden imports + entry: python tools/pre_commit/check_forbidden_imports.py + language: python + types: [python] + additional_dependencies: [regex] + + - id: check-torch-cuda-call + name: Prevent new torch.cuda API calls + entry: python tools/pre_commit/check_torch_cuda.py + language: python + types: [python] + additional_dependencies: [regex] + + - id: check-buildkite + name: Validate Buildkite Pipelines + entry: python tools/pre_commit/check_buildkite.py + language: python + files: ^\.buildkite/.*\.ya?ml$ + additional_dependencies: [pyyaml, check-jsonschema==0.38.0] + + # Keep `suggestion` as the final hook. Run after autofix hooks so the + # bypass/SKIP tip is not displaced. - id: suggestion name: Suggestion entry: bash -c 'echo "To bypass all the pre-commit hooks, add --no-verify to git commit. To skip a specific hook, prefix the commit command with SKIP=."' @@ -79,10 +168,3 @@ repos: verbose: true pass_filenames: false # Insert new entries above the `suggestion` entry - - - id: check-pickle-imports - name: Prevent new pickle/cloudpickle imports - entry: python tools/pre_commit/check_pickle_imports.py - language: python - types: [python] - additional_dependencies: [regex] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4ace89fd6..36cefcf0d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,11 @@ # Contributing to vLLM-Omni -You may find information about contributing to vLLM-Omni on [Contributing](https://vllm-omni.readthedocs.io/en/latest/contributing/) +You may find information about contributing to vLLM-Omni on [Contributing](https://vllm-omni.readthedocs.io/en/latest/contributing/). + +Local `pre-commit` gates (SPDX, forbidden imports, `torch.cuda`, shellcheck, +mypy, test marks, TTS adapter ratchet, markdownlint, Buildkite schema, and +how to extend allowlists) are documented in the +[Linting](https://vllm-omni.readthedocs.io/en/latest/contributing/#linting) +section of that guide. Before submitting a PR, run the [precheck-pr skill](https://github.com/vllm-project/vllm-omni/blob/main/.claude/skills/precheck-pr/SKILL.md) with the code agent for a self-check against project conventions. diff --git a/docs/contributing/README.md b/docs/contributing/README.md index 2224c667e..43afd3e5c 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -44,6 +44,149 @@ vLLM-Omni's `pre-commit` hooks will now run automatically every time you commit. pre-commit run --show-diff-on-failure --color=always --all-files # runs on all files (short for --all-files) ``` +!!! warning + GitHub Actions `pre-commit` **skips** several local gates so a whole-tree + `--all-files` run stays green while historical debt is cleaned up. The + current `SKIP` list in + [`.github/workflows/pre-commit.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.github/workflows/pre-commit.yml) + is `check-test-ci-coverage`, `markdownlint-cli2`, `shellcheck`, + `check-spdx-header`, and `mypy-3.10`. Those hooks still run on **your + commit** for changed files. A passing GitHub pre-commit check does not mean + they passed locally. Hooks **not** on that list (forbidden imports, + `torch.cuda`, TTS adapter ratchet, Buildkite schema, Ruff, typos, …) run in + both places. + +The hooks below are new relative to the previous Omni config (Ruff, typos, +actionlint, YAML/whitespace, DCO sign-off, and the pickle-only checker were +already there). `check-pickle-imports` is gone: pickle is now one rule inside +`check-forbidden-imports`. + +| Hook id | Enforces | GitHub Actions | +| ------- | -------- | -------------- | +| `markdownlint-cli2` | Markdown in `docs/`, `recipes/`, `README.md`, `CONTRIBUTING.md` (not `.claude/` / `.cursor/` / `CLAUDE.md`) | skipped | +| `mypy-3.10` | Type-check changed `vllm_omni/` files (model trees excluded). `tests/` uses `--follow-imports skip` | skipped | +| `mypy-3.11` / `3.12` / `3.13` | Same checker, extra Python versions | not installed; `pre-commit run --hook-stage manual mypy-3.12` | +| `check-test-ci-coverage` | Every `tests/**/test_*.py` has a CI level mark and a hardware mark/helper | skipped | +| `check-tts-adapter-migration` | `self._tts_model_type` branches in `serving_speech.py` must not increase | runs | +| `shellcheck` | `*.sh` quoting / undefined vars | skipped | +| `check-spdx-header` | Omni SPDX header on `.py` / `.pyi` / `.sh` / `.rs` / `.proto` | skipped | +| `check-forbidden-imports` | pickle, stdlib `re`/`base64`, Triton, TileLang, Hugging Face Hub download/API | runs | +| `check-torch-cuda-call` | Direct `torch.cuda.*` helpers outside platform adapters | runs | +| `check-buildkite` | `.buildkite/*.yml` against the Buildkite schema | runs | + +Related config (not a new hook id): Ruff `TID251` also bans `librosa` and several +`torch.cuda.*` names; typos is pinned and ignores git-describe hashes (`+g` + +hex) and NumPy `writeable`. Keep the `suggestion` hook last in +`.pre-commit-config.yaml` so autofix output does not bury the SKIP tip. + +#### SPDX headers + +Source files must start with: + +```text +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +``` + +Rust and proto files use `//` instead of `#`. Empty files are exempt. A stale +`Copyright contributors to the vLLM project` line is **rewritten in place** to +the Omni copyright; the hook then exits non-zero so you restage the rewrite. + +#### Forbidden imports + +Under `vllm_omni/`, do not use: + +- stdlib `re` → `import regex as re` +- stdlib `base64` → `import pybase64` (or `import pybase64 as base64`) +- `pickle` / `cloudpickle` (also banned in most tests) +- `from huggingface_hub import snapshot_download` / `hf_hub_download` / `HfApi` + / similar → `vllm.transformers_utils.repo_utils` +- direct `triton` / `tilelang` → `vllm.triton_utils` / `vllm.tilelang_utils` + +`examples/`, `tests/`, `benchmarks/`, `apps/`, `docs/`, `tools/`, `scripts/`, +`.buildkite/`, and `.github/` may keep stdlib `re` and `base64` +(`_NON_LIBRARY_DIRS`). Pickle is stricter: tests are not exempt unless the +file is on the pickle allowlist. + +#### `torch.cuda` call sites + +Prefer `current_omni_platform` / `OmniPlatform`. Platform adapters under +`vllm_omni/platforms/` are allowed; other files must not add new +`torch.cuda.*` helpers (`device_count`, `synchronize`, `empty_cache`, +`device()`, `manual_seed`, and similar). `import torch` is fine. + +#### shellcheck + +The hook does **not** download a binary. Install `shellcheck` with a signed +package manager, then re-run: + +- Debian/Ubuntu/WSL: `sudo apt-get install shellcheck` +- Fedora: `sudo dnf install ShellCheck` +- macOS: `brew install shellcheck` +- Git Bash (MINGW): `scoop install shellcheck` so `shellcheck.exe` is on PATH. + WSL is Linux: use `apt-get`, not a Windows `.exe`. + +If no native binary is found, the wrapper prints those commands (and +[the upstream install page](https://github.com/koalaman/shellcheck?tab=readme-ov-file#installing)) +and exits 1. AMD and other vendor scripts are **not** excluded. + +#### mypy + +`mypy-3.10` runs on commit for changed files. `[tool.mypy]` uses Python 3.12, +`ignore_missing_imports = true`, and `follow_imports = "silent"`. Wrapper +`tools/pre_commit/mypy.py` skips `vllm_omni/model_executor/models/` and +`vllm_omni/diffusion/models/`, and type-checks `tests/` with +`--follow-imports skip`. Extra versions: + +```bash +pre-commit run --hook-stage manual mypy-3.12 +``` + +#### Test CI marks + +`check-test-ci-coverage` requires each collected test module under `tests/` to +have at least one CI **level** mark (`core_model`, `advanced_model`, +`full_model`, `local_model`, or `slow`) and a **hardware** mark (`cpu`, `cuda`, +`H100`, …) or helper (`hardware_test(` / `hardware_marks(`). GitHub Actions +skips this hook; local commit does not. See the +[test writing guide](./ci/test_writing_guide.md). + +#### TTS adapter ratchet + +`check-tts-adapter-migration` counts `self._tts_model_type` comparisons in +`vllm_omni/entrypoints/openai/serving_speech.py`. The count must stay at or +below `MAX_MODEL_TYPE_BRANCHES` in +[`tools/pre_commit/check_tts_adapter.py`](https://github.com/vllm-project/vllm-omni/blob/main/tools/pre_commit/check_tts_adapter.py). +New per-model behavior belongs in +`vllm_omni/entrypoints/openai/tts_adapters/`. Removing branches **must** lower +the budget in the same change; raising it is a reviewable policy edit. + +#### Markdownlint and Buildkite + +`markdownlint-cli2` auto-fixes `docs/`, `recipes/`, and the root README / +CONTRIBUTING (rules in `.markdownlint.yaml`). Skill files under `.claude/` and +`.cursor/` are excluded. + +`check-buildkite` validates `.buildkite/*.yml` against the official schema +after the same expansion `upload_pipeline.py` uses. Omni-only keys such as +`mirror_hardwares` are stripped or rendered first. A few files are skipped in +`SKIP_FILES` inside `tools/pre_commit/check_buildkite.py`. + +#### Extending allowlists and budgets + +Prefer fixing the call site. Growing an allowlist or raising a budget is a +policy change and needs review — do not edit these just to make the hook pass. + +| Gate | File | What to edit | +| ---- | ---- | ------------ | +| pickle, `re`, `base64`, Hugging Face Hub, Triton/TileLang | [`tools/pre_commit/check_forbidden_imports.py`](https://github.com/vllm-project/vllm-omni/blob/main/tools/pre_commit/check_forbidden_imports.py) | `CHECK_IMPORTS[].allowed_files`. Use `allowed_dirs` only when a whole tree should stay exempt. | +| `torch.cuda` | [`tools/pre_commit/check_torch_cuda.py`](https://github.com/vllm-project/vllm-omni/blob/main/tools/pre_commit/check_torch_cuda.py) | `ALLOWED_FILES`, or `ALLOWED_PREFIXES` for a platform adapter tree. | +| TTS `_tts_model_type` branches | [`tools/pre_commit/check_tts_adapter.py`](https://github.com/vllm-project/vllm-omni/blob/main/tools/pre_commit/check_tts_adapter.py) | `MAX_MODEL_TYPE_BRANCHES` (down only, unless the PR justifies a raise). | +| Buildkite skip list | [`tools/pre_commit/check_buildkite.py`](https://github.com/vllm-project/vllm-omni/blob/main/tools/pre_commit/check_buildkite.py) | `SKIP_FILES` for pipelines that cannot be expanded. | + +To skip one hook for a single commit (discouraged): `SKIP= git commit`. +To bypass every hook: `git commit --no-verify` (also discouraged). + ### Documentation MkDocs is a fast, simple and downright gorgeous static site generator that's geared towards building project documentation. Documentation source files are written in Markdown, and configured with a single YAML configuration file, `mkdocs.yml`. @@ -162,12 +305,14 @@ Only specific types of PRs will be reviewed. The PR title is prefixed appropriat Before submitting a PR, run the [precheck-pr skill](https://github.com/vllm-project/vllm-omni/blob/main/.claude/skills/precheck-pr/SKILL.md) with the code agent for a self-review against project conventions: The skill offers two modes: + - **Quick (~3 min):** catches showstoppers — PR title format, missing benchmark claims, rebase status - **Full (~10 min):** thorough maintainer-grade review — dead code scan, copy-paste detection, import hygiene The precheck covers five PR types: Bug Fix, Performance, New Model, Diffusion Model, and General. Each type has a tailored checklist that validates evidence quality (repro steps, A/B benchmarks, registry entries, etc.). See the [precheck-pr skill](https://github.com/vllm-project/vllm-omni/blob/main/.claude/skills/precheck-pr/SKILL.md) for the full checklist. ### Local Test + Please run the L1 and L2 test cases locally first and attach the results before contacting us to add the "ready" label. Please refer to the [test instructions](./ci/test_execution_guide.md) for running the test cases. ### Automatic skip-ci (docs and pytest skip marks) diff --git a/docs/contributing/ci/failures.md b/docs/contributing/ci/failures.md index 9743097cf..e4b846cca 100644 --- a/docs/contributing/ci/failures.md +++ b/docs/contributing/ci/failures.md @@ -6,7 +6,7 @@ When you open a PR against vLLM-Omni, several CI checks run automatically: | Check | Platform | What it does | | ----- | -------- | ------------ | -| **pre-commit** | GitHub Actions | Runs linting (Ruff), formatting, spell-checking (typos), and YAML validation. | +| **pre-commit** | GitHub Actions | Runs Ruff, formatting, typos, YAML validation, forbidden imports, `torch.cuda`, TTS adapter ratchet, and Buildkite schema. Local-only gates (SPDX, shellcheck, markdownlint, mypy-3.10, test marks) are in the workflow `SKIP` env — see [Linting](../README.md#linting). | | **Build Wheel** | GitHub Actions | Builds Python wheels for Python 3.11 and 3.12 on Ubuntu. Skipped for docs-only or Markdown-only changes (controlled by `paths-ignore` in the workflow). | | **DCO** | GitHub | Verifies every commit has a `Signed-off-by` line. | | **docs/readthedocs.org:vllm-omni** | Read the Docs | Builds the MkDocs documentation site. | @@ -29,6 +29,13 @@ pre-commit run --all-files Then commit the fixes and push. +A green GitHub pre-commit job can still miss SPDX, shellcheck, mypy-3.10, +markdownlint, and test-mark coverage because CI skips those IDs. If your +**local** commit hook fails, follow [Linting](../README.md#linting) for the +full hook list (forbidden imports, `torch.cuda`, TTS adapter ratchet, and +Buildkite schema still run in GHA). Do not grow allowlists or raise +`MAX_MODEL_TYPE_BRANCHES` without review. + ### DCO failures Every commit must include a `Signed-off-by` line. If you forgot, amend your commits: diff --git a/docs/contributing/ci/test_writing_guide.md b/docs/contributing/ci/test_writing_guide.md index 3b422ec9f..7912fc4a1 100644 --- a/docs/contributing/ci/test_writing_guide.md +++ b/docs/contributing/ci/test_writing_guide.md @@ -24,38 +24,39 @@ Deploy YAMLs: `vllm_omni/deploy/` via `tests.helpers.stage_config.get_deploy_con By adding markers before test functions, tests can later be executed uniformly by simply declaring the corresponding marker type. ### Current Markers + Defined in `pyproject.toml`: -| Marker | Description | -| ------------------ | --------------------------------------------------------- | -| `core_model` | L1&L2 tests (run in each PR) | -| `advanced_model` | L3 tests (run on each merge to main) | -| `full_model` | L4 tests (run nightly) | -| `diffusion` | Diffusion model tests | -| `omni` | Omni multimodal model tests | -| `tts` | TTS model tests | -| `cache` | Cache backend tests | -| `parallel` | Parallelism/distributed tests | -| `cpu` | Tests that run on CPU | -| `gpu` | Tests that run on GPU * | -| `cuda` | Tests that run on CUDA * | -| `rocm` | Tests that run on AMD/ROCm * | -| `xpu` | Tests that run on Intel XPU * | -| `npu` | Tests that run on NPU/Ascend * | -| `H100` | Tests that require H100 GPU * | -| `L4` | Tests that require L4 GPU * | -| `MI325` | Tests that require MI325 GPU (AMD/ROCm) * | -| `A2` | Tests that require A2 NPU * | -| `A3` | Tests that require A3 NPU * | -| `distributed_cuda` | Tests that require multi cards on CUDA platform * | -| `distributed_rocm` | Tests that require multi cards on ROCm platform * | -| `distributed_npu` | Tests that require multi cards on NPU platform * | -| `skipif_cuda` | Skip if the num of CUDA cards is less than the required * | -| `skipif_rocm` | Skip if the num of ROCm cards is less than the required * | -| `skipif_npu` | Skip if the num of NPU cards is less than the required * | -| `slow` | Slow tests (may skip in quick CI) | +| Marker | Description | +| ------------------ | ------------------------------------------------------------------------------------------------------- | +| `core_model` | L1&L2 tests (run in each PR) | +| `advanced_model` | L3 tests (run on each merge to main) | +| `full_model` | L4 tests (run nightly) | +| `diffusion` | Diffusion model tests | +| `omni` | Omni multimodal model tests | +| `tts` | TTS model tests | +| `cache` | Cache backend tests | +| `parallel` | Parallelism/distributed tests | +| `cpu` | Tests that run on CPU | +| `gpu` | Tests that run on GPU * | +| `cuda` | Tests that run on CUDA * | +| `rocm` | Tests that run on AMD/ROCm * | +| `xpu` | Tests that run on Intel XPU * | +| `npu` | Tests that run on NPU/Ascend * | +| `H100` | Tests that require H100 GPU * | +| `L4` | Tests that require L4 GPU * | +| `MI325` | Tests that require MI325 GPU (AMD/ROCm) * | +| `A2` | Tests that require A2 NPU * | +| `A3` | Tests that require A3 NPU * | +| `distributed_cuda` | Tests that require multi cards on CUDA platform * | +| `distributed_rocm` | Tests that require multi cards on ROCm platform * | +| `distributed_npu` | Tests that require multi cards on NPU platform * | +| `skipif_cuda` | Skip if the num of CUDA cards is less than the required * | +| `skipif_rocm` | Skip if the num of ROCm cards is less than the required * | +| `skipif_npu` | Skip if the num of NPU cards is less than the required * | +| `slow` | Slow tests (may skip in quick CI) | | `benchmark` | Benchmark tests (decorator on runner test functions; perf JSON uses `full_model` + type marker instead) | -| `local_model` | Tests requiring local / non-HF-hub model weights | +| `local_model` | Tests requiring local / non-HF-hub model weights | \* Means those markers are auto-added by `@hardware_test` (parametrization decorator) or `hardware_marks` (only returning the list of marks for flexibility). @@ -81,13 +82,13 @@ This decorator is intended to make hardware-aware, cross-platform test authoring 1. **Applies platform and resource markers** Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `MI325`, `B60`, `A2`, `A3`). - ``` + ```python @pytest.mark.cuda @pytest.mark.L4 ``` 2. **Handles multi-card (distributed) scenarios** For tests requiring multiple cards, it automatically adds distributed markers such as `distributed_cuda`, `distributed_rocm`, or `distributed_npu`. - ``` + ```python @pytest.mark.distributed_cuda(num_cards=num_cards) ``` 3. **Supports flexible card requirements** @@ -97,11 +98,11 @@ This decorator is intended to make hardware-aware, cross-platform test authoring On CUDA, adds a skip marker (`skipif_cuda`) if the system does not have the required number of devices. Support for `skipif_rocm` and `skipif_npu` will be implemented later. - 5. **Works with pytest filtering** Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "distributed_cuda and L4"`). ##### Example usage for decorator + - Single call for multiple platforms: ```python @hardware_test( @@ -220,6 +221,7 @@ If you want to add support for a new platform (e.g., "tpu" for a new accelerator ``` **Summary**: + - Add pytest markers for your new platform/resources - Implement a marker function (`xxx_marks`) - Plug into `hardware_marks` @@ -237,47 +239,46 @@ L1 and L2 level testing form the foundation of the quality assurance system. L1 L2 level testing builds upon L1 by introducing GPU resources and verifying that the end-to-end (E2E) process of the model in basic deployment scenarios is smooth. For example, it uses dummy models to confirm that core interfaces like the inference pipeline, output format, and streaming response work properly. The common goal of these two levels is to provide developers with rapid feedback, discovering and fixing issues early in the development cycle. - - #### 1.2 Testing Content and Scope -- ***L1 (Unit & Logic Testing)***: -- - ***Scope***: Tests internal functions and methods of core components such as `entrypoints`, `models`. - - ***Focus***: Branch coverage, exception handling, algorithm logic correctness. Does not involve external dependencies or the complete service stack. - - ***Time Cost***: Execution time is controlled within ***15 minutes*** to ensure fast feedback. -- ***L2 (Basic End-to-End Testing)***: -- - ***Scope***: Covers two basic deployment scenarios: `online` (serving) and `offline` (inference). - - ***Focus***: Uses `dummy` weights (via deploy YAML patching at `core_model`) or lightweight real models to verify that the entire chain from request input to result output works normally, including output data structure, streaming (stream) support, and **cheap payload checks** at `--run-level core_model` (see below). Also includes some unit tests that require launching independent service instances. - - ***L2 response validation (`core_model`)***: Implemented in `tests/helpers/assertions.py` and invoked by `OpenAIClientHandler` based on `--run-level`. At L2 we require **request success** plus minimal output sanity—not full accuracy: - - **Speech / TTS** (`assert_audio_speech_response`): decoded audio must be present and non-empty (or exceed `min_audio_bytes` when set in `request_config`); `response_format` must match the returned content-type (e.g. `wav`, `pcm`). Whisper transcript similarity, PCM HNR, and preset-voice gender checks run only at L3+. - - **Diffusion** (`assert_diffusion_response`): at least one non-empty image, video, or audio artifact. Resolution/frame-count parity and other deep checks run only at L3+. - - **Omni multimodal** (`assert_omni_response`): L2 asserts successful completion; keyword, transcript, and cross-modal similarity checks run only at L3+. - - ***Characteristic***: Requires ***GPU*** resources to perform model computations. +- ***L1 (Unit & Logic Testing)***: + - ***Scope***: Tests internal functions and methods of core components such as `entrypoints`, `models`. + - ***Focus***: Branch coverage, exception handling, algorithm logic correctness. Does not involve external dependencies or the complete service stack. + - ***Time Cost***: Execution time is controlled within ***15 minutes*** to ensure fast feedback. +- ***L2 (Basic End-to-End Testing)***: + - ***Scope***: Covers two basic deployment scenarios: `online` (serving) and `offline` (inference). + - ***Focus***: Uses `dummy` weights (via deploy YAML patching at `core_model`) or lightweight real models to verify that the entire chain from request input to result output works normally, including output data structure, streaming (stream) support, and **cheap payload checks** at `--run-level core_model` (see below). Also includes some unit tests that require launching independent service instances. + - ***L2 response validation (`core_model`)***: Implemented in `tests/helpers/assertions.py` and invoked by `OpenAIClientHandler` based on `--run-level`. At L2 we require **request success** plus minimal output sanity—not full accuracy: + - **Speech / TTS** (`assert_audio_speech_response`): decoded audio must be present and non-empty (or exceed `min_audio_bytes` when set in `request_config`); `response_format` must match the returned content-type (e.g. `wav`, `pcm`). Whisper transcript similarity, PCM HNR, and preset-voice gender checks run only at L3+. + - **Diffusion** (`assert_diffusion_response`): at least one non-empty image, video, or audio artifact. Resolution/frame-count parity and other deep checks run only at L3+. + - **Omni multimodal** (`assert_omni_response`): L2 asserts successful completion; keyword, transcript, and cross-modal similarity checks run only at L3+. + - ***Characteristic***: Requires ***GPU*** resources to perform model computations. #### 1.3 Test Directory and Execution Files A clear directory structure is key to managing test cases efficiently. See the **Test Dir** column in [Test System Overview](./test_system_overview.md) for the canonical placement rules. -- ***L1 (component / unit)***: `tests/{component_name}/…` mirroring `vllm_omni/{component_name}/` (e.g. `tests/diffusion/`, `tests/entrypoints/`). Do **not** invent new top-level `tests/` folders unrelated to a component. -- ***L2 (basic model E2E)***: - - Online serving: `tests/e2e/online_serving/test_{model_name}.py` - - Offline inference: `tests/e2e/offline_inference/test_{model_name}.py` -- ***Feature-level integration*** (cross-cutting, not a single-model smoke): `tests/e2e/features//` (e.g. `fullduplex/`, `custom_pipeline/`, `rlhf_test/`). +- ***L1 (component / unit)***: `tests/{component_name}/…` mirroring `vllm_omni/{component_name}/` (e.g. `tests/diffusion/`, `tests/entrypoints/`). Do **not** invent new top-level `tests/` folders unrelated to a component. +- ***L2 (basic model E2E)***: + - Online serving: `tests/e2e/online_serving/test_{model_name}.py` + - Offline inference: `tests/e2e/offline_inference/test_{model_name}.py` +- ***Feature-level integration*** (cross-cutting, not a single-model smoke): `tests/e2e/features//` (e.g. `fullduplex/`, `custom_pipeline/`, `rlhf_test/`). #### 1.4 Execution Method and Example -- ***Trigger Timing***: **`PR with ready label`**. That is, when a developer adds a "ready for review" or similar label to a PR on platforms like GitHub, L1 and L2 tests are automatically triggered. -- ***Diff-aware step skipping***: On L2, **E2E Test** jobs may be omitted at pipeline upload when the PR diff does not touch their [`source_file_dependencies`](ci_settings.md#step-filtering) prefixes; non-E2E groups still always upload. See [CI Settings — Diff-aware CI](ci_settings.md#diff-aware-ci). -- ***Execution Environment***: L1 uses ***CPU*** environment; L2 requires ***GPU*** environment. -- ***Script Example***: +- ***Trigger Timing***: **`PR with ready label`**. That is, when a developer adds a "ready for review" or similar label to a PR on platforms like GitHub, L1 and L2 tests are automatically triggered. +- ***Diff-aware step skipping***: On L2, **E2E Test** jobs may be omitted at pipeline upload when the PR diff does not touch their [`source_file_dependencies`](ci_settings.md#step-filtering) prefixes; non-E2E groups still always upload. See [CI Settings — Diff-aware CI](ci_settings.md#diff-aware-ci). +- ***Execution Environment***: L1 uses ***CPU*** environment; L2 requires ***GPU*** environment. +- ***Script Example***:

L1 Test Examples Examples from `tests/model_executor/models/qwen2_5_omni/test_audio_length.py` + ```python # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project import pytest @@ -331,6 +332,7 @@ def test_cap_and_align_mel_length_no_mismatch(repeats, code_len, max_mel_frames) if max_mel_frames is not None and int(max_mel_frames) > 0 and int(max_mel_frames) >= repeats: assert target_mel_len <= int(max_mel_frames) ``` +
@@ -338,7 +340,7 @@ def test_cap_and_align_mel_length_no_mismatch(repeats, code_len, max_mel_frames) You can refer to Test Examples in the L3 section to see example test cases that incorporate both L2 and L3 testing logic.
-- - ***Run Command***: +- ***Run Command***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py` `pytest -s -v -m 'core_model and cpu' --run-level=core_model` @@ -351,36 +353,34 @@ L3 level testing executes after code is merged into the main branch. Its core pu . It acts as the "quality gatekeeper" for the main branch, ensuring that no merge breaks the core capabilities of the model service. Testing needs to provide clear conclusions within a relatively short time (<30min), balancing test depth with feedback speed. - - #### 2.2 Testing Content and Scope -- ***Deployment Scenarios***: Covers richer `online` and `offline` deployment configurations, which may include different hardware configurations, batch sizes, concurrency levels, etc. -- ***Core Verification***: -- 1. ***Inference Functionality***: Ensures real models can perform forward computation normally and return results. - 2. ***Accuracy Compliance***: Verifies that the model's evaluation metrics (e.g., accuracy) meet the expected baseline, preventing code changes from introducing accuracy issues. - 3. ***Important Performance***: Verifies whether performance (e.g., P99 latency, throughput) in core scenarios meets preset thresholds. -- ***L3 response validation (`advanced_model`)***: At `--run-level advanced_model`, `tests/helpers/assertions.py` adds semantic checks on top of L2 payload gates (see L1 & L2 above): - - **Speech / TTS** (`assert_audio_speech_response`): Whisper transcript of returned audio vs `request_config["input"]` (cosine similarity > 0.9 when input text is set); optional `min_audio_bytes` floor; PCM harmonic-to-noise ratio when `response_format` is `pcm`; **preset voice gender** via `_assert_preset_voice_gender_from_audio` when `voice` matches a known preset in `_PRESET_VOICE_GENDER_MAP` (pitch/F0-based classifier; skipped for unknown voices or `pcm` output). - - **Omni multimodal** (`assert_omni_response`): non-empty text/audio outputs; `key_words` in transcript or text; text–audio similarity / containment; preset **`speaker`** gender check (same helper as TTS). - - **Diffusion** (`assert_*_diffusion_response`): image/video dimension and frame-count parity with request parameters where configured. - - **Accuracy suites**: pixel/video similarity and metric baselines under `/tests/e2e/accuracy/` (separate from inline helper assertions). +- ***Deployment Scenarios***: Covers richer `online` and `offline` deployment configurations, which may include different hardware configurations, batch sizes, concurrency levels, etc. +- ***Core Verification***: + 1. ***Inference Functionality***: Ensures real models can perform forward computation normally and return results. + 2. ***Accuracy Compliance***: Verifies that the model's evaluation metrics (e.g., accuracy) meet the expected baseline, preventing code changes from introducing accuracy issues. + 3. ***Important Performance***: Verifies whether performance (e.g., P99 latency, throughput) in core scenarios meets preset thresholds. +- ***L3 response validation (`advanced_model`)***: At `--run-level advanced_model`, `tests/helpers/assertions.py` adds semantic checks on top of L2 payload gates (see L1 & L2 above): + - **Speech / TTS** (`assert_audio_speech_response`): Whisper transcript of returned audio vs `request_config["input"]` (cosine similarity > 0.9 when input text is set); optional `min_audio_bytes` floor; PCM harmonic-to-noise ratio when `response_format` is `pcm`; **preset voice gender** via `_assert_preset_voice_gender_from_audio` when `voice` matches a known preset in `_PRESET_VOICE_GENDER_MAP` (pitch/F0-based classifier; skipped for unknown voices or `pcm` output). + - **Omni multimodal** (`assert_omni_response`): non-empty text/audio outputs; `key_words` in transcript or text; text–audio similarity / containment; preset **`speaker`** gender check (same helper as TTS). + - **Diffusion** (`assert_*_diffusion_response`): image/video dimension and frame-count parity with request parameters where configured. + - **Accuracy suites**: pixel/video similarity and metric baselines under `/tests/e2e/accuracy/` (separate from inline helper assertions). #### 2.3 Test Directory and Execution Files -- ***Functional / model E2E***: - - Online serving: `tests/e2e/online_serving/test_{model_name}_expansion.py` - - Offline inference: `tests/e2e/offline_inference/test_{model_name}_expansion.py` - - Accuracy / similarity: `tests/e2e/accuracy/` - - (`_expansion.py` holds broader scenario coverage than the L2 smoke file.) -- ***Feature-level integration***: `tests/e2e/features//` (not under a new top-level `tests/` directory). +- ***Functional / model E2E***: + - Online serving: `tests/e2e/online_serving/test_{model_name}_expansion.py` + - Offline inference: `tests/e2e/offline_inference/test_{model_name}_expansion.py` + - Accuracy / similarity: `tests/e2e/accuracy/` + - (`_expansion.py` holds broader scenario coverage than the L2 smoke file.) +- ***Feature-level integration***: `tests/e2e/features//` (not under a new top-level `tests/` directory). #### 2.4 Execution Method and Example -- ***Trigger Timing***: **`PR Merged`**. Automatically triggered after code review is approved and merged into the main branch (typically via `merge-test` label on the PR before merge). -- ***Diff-aware step skipping***: Same [`source_file_dependencies`](ci_settings.md#step-filtering) mechanism as L2—**all E2E Test** leaf steps are diff-gated; other L3 groups always upload. See [CI Settings — Diff-aware CI](ci_settings.md#diff-aware-ci). -- ***Execution Environment***: ***GPU*** servers. -- ***Script Example***: +- ***Trigger Timing***: **`PR Merged`**. Automatically triggered after code review is approved and merged into the main branch (typically via `merge-test` label on the PR before merge). +- ***Diff-aware step skipping***: Same [`source_file_dependencies`](ci_settings.md#step-filtering) mechanism as L2—**all E2E Test** leaf steps are diff-gated; other L3 groups always upload. See [CI Settings — Diff-aware CI](ci_settings.md#diff-aware-ci). +- ***Execution Environment***: ***GPU*** servers. +- ***Script Example***: ???+ example "Test Examples" @@ -496,9 +496,9 @@ L3 level testing executes after code is merged into the main branch. Its core pu **Audio output debugging**: Deep validation may transcribe returned audio via `convert_audio_bytes_to_text` (Whisper). If an audio keyword or text–audio similarity assertion fails, set `VLLM_OMNI_KEEP_REQUEST_MEDIA=1` before running pytest to keep the intermediate WAV files for inspection (see [Test helper environment variables](test_system_overview.md#test-helper-environment-variables)). -- ***Run Command (L3 merge)***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py -m advanced_model --run-level=advanced_model` +- ***Run Command (L3 merge)***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py -m advanced_model --run-level=advanced_model` -- ***Run Command (L4 nightly expansion)***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}_expansion.py -m full_model --run-level=full_model` +- ***Run Command (L4 nightly expansion)***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}_expansion.py -m full_model --run-level=full_model` ### L4 Level Testing - Full Functionality, Performance, and Documentation Testing @@ -506,27 +506,25 @@ L3 level testing executes after code is merged into the main branch. Its core pu L4 level testing is a comprehensive quality audit before a version release. It expands upon L3, executing ***full*** functional scenarios, conducting systematic ***performance stress tests***, and simultaneously verifying the correctness of accompanying ***example documentation***. Its purpose is to perform deep validation of the system during off-peak nighttime hours, providing quality trend reports for daytime development and data support for release decisions. - - #### 3.2 Testing Content and Scope -- ***Full Functionality Testing***: Executes all test cases defined in `test_{model_name}_expansion.py`, covering all implemented features, positive flows, boundary conditions, and exception handling. -- ***Performance Testing***: Uses `tests/dfx/perf/tests/test_qwen3_omni_*.json` (Omni), `test_tts.json` / `test_voxcpm2.json` / `test_higgs_audio_v3.json` (TTS), and diffusion configs `tests/dfx/perf/tests/test_*_vllm_omni.json` (passed to `run_benchmark.py` or `run_diffusion_benchmark.py` via `--test-config-file` in nightly **Perf Test** steps) to drive throughput, latency, and memory benchmarks. Each JSON **case** may declare an optional top-level **`mark`** array: exactly one ``hardware_marks`` object plus pytest marker name strings (`full_model`, `omni` / `tts` / `diffusion`, …). Runners attach those marks to each parametrized `(server, benchmark index)` pair so **local** bulk runs can filter with `-m` (for example `-m "full_model and H100 and diffusion"`). Nightly CI perf jobs select workloads by **`--test-config-file`**, not `-m`. Details are in the Performance Tests example below. -- ***Documentation Testing***: Verifies whether the example code provided to users is runnable and its results match the description. +- ***Full Functionality Testing***: Executes all test cases defined in `test_{model_name}_expansion.py`, covering all implemented features, positive flows, boundary conditions, and exception handling. +- ***Performance Testing***: Uses `tests/dfx/perf/tests/test_qwen3_omni_*.json` (Omni), `test_tts.json` / `test_voxcpm2.json` / `test_higgs_audio_v3.json` (TTS), and diffusion configs `tests/dfx/perf/tests/test_*_vllm_omni.json` (passed to `run_benchmark.py` or `run_diffusion_benchmark.py` via `--test-config-file` in nightly **Perf Test** steps) to drive throughput, latency, and memory benchmarks. Each JSON **case** may declare an optional top-level **`mark`** array: exactly one ``hardware_marks`` object plus pytest marker name strings (`full_model`, `omni` / `tts` / `diffusion`, …). Runners attach those marks to each parametrized `(server, benchmark index)` pair so **local** bulk runs can filter with `-m` (for example `-m "full_model and H100 and diffusion"`). Nightly CI perf jobs select workloads by **`--test-config-file`**, not `-m`. Details are in the Performance Tests example below. +- ***Documentation Testing***: Verifies whether the example code provided to users is runnable and its results match the description. #### 3.3 Test Directory and Execution Files -- ***Functional Testing***: Same directories as L3. -- ***Performance Test Configuration***: `tests/dfx/perf/tests/test_qwen3_omni_*.json`, `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json`, and diffusion configs `tests/dfx/perf/tests/test_*_vllm_omni.json` (e.g. `test_qwen_image_vllm_omni.json`, `test_cosmos3_vllm_omni.json`). Optional per-case `mark` for local `-m` filtering is documented in Section 3.4 Performance Tests. -- ***Documentation Example Tests***: - - `tests/examples/online_serving/test_{model_name}.py` - - `tests/examples/offline_inference/test_{model_name}.py` +- ***Functional Testing***: Same directories as L3. +- ***Performance Test Configuration***: `tests/dfx/perf/tests/test_qwen3_omni_*.json`, `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json`, and diffusion configs `tests/dfx/perf/tests/test_*_vllm_omni.json` (e.g. `test_qwen_image_vllm_omni.json`, `test_cosmos3_vllm_omni.json`). Optional per-case `mark` for local `-m` filtering is documented in Section 3.4 Performance Tests. +- ***Documentation Example Tests***: + - `tests/examples/online_serving/test_{model_name}.py` + - `tests/examples/offline_inference/test_{model_name}.py` #### 3.4 Execution Method and Example -- ***Trigger Timing***: **`Nightly`**, automatically executed every night. -- ***Execution Environment***: ***GPU*** server clusters to meet the resource demands of performance testing. -- ***Script Example***: +- ***Trigger Timing***: **`Nightly`**, automatically executed every night. +- ***Execution Environment***: ***GPU*** server clusters to meet the resource demands of performance testing. +- ***Script Example***: ??? example "Test Examples: Documentation Example Tests" @@ -540,7 +538,7 @@ L4 level testing is a comprehensive quality audit before a version release. It e --8<-- "docs/contributing/ci/test_examples/l4_functionality_tests.inc.md" -- ***Run Command***: (Specific commands would depend on the performance testing tool and configuration defined in `nightly.json`). +- ***Run Command***: (Specific commands would depend on the performance testing tool and configuration defined in `nightly.json`). ### L5 Level Testing - Stability and Reliability Testing @@ -548,33 +546,36 @@ L4 level testing is a comprehensive quality audit before a version release. It e L5 level testing focuses on the performance of model services under ***long-running*** and ***abnormal fault*** scenarios. It aims to uncover deep-seated issues that only manifest under sustained pressure or extreme conditions, such as memory leaks, resource contention, gradual performance degradation, and lack of fault tolerance mechanisms. This is the final, yet crucial, line of defense for ensuring service high availability and production environment robustness. - - #### 4.2 Testing Content and Scope -- ***Long-term Stability (Stability) Testing***: Uses JSON under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` and `test_wan22.json`) to run the service under moderate load for an extended period (e.g., over 12 hours), monitoring whether metrics like memory/VRAM usage, response time, and throughput degrade over time, and whether the service process remains stable. -- ***Reliability Testing***: Uses pytest suites under `tests/dfx/reliability/` to inject controlled faults against a **live** `vllm_omni serve` instance (same **`omni_server` / `omni_server_function`** fixture style as E2E). Current suites emphasize **GPU memory pressure** (CUDA sidecar “memory hog”), **worker / runtime process kill** (`SIGKILL` on `VLLM::Worker` for Qwen3-Omni, `multiprocessing.spawn` for Wan2.2 video workers, or `vLLM-Omni::` for HunyuanImage DiT workers), **large multimodal chat**, **`/v1/videos`**, **`/v1/images/generations`**, or **`/v1/audio/speech`** jobs under OOM, **`/health` → 503** and **fast-fail / non-hanging concurrent** requests after kill, and **OpenAI-style 5xx error contracts** (e.g. text vs text+audio under OOM). **Post-fault recovery** checks exist where enabled (some cases may be `skip` while issues are tracked). See the Reliability `
` block in Section 4.4 for file-level responsibilities and CI markers (`slow`, `hardware_test`, POSIX-only kill). +- ***Long-term Stability (Stability) Testing***: Uses JSON under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` and `test_wan22.json`) to run the service under moderate load for an extended period (e.g., over 12 hours), monitoring whether metrics like memory/VRAM usage, response time, and throughput degrade over time, and whether the service process remains stable. +- ***Reliability Testing***: Uses pytest suites under `tests/dfx/reliability/` to inject controlled faults against a **live** `vllm_omni serve` instance (same **`omni_server` / `omni_server_function`** fixture style as E2E). Current suites emphasize **GPU memory pressure** (CUDA sidecar “memory hog”), **worker / runtime process kill** (`SIGKILL` on `VLLM::Worker` for Qwen3-Omni, `multiprocessing.spawn` for Wan2.2 video workers, or `vLLM-Omni::` for HunyuanImage DiT workers), **large multimodal chat**, **`/v1/videos`**, **`/v1/images/generations`**, or **`/v1/audio/speech`** jobs under OOM, **`/health` → 503** and **fast-fail / non-hanging concurrent** requests after kill, and **OpenAI-style 5xx error contracts** (e.g. text vs text+audio under OOM). **Post-fault recovery** checks exist where enabled (some cases may be `skip` while issues are tracked). See Section 4.3 for file-level responsibilities and Section 4.4 Test Examples (Reliability) for CI markers (`slow`, `hardware_test`, POSIX-only kill). #### 4.3 Test Directory and Execution Files -- ***Stability Test Configuration***: `tests/dfx/stability/tests/test_qwen3_omni.json`, `tests/dfx/stability/tests/test_wan22.json` (one JSON per model / runner family) -- ***Reliability Test Suite*** (`tests/dfx/reliability/`): - - `test_reliability_qwen3_omni.py` — Qwen3-Omni chat / multimodal reliability (GPU OOM, process kill, recovery, error contract under `--async-chunk` vs default). - - `test_reliability_wan22.py` — Wan2.2 I2V video API reliability (`/v1/videos` under OOM and process kill, recovery). - - `test_reliability_hunyuan_image.py` — HunyuanImage-3.0-Instruct DiT-only reliability (`/v1/images/generations` under OOM and process kill; deploy `hunyuan_image3_dit.yaml`, H100 × 4). - - `helpers.py` — Shared primitives used by current suites: raw HTTP probes for `/v1/chat/completions` and `/health`, OpenAI-style error parsing, GPU OOM sidecar (`inject_gpu_oom` / `stop_gpu_oom_hogs`), and `pgrep`-based process-kill injector construction (`make_process_kill_fault_injector`). - - `conftest.py` — `fault_injector` and `omni_server_after_fault` / `omni_server_after_fault_function` fixtures to run a callable **after** the server is ready. - - `README.md` — Short local run commands for this directory. +- ***Stability Test Configuration***: `tests/dfx/stability/tests/test_qwen3_omni.json`, `tests/dfx/stability/tests/test_wan22.json` (one JSON per model / runner family) +- ***Reliability Test Suite*** (`tests/dfx/reliability/`): + - `test_reliability_qwen3_omni.py` — Qwen3-Omni chat / multimodal reliability (GPU OOM, process kill, recovery, error contract under `--async-chunk` vs default). + - `test_reliability_wan22.py` — Wan2.2 I2V video API reliability (`/v1/videos` under OOM and process kill, recovery). + - `test_reliability_hunyuan_image.py` — HunyuanImage-3.0-Instruct DiT-only reliability (`/v1/images/generations` under OOM and process kill; deploy `hunyuan_image3_dit.yaml`, H100 × 4). + - `helpers.py` — Shared primitives used by current suites: raw HTTP probes for `/v1/chat/completions` and `/health`, OpenAI-style error parsing, GPU OOM sidecar (`inject_gpu_oom` / `stop_gpu_oom_hogs`), and `pgrep`-based process-kill injector construction (`make_process_kill_fault_injector`). + - `conftest.py` — `fault_injector` and `omni_server_after_fault` / `omni_server_after_fault_function` fixtures to run a callable **after** the server is ready. + - `README.md` — Short local run commands for this directory. #### 4.4 Execution Method and Example -- ***Trigger Timing***: **`Weekly`** (weekly) or **`Days before Release`** (several days before a major release). Due to long execution times, the frequency is lower. -- ***Execution Environment***: ***GPU*** servers, requiring a stable and exclusive testing environment. -- ***Script Example***: +- ***Trigger Timing***: **`Weekly`** (weekly) or **`Days before Release`** (several days before a major release). Due to long execution times, the frequency is lower. +- ***Run Command***: + - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_.py` alongside a matching JSON config) + - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`). Weekly CI (`.buildkite/cuda/test-weekly.yml`) runs one step per suite (`WEEKLY=1` or PR label `weekly-test`) +- ***Script Example***: +
Test Examples -When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). The following illustrates the Qwen3-Omni shape: +##### Stability + +When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). Pair the JSON with `tests/dfx/stability/scripts/test_stability_.py`. The following illustrates the Qwen3-Omni shape: ```json { @@ -600,7 +601,7 @@ When you want to add L5-level stability test cases, add or extend the appropriat } ``` -##### Parameter Explanation +###### Parameter Explanation ***Overview*** @@ -610,91 +611,47 @@ When you want to add L5-level stability test cases, add or extend the appropriat | server_params | Yes | Server-side configuration parameters (model, deploy configuration, etc.) | | benchmark_params | Yes | Stability benchmark running parameters (supports multiple configurations) | -##### server_params Configuration +###### server_params Configuration -###### Basic Parameters +Basic parameters: | Parameter | Required | Example | Description | | ----------------- | -------- | ---------------------------------- | ----------------------------------- | | model | Yes | "Qwen/Qwen3-Omni-30B-A3B-Instruct" | Model name or path | | stage_config_name | Yes | "qwen3_omni_moe.yaml" | Deploy configuration file name | -###### Dynamic Configuration (update/delete) - -Supports incremental modifications based on the basic configuration: +Dynamic configuration (`update` / `delete`) supports incremental modifications based on the basic configuration: | Operation | Description | | --------- | ------------------------------------ | | update | Update or add configuration items | | delete | Delete specified configuration items | -***Example***: -You can refer to Test Examples in L4 §3.4 +***Example***: You can refer to Test Examples in L4 §3.4. -##### benchmark_params Configuration +###### benchmark_params Configuration For stability testing, the key parameters are: -- **duration_sec**: Total duration (in seconds) during which benchmark traffic is sent. The stability benchmark will keep sending batches until this duration is reached. -- **request_rate** / **max_concurrency**: Exactly one of them must be specified. They control how the traffic is generated for each batch: - - `request_rate`: Number of requests per second. The benchmark will send `num_prompts_per_batch` requests at the given rate. - - `max_concurrency`: Maximum number of concurrent requests. When this is used, `request_rate` is set to `inf` internally. -- **num_prompts_per_batch**: Number of prompts sent in each batch. Multiple batches will be executed sequentially within `duration_sec`. +- **duration_sec**: Total duration (in seconds) during which benchmark traffic is sent. The stability benchmark will keep sending batches until this duration is reached. +- **request_rate** / **max_concurrency**: Exactly one of them must be specified. They control how the traffic is generated for each batch: + - `request_rate`: Number of requests per second. The benchmark will send `num_prompts_per_batch` requests at the given rate. + - `max_concurrency`: Maximum number of concurrent requests. When this is used, `request_rate` is set to `inf` internally. +- **num_prompts_per_batch**: Number of prompts sent in each batch. Multiple batches will be executed sequentially within `duration_sec`. All other optional parameters follow the same rules as in L4 §3.4. -
+##### Reliability -
- Reliability test suite (tests/dfx/reliability) +Add pytest modules under `tests/dfx/reliability/` (for example `test_reliability_qwen3_omni.py`, `test_reliability_wan22.py`, `test_reliability_hunyuan_image.py`). Reuse `helpers.py` and `conftest.py` rather than duplicating OOM / process-kill / raw HTTP plumbing. -##### Purpose and relationship to stability - -Reliability tests are **short fault-injection** integration runs (L5 **(b)** in `tests/dfx/reliability/README.md`). They complement **stability** JSON-driven long runs: instead of hours of steady traffic, they **perturb** the server (GPU OOM sidecar, fatal signals on selected processes) and check **failure mode** and **latency bounds** (e.g. chat or `/v1/videos` must not hang under concurrent fault-time load). - -##### Directory layout - -| Path | Responsibility | -| ---- | -------------- | -| `helpers.py` | Shared helpers used by current reliability suites: raw `POST`/`GET` probes (`/v1/chat/completions`, `/health`), OpenAI error parsing (`extract_openai_error_contract_from_bytes`), GPU OOM sidecar lifecycle (`inject_gpu_oom`, `stop_gpu_oom_hogs`), and process-kill injector builder (`make_process_kill_fault_injector`). | -| `conftest.py` | Pytest fixtures: indirect `fault_injector`, `omni_server_after_fault` / `omni_server_after_fault_function` (run injector after server is ready, then yield server). | -| `test_reliability_qwen3_omni.py` | Qwen3-Omni: OOM vs **text vs text+audio** error contract, large multimodal chat under OOM, concurrent pressure, **SIGKILL** on `VLLM::Worker`, `/health` → 503 + fast-fail + concurrent chat; optional OOM recovery scenario (may be skipped while tracked in issues). | -| `test_reliability_wan22.py` | Wan2.2 I2V: large `/v1/videos` under OOM, **SIGKILL** on `multiprocessing.spawn` chain, health / fast-fail / concurrent video requests; optional recovery test (may be skipped). | -| `test_reliability_hunyuan_image.py` | HunyuanImage DiT-only: large `/v1/images/generations` under OOM, **SIGKILL** on `vLLM-Omni::` workers and serve/tree targets, health / fast-fail / concurrent image requests; some OOM/recovery cases may be skipped while tracked in issues. | -| `README.md` | Minimal run / collect examples. | - -##### Parametrization and markers +###### Parametrization and markers - Each test module defines a **`RELIABILITY_SCENARIOS`** list (`test_name`, `server_params`: model, `stage_config_name` or diffusion `server_args`, etc.). **`create_reliability_omni_server_params()`** in `tests/dfx/conftest.py` resolves stage paths (including XPU substitutions where applicable) and builds **`OmniServerParams`** lists consumed by **`@pytest.mark.parametrize(..., indirect=True)`** on `omni_server` or `omni_server_function`. - Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** (Qwen3-Omni **2**× H100; Wan2.2 **1**× H100; HunyuanImage DiT **4**× H100). - **Process-kill** tests use **`@pytest.mark.skipif(os.name == "nt", ...)`** because injection uses POSIX **`pgrep` / `kill`**. -##### CI trigger - -Weekly Buildkite (`.buildkite/cuda/test-weekly.yml`) runs one step per model suite (trigger: `WEEKLY=1` or PR label `weekly-test`), for example: - -| Buildkite step | Test file | CI hardware | -| -------------- | --------- | ----------- | -| Reliability Test - qwen3-omni | `test_reliability_qwen3_omni.py` | H100 × 2 (`mithril-h100-pool`) | -| Reliability Test - wan22 | `test_reliability_wan22.py` | H100 × 2 (`mithril-h100-pool`) | -| Reliability Test - hunyuan-image | `test_reliability_hunyuan_image.py` | H100 × 4 (`mithril-h100-pool`) | - -```bash -pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow" -pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow" -pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow" -``` - -##### Local commands - -```bash -pytest --collect-only tests/dfx/reliability -pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow -pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow -pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow -``` - -##### Adding a new model suite +###### Adding a new model suite 1. Add `test_reliability_.py` under `tests/dfx/reliability/`. 2. Define **`RELIABILITY_SCENARIOS`** and pass them through **`create_reliability_omni_server_params()`** with the correct deploy or e2e deploy-config directory (same pattern as existing files). @@ -702,6 +659,3 @@ pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow 4. Register **`slow`** (and **`hardware_test`** if needed); extend **`.buildkite/cuda/test-weekly.yml`** when the suite should run in weekly L5.
- -- - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_.py` alongside a matching JSON config) - - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`; add `test_reliability_.py` and a matching step in `.buildkite/cuda/test-weekly.yml` for new models) diff --git a/docs/contributing/model/adding_tts_model.md b/docs/contributing/model/adding_tts_model.md index 76aab4291..0b8e07c9c 100644 --- a/docs/contributing/model/adding_tts_model.md +++ b/docs/contributing/model/adding_tts_model.md @@ -78,7 +78,7 @@ Profile before optimizing. new TTS model must pass all three levels: | Layer | Catches | Tool | -|-------|---------|------| +| ------- | --------- | ------ | | Offline RTF / duration | Throughput regressions, missing audio, wrong sample rate | `end2end.py`, pytest e2e | | Browser streaming playback | Delta-vs-cumulative bugs, chunk boundary glitches, TTFP regressions | Gradio demo over `/v1/audio/speech?stream=true` | | Concurrent requests | Per-request state leaks, codec window round-robin gaps | `max_num_seqs>1` smoke with 4+ parallel prompts | @@ -94,10 +94,10 @@ the symptom is crosstalk or truncation under load, nothing in single-request tes vLLM-Omni supports TTS models as multi-stage pipelines where each stage runs independently and can be placed on different devices. Qwen3-TTS has two stages: -| Stage | Name | Input | Output | -|-------|------|-------|--------| -| 0 | Code Predictor (AR) | Text tokens | Discrete RVQ codec codes | -| 1 | Code2Wav (Decoder) | RVQ codec codes | Audio waveform | +| Stage | Name | Input | Output | +| ----- | ------------------- | --------------- | ------------------------ | +| 0 | Code Predictor (AR) | Text tokens | Discrete RVQ codec codes | +| 1 | Code2Wav (Decoder) | RVQ codec codes | Audio waveform | Each stage is a separate model class configured independently via YAML. The two stages are connected by the `async_chunk` framework, which enables inter-stage streaming for @@ -170,7 +170,7 @@ and Qwen3-Omni). When adding a new TTS model, create the following structure: -``` +```text vllm_omni/model_executor/models/ your_model_name/ __init__.py @@ -201,7 +201,7 @@ it would not be picked up — the hub README is the documented surface. **Qwen3-TTS reference files:** | File | Purpose | -|------|---------| +| ------ | --------- | | `models/qwen3_tts/qwen3_tts.py` | Unified model class | | `models/qwen3_tts/qwen3_tts_code_predictor_vllm.py` | Stage 0 - optimized AR | | `models/qwen3_tts/qwen3_tts_code2wav.py` | Stage 1 - decoder | @@ -528,9 +528,9 @@ Understanding what's available in stage outputs: - `stage_list[source_id].engine_outputs` - list of `EngineCoreOutput` objects - Each `EngineCoreOutput` has `outputs` - list of `RequestOutput` objects - Each `RequestOutput` has: - - `token_ids` - generated token IDs - - `multimodal_output` - dict with keys matching your model's `OmniOutput.multimodal_outputs` - - `prompt_token_ids` - original prompt token IDs + - `token_ids` - generated token IDs + - `multimodal_output` - dict with keys matching your model's `OmniOutput.multimodal_outputs` + - `prompt_token_ids` - original prompt token IDs ### Batch mode (non-streaming) @@ -610,6 +610,7 @@ def ar2decoder_async_chunk( ``` Key points: + - `transfer_manager` is the `OmniChunkTransferAdapter` that owns the chunk lifecycle - Each call appends one AR decode step's output; a chunk is emitted every `chunk_size` steps - The final (possibly partial) chunk is flushed when `is_finished` is true @@ -664,6 +665,7 @@ Create `vllm_omni/entrypoints/openai/tts_adapters/your_model.py`: ```python # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """YourModel serving adapter.""" from typing import TYPE_CHECKING @@ -802,7 +804,7 @@ directly from the model's own generator. ### Directory structure -``` +```text vllm_omni/model_executor/models/your_model_name/ __init__.py modeling_your_model_name.py # unified class: load_weights + forward + streaming @@ -950,10 +952,17 @@ is correct. Stage the modified files and commit again — do not revert the chan Common failures and fixes: | Check | Cause | Fix | -|-------|-------|-----| +| ------- | ------- | ----- | | `ruff F841` | Local variable assigned but never used | Remove the extraction or forward it to the model call | | `ruff E402` | Module-level import not at top of file | Move import to the top-level import block | | `ruff format` | Line length, spacing, or quote style | Accept the auto-fix, stage, and re-commit | +| `check-spdx-header` | Missing header, or copyright still says `vLLM project` | Use `Copyright contributors to the vLLM-Omni project`; restage the rewrite | +| `check-forbidden-imports` | Stdlib `re`/`base64`, pickle, Hugging Face Hub API, or direct Triton/TileLang | `import regex as re` and `pybase64`; do not grow the allowlist without review | +| `check-torch-cuda-call` | New `torch.cuda.*` call site | Use `current_omni_platform`; do not grow `ALLOWED_FILES` without review | +| `check-tts-adapter-migration` | New `_tts_model_type` branch in `serving_speech.py` | Put logic in `tts_adapters/`; lower `MAX_MODEL_TYPE_BRANCHES` when removing branches | +| `check-test-ci-coverage` | Test file missing level or hardware mark | Add `core_model`/… plus `cpu`/`cuda`/`hardware_test(` | +| `shellcheck` | Native `shellcheck` missing, or script warning | Install via apt/dnf/brew (or `shellcheck.exe` on Windows). See [Linting](../README.md#linting) | +| `mypy-3.10` / `markdownlint-cli2` / `check-buildkite` | Types, docs markdown, or Buildkite YAML | See [Linting](../README.md#linting) | ### DCO sign-off @@ -1008,7 +1017,7 @@ Adding a TTS model to vLLM-Omni involves: ### Qwen3-TTS Reference Files | File | Purpose | -|------|---------| +| ------ | --------- | | `models/qwen3_tts/qwen3_tts.py` | Unified model class | | `models/qwen3_tts/qwen3_tts_code_predictor_vllm.py` | AR stage with vLLM fused ops | | `models/qwen3_tts/qwen3_tts_code2wav.py` | Decoder stage with `chunked_decode_streaming()` | diff --git a/pyproject.toml b/pyproject.toml index 160c6bc2f..e9bdd3b29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -225,19 +225,14 @@ ignore = [ "torch.cuda.synchronize".msg = "Use torch.accelerator.synchronize" [tool.mypy] -python_version = "3.12, 3.13" -warn_return_any = true -warn_unused_configs = true -disallow_untyped_defs = true -disallow_incomplete_defs = true +python_version = "3.12" +# Align with vLLM: silent follow for library code; missing third-party stubs +# are ignored everywhere (including tests). tests/ still get +# --follow-imports skip from tools/pre_commit/mypy.py. +plugins = ["pydantic.mypy"] +ignore_missing_imports = true check_untyped_defs = true -disallow_untyped_decorators = true -no_implicit_optional = true -warn_redundant_casts = true -warn_unused_ignores = true -warn_no_return = true -warn_unreachable = true -strict_equality = true +follow_imports = "silent" [tool.pytest.ini_options] asyncio_mode = "auto" @@ -343,6 +338,8 @@ extend-ignore-identifiers-re = [ ] extend-ignore-re = [ "\\.\\.\\.n,d->\\.\\.\\.nd", + # setuptools-scm / git-describe local versions, e.g. 0.26.1.dev14+gae6577ea + "\\+g[0-9a-f]{7,}", ] [tool.typos.default.extend-words] ue = "ue" @@ -367,3 +364,5 @@ Leary = "Leary" fo = "fo" ba = "ba" SER = "SER" +# NumPy ndarray.flags.writeable (canonical flag name, not "writable"). +writeable = "writeable" diff --git a/tests/diffusion/test_diffusion_ipc.py b/tests/diffusion/test_diffusion_ipc.py index 546aa11e7..f24767e18 100644 --- a/tests/diffusion/test_diffusion_ipc.py +++ b/tests/diffusion/test_diffusion_ipc.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project import contextlib import multiprocessing as mp @@ -7,6 +7,7 @@ import queue import threading import time from multiprocessing import shared_memory +from multiprocessing.connection import Connection import numpy as np import pytest @@ -29,7 +30,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] def _large_numel(dtype: torch.dtype) -> int: - return (_SHM_TENSOR_THRESHOLD // torch.empty((), dtype=dtype).element_size()) + 1 + return int(_SHM_TENSOR_THRESHOLD // torch.empty((), dtype=dtype).element_size()) + 1 def _cleanup_shm_handle(value: object) -> None: @@ -38,7 +39,7 @@ def _cleanup_shm_handle(value: object) -> None: _unpack_if_shm_handle(value) -def _result_queue_worker(connection, rank: int) -> None: +def _result_queue_worker(connection: Connection, rank: int) -> None: """Send one nested NumPy result through a worker-owned MessageQueue.""" result_mq = MessageQueue(n_reader=1, n_local_reader=1, local_reader_ranks=[0]) try: @@ -309,11 +310,9 @@ def test_per_request_views_of_a_batch_tensor_do_not_ship_whole_storage() -> None """Regression: per-request slices used to pickle the whole batch storage. Each view is under the packing threshold, but pickle writes the entire - shared storage per view, so a batch's message grew by the full batch - tensor once per request and overflowed the ring buffer. + shared storage per view. Packing must key off storage nbytes so those + views become SHM handles instead of going on the wire. """ - import pickle - num_requests = 16 # Each row stays under the threshold; the shared storage is well over it. row_numel = _SHM_TENSOR_THRESHOLD // (2 * torch.empty((), dtype=torch.float32).element_size()) @@ -322,6 +321,13 @@ def test_per_request_views_of_a_batch_tensor_do_not_ship_whole_storage() -> None assert per_request.nelement() * per_request.element_size() <= _SHM_TENSOR_THRESHOLD assert per_request.untyped_storage().nbytes() > _SHM_TENSOR_THRESHOLD + packed_view = _pack_value_if_large(per_request) + try: + assert isinstance(packed_view, dict) + assert packed_view["__tensor_shm__"] is True + finally: + _cleanup_shm_handle(packed_view) + output = BatchRunnerOutput.from_list( [ RunnerOutput( @@ -332,16 +338,10 @@ def test_per_request_views_of_a_batch_tensor_do_not_ship_whole_storage() -> None for i in range(num_requests) ] ) - unpacked_size = len(pickle.dumps(output, protocol=pickle.HIGHEST_PROTOCOL)) pack_diffusion_output_shm(output) - packed_size = len(pickle.dumps(output, protocol=pickle.HIGHEST_PROTOCOL)) assert output.runner_outputs[0].result.trajectory_latents["__tensor_shm__"] is True - # Whole storage per request before; handles only after. - storage_bytes = per_request.untyped_storage().nbytes() - assert unpacked_size > num_requests * storage_bytes / 2 - assert packed_size < _SHM_TENSOR_THRESHOLD unpack_diffusion_output_shm(output) for i in range(num_requests): @@ -380,31 +380,37 @@ def test_pack_value_recurses_nested_dicts_and_lists_without_mutating_inline_valu large = torch.arange(_large_numel(torch.float32), dtype=torch.float32) small = torch.arange(8, dtype=torch.float32) list_tensor = torch.arange(_large_numel(torch.float32), dtype=torch.float32) + original_list = [list_tensor] payload = { "media": { "large": large, "small": small, }, - "list_value": [list_tensor], + "list_value": original_list, "metadata": {"prompt": "keep inline"}, } packed = _pack_value_if_large(payload) try: + assert isinstance(packed, dict) assert packed is not payload - assert packed["media"] is not payload["media"] - assert packed["media"]["large"]["__tensor_shm__"] is True - assert packed["media"]["small"] is small + media = packed["media"] + list_value = packed["list_value"] + assert isinstance(media, dict) + assert isinstance(list_value, list) + assert media is not payload["media"] + assert media["large"]["__tensor_shm__"] is True + assert media["small"] is small # Lists are recursed too: the large tensor inside is packed and a new # list is returned, while the input payload is left untouched. - assert packed["list_value"] is not payload["list_value"] - assert packed["list_value"][0]["__tensor_shm__"] is True - assert payload["list_value"][0] is list_tensor + assert list_value is not original_list + assert list_value[0]["__tensor_shm__"] is True + assert original_list[0] is list_tensor assert packed["metadata"] == {"prompt": "keep inline"} - torch.testing.assert_close(_unpack_if_shm_handle(packed["media"]["large"]), large) - torch.testing.assert_close(_unpack_if_shm_handle(packed["list_value"][0]), list_tensor) + torch.testing.assert_close(_unpack_if_shm_handle(media["large"]), large) + torch.testing.assert_close(_unpack_if_shm_handle(list_value[0]), list_tensor) finally: if isinstance(packed, dict): _cleanup_shm_handle(packed.get("media", {}).get("large")) diff --git a/tools/pre_commit/check_buildkite.py b/tools/pre_commit/check_buildkite.py new file mode 100644 index 000000000..362253494 --- /dev/null +++ b/tools/pre_commit/check_buildkite.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Validate .buildkite YAML against the official Buildkite schema. + +Test pipelines (test-ready.yml, etc.) use Omni-only keys such as +``mirror_hardwares`` and ``source_file_dependencies``. Those are expanded or +stripped the same way ``upload_pipeline.py`` does before upload, then the +rendered YAML is checked with check-jsonschema. +""" + +from __future__ import annotations + +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / ".buildkite" / "common" / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +from upload_pipeline import _render_test_pipeline # noqa: E402 + +SKIP_FILES = { + ".buildkite/common/ci_mirror_hardwares.yml", + ".buildkite/npu/pipeline-npu-a3.yml", +} + +# AMD pipelines are uploaded by a different path and use list-valued +# mirror_hardwares plus agent_pool/grade, which CUDA's expander rejects. +AMD_STRIP_KEYS = frozenset({"mirror_hardwares", "agent_pool", "grade", "source_file_dependencies"}) + + +def _normalize(path: str) -> str: + return path.replace("\\", "/") + + +def _strip_keys(obj: Any, keys: frozenset[str]) -> Any: + if isinstance(obj, dict): + return {k: _strip_keys(v, keys) for k, v in obj.items() if k not in keys} + if isinstance(obj, list): + return [_strip_keys(item, keys) for item in obj] + return obj + + +def _prepare_pipeline(path: Path) -> dict[str, Any]: + rel = _normalize(str(path.relative_to(ROOT) if path.is_absolute() else path)) + doc = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(doc, dict): + raise ValueError(f"{rel}: expected a mapping (pipeline YAML), got {type(doc).__name__}") + + if "/amd/" in f"/{rel}": + return _strip_keys(doc, AMD_STRIP_KEYS) + + return _render_test_pipeline(doc, changed_files=None) + + +def _validate(path: Path, doc: dict[str, Any]) -> int: + with tempfile.NamedTemporaryFile( + mode="w", + suffix=".yml", + encoding="utf-8", + delete=False, + ) as handle: + yaml.safe_dump(doc, handle, sort_keys=False) + tmp = Path(handle.name) + + try: + result = subprocess.run( + [ + sys.executable, + "-m", + "check_jsonschema", + "--builtin-schema", + "vendor.buildkite", + str(tmp), + ], + capture_output=True, + text=True, + ) + finally: + tmp.unlink(missing_ok=True) + + if result.returncode == 0: + return 0 + + rel = _normalize(str(path)) + sys.stderr.write(f"{rel}: Buildkite schema validation failed\n") + sys.stderr.write(result.stdout or "") + sys.stderr.write(result.stderr or "") + return 1 + + +def main(argv: list[str]) -> int: + rc = 0 + for raw in argv: + path = Path(raw) + rel = _normalize(str(path)) + if rel in SKIP_FILES: + continue + if not path.is_file(): + path = ROOT / path + try: + doc = _prepare_pipeline(path) + except Exception as exc: # noqa: BLE001 - surface expander errors as hook failures + sys.stderr.write(f"{rel}: {exc}\n") + rc = 1 + continue + rc |= _validate(path, doc) + return rc + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tools/pre_commit/check_forbidden_imports.py b/tools/pre_commit/check_forbidden_imports.py new file mode 100644 index 000000000..a3f5e320a --- /dev/null +++ b/tools/pre_commit/check_forbidden_imports.py @@ -0,0 +1,368 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Ban dangerous / policy-violating imports unless a file is allowlisted. + +Ported from vllm/tools/pre_commit/check_forbidden_imports.py. Existing +vllm_omni/ usages are grandfathered; add a path to the matching +allowed_files set only after a security/policy review. +""" + +import sys +from dataclasses import dataclass, field + +import regex as re + +# Hub entry points that must go through the vLLM-tagged repo_utils helpers. +_HF_NAMES = ( + r"HfApi|HfFileSystem|hf_hub_download|snapshot_download" + r"|list_repo_files|file_exists|try_to_load_from_cache" + r"|list_repo_refs|repo_exists" +) + +# Non-library trees: examples, tests, and tooling may keep stdlib imports. +_NON_LIBRARY_DIRS = { + "examples/", + "tests/", + "benchmarks/", + "apps/", + "docs/", + "tools/", + "scripts/", + ".buildkite/", + ".github/", +} + + +@dataclass +class ForbiddenImport: + pattern: str + tip: str + allowed_pattern: re.Pattern = re.compile(r"^$") # matches nothing by default + allowed_files: set[str] = field(default_factory=set) + allowed_dirs: set[str] = field(default_factory=set) + + +CHECK_IMPORTS = { + "pickle/cloudpickle": ForbiddenImport( + pattern=( + r"^\s*(import\s+(?:[\w.]+\s*(?:as\s+\w+)?\s*,\s*)*(pickle|cloudpickle)\b" + r"|from\s+(pickle|cloudpickle)\s+import\b)" + ), + tip=("Avoid using pickle or cloudpickle or add this file to tools/pre_commit/check_forbidden_imports.py."), + allowed_files={ + # STOP AND READ BEFORE YOU ADD ANYTHING ELSE TO THIS LIST: + # pickle/cloudpickle are unsafe when deserializing untrusted data. + "tests/diffusion/attention/test_attention_sp.py", + "tests/helpers/process.py", + "vllm_omni/diffusion/distributed/group_coordinator.py", + }, + ), + "base64": ForbiddenImport( + pattern=r"^\s*(?:import\s+base64(?:$|\s|,)|from\s+base64\s+import)", + tip=("Replace 'import base64' with 'import pybase64' or 'import pybase64 as base64'."), + allowed_pattern=re.compile(r"^\s*import\s+pybase64(\s*|\s+as\s+base64\s*)$"), + allowed_dirs=_NON_LIBRARY_DIRS, + allowed_files={ + "vllm_omni/benchmarks/data_modules/daily_omni_dataset.py", + "vllm_omni/benchmarks/data_modules/random_multi_modal_dataset.py", + "vllm_omni/benchmarks/data_modules/seed_tts_dataset.py", + "vllm_omni/entrypoints/openai/api_server.py", + "vllm_omni/entrypoints/openai/audio_utils_mixin.py", + "vllm_omni/entrypoints/openai/image_api_utils.py", + "vllm_omni/entrypoints/openai/protocol/images.py", + "vllm_omni/entrypoints/openai/realtime_connection.py", + "vllm_omni/entrypoints/openai/serving_chat.py", + "vllm_omni/entrypoints/openai/serving_speech.py", + "vllm_omni/entrypoints/openai/serving_speech_stream.py", + "vllm_omni/entrypoints/openai/video_api_utils.py", + "vllm_omni/entrypoints/openai/video_stream_base.py", + "vllm_omni/entrypoints/openai/video_stream_session.py", + "vllm_omni/experimental/fullduplex/client.py", + "vllm_omni/experimental/fullduplex/engine/contracts.py", + "vllm_omni/experimental/fullduplex/joyvl/memory/memory.py", + "vllm_omni/experimental/fullduplex/minicpmo45/adapter.py", + "vllm_omni/experimental/fullduplex/minicpmo45/input.py", + "vllm_omni/experimental/fullduplex/minicpmo45/runtime.py", + "vllm_omni/experimental/fullduplex/minicpmo45/stage0.py", + "vllm_omni/experimental/fullduplex/openai/audio.py", + "vllm_omni/experimental/fullduplex/openai/realtime_input.py", + "vllm_omni/experimental/fullduplex/openai/runtime_bridge.py", + "vllm_omni/experimental/fullduplex/openai/serving.py", + "vllm_omni/experimental/fullduplex/personaplex/input.py", + "vllm_omni/experimental/fullduplex/personaplex/runtime_extension.py", + "vllm_omni/experimental/fullduplex/personaplex/stage0.py", + "vllm_omni/model_executor/models/indextts2/tokenizer_v2_5.py", + "vllm_omni/model_executor/models/qwen3_tts/prompt_embeds_builder.py", + "vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_tokenizer.py", + "vllm_omni/tokenizers/mammoth_moda2_tokenizer.py", + "vllm_omni/transformers_utils/processors/audiox.py", + }, + ), + "re": ForbiddenImport( + pattern=r"^\s*(?:import\s+re(?:$|\s|,)|from\s+re\s+import)", + tip="Replace 'import re' with 'import regex as re' or 'import regex'.", + allowed_pattern=re.compile(r"^\s*import\s+regex(\s*|\s+as\s+re\s*)$"), + allowed_dirs=_NON_LIBRARY_DIRS, + allowed_files={ + "setup.py", + "vllm_omni/benchmarks/data_modules/daily_omni_eval.py", + "vllm_omni/config/stage_config.py", + "vllm_omni/diffusion/layers/mot/ops/mot_gemm.py", + "vllm_omni/diffusion/model_loader/diffusers_loader.py", + "vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py", + "vllm_omni/diffusion/models/dreamid_omni/fusion.py", + "vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py", + "vllm_omni/diffusion/models/glm_image/pipeline_glm_image.py", + "vllm_omni/diffusion/models/gr00t/modeling/processing_gr00t_n1d7.py", + "vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5.py", + "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", + "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py", + "vllm_omni/diffusion/models/longcat_video/pipeline_longcat_video_avatar.py", + "vllm_omni/diffusion/models/minimax_h3/encoder.py", + "vllm_omni/diffusion/models/nextstep_1_1/pipeline_nextstep_1_1.py", + "vllm_omni/diffusion/models/omnivoice/pipeline_omnivoice.py", + "vllm_omni/diffusion/models/soulx_singer/modules/preprocess/asr.py", + "vllm_omni/diffusion/models/soulx_singer/preprocess/g2p.py", + "vllm_omni/diffusion/profiler/diffusion_pipeline_profiler.py", + "vllm_omni/distributed/omni_connectors/utils/env.py", + "vllm_omni/entrypoints/cli/logo.py", + "vllm_omni/entrypoints/openai/serving_speech.py", + "vllm_omni/experimental/fullduplex/joyvl/memory/memory.py", + "vllm_omni/model_executor/models/common/whisper_vq.py", + "vllm_omni/model_executor/models/fish_speech/prompt_utils.py", + "vllm_omni/model_executor/models/glm_tts/text_frontend.py", + "vllm_omni/model_executor/models/higgs_audio_v2/higgs_audio_v2_tokenizer.py", + "vllm_omni/model_executor/models/indextts2/indextts2_talker.py", + "vllm_omni/model_executor/models/indextts2/text_processing_v2_5.py", + "vllm_omni/model_executor/models/indextts2/utils/common.py", + "vllm_omni/model_executor/models/ming_flash_omni/text_processing.py", + "vllm_omni/model_executor/models/ming_flash_omni/vision_encoder.py", + "vllm_omni/model_executor/models/ming_tts/prompt_assembly.py", + "vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/ear_tts_commons.py", + "vllm_omni/model_executor/stage_input_processors/ming_flash_omni.py", + "vllm_omni/outputs/output_modality.py", + "vllm_omni/quantization/mxfp4_config.py", + "vllm_omni/quantization/tools/merge_mxfp4_dualscale_checkpoint.py", + "vllm_omni/utils/custom_voice_io.py", + }, + ), + "triton": ForbiddenImport( + pattern=r"^(from|import)\s+triton(\s|\.|$)", + tip="Use 'from vllm.triton_utils import triton' instead.", + allowed_pattern=re.compile(r"from vllm.triton_utils import (triton|tl|tl, triton)"), + ), + "tilelang": ForbiddenImport( + pattern=r"^(from|import)\s+tilelang(\s|\.|$)", + tip="Use 'from vllm.tilelang_utils import tilelang, T' instead.", + allowed_pattern=re.compile( + r"from\s+vllm\.tilelang_utils\s+import\s+" + r"(tilelang|T|T, tilelang|tilelang, T)\b" + ), + ), + "huggingface_hub repo API": ForbiddenImport( + # Catch `from huggingface_hub import `, including parenthesized, + # multi-line imports. + pattern=( + r"^\s*from\s+huggingface_hub\s+import\s*\([^)]*\b(?:" + _HF_NAMES + r")\b" + r"|" + r"^\s*from\s+huggingface_hub\s+import\b[^\n]*\b(?:" + _HF_NAMES + r")\b" + ), + tip=( + "Use the shared, vLLM-tagged helpers from " + "vllm.transformers_utils.repo_utils (e.g. hf_api(), hf_fs(), " + "list_repo_files, file_exists) instead of calling " + "huggingface_hub directly." + ), + allowed_dirs=_NON_LIBRARY_DIRS, + allowed_files={ + "vllm_omni/benchmarks/data_modules/daily_omni_dataset.py", + "vllm_omni/benchmarks/data_modules/seed_tts_dataset.py", + "vllm_omni/benchmarks/data_modules/seed_tts_eval.py", + "vllm_omni/diffusion/lora/loader.py", + "vllm_omni/diffusion/model_loader/hub_prefetch.py", + "vllm_omni/diffusion/models/cosmos3/sound_tokenizer.py", + "vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py", + "vllm_omni/diffusion/models/helios/pipeline_helios.py", + "vllm_omni/diffusion/models/longcat_video/pipeline_longcat_video_avatar.py", + "vllm_omni/diffusion/models/ltx2/ltx2_components.py", + "vllm_omni/diffusion/models/magi_human/pipeline_magi_human.py", + "vllm_omni/diffusion/models/omnivoice/pipeline_omnivoice.py", + "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", + "vllm_omni/diffusion/models/sensenova_u1/pipeline_sensenova_u1.py", + "vllm_omni/diffusion/models/soulx_singer/utils.py", + "vllm_omni/diffusion/models/utils.py", + "vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py", + "vllm_omni/engine/arg_utils.py", + "vllm_omni/engine/stage_init_utils.py", + "vllm_omni/entrypoints/openai/api_server.py", + "vllm_omni/entrypoints/openai/serving_speech.py", + "vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py", + "vllm_omni/experimental/fullduplex/personaplex/runtime.py", + "vllm_omni/experimental/fullduplex/personaplex/serving/server.py", + "vllm_omni/model_executor/model_loader/weight_utils.py", + "vllm_omni/model_executor/models/audex/checkpoint.py", + "vllm_omni/model_executor/models/bagel/bagel.py", + "vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py", + "vllm_omni/model_executor/models/covo_audio/covo_audio_code2wav.py", + "vllm_omni/model_executor/models/dots_tts/dots_tts_talker.py", + "vllm_omni/model_executor/models/dynin_omni/dynin_omni_common.py", + "vllm_omni/model_executor/models/dynin_omni/dynin_omni_token2audio.py", + "vllm_omni/model_executor/models/glm_tts/glm_tts.py", + "vllm_omni/model_executor/models/higgs_audio_v2/higgs_audio_v2_code2wav.py", + "vllm_omni/model_executor/models/higgs_audio_v2/higgs_audio_v2_tokenizer.py", + "vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_code2wav.py", + "vllm_omni/model_executor/models/indextts2/preprocess_utils.py", + "vllm_omni/model_executor/models/indextts2/s2mel/modules/bigvgan.py", + "vllm_omni/model_executor/models/indextts2/tokenizer_v2_5.py", + "vllm_omni/model_executor/models/ming_tts/speaker_extractor.py", + "vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py", + "vllm_omni/model_executor/models/omnivoice/omnivoice.py", + "vllm_omni/model_executor/models/personaplex/personaplex_mimi.py", + "vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py", + "vllm_omni/model_executor/models/voxtral_tts/voxtral_tts.py", + "vllm_omni/transformers_utils/configs/higgs_audio_v3.py", + }, + ), +} + + +def _normalize_path(path: str) -> str: + return path.replace("\\", "/") + + +def check_file(path: str) -> int: + path = _normalize_path(path) + with open(path, encoding="utf-8") as f: + content = f.read() + return_code = 0 + # Check all patterns in the whole file + for import_name, forbidden_import in CHECK_IMPORTS.items(): + # Skip files that are allowed for this import + if path in forbidden_import.allowed_files: + continue + # Skip directories that are allowed for this import + if any(path.startswith(prefix) for prefix in forbidden_import.allowed_dirs): + continue + # Search for forbidden imports + for match in re.finditer(forbidden_import.pattern, content, re.MULTILINE): + # Check if it's allowed + if forbidden_import.allowed_pattern.match(match.group()): + continue + # Skip matches inside a comment + line_start = content.rfind("\n", 0, match.start()) + 1 + if "#" in content[line_start : match.start()]: + continue + # Calculate line number from match position + line_num = content[: match.start() + 1].count("\n") + 1 + print( + f"{path}:{line_num}: " + "\033[91merror:\033[0m " # red color + f"Found forbidden import: {import_name}. {forbidden_import.tip}" + ) + return_code = 1 + return return_code + + +def main(): + returncode = 0 + for path in sys.argv[1:]: + returncode |= check_file(path) + return returncode + + +def test_regex(): + def matches(rule: str, content: str) -> bool: + return bool(re.search(CHECK_IMPORTS[rule].pattern, content, re.MULTILINE)) + + pickle_cases = [ + # Should match + ("import pickle", True), + ("import cloudpickle", True), + ("import pickle as pkl", True), + ("import cloudpickle as cpkl", True), + ("from pickle import *", True), + ("from cloudpickle import dumps", True), + ("from pickle import dumps, loads", True), + ("from cloudpickle import (dumps, loads)", True), + (" import pickle", True), + ("\timport cloudpickle", True), + ("from pickle import loads", True), + ("import pickle, os", True), + ("import os, pickle", True), + ("import os, pickle as pkl", True), + ("import pickle as pkl, os", True), + ("import os.path, cloudpickle", True), + ("import cloudpickle, json", True), + # Should not match + ("import somethingelse", False), + ("from somethingelse import pickle", False), + ("# import pickle", False), + ("print('import pickle')", False), + ("import pickleas as asdf", False), + ] + for i, (content, should_match) in enumerate(pickle_cases): + result = matches("pickle/cloudpickle", content) + assert result == should_match, f"pickle case {i} failed: {content!r} (expected {should_match}, got {result})" + + tilelang_cases = [ + # Should match + ("import tilelang", True), + ("import tilelang.language as T", True), + ("from tilelang.jit import JITImpl", True), + ("from tilelang.jit.kernel import JITKernel", True), + # Should not match: indented (local) imports are allowed, mirroring + # the "triton" rule, so mocked-module test imports are not flagged. + (" import tilelang", False), + (" from tilelang.jit import JITImpl", False), + ("from vllm.tilelang_utils import tilelang", False), + ("from vllm.tilelang_utils import T", False), + ("from vllm.tilelang_utils import T, tilelang", False), + ("from vllm.tilelang_utils import tilelang, T", False), + ("import tilelang_kernels", False), + ] + for i, (content, should_match) in enumerate(tilelang_cases): + result = matches("tilelang", content) + assert result == should_match, f"tilelang case {i} failed: {content!r} (expected {should_match}, got {result})" + + hf_cases = [ + # Should match + ("from huggingface_hub import snapshot_download", True), + ("from huggingface_hub import hf_hub_download", True), + ("from huggingface_hub import HfApi", True), + ("from huggingface_hub import HfFileSystem", True), + ("from huggingface_hub import list_repo_files", True), + ("from huggingface_hub import try_to_load_from_cache", True), + (" from huggingface_hub import snapshot_download", True), + ("from huggingface_hub import PyTorchModelHubMixin, hf_hub_download", True), + ("from huggingface_hub import (snapshot_download)", True), + # Parenthesized multi-line import must not bypass the hook + ("from huggingface_hub import (\n snapshot_download,\n)", True), + ( + "from huggingface_hub import (\n PyTorchModelHubMixin,\n HfApi,\n)", + True, + ), + # Should not match + ("import huggingface_hub", False), + ("import huggingface_hub as hf", False), + ("from huggingface_hub import PyTorchModelHubMixin", False), + ("from huggingface_hub.constants import HF_HUB_CACHE", False), + ("from huggingface_hub.utils import EntryNotFoundError", False), + ("from vllm.transformers_utils.repo_utils import hf_api", False), + ("from huggingface_hub import (\n PyTorchModelHubMixin,\n)", False), + ("# from huggingface_hub import snapshot_download", False), + ] + for i, (content, should_match) in enumerate(hf_cases): + result = matches("huggingface_hub repo API", content) + assert result == should_match, ( + f"huggingface_hub case {i} failed: {content!r} (expected {should_match}, got {result})" + ) + + print("All regex tests passed.") + + +if __name__ == "__main__": + if "--test-regex" in sys.argv: + test_regex() + else: + sys.exit(main()) diff --git a/tools/pre_commit/check_pickle_imports.py b/tools/pre_commit/check_pickle_imports.py deleted file mode 100644 index f8f0ef8f2..000000000 --- a/tools/pre_commit/check_pickle_imports.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import sys - -import regex as re - -# List of files (relative to repo root) that are allowed to import pickle or -# cloudpickle -# -# STOP AND READ BEFORE YOU ADD ANYTHING ELSE TO THIS LIST: -# The pickle and cloudpickle modules are known to be unsafe when deserializing -# data from potentially untrusted parties. They have resulted in multiple CVEs -# for vLLM and numerous vulnerabilities in the Python ecosystem more broadly. -# Before adding new uses of pickle/cloudpickle, please consider safer -# alternatives like msgpack or pydantic that are already in use in vLLM. Only -# add to this list if absolutely necessary and after careful security review. -ALLOWED_FILES = { - "tests/helpers/process.py", - "vllm_omni/diffusion/distributed/group_coordinator.py", - "tests/diffusion/attention/test_attention_sp.py", - # Measures the wire size of diffusion outputs. The transport pickles them, - # and only pickle reproduces the per-view storage duplication under test - # (torch.save deduplicates shared storages). No untrusted data is loaded. - "tests/diffusion/test_diffusion_ipc.py", -} - -PICKLE_RE = re.compile( - r"^\s*(import\s+(pickle|cloudpickle)(\s|$|\sas)" - r"|from\s+(pickle|cloudpickle)\s+import\b)" -) - - -def scan_file(path: str) -> int: - with open(path, encoding="utf-8") as f: - for i, line in enumerate(f, 1): - if PICKLE_RE.match(line): - print( - f"{path}:{i}: " - "\033[91merror:\033[0m " # red color - "Found pickle/cloudpickle import" - ) - return 1 - return 0 - - -def main(): - returncode = 0 - for filename in sys.argv[1:]: - if filename in ALLOWED_FILES: - continue - returncode |= scan_file(filename) - return returncode - - -def test_regex(): - test_cases = [ - # Should match - ("import pickle", True), - ("import cloudpickle", True), - ("import pickle as pkl", True), - ("import cloudpickle as cpkl", True), - ("from pickle import *", True), - ("from cloudpickle import dumps", True), - ("from pickle import dumps, loads", True), - ("from cloudpickle import (dumps, loads)", True), - (" import pickle", True), - ("\timport cloudpickle", True), - ("from pickle import loads", True), - # Should not match - ("import somethingelse", False), - ("from somethingelse import pickle", False), - ("# import pickle", False), - ("print('import pickle')", False), - ("import pickleas as asdf", False), - ] - for i, (line, should_match) in enumerate(test_cases): - result = bool(PICKLE_RE.match(line)) - assert result == should_match, f"Test case {i} failed: '{line}' (expected {should_match}, got {result})" - print("All regex tests passed.") - - -if __name__ == "__main__": - if "--test-regex" in sys.argv: - test_regex() - else: - sys.exit(main()) diff --git a/tools/pre_commit/check_spdx_header.py b/tools/pre_commit/check_spdx_header.py new file mode 100644 index 000000000..5bfb26034 --- /dev/null +++ b/tools/pre_commit/check_spdx_header.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Require SPDX license headers on source files. + +Ported from vllm/tools/pre_commit/check_spdx_header.py. The copyright line +uses the vLLM-Omni project name rather than upstream vLLM. Shell scripts +(``.sh``) are included so wrappers such as ``shellcheck.sh`` cannot keep a +stale upstream copyright that Python/Rust/proto files would have rewritten. +``.pyi`` stubs use the same ``#`` header style as ``.py``. +""" + +import sys +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + +# Older files still carry the upstream vLLM copyright line. Treat it as stale +# and rewrite it to the Omni line instead of inserting a duplicate header. +LEGACY_COPYRIGHT_TEXT = "SPDX-FileCopyrightText: Copyright contributors to the vLLM project" + + +@dataclass(frozen=True) +class HeaderStyle: + """Comment syntax and preamble handling for an SPDX header.""" + + comment_prefix: str + preserve_shebang: bool = False + + +class SPDXStatus(Enum): + """SPDX header status enumeration""" + + EMPTY = "empty" # empty __init__.py + COMPLETE = "complete" + MISSING_LICENSE = "missing_license" # Only has copyright line + MISSING_COPYRIGHT = "missing_copyright" # Only has license line + MISSING_BOTH = "missing_both" # Completely missing + + +LICENSE_TEXT = "SPDX-License-Identifier: Apache-2.0" +COPYRIGHT_TEXT = "SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project" +FILE_STYLES = { + ".py": HeaderStyle("#", preserve_shebang=True), + ".pyi": HeaderStyle("#"), + ".sh": HeaderStyle("#", preserve_shebang=True), + ".rs": HeaderStyle("//"), + ".proto": HeaderStyle("//"), +} + + +def file_style(file_path): + """Return the declared header style for a file.""" + suffix = Path(file_path).suffix + try: + return FILE_STYLES[suffix] + except KeyError: + raise ValueError(f"Unsupported file type: {file_path}") from None + + +def spdx_header(style): + """Return the SPDX header for a file style.""" + license_line = f"{style.comment_prefix} {LICENSE_TEXT}" + copyright_line = f"{style.comment_prefix} {COPYRIGHT_TEXT}" + return license_line, copyright_line + + +def legacy_copyright_line(style): + """Return the stale upstream vLLM copyright line for a file style.""" + return f"{style.comment_prefix} {LEGACY_COPYRIGHT_TEXT}" + + +def header_insertion_index(style, lines): + """Return the line index where a missing header should be inserted.""" + if style.preserve_shebang and lines and lines[0].startswith("#!"): + return 1 + return 0 + + +def check_spdx_header_status(file_path): + """Check SPDX header status of the file""" + style = file_style(file_path) + license_line, copyright_line = spdx_header(style) + with open(file_path, encoding="UTF-8") as file: + lines = file.readlines() + if not lines: + # Empty file + return SPDXStatus.EMPTY + + has_license = False + has_copyright = False + + # Check all lines for SPDX headers (not just the first two) + for raw_line in lines: + line = raw_line.strip() + if line == license_line: + has_license = True + elif line == copyright_line: + has_copyright = True + + # Determine status based on what we found + if has_license and has_copyright: + return SPDXStatus.COMPLETE + elif has_license and not has_copyright: + # Only has license line (or a stale vLLM copyright line) + return SPDXStatus.MISSING_COPYRIGHT + elif not has_license and has_copyright: + # Only has copyright line + return SPDXStatus.MISSING_LICENSE + else: + # Completely missing both lines + return SPDXStatus.MISSING_BOTH + + +def add_header(file_path, status): + """Add or supplement SPDX header based on status""" + style = file_style(file_path) + license_line, copyright_line = spdx_header(style) + stale_copyright_line = legacy_copyright_line(style) + full_spdx_header = f"{license_line}\n{copyright_line}" + with open(file_path, "r+", encoding="UTF-8") as file: + lines = file.readlines() + file.seek(0, 0) + file.truncate() + + if status == SPDXStatus.MISSING_BOTH: + # Completely missing, add complete header + insertion_index = header_insertion_index(style, lines) + file.writelines(lines[:insertion_index]) + file.write(full_spdx_header + "\n") + remaining_lines = lines[insertion_index:] + if remaining_lines and remaining_lines[0].strip(): + file.write("\n") + file.writelines(remaining_lines) + + elif status == SPDXStatus.MISSING_COPYRIGHT: + # Only has license line, need to add copyright line. + # Rewrite a stale upstream vLLM copyright line in place. + replaced = False + for i, line in enumerate(lines): + if line.strip() == stale_copyright_line: + lines[i] = f"{copyright_line}\n" + replaced = True + break + if not replaced: + for i, line in enumerate(lines): + if line.strip() == license_line: + lines.insert(i + 1, f"{copyright_line}\n") + break + + file.writelines(lines) + + elif status == SPDXStatus.MISSING_LICENSE: + # Only has copyright line, need to add license line + # Find the copyright line and add license line before it + for i, line in enumerate(lines): + if line.strip() == copyright_line: + # Insert license line before copyright line + lines.insert(i, f"{license_line}\n") + break + file.writelines(lines) + + +def main(): + """Main function""" + files_missing_both = [] + files_missing_copyright = [] + files_missing_license = [] + + for file_path in sys.argv[1:]: + if Path(file_path).suffix not in FILE_STYLES: + continue + status = check_spdx_header_status(file_path) + + if status == SPDXStatus.MISSING_BOTH: + files_missing_both.append(file_path) + elif status == SPDXStatus.MISSING_COPYRIGHT: + files_missing_copyright.append(file_path) + elif status == SPDXStatus.MISSING_LICENSE: + files_missing_license.append(file_path) + else: + continue + + # Collect all files that need fixing + all_files_to_fix = files_missing_both + files_missing_copyright + files_missing_license + if all_files_to_fix: + print("The following files are missing the SPDX header:") + if files_missing_both: + for file_path in files_missing_both: + print(f" {file_path}") + add_header(file_path, SPDXStatus.MISSING_BOTH) + + if files_missing_copyright: + for file_path in files_missing_copyright: + print(f" {file_path}") + add_header(file_path, SPDXStatus.MISSING_COPYRIGHT) + if files_missing_license: + for file_path in files_missing_license: + print(f" {file_path}") + add_header(file_path, SPDXStatus.MISSING_LICENSE) + + sys.exit(1 if all_files_to_fix else 0) + + +if __name__ == "__main__": + main() diff --git a/tools/pre_commit/check_torch_cuda.py b/tools/pre_commit/check_torch_cuda.py new file mode 100644 index 000000000..a298baae5 --- /dev/null +++ b/tools/pre_commit/check_torch_cuda.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Ban direct torch.cuda.* usage outside an allowlist. + +Modeled on vllm/tools/pre_commit/check_torch_cuda.py. Call sites should use +current_omni_platform / OmniPlatform instead of CUDA-only torch.cuda APIs. +""" + +import sys + +import regex as re + +# Match `torch.cuda.xxx` device/memory/seed helpers that have portable Omni +# platform equivalents. CUDAGraph / is_available / stream capture are not in +# this list (same split as upstream vLLM). +_TORCH_CUDA_PATTERNS = [ + r"\btorch\.cuda\.(empty_cache|synchronize|device_count|current_device|memory_reserved|memory_allocated|max_memory_allocated|max_memory_reserved|reset_peak_memory_stats|memory_stats|mem_get_info|set_device)\b", + # Match torch.cuda.device(...) including args. `device\()\b` misses + # `device()` (no word char after `)`) and `device(0)`. + r"\btorch\.cuda\.device\s*\(", + r"\btorch\.cuda\.(manual_seed|manual_seed_all)\b", + r"\bwith\storch\.cuda\.device\b", + # Calls torch.cuda.{_is_compiled/_device_count_amdsmi/_device_count_nvml} internally + r"\bcuda_device_count_stateless\(\)\b", +] + +# Platform adapters must call native torch.cuda / torch.npu / torch.xpu APIs. +ALLOWED_PREFIXES = { + "vllm_omni/platforms/", +} + +# Grandfathered call sites. Remove a path after migrating it to +# current_omni_platform / OmniPlatform. Do not add new files without review. +ALLOWED_FILES = { + "benchmarks/kernels/mot_linear_benchmarks.py", + "examples/online_serving/qwen2_5_omni/gradio_demo.py", + "examples/online_serving/qwen3_omni/gradio_demo.py", + "tests/dfx/reliability/helpers.py", + "tests/diffusion/distributed/test_cfg_parallel.py", + "tests/diffusion/distributed/test_pipeline_parallel.py", + "tests/e2e/accuracy/wan22_i2v/run_wan22_i2v_diffusers_cp.py", + "tests/e2e/features/custom_pipeline/test_async_omni_collective_rpc.py", + "tests/e2e/features/fullduplex/test_personaplex_elastic.py", + "tests/entrypoints/test_omni_sleep_mode.py", + "tools/configure_stage_memory.py", + "vllm_omni/diffusion/models/bagel/pipeline_bagel.py", + "vllm_omni/diffusion/models/lance/pipeline_lance.py", + "vllm_omni/diffusion/models/magi_human/pipeline_magi_human.py", + "vllm_omni/diffusion/models/nextstep_1_1/pipeline_nextstep_1_1.py", + "vllm_omni/distributed/omni_connectors/connectors/mooncake_transfer_engine_connector.py", + "vllm_omni/distributed/omni_connectors/connectors/mori_transfer_engine_connector.py", + "vllm_omni/experimental/ar_diffusion/runner.py", + "vllm_omni/model_executor/models/moss_tts/cuda_graph_streaming_decoder_wrapper.py", + "vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py", + "vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py", + "vllm_omni/worker/gpu_ar_model_runner.py", +} + +_GENERIC_TIP = ( + "Found torch.cuda API call. Use current_omni_platform / OmniPlatform " + "(empty_cache, synchronize, get_device_count, current_device, " + "get_device_memory, set_device) instead." +) +_SEED_TIP = ( + "Found CUDA manual-seed API call. Use " + "torch.Generator(device=current_omni_platform.device_type).manual_seed(...) " + "instead." +) + + +def _normalize_path(path: str) -> str: + return path.replace("\\", "/") + + +def _is_allowed(path: str) -> bool: + if any(path.startswith(prefix) for prefix in ALLOWED_PREFIXES): + return True + return path in ALLOWED_FILES + + +def scan_file(path: str) -> int: + with open(path, encoding="utf-8") as f: + content = f.read() + for pattern in _TORCH_CUDA_PATTERNS: + for match in re.finditer(pattern, content, re.MULTILINE): + line_start = content.rfind("\n", 0, match.start()) + 1 + if "#" in content[line_start : match.start()]: + continue + line_num = content[: match.start() + 1].count("\n") + 1 + matched_text = match.group(0) + tip = _SEED_TIP if "manual_seed" in matched_text else _GENERIC_TIP + print(f"{path}:{line_num}: \033[91merror:\033[0m {tip}") + return 1 + return 0 + + +def main(): + returncode = 0 + for filename in sys.argv[1:]: + path = _normalize_path(filename) + if _is_allowed(path): + continue + returncode |= scan_file(path) + return returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py new file mode 100644 index 000000000..e24f62444 --- /dev/null +++ b/tools/pre_commit/mypy.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Run mypy on changed files. + +Ported from vllm/tools/pre_commit/mypy.py. Group files so tests can use +``follow_imports=skip`` without pulling that setting into library code. + +Usage: + python tools/pre_commit/mypy.py + +Args: + python_version: Python version to use (e.g. "3.10") or "local" to use + the interpreter running this script. + changed_files: List of changed files to check. +""" + +from __future__ import annotations + +import subprocess +import sys + +import regex as re + +# tests/ uses follow_imports=skip so third-party and library errors do not +# leak into test type-checking. After a directory is clean under skip, it +# can join the default group (follow_imports from pyproject.toml). +SEPARATE_GROUPS = [ + "tests", +] + +# Model trees are large third-party ports. Drop a prefix from this list when +# that directory is ready to be type-checked (same ratchet as upstream vLLM). +EXCLUDE = [ + r"vllm_omni/model_executor/models/", + r"vllm_omni/diffusion/models/", +] + + +def _normalize_path(path: str) -> str: + return path.replace("\\", "/") + + +def group_files(changed_files: list[str]) -> dict[str, list[str]]: + """Group changed files into different mypy calls.""" + exclude_pattern = re.compile(f"^{'|'.join(EXCLUDE)}.*") + file_groups: dict[str, list[str]] = {"": []} + file_groups.update({k: [] for k in SEPARATE_GROUPS}) + # Longest path first so a sub-directory is not shadowed by its parent. + separate_groups = sorted(SEPARATE_GROUPS, key=len, reverse=True) + for changed_file in changed_files: + changed_file = _normalize_path(changed_file) + if exclude_pattern.match(changed_file): + continue + for directory in separate_groups: + if re.match(f"^{directory}.*", changed_file): + file_groups[directory].append(changed_file) + break + else: + if changed_file.startswith(("vllm_omni/", "tests/")): + file_groups[""].append(changed_file) + return file_groups + + +def mypy( + targets: list[str], + python_version: str | None, + follow_imports: str | None, + file_group: str, +) -> int: + """Run mypy on the given targets.""" + args = ["mypy"] + if python_version is not None: + args += ["--python-version", python_version] + if follow_imports is not None: + args += ["--follow-imports", follow_imports] + print(f"$ {' '.join(args)} {file_group}") + return subprocess.run(args + targets, check=False).returncode + + +def main() -> int: + python_version = sys.argv[1] + file_groups = group_files(sys.argv[2:]) + + if python_version == "local": + python_version = f"{sys.version_info.major}.{sys.version_info.minor}" + + returncode = 0 + for file_group, changed_files in file_groups.items(): + follow_imports = None if file_group == "" else "skip" + if changed_files: + returncode |= mypy(changed_files, python_version, follow_imports, file_group) + return returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/pre_commit/shellcheck.sh b/tools/pre_commit/shellcheck.sh new file mode 100755 index 000000000..7794ffe51 --- /dev/null +++ b/tools/pre_commit/shellcheck.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +# +# Lint bash scripts for undefined vars, quoting, and similar bugs. +# Do not download a binary: use a distro/package-manager install so the +# artifact is signed (apt/dnf/brew) or at least an explicit local install. +set -euo pipefail + +is_windows_exe() { + [[ "$1" == *.exe ]] +} + +is_git_bash() { + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) return 0 ;; + *) return 1 ;; + esac +} + +find_native_shellcheck() { + local cand + cand="$(command -v shellcheck 2>/dev/null || true)" + if [ -n "$cand" ] && ! is_windows_exe "$cand"; then + echo "$cand" + return 0 + fi + return 1 +} + +install_hint() { + echo "Please install shellcheck with your package manager, then re-run:" + echo " Debian/Ubuntu/WSL: sudo apt-get install shellcheck" + echo " Fedora: sudo dnf install ShellCheck" + echo " macOS: brew install shellcheck" + echo " Git Bash: scoop install shellcheck" + echo " https://github.com/koalaman/shellcheck?tab=readme-ov-file#installing" +} + +SHELLCHECK_BIN="" +if SHELLCHECK_BIN="$(find_native_shellcheck)"; then + : +elif is_git_bash && SHELLCHECK_BIN="$(command -v shellcheck.exe 2>/dev/null || true)" && [ -n "$SHELLCHECK_BIN" ]; then + : +else + install_hint + exit 1 +fi + +should_lint() { + local f="${1//\\//}" + f="${f#./}" + case "$f" in + *.sh) ;; + *) return 1 ;; + esac + git check-ignore -q "$f" && return 1 + return 0 +} + +run_shellcheck() { + local f + for f in "$@"; do + if should_lint "$f"; then + "$SHELLCHECK_BIN" -s bash "$f" + fi + done +} + +if [ "$#" -gt 0 ]; then + run_shellcheck "$@" + exit 0 +fi + +# Direct invocation with no args: lint every tracked *.sh. +while IFS= read -r -d '' f || [ -n "$f" ]; do + git check-ignore -q "$f" && continue + "$SHELLCHECK_BIN" -s bash "$f" +done < <(find . -path ./.git -prune -o -name "*.sh" -print0) diff --git a/vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py b/vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py index 53d3eb9b9..08e9e0e4a 100644 --- a/vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py +++ b/vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Fused MiniMax H3 modulation with FP32 accumulation.""" from __future__ import annotations import torch -import triton -import triton.language as tl +from vllm.triton_utils import tl, triton @triton.jit diff --git a/vllm_omni/model_executor/models/indextts2/gpt/conformer_encoder.py b/vllm_omni/model_executor/models/indextts2/gpt/conformer_encoder.py index 67eebbab3..6ba4ff00b 100644 --- a/vllm_omni/model_executor/models/indextts2/gpt/conformer_encoder.py +++ b/vllm_omni/model_executor/models/indextts2/gpt/conformer_encoder.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project import torch import torch.nn as nn @@ -175,7 +175,7 @@ class ConformerEncoderLayer(nn.Module): instance. `PositionwiseFeedForward` instance can be used as the argument. conv_module (torch.nn.Module): Convolution module instance. - `ConvlutionModule` instance can be used as the argument. + `ConvolutionModule` instance can be used as the argument. dropout_rate (float): Dropout rate. normalize_before (bool): True: use layer_norm before each sub-block. diff --git a/vllm_omni/model_executor/models/minimax_music3/prompt.py b/vllm_omni/model_executor/models/minimax_music3/prompt.py index b9093eae9..25e18fd7e 100644 --- a/vllm_omni/model_executor/models/minimax_music3/prompt.py +++ b/vllm_omni/model_executor/models/minimax_music3/prompt.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project # # Prompt normalization follows the MiniMaxAI/MiniMax-Music3 input contract. # The transformations are reproduced byte-for-byte because any divergence @@ -19,9 +19,10 @@ Both rows therefore have identical length and identical audio-start position. from __future__ import annotations -import re from typing import Any +import regex as re + # Pinned by ``validate_tokenizer_ids``: a checkpoint whose tokenizer disagrees # is not this model, and silently producing noise is worse than failing. SPECIAL_TOKEN_IDS: dict[str, int] = { From 718d9a4cd7505f0a7e11d8a1de9d177bb01c556f Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Thu, 20 Aug 2026 17:28:55 +0200 Subject: [PATCH 088/267] [Bugfix] Return diffusion metrics for image-edit endpoint (#5999) Signed-off-by: Kristoffer Signed-off-by: kTorp Co-authored-by: Hongsheng Liu --- docs/serving/image_edit_api.md | 9 +- .../openai_api/test_image_server.py | 116 +++++++++++++++++- vllm_omni/entrypoints/openai/api_server.py | 8 +- 3 files changed, 128 insertions(+), 5 deletions(-) diff --git a/docs/serving/image_edit_api.md b/docs/serving/image_edit_api.md index fef28142e..a38512342 100644 --- a/docs/serving/image_edit_api.md +++ b/docs/serving/image_edit_api.md @@ -123,7 +123,14 @@ When `stream=false` or omitted, the endpoint returns the standard image edit res } ], "output_format": null, - "size": null + "size": null, + "metrics": { + "stage_durations": { + "queue_wait_ms": 1.0, + "stage_0_gen_ms": 2.0 + }, + "peak_memory_mb": 1024.0 + } } ``` diff --git a/tests/entrypoints/openai_api/test_image_server.py b/tests/entrypoints/openai_api/test_image_server.py index 2136d84f8..b2eb2edfc 100644 --- a/tests/entrypoints/openai_api/test_image_server.py +++ b/tests/entrypoints/openai_api/test_image_server.py @@ -117,10 +117,23 @@ def test_encode_image_base64(): class MockGenerationResult: """Mock result object compatible with current diffusion output shape.""" - def __init__(self, images): + def __init__(self, images, stage_durations=None, peak_memory_mb=0.0): self.images = images - self.stage_durations = {} - self.peak_memory_mb = 0.0 + self.stage_durations = {} if stage_durations is None else stage_durations + self.peak_memory_mb = peak_memory_mb + + +def _stub_engine_generate_with_metrics(engine, *, stage_durations, peak_memory_mb): + """Stub engine.generate to return profiler metrics.""" + + async def generate(*args, **kwargs): + yield MockGenerationResult( + [Image.new("RGB", (16, 16), color="blue")], + stage_durations=stage_durations, + peak_memory_mb=peak_memory_mb, + ) + + engine.generate = generate class MockStageResult: @@ -972,6 +985,53 @@ def test_image_edits_streaming_rejects_single_stage_before_loading_url(test_clie assert "multi-stage" in response.json()["detail"] +def test_generate_images_returns_metrics_single_stage(test_client): + """Single-stage /v1/images/generations copies getattr metrics onto the response.""" + stage_durations = {"queue_wait_ms": 1.0, "stage_0_gen_ms": 2.0} + peak_memory_mb = 1024.0 + _stub_engine_generate_with_metrics( + test_client.app.state.engine_client, + stage_durations=stage_durations, + peak_memory_mb=peak_memory_mb, + ) + + response = test_client.post( + "/v1/images/generations", + json={"prompt": "a cat", "n": 1, "size": "256x256"}, + ) + assert response.status_code == 200 + assert response.json()["metrics"] == { + "stage_durations": stage_durations, + "peak_memory_mb": peak_memory_mb, + } + + +def test_generate_images_returns_metrics_multistage(async_omni_test_client): + """Multi-stage /v1/images/generations unpacks generate_diffusion_images metrics.""" + stage_durations = { + "queue_wait_ms": 1.0, + "preprocess_ms": 2.0, + "stage_0_gen_ms": 3.0, + "stage_1_gen_ms": 4.0, + } + peak_memory_mb = 2048.0 + _stub_engine_generate_with_metrics( + async_omni_test_client.app.state.engine_client, + stage_durations=stage_durations, + peak_memory_mb=peak_memory_mb, + ) + + response = async_omni_test_client.post( + "/v1/images/generations", + json={"prompt": "a cat", "n": 1, "size": "256x256"}, + ) + assert response.status_code == 200 + assert response.json()["metrics"] == { + "stage_durations": stage_durations, + "peak_memory_mb": peak_memory_mb, + } + + def test_generate_images_max_size_rejected(async_omni_test_client): """Test that a size exceeding max_generated_image_size returns 400.""" response = async_omni_test_client.post( @@ -1987,6 +2047,56 @@ def test_image_edit_with_seed_zero(async_omni_test_client): ) +def test_image_edits_returns_metrics_single_stage(test_client): + """Single-stage /v1/images/edits copies getattr metrics onto the response.""" + stage_durations = {"queue_wait_ms": 1.0, "stage_0_gen_ms": 2.0} + peak_memory_mb = 1024.0 + _stub_engine_generate_with_metrics( + test_client.app.state.engine_client, + stage_durations=stage_durations, + peak_memory_mb=peak_memory_mb, + ) + + response = test_client.post( + "/v1/images/edits", + files=[("image", make_test_image_bytes((16, 16)))], + data={"prompt": "edit me"}, + ) + assert response.status_code == 200 + assert response.json()["metrics"] == { + "stage_durations": stage_durations, + "peak_memory_mb": peak_memory_mb, + } + + +def test_image_edits_returns_metrics_multistage(async_omni_test_client): + """Multi-stage /v1/images/edits unpacks generate_diffusion_images metrics.""" + stage_durations = { + "queue_wait_ms": 1.0, + "preprocess_ms": 2.0, + "ar2diffusion_ms": 3.0, + "stage_0_gen_ms": 4.0, + "stage_1_gen_ms": 5.0, + } + peak_memory_mb = 2048.0 + _stub_engine_generate_with_metrics( + async_omni_test_client.app.state.engine_client, + stage_durations=stage_durations, + peak_memory_mb=peak_memory_mb, + ) + + response = async_omni_test_client.post( + "/v1/images/edits", + files=[("image", make_test_image_bytes((16, 16)))], + data={"prompt": "edit me"}, + ) + assert response.status_code == 200 + assert response.json()["metrics"] == { + "stage_durations": stage_durations, + "peak_memory_mb": peak_memory_mb, + } + + def test_image_edit_with_seed_zero_single_stage(test_client): """Test that seed=0 is correctly handled in image editing (single stage). diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 71020bd84..57adaad08 100755 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -2377,7 +2377,7 @@ async def edit_images( status_code=generation_result.error.code if generation_result.error else 400, detail=generation_result.message, ) - images, _, _, cot_output = generation_result + images, stage_durations, peak_memory_mb, cot_output = generation_result else: # Single-stage diffusion: use the direct path. result = await _generate_with_async_omni( @@ -2388,6 +2388,8 @@ async def edit_images( request_id=request_id, ) images = _extract_images_from_result(result) + stage_durations = getattr(result, "stage_durations", None) + peak_memory_mb = getattr(result, "peak_memory_mb", None) logger.debug(f"Successfully generated {len(images)} image(s)") @@ -2408,6 +2410,10 @@ async def edit_images( output_format=output_format, size=size_str, cot_output=cot_output, + metrics={ + "stage_durations": stage_durations or None, + "peak_memory_mb": float(peak_memory_mb) if peak_memory_mb else None, + }, ) except (EngineGenerateError, EngineDeadError) as exc: From cd3867c373282c5c1dfda9b2f27d44ee33759b1b Mon Sep 17 00:00:00 2001 From: Inesh Reddy Chappidi Date: Thu, 20 Aug 2026 12:22:37 -0400 Subject: [PATCH 089/267] =?UTF-8?q?[Model]=20Suppress=20silence=20codec=20?= =?UTF-8?q?tokens=20for=20the=20first=20N=20Qwen3-TTS=20decod=E2=80=A6=20(?= =?UTF-8?q?#5048)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Inesh Reddy --- tests/config/test_omni_config.py | 1 + tests/engine/test_stage_engine_args.py | 4 + .../test_qwen3_tts_talker_silence_ban.py | 322 ++++++++++++++++++ vllm_omni/config/model.py | 1 + vllm_omni/config/omni_config.py | 2 + vllm_omni/config/stage_config.py | 1 + vllm_omni/deploy/qwen3_tts.yaml | 4 + vllm_omni/engine/arg_utils.py | 2 + .../models/qwen3_tts/prompt_embeds_builder.py | 31 ++ .../models/qwen3_tts/qwen3_tts_talker.py | 162 ++++++++- .../npu/worker/npu_ar_model_runner.py | 9 + vllm_omni/worker/gpu_ar_model_runner.py | 9 + 12 files changed, 547 insertions(+), 1 deletion(-) create mode 100644 tests/model_executor/models/qwen3_tts/test_qwen3_tts_talker_silence_ban.py diff --git a/tests/config/test_omni_config.py b/tests/config/test_omni_config.py index 3ee1d40a1..cee4a8214 100644 --- a/tests/config/test_omni_config.py +++ b/tests/config/test_omni_config.py @@ -551,6 +551,7 @@ def test_sub_config_fields_match_structured_scopes(): "enable_sleep_mode", "default_sampling_params", "subtalker_sampling_params", + "silence_ban_frames", "has_sampling_extra_args", "custom_voice_dir", "task_type", diff --git a/tests/engine/test_stage_engine_args.py b/tests/engine/test_stage_engine_args.py index e7c9b2311..9ec7e520a 100644 --- a/tests/engine/test_stage_engine_args.py +++ b/tests/engine/test_stage_engine_args.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations import copy @@ -87,6 +90,7 @@ _OMNI_ONLY_LLM_STAGE_ENGINE_FIELDS = frozenset( "num_weight_load_threads", "omni_kv_config", "parallel_config", + "silence_ban_frames", "subtalker_sampling_params", "task_type", "tokenizer_subdir", diff --git a/tests/model_executor/models/qwen3_tts/test_qwen3_tts_talker_silence_ban.py b/tests/model_executor/models/qwen3_tts/test_qwen3_tts_talker_silence_ban.py new file mode 100644 index 000000000..3552c5356 --- /dev/null +++ b/tests/model_executor/models/qwen3_tts/test_qwen3_tts_talker_silence_ban.py @@ -0,0 +1,322 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for the Qwen3-TTS leading-silence ban (#4966, #5048). + +``compute_logits`` suppresses the checkpoint-derived silence codec vocabulary +for the first ``silence_ban_frames`` decode frames of a request, and only for +requests running Base voice-clone in x-vector-only mode. ICL onsets are +legitimate audio and must not be masked. + +Two per-request gates guard the mask, and both index by batch position: + +* the decode step, from ``sampling_metadata.output_token_ids[i]`` -- the + request's generated-token history, so the window closes on its own and a + preempted request cannot re-enter it; +* the generation mode, recorded at prefill in ``preprocess_batch`` and looked + up through the batch req_id order the runner supplies via + ``set_batch_req_ids``. + +These tests drive the methods directly against a talker built with ``__new__``: +no config, no checkpoint, no device. Only the attributes the masking path reads +are populated, and ``logits_processor`` is stubbed to return a fixed tensor. +""" + +from types import SimpleNamespace + +import pytest +import torch + +from vllm_omni.model_executor.models.qwen3_tts.prompt_embeds_builder import resolve_x_vector_only +from vllm_omni.model_executor.models.qwen3_tts.qwen3_tts_talker import ( + Qwen3TTSTalkerForConditionalGeneration, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_VOCAB = 32 +_SILENCE_IDS = (3, 7, 11) + + +def _make_talker(*, ban_frames: int, logits: torch.Tensor, modes=None): + """Talker stub exposing only what ``compute_logits`` touches. + + ``modes`` is one entry per batch row: ``True`` for x-vector-only, ``False`` + for ICL, ``None`` for a request whose mode was never recorded. Defaults to + every row in scope, so a test that says nothing about mode exercises the + step gate alone. + """ + talker = Qwen3TTSTalkerForConditionalGeneration.__new__(Qwen3TTSTalkerForConditionalGeneration) + + silence_mask = torch.zeros((_VOCAB,), dtype=torch.bool) + for token in _SILENCE_IDS: + silence_mask[token] = True + + if modes is None: + modes = [True] * logits.shape[0] + req_ids = [f"req{i}" for i in range(len(modes))] + + object.__setattr__(talker, "_silence_ban_frames", ban_frames) + object.__setattr__(talker, "_silence_mask", silence_mask) + object.__setattr__(talker, "_codec_disallowed_mask", torch.zeros((_VOCAB,), dtype=torch.bool)) + object.__setattr__(talker, "_batch_req_ids", req_ids) + object.__setattr__( + talker, + "_req_x_vector_only", + {rid: mode for rid, mode in zip(req_ids, modes, strict=True) if mode is not None}, + ) + object.__setattr__(talker, "lm_head", None) + object.__setattr__(talker, "logits_processor", lambda _head, _hidden: logits.clone()) + return talker + + +def _metadata(*steps: int): + """sampling_metadata whose output_token_ids encode one step per request.""" + return SimpleNamespace(output_token_ids=[[0] * step for step in steps]) + + +def _banned(row: torch.Tensor) -> set[int]: + return {int(i) for i in torch.nonzero(torch.isinf(row) & (row < 0)).flatten()} + + +# -------------------- the step gate -------------------- + + +def test_ban_disabled_leaves_logits_unchanged() -> None: + logits = torch.zeros((1, _VOCAB)) + talker = _make_talker(ban_frames=0, logits=logits) + out = talker.compute_logits(torch.zeros((1, 4)), _metadata(0)) + assert _banned(out[0]) == set(), "ban_frames=0 must be a no-op" + + +@pytest.mark.parametrize("step", [0, 1, 2]) +def test_early_steps_mask_only_the_silence_vocabulary(step: int) -> None: + logits = torch.zeros((1, _VOCAB)) + talker = _make_talker(ban_frames=3, logits=logits) + out = talker.compute_logits(torch.zeros((1, 4)), _metadata(step)) + assert _banned(out[0]) == set(_SILENCE_IDS), f"step {step} must mask exactly the silence ids" + + +@pytest.mark.parametrize("step", [3, 4, 50]) +def test_steps_past_the_window_are_untouched(step: int) -> None: + logits = torch.zeros((1, _VOCAB)) + talker = _make_talker(ban_frames=3, logits=logits) + out = talker.compute_logits(torch.zeros((1, 4)), _metadata(step)) + assert _banned(out[0]) == set(), f"step {step} is past the window; nothing may be masked" + + +def test_mixed_step_batch_masks_only_the_early_request() -> None: + logits = torch.zeros((2, _VOCAB)) + talker = _make_talker(ban_frames=3, logits=logits) + out = talker.compute_logits(torch.zeros((2, 4)), _metadata(0, 3)) + assert _banned(out[0]) == set(_SILENCE_IDS), "the step-0 request must be masked" + assert _banned(out[1]) == set(), "the step-3 request must not be masked" + + +# -------------------- the mode gate -------------------- + + +def test_icl_request_is_never_masked() -> None: + logits = torch.zeros((1, _VOCAB)) + talker = _make_talker(ban_frames=3, logits=logits, modes=[False]) + out = talker.compute_logits(torch.zeros((1, 4)), _metadata(0)) + assert _banned(out[0]) == set(), "an ICL onset is real audio and must not be suppressed" + + +def test_mixed_mode_batch_masks_only_the_x_vector_request() -> None: + logits = torch.zeros((2, _VOCAB)) + talker = _make_talker(ban_frames=3, logits=logits, modes=[True, False]) + out = talker.compute_logits(torch.zeros((2, 4)), _metadata(0, 0)) + assert _banned(out[0]) == set(_SILENCE_IDS), "the x-vector-only request must be masked" + assert _banned(out[1]) == set(), "the ICL request in the same batch must not be" + + +def test_unrecorded_mode_is_left_alone() -> None: + logits = torch.zeros((2, _VOCAB)) + talker = _make_talker(ban_frames=3, logits=logits, modes=[True, None]) + out = talker.compute_logits(torch.zeros((2, 4)), _metadata(0, 0)) + assert _banned(out[0]) == set(_SILENCE_IDS) + assert _banned(out[1]) == set(), "a request that cannot be classified must not be masked" + + +def test_stale_batch_req_ids_skip_the_ban() -> None: + logits = torch.zeros((2, _VOCAB)) + talker = _make_talker(ban_frames=3, logits=logits) + # One req_id for a two-row batch: the recorded order no longer indexes the + # logits, so masking by position would hit the wrong request. + object.__setattr__(talker, "_batch_req_ids", ["req0"]) + out = talker.compute_logits(torch.zeros((2, 4)), _metadata(0, 0)) + assert _banned(out[0]) == set(), "a req_id/logits length mismatch must disable the ban" + assert _banned(out[1]) == set() + + +# -------------------- metadata plumbing -------------------- + + +def test_missing_sampling_metadata_does_not_crash() -> None: + logits = torch.zeros((1, _VOCAB)) + talker = _make_talker(ban_frames=3, logits=logits) + out = talker.compute_logits(torch.zeros((1, 4)), None) + assert _banned(out[0]) == set() + + +@pytest.mark.parametrize( + "metadata", + [SimpleNamespace(output_token_ids=None), SimpleNamespace(), SimpleNamespace(output_token_ids=[])], +) +def test_absent_output_token_ids_warns_without_masking(metadata) -> None: + logits = torch.zeros((1, _VOCAB)) + talker = _make_talker(ban_frames=3, logits=logits) + out = talker.compute_logits(torch.zeros((1, 4)), metadata) + assert _banned(out[0]) == set(), "the ban must skip, not crash, when history is unavailable" + + +def test_batch_length_mismatch_skips_the_ban() -> None: + logits = torch.zeros((2, _VOCAB)) + talker = _make_talker(ban_frames=3, logits=logits) + out = talker.compute_logits(torch.zeros((2, 4)), _metadata(0)) + assert _banned(out[0]) == set(), "a runner shape mismatch must disable the ban, not misindex it" + assert _banned(out[1]) == set() + + +# -------------------- mode recording and eviction -------------------- + + +def _mode_talker(ban_frames: int = 3): + talker = Qwen3TTSTalkerForConditionalGeneration.__new__(Qwen3TTSTalkerForConditionalGeneration) + object.__setattr__(talker, "_silence_ban_frames", ban_frames) + object.__setattr__(talker, "_req_x_vector_only", {}) + object.__setattr__(talker, "_batch_req_ids", []) + object.__setattr__(talker, "_prompt_builder", SimpleNamespace(preprocess_batch=lambda **_kwargs: None)) + return talker + + +def _info(*, task_type: str = "Base", xvec: bool = True, icl: bool | None = None) -> dict: + info: dict = {"task_type": [task_type], "x_vector_only_mode": [xvec]} + if icl is not None: + info["voice_clone_prompt"] = [{"icl_mode": icl}] + return info + + +def test_preprocess_batch_records_the_generation_mode() -> None: + talker = _mode_talker() + buffer = { + "req0": _info(xvec=True), + "req1": _info(xvec=False), + "req2": _info(task_type="CustomVoice"), + } + talker.preprocess_batch(req_ids=list(buffer), model_intermediate_buffer=buffer, device=torch.device("cpu")) + assert talker._req_x_vector_only == {"req0": True, "req1": False, "req2": None} + + +def test_recorded_mode_matches_the_prompt_builder_resolution() -> None: + """The recording path must agree with the resolution the prefill path uses; + a divergence here would mask requests the builder treats as in-context.""" + infos = [ + _info(xvec=True), + _info(xvec=False), + _info(xvec=True, icl=True), + _info(xvec=False, icl=False), + _info(task_type="VoiceDesign"), + ] + talker = _mode_talker() + buffer = {f"req{i}": info for i, info in enumerate(infos)} + talker.preprocess_batch(req_ids=list(buffer), model_intermediate_buffer=buffer, device=torch.device("cpu")) + for req_id, info in buffer.items(): + assert talker._req_x_vector_only[req_id] == resolve_x_vector_only(info) + + +def test_preprocess_batch_records_nothing_when_the_ban_is_off() -> None: + talker = _mode_talker(ban_frames=0) + buffer = {"req0": _info(xvec=True)} + talker.preprocess_batch(req_ids=["req0"], model_intermediate_buffer=buffer, device=torch.device("cpu")) + assert talker._req_x_vector_only == {} + + +def test_set_batch_req_ids_evicts_departed_requests() -> None: + talker = _mode_talker() + object.__setattr__(talker, "_req_x_vector_only", {"req0": True, "req1": False, "req2": True}) + talker.set_batch_req_ids(["req2", "req0"]) + assert talker._batch_req_ids == ["req2", "req0"], "batch order must be preserved, not sorted" + assert talker._req_x_vector_only == {"req0": True, "req2": True}, "req1 has left the batch" + + +# -------------------- _init_silence_mask sanity checks -------------------- +# +# The silence vocabulary is derived from the checkpoint at load time rather than +# hardcoded. A derivation that comes back empty, oversized, or out of range means +# the encode did not produce silence codes, so the ban disables itself instead of +# masking a guess. + +_TALKER_VOCAB = 2048 + + +def _make_init_talker(*, codes, ban_frames: int = 3, vocab: int = _TALKER_VOCAB): + """Talker stub exposing only what ``_init_silence_mask`` touches. + + ``codes`` is either the list ``_encode_ref_audio_batch`` should return or an + exception instance it should raise. + """ + talker = Qwen3TTSTalkerForConditionalGeneration.__new__(Qwen3TTSTalkerForConditionalGeneration) + + def _encode(_wavs, _sr, *, device): + if isinstance(codes, Exception): + raise codes + return codes + + def _parameters(): + return iter([torch.zeros(1)]) + + object.__setattr__(talker, "_silence_ban_frames", ban_frames) + object.__setattr__(talker, "_silence_mask", torch.zeros((vocab,), dtype=torch.bool)) + object.__setattr__(talker, "talker_config", SimpleNamespace(vocab_size=vocab)) + object.__setattr__(talker, "_encode_ref_audio_batch", _encode) + object.__setattr__(talker, "parameters", _parameters) + return talker + + +def _codes_from(ids) -> list[torch.Tensor]: + """One [T, Q] codec tensor whose codebook-0 column holds ``ids``.""" + cb0 = torch.tensor(list(ids), dtype=torch.long).unsqueeze(1) + return [torch.cat([cb0, torch.zeros_like(cb0)], dim=1)] + + +def test_plausible_derivation_populates_the_mask() -> None: + talker = _make_init_talker(codes=_codes_from([4, 4, 9, 17])) + talker._init_silence_mask() + assert talker._silence_ban_frames == 3 + assert set(torch.nonzero(talker._silence_mask).flatten().tolist()) == {4, 9, 17} + + +def test_empty_derivation_disables_the_ban() -> None: + talker = _make_init_talker(codes=[]) + talker._init_silence_mask() + assert talker._silence_ban_frames == 0, "an all-False mask must not stay enabled" + assert not talker._silence_mask.any() + + +def test_oversized_derivation_disables_the_ban() -> None: + limit = max(64, _TALKER_VOCAB // 20) + talker = _make_init_talker(codes=_codes_from(range(limit + 1))) + talker._init_silence_mask() + assert talker._silence_ban_frames == 0, "a speech-sized vocabulary must not be masked" + assert not talker._silence_mask.any() + + +def test_out_of_range_derivation_disables_the_ban() -> None: + talker = _make_init_talker(codes=_codes_from([4, 9, _TALKER_VOCAB + 1])) + talker._init_silence_mask() + assert talker._silence_ban_frames == 0, "a vocabulary-size disagreement must not be half-applied" + assert not talker._silence_mask.any() + + +def test_encoder_failure_disables_the_ban() -> None: + talker = _make_init_talker(codes=RuntimeError("encoder unavailable")) + talker._init_silence_mask() + assert talker._silence_ban_frames == 0 + assert not talker._silence_mask.any() + + +def test_disabled_ban_skips_derivation_entirely() -> None: + talker = _make_init_talker(codes=RuntimeError("must not be called"), ban_frames=0) + talker._init_silence_mask() + assert talker._silence_ban_frames == 0 diff --git a/vllm_omni/config/model.py b/vllm_omni/config/model.py index ef08dde8d..736a504bd 100644 --- a/vllm_omni/config/model.py +++ b/vllm_omni/config/model.py @@ -141,6 +141,7 @@ class OmniModelConfig(ModelConfig): } ) subtalker_sampling_params: dict[str, Any] | None = None + silence_ban_frames: int = 0 omni_kv_config: dict | None = None codec_frame_rate_hz: float | None = None task_type: str | None = None diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index 8dd9c3db6..c1e24ea4b 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -152,6 +152,7 @@ class _ModelEngineOverrides(TypedDict, total=False): active_stream_window: int enable_sleep_mode: bool subtalker_sampling_params: dict[str, Any] + silence_ban_frames: int has_sampling_extra_args: bool custom_voice_dir: str task_type: str @@ -387,6 +388,7 @@ class OmniStageModelConfig: enable_sleep_mode: bool = False default_sampling_params: dict[str, Any] | None = None subtalker_sampling_params: dict[str, Any] | None = None + silence_ban_frames: int = 0 has_sampling_extra_args: bool = False custom_voice_dir: str | None = None task_type: str | None = None diff --git a/vllm_omni/config/stage_config.py b/vllm_omni/config/stage_config.py index 0e4457733..2b1a9f479 100644 --- a/vllm_omni/config/stage_config.py +++ b/vllm_omni/config/stage_config.py @@ -301,6 +301,7 @@ class StageDeployConfig: default_sampling_params: dict[str, Any] | None = None default_pooling_params: dict[str, Any] | None = None subtalker_sampling_params: dict[str, Any] | None = None + silence_ban_frames: int = 0 # === Generic stage engine fields === # Parallelism, scheduler, and memory-capacity controls. diff --git a/vllm_omni/deploy/qwen3_tts.yaml b/vllm_omni/deploy/qwen3_tts.yaml index 08531869a..58b191f77 100644 --- a/vllm_omni/deploy/qwen3_tts.yaml +++ b/vllm_omni/deploy/qwen3_tts.yaml @@ -73,6 +73,10 @@ stages: # audio (HTTP 400). See #4962. min_tokens: 2 repetition_penalty: 1.05 + # Suppress codebook-0 silence tokens for the first N decode frames of a + # request (#4966). Applies to every request on this stage, so intended + # for x-vector-only (Base voice-clone) deploys. 0 = off. + silence_ban_frames: 0 subtalker_sampling_params: do_sample: true temperature: 0.9 diff --git a/vllm_omni/engine/arg_utils.py b/vllm_omni/engine/arg_utils.py index fcd11b173..152e5b3ee 100644 --- a/vllm_omni/engine/arg_utils.py +++ b/vllm_omni/engine/arg_utils.py @@ -184,6 +184,7 @@ class OmniEngineArgs(EngineArgs): custom_process_next_stage_input_func: str | None = None stage_connector_spec: dict[str, Any] = field(default_factory=dict) subtalker_sampling_params: dict[str, Any] | None = None + silence_ban_frames: int = 0 async_chunk: bool = False retains_state_across_chunks: bool = False # WS-A: Stage-1 active stream slots. 0 = legacy preempt-everything. @@ -407,6 +408,7 @@ class OmniEngineArgs(EngineArgs): custom_process_next_stage_input_func=self.custom_process_next_stage_input_func, stage_connector_config=stage_connector_config, subtalker_sampling_params=self.subtalker_sampling_params, + silence_ban_frames=self.silence_ban_frames, omni_kv_config=self.omni_kv_config, task_type=self.task_type, has_sampling_extra_args=self.has_sampling_extra_args, diff --git a/vllm_omni/model_executor/models/qwen3_tts/prompt_embeds_builder.py b/vllm_omni/model_executor/models/qwen3_tts/prompt_embeds_builder.py index 0d8f03294..4053ddae4 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/prompt_embeds_builder.py +++ b/vllm_omni/model_executor/models/qwen3_tts/prompt_embeds_builder.py @@ -85,6 +85,37 @@ def build_instruct_text(instruct: str) -> str: # --------------------------------------------------------------------------- +def resolve_x_vector_only(info_dict: dict) -> bool | None: + """Resolve whether a request runs Base voice-clone in x-vector-only mode. + + Mirrors the resolution in :meth:`Qwen3TTSPromptEmbedsBuilder.build_prompt_embeds` + for the ``task_type == "Base"`` branch: the ``x_vector_only_mode`` flag, then + the ``voice_clone_prompt.icl_mode`` override when present. + + Returns ``None`` when the mode does not apply (any non-Base task), so callers + can distinguish "in-context" from "not a voice-clone request at all". + """ + task_type = first_value(info_dict.get("task_type"), "CustomVoice") + if task_type != "Base": + return None + + xvec_only = bool((info_dict.get("x_vector_only_mode") or [False])[0]) + + raw = info_dict.get("voice_clone_prompt") + if isinstance(raw, list): + raw = raw[0] if raw else None + if isinstance(raw, list) and raw and isinstance(raw[0], dict): + raw = raw[0] + if isinstance(raw, dict) and "icl_mode" in raw: + icl_flag = raw.get("icl_mode") + if isinstance(icl_flag, list): + icl_flag = icl_flag[0] if icl_flag else None + if isinstance(icl_flag, bool): + xvec_only = not icl_flag + + return xvec_only + + def first_value(value: object, default: object = None) -> object: if isinstance(value, list): return value[0] if value else default diff --git a/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py b/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py index 3daeb4cf2..535a447d3 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py +++ b/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py @@ -30,7 +30,7 @@ from vllm_omni.utils.speaker_cache import ( ) from .configuration_qwen3_tts import Qwen3TTSConfig, Qwen3TTSSpeakerEncoderConfig, Qwen3TTSTalkerConfig -from .prompt_embeds_builder import PRECOMPUTED_TEXT_IDS_KEY, Qwen3TTSPromptEmbedsBuilder +from .prompt_embeds_builder import PRECOMPUTED_TEXT_IDS_KEY, Qwen3TTSPromptEmbedsBuilder, resolve_x_vector_only from .qwen3_tts_code_predictor_vllm import Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM from .tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2 import Qwen3TTSTokenizerV2Config from .tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import Qwen3TTSTokenizerV2Encoder @@ -421,6 +421,27 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): codec_mask[self._codec_eos_token_id] = True self.register_buffer("_codec_disallowed_mask", ~codec_mask, persistent=False) + # Silence-region codec tokens (#4966). The talker has real probability mass on + # these in the opening decode frames; how many it draws is a sampling outcome, + # which is the source of both the leading silence and its seed-to-seed variance. + # The vocabulary is checkpoint-specific, so it is derived at load time in + # :meth:`_init_silence_mask`; this is only the placeholder. + self._silence_ban_frames = max(0, int(getattr(vllm_config.model_config, "silence_ban_frames", 0) or 0)) + # vLLM only fills sampling_metadata.output_token_ids when penalties, bad + # words, or a logits processor need history. compute_logits reads it for + # the per-request decode step, so request it when the ban is on. + self.logitsprocs_need_output_token_ids = self._silence_ban_frames > 0 + self.register_buffer("_silence_mask", torch.zeros((vocab,), dtype=torch.bool), persistent=False) + + # Per-request generation mode for the silence ban (#4966). The ban is a + # Base voice-clone (x-vector-only) fix; ICL onsets are legitimate audio + # and must not be masked. Mode is only knowable at prefill, so it is + # recorded in preprocess_batch and read back at decode. The batch order + # comes from set_batch_req_ids, which the runner calls each step with + # the same req_id ordering that indexes sampling_metadata. + self._req_x_vector_only: dict[str, bool | None] = {} + self._batch_req_ids: list[str] = [] + # Keys that should stay on GPU in model_intermediate_buffer to avoid # CPU-to-GPU round-trips on every decode step. self.gpu_resident_buffer_keys: set[tuple[str, str]] = { @@ -560,6 +581,48 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): # Mask out invalid codec ids using the pre-built constant buffer. logits = logits.masked_fill(self._codec_disallowed_mask, float("-inf")) + # Suppress silence-region tokens for the first N decode frames (#4966). The + # per-request decode step is the length of its generated-token history. + ban_n = self._silence_ban_frames + if ban_n > 0 and sampling_metadata is not None: + output_token_ids = getattr(sampling_metadata, "output_token_ids", None) + if output_token_ids is None or len(output_token_ids) != logits.shape[0]: + logger.warning_once( + "Silence ban is enabled but output_token_ids (%s) does not match the " + "logits batch (%d); the ban will not take effect.", + "None" if output_token_ids is None else len(output_token_ids), + logits.shape[0], + ) + else: + # Both gates are tiny [B] checks, so keep them on CPU: once no + # request is both in scope and inside the window, skip the device + # work rather than paying a host-to-device copy and a [B, vocab] + # mask on every decode frame. + # + # Scope the ban to x-vector-only requests (#4966). A request whose + # mode was never recorded is left alone: masking a request that + # cannot be classified is the risk this gate exists to remove. + batch_req_ids = self._batch_req_ids + if len(batch_req_ids) == logits.shape[0]: + in_scope = [self._req_x_vector_only.get(rid) is True for rid in batch_req_ids] + else: + logger.warning_once( + "Silence ban is enabled but the recorded batch req_ids (%d) do not match " + "the logits batch (%d); the ban will not take effect.", + len(batch_req_ids), + logits.shape[0], + ) + in_scope = [False] * logits.shape[0] + + steps = [len(t) for t in output_token_ids] + if any(scoped and step < ban_n for scoped, step in zip(in_scope, steps, strict=True)): + early = torch.tensor( + [scoped and step < ban_n for scoped, step in zip(in_scope, steps, strict=True)], + device=logits.device, + dtype=torch.bool, + ).unsqueeze(1) + logits = logits.masked_fill(early & self._silence_mask, float("-inf")) + return logits # -------------------- Omni multimodal output plumbing -------------------- @@ -944,12 +1007,30 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): device: torch.device, ) -> None: """Delegate batched preprocess to :class:`Qwen3TTSPromptEmbedsBuilder`.""" + if self._silence_ban_frames > 0: + for req_id in req_ids: + info_dict = model_intermediate_buffer.get(req_id) + if isinstance(info_dict, dict): + self._req_x_vector_only[req_id] = resolve_x_vector_only(info_dict) self._prompt_builder.preprocess_batch( req_ids=req_ids, model_intermediate_buffer=model_intermediate_buffer, device=device, ) + def set_batch_req_ids(self, req_ids: Sequence[str]) -> None: + """Record the current batch's req_ids, in the order that indexes logits. + + Called by the runner after the forward and before ``compute_logits``, + with the same ``input_batch`` ordering that ``sampling_metadata`` + uses. Doubles as the eviction point: any request no longer in the batch + has finished or been aborted, so its recorded mode is dropped. + """ + self._batch_req_ids = list(req_ids) + if len(self._req_x_vector_only) > len(self._batch_req_ids): + live = set(self._batch_req_ids) + self._req_x_vector_only = {k: v for k, v in self._req_x_vector_only.items() if k in live} + def _encode_ref_audio_batch(self, wavs: list[np.ndarray], sr: int, *, device: torch.device) -> list[torch.Tensor]: fe = self._encoder_feature_extractor target_sr = int(fe.sampling_rate) @@ -1042,6 +1123,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): self.encoder.to(device=device, dtype=torch.bfloat16) self._init_runtime_buffers() + self._init_silence_mask() logger.info("Loaded %d weights for Qwen3TTSTalkerForConditionalGeneration", len(loaded)) self._build_stacked_codec_embed() @@ -1056,6 +1138,84 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): device=w.device, dtype=w.dtype ) + @torch.no_grad() + def _init_silence_mask(self) -> None: + """Derive the silence-region codec vocabulary from the checkpoint. + + Encodes silence at several durations and amplitudes and collects the + codebook-0 ids the tokenizer emits. These are the tokens the talker draws + when it produces leading silence (#4966). Derived rather than hardcoded + because the codec vocabulary is checkpoint-specific. + """ + if self._silence_ban_frames <= 0: + return + device = next(self.parameters()).device + sr = 24000 + try: + rng = np.random.RandomState(0) + wavs = [ + (amp * rng.randn(int(dur * sr))).astype(np.float32) + for amp in (0.0, 1e-5, 1e-4, 1e-3) + for dur in (0.3, 0.5, 1.0) + ] + codes = self._encode_ref_audio_batch(wavs, sr, device=device) + ids: set[int] = set() + for code in codes: + code = code.detach().cpu() + cb0 = code[:, 0] if code.ndim == 2 else code + ids.update(int(t) for t in cb0.tolist()) + except Exception as exc: + logger.warning("Could not derive the silence codec vocabulary; disabling the ban: %s", exc) + self._silence_ban_frames = 0 + return + vocab = int(self.talker_config.vocab_size) + in_range = sorted(t for t in ids if 0 <= t < vocab) + num_out_of_range = len(ids) - len(in_range) + if num_out_of_range: + # Every derived id should be a codebook-0 index for this checkpoint. + # Ids outside the talker vocabulary mean the encoder and the talker + # disagree on vocabulary size, so the rest of the derivation is not + # trustworthy either. + logger.warning( + "%d of %d derived silence codec tokens fall outside the talker vocabulary " + "(size %d); disabling the ban.", + num_out_of_range, + len(ids), + vocab, + ) + self._silence_ban_frames = 0 + return + if not in_range: + # An empty derivation would leave an all-False mask: the ban would + # suppress nothing while still paying the per-frame check. + logger.warning("Derived no silence codec tokens; disabling the ban.") + self._silence_ban_frames = 0 + return + # The silence region is a small corner of codebook 0 (12 tokens on + # Qwen3-TTS-12Hz-1.7B-Base). A derivation this large means the encode + # returned speech-like codes, and masking that much of the vocabulary + # would distort generation rather than trim the onset. + max_silence_tokens = max(64, vocab // 20) + if len(in_range) > max_silence_tokens: + logger.warning( + "Derived %d silence codec tokens, above the %d sanity limit for a vocabulary of %d; disabling the ban.", + len(in_range), + max_silence_tokens, + vocab, + ) + self._silence_ban_frames = 0 + return + mask = torch.zeros((vocab,), dtype=torch.bool) + for token in in_range: + mask[token] = True + self._silence_mask.copy_(mask.to(self._silence_mask.device)) + logger.info( + "Derived %d silence codec tokens, suppressed for the first %d decode frames: %s", + len(in_range), + self._silence_ban_frames, + in_range, + ) + @torch.no_grad() def _init_runtime_buffers(self) -> None: """Populate request-independent runtime buffers from frozen weights. diff --git a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py index 1829fafa2..c43a4f8d0 100644 --- a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py @@ -778,6 +778,15 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin, Duplex if callable(flush_pending_metadata): flush_pending_metadata(req_ids[:num_reqs]) + # [Omni] Hand the model the batch's req_ids in logits order, for + # models that gate logits per request. Mirrors gpu_ar_model_runner. + # Only valid without spec decode: there logits_indices carries several + # rows per request, so row i no longer corresponds to req_ids[i]. + if spec_decode_metadata is None: + set_batch_req_ids = getattr(self.model, "set_batch_req_ids", None) + if callable(set_batch_req_ids): + set_batch_req_ids(req_ids[:num_reqs]) + hidden_states, multimodal_outputs = self.extract_multimodal_outputs(hidden_states) if multimodal_outputs is not None: diff --git a/vllm_omni/worker/gpu_ar_model_runner.py b/vllm_omni/worker/gpu_ar_model_runner.py index c87d17cc2..1197f5b44 100644 --- a/vllm_omni/worker/gpu_ar_model_runner.py +++ b/vllm_omni/worker/gpu_ar_model_runner.py @@ -1248,6 +1248,15 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin, Duplex flush_pending_metadata = getattr(self.model, "flush_pending_metadata", None) if callable(flush_pending_metadata): flush_pending_metadata(req_ids[:num_reqs]) + + # [Omni] Hand the model the batch's req_ids in logits order, for + # models that gate logits per request. Only valid without spec + # decode: there logits_indices carries several rows per request, + # so row i no longer corresponds to req_ids[i]. + if spec_decode_metadata is None: + set_batch_req_ids = getattr(self.model, "set_batch_req_ids", None) + if callable(set_batch_req_ids): + set_batch_req_ids(req_ids[:num_reqs]) finally: if runner_assisted_context_enabled: self._set_runner_assisted_full_attention_metadata_context(enabled=False) From 8d34d3b5deed95a3cdfa4ed3b58fdd86ddddca80 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Fri, 21 Aug 2026 00:29:21 +0800 Subject: [PATCH 090/267] [Perf][Hunyuan] Optimize for vae and patch_embed (#6306) Signed-off-by: dengyunyang <584797741@qq.com> --- .../model_executor/models/common/__init__.py | 0 .../test_fused_adaptive_group_norm_silu.py | 292 +++++++++++++++ .../common/test_fused_group_norm_silu.py | 338 ++++++++++++++++++ .../models/hunyuan_image3/__init__.py | 6 +- .../models/hunyuan_image3/autoencoder.py | 104 +----- .../hunyuan_image3_transformer.py | 138 +------ .../models/hunyuan_image3/layers/__init__.py | 22 ++ .../hunyuan_image3/layers/common/__init__.py | 11 + .../hunyuan_image3/layers/common/utils.py | 51 +++ .../hunyuan_image3/layers/native/__init__.py | 9 + .../layers/native/autoencoder_blocks.py | 44 +++ .../layers/native/transformer_blocks.py | 110 ++++++ .../hunyuan_image3/layers/nvidia/__init__.py | 9 + .../hunyuan_image3/layers/nvidia/_cudnn.py | 27 ++ .../layers/nvidia/autoencoder_blocks.py | 56 +++ .../layers/nvidia/transformer_blocks.py | 138 +++++++ .../models/common/ops/__init__.py | 17 + .../ops/fused_adaptive_group_norm_silu.py | 222 ++++++++++++ .../common/ops/fused_group_norm_silu.py | 207 +++++++++++ 19 files changed, 1583 insertions(+), 218 deletions(-) create mode 100644 tests/model_executor/models/common/__init__.py create mode 100644 tests/model_executor/models/common/test_fused_adaptive_group_norm_silu.py create mode 100644 tests/model_executor/models/common/test_fused_group_norm_silu.py create mode 100644 vllm_omni/diffusion/models/hunyuan_image3/layers/__init__.py create mode 100644 vllm_omni/diffusion/models/hunyuan_image3/layers/common/__init__.py create mode 100644 vllm_omni/diffusion/models/hunyuan_image3/layers/common/utils.py create mode 100644 vllm_omni/diffusion/models/hunyuan_image3/layers/native/__init__.py create mode 100644 vllm_omni/diffusion/models/hunyuan_image3/layers/native/autoencoder_blocks.py create mode 100644 vllm_omni/diffusion/models/hunyuan_image3/layers/native/transformer_blocks.py create mode 100644 vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/__init__.py create mode 100644 vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/_cudnn.py create mode 100644 vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/autoencoder_blocks.py create mode 100644 vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/transformer_blocks.py create mode 100644 vllm_omni/model_executor/models/common/ops/__init__.py create mode 100644 vllm_omni/model_executor/models/common/ops/fused_adaptive_group_norm_silu.py create mode 100644 vllm_omni/model_executor/models/common/ops/fused_group_norm_silu.py diff --git a/tests/model_executor/models/common/__init__.py b/tests/model_executor/models/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/model_executor/models/common/test_fused_adaptive_group_norm_silu.py b/tests/model_executor/models/common/test_fused_adaptive_group_norm_silu.py new file mode 100644 index 000000000..e202321dc --- /dev/null +++ b/tests/model_executor/models/common/test_fused_adaptive_group_norm_silu.py @@ -0,0 +1,292 @@ +"""Unit tests for fused_adaptive_group_norm_silu operator. + +Tests numeric correctness against PyTorch native implementation: + F.silu(F.group_norm(x, num_groups, weight, bias, eps) * (1 + scale) + shift) +""" + +import pytest +import torch +import torch.nn.functional as F + +from vllm_omni.model_executor.models.common.ops import fused_adaptive_group_norm_silu + +# Skip tests if CUDA not available +pytestmark = [ + pytest.mark.core_model, + pytest.mark.cuda, + pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required for Triton kernels"), +] + + +def _reference(x, weight, bias, scale, shift, num_groups, eps): + """AdaGN reference, computed in fp32. + + As with fused_group_norm_silu, a same-dtype reference is unusable: the + GroupNorm affine can cancel to near zero, and rounding its O(1) terms to + bf16 wipes out the remaining significant digits. + """ + B, C = x.shape[:2] + normed = F.group_norm(x.float(), num_groups, weight.float(), bias.float(), eps) + scale_b = scale.float().view(B, C, *([1] * (x.ndim - 2))) + shift_b = shift.float().view(B, C, *([1] * (x.ndim - 2))) + return F.silu(normed * (1 + scale_b) + shift_b) + + +def _tolerance(dtype): + if dtype == torch.float32: + return 1e-5, 1e-5 + if dtype == torch.float16: + return 2e-3, 2e-3 + return 1e-2, 1e-2 # bfloat16 + + +@pytest.mark.parametrize("batch_size", [1, 2]) +@pytest.mark.parametrize("channels", [32, 64, 128]) +@pytest.mark.parametrize("spatial_size", [(16, 16), (32, 32)]) +@pytest.mark.parametrize("num_groups", [8, 32]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +def test_fused_adaptive_group_norm_silu_correctness(batch_size, channels, spatial_size, num_groups, dtype): + """Test numeric correctness against PyTorch native ops.""" + if channels % num_groups != 0: + pytest.skip(f"channels {channels} not divisible by num_groups {num_groups}") + + H, W = spatial_size + eps = 1e-6 + torch.manual_seed(42) + kw = dict(device="cuda", dtype=dtype) + + x = torch.randn(batch_size, channels, H, W, **kw) + weight = torch.randn(channels, **kw) + bias = torch.randn(channels, **kw) + scale = torch.randn(batch_size, channels, **kw) + shift = torch.randn(batch_size, channels, **kw) + + ref_out = _reference(x, weight, bias, scale, shift, num_groups, eps).to(dtype) + fused_out = fused_adaptive_group_norm_silu(x, weight, bias, scale, shift, num_groups, eps) + + rtol, atol = _tolerance(dtype) + torch.testing.assert_close( + fused_out, + ref_out, + rtol=rtol, + atol=atol, + msg=f"Mismatch for batch={batch_size}, C={channels}, HW={spatial_size}, groups={num_groups}, dtype={dtype}", + ) + + +def test_broadcast_scale_shift_shapes(): + """scale/shift may arrive as (B, C) or already broadcast to (B, C, 1, 1).""" + B, C, H, W = 2, 64, 16, 16 + kw = dict(device="cuda", dtype=torch.float32) + x = torch.randn(B, C, H, W, **kw) + weight = torch.randn(C, **kw) + bias = torch.randn(C, **kw) + scale = torch.randn(B, C, **kw) + shift = torch.randn(B, C, **kw) + + flat = fused_adaptive_group_norm_silu(x, weight, bias, scale, shift, 32, 1e-6) + expanded = fused_adaptive_group_norm_silu(x, weight, bias, scale.view(B, C, 1, 1), shift.view(B, C, 1, 1), 32, 1e-6) + torch.testing.assert_close(flat, expanded, rtol=0, atol=0) + + +def test_output_dtype_matches_eager(): + """Output dtype must match what eager GroupNorm would produce.""" + for dtype in [torch.float32, torch.float16, torch.bfloat16]: + B, C = 2, 64 + kw = dict(device="cuda", dtype=dtype) + x = torch.randn(B, C, 16, 16, **kw) + weight = torch.randn(C, **kw) + bias = torch.randn(C, **kw) + scale = torch.randn(B, C, **kw) + shift = torch.randn(B, C, **kw) + + eager_dtype = F.silu(F.group_norm(x, 32, weight, bias, 1e-6)).dtype + out = fused_adaptive_group_norm_silu(x, weight, bias, scale, shift, 32, 1e-6) + + assert out.dtype == eager_dtype, ( + f"Output dtype {out.dtype} != eager dtype {eager_dtype} for input dtype {dtype}" + ) + assert out.shape == x.shape + + +@pytest.mark.parametrize( + "autocast_dtype, input_dtype", + [ + (torch.float16, torch.float16), + (torch.bfloat16, torch.bfloat16), + (torch.float16, torch.float32), + (torch.bfloat16, torch.float32), + ], +) +def test_autocast_matches_eager(autocast_dtype, input_dtype): + """Inside autocast the fused op must stay a drop-in replacement. + + Also pins the fp32-activation corner so the inlined ``is_autocast_enabled + -> fp32`` branch cannot silently downcast an already-fp32 activation. + """ + B, C = 2, 64 + x = torch.randn(B, C, 16, 16, device="cuda", dtype=input_dtype) + weight = torch.randn(C, device="cuda", dtype=torch.float32) + bias = torch.randn(C, device="cuda", dtype=torch.float32) + scale = torch.randn(B, C, device="cuda", dtype=torch.float32) + shift = torch.randn(B, C, device="cuda", dtype=torch.float32) + + with torch.autocast("cuda", dtype=autocast_dtype): + eager_out = F.silu( + F.group_norm(x, 32, weight, bias, 1e-6) * (1 + scale.view(B, C, 1, 1)) + shift.view(B, C, 1, 1) + ) + fused_out = fused_adaptive_group_norm_silu(x, weight, bias, scale, shift, 32, 1e-6) + + assert eager_out.dtype == torch.float32, "sanity check: eager group_norm is expected to return fp32 under autocast" + assert fused_out.dtype == eager_out.dtype, ( + f"Output dtype {fused_out.dtype} != eager dtype {eager_out.dtype} " + f"under autocast({autocast_dtype}) for input {input_dtype}" + ) + torch.testing.assert_close(fused_out, eager_out, rtol=1e-5, atol=1e-5) + + +def test_invalid_channels_not_divisible(): + """Test that assertion fires when channels not divisible by num_groups.""" + B, C = 1, 33 + kw = dict(device="cuda", dtype=torch.float32) + x = torch.randn(B, C, 16, 16, **kw) + weight = torch.randn(C, **kw) + bias = torch.randn(C, **kw) + scale = torch.randn(B, C, **kw) + shift = torch.randn(B, C, **kw) + + with pytest.raises(AssertionError, match="must be divisible by num_groups"): + fused_adaptive_group_norm_silu(x, weight, bias, scale, shift, 32, 1e-6) + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("shape", [(2, 64, 3, 16, 16), (1, 32, 8, 8), (2, 64, 128)]) +def test_non_2d_spatial_matches_eager(dtype, shape): + """Any spatial rank must work; the wrapper collapses the spatial axes. + + GroupNorm reduces over the whole channel group and every spatial position, + so flattening is exact rather than an approximation. + """ + torch.manual_seed(0) + B, C = shape[:2] + kw = dict(device="cuda", dtype=dtype) + x = torch.randn(*shape, **kw) + weight = torch.randn(C, **kw) + bias = torch.randn(C, **kw) + scale = torch.randn(B, C, **kw) + shift = torch.randn(B, C, **kw) + + fused_out = fused_adaptive_group_norm_silu(x, weight, bias, scale, shift, 32, 1e-6) + ref_out = _reference(x, weight, bias, scale, shift, 32, 1e-6).to(dtype) + + assert fused_out.shape == x.shape + rtol, atol = _tolerance(dtype) + torch.testing.assert_close(fused_out, ref_out, rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("batch_size", [1, 2, 3]) +def test_scale_shift_from_chunk_are_not_contiguous(batch_size): + """Regression: scale/shift arrive as halves of a torch.chunk. + + ResBlock builds them as ``torch.chunk(emb_out, 2, dim=1)`` where emb_out is + (B, 2C, 1, 1). For B > 1 each half is a strided view, so the wrapper must + ``reshape`` (which copies) instead of ``view`` (which raises). + """ + C, H, W = 64, 16, 16 + kw = dict(device="cuda", dtype=torch.float32) + x = torch.randn(batch_size, C, H, W, **kw) + weight = torch.randn(C, **kw) + bias = torch.randn(C, **kw) + + emb_out = torch.randn(batch_size, 2 * C, 1, 1, **kw) + scale, shift = torch.chunk(emb_out, 2, dim=1) + if batch_size > 1: + assert not scale.is_contiguous(), "test premise: chunk half is strided" + + fused_out = fused_adaptive_group_norm_silu(x, weight, bias, scale, shift, 32, 1e-6) + ref_out = F.silu(F.group_norm(x, 32, weight, bias, 1e-6) * (1.0 + scale) + shift) + + torch.testing.assert_close(fused_out, ref_out, rtol=1e-5, atol=1e-5) + + +def test_non_contiguous_input_matches_eager(): + """A channels-last / permuted activation must still be handled correctly.""" + B, C, H, W = 2, 64, 16, 16 + kw = dict(device="cuda", dtype=torch.float32) + x = torch.randn(B, H, W, C, **kw).permute(0, 3, 1, 2) + assert not x.is_contiguous(), "test premise: input is strided" + weight = torch.randn(C, **kw) + bias = torch.randn(C, **kw) + scale = torch.randn(B, C, **kw) + shift = torch.randn(B, C, **kw) + + fused_out = fused_adaptive_group_norm_silu(x, weight, bias, scale, shift, 32, 1e-6) + ref_out = F.silu(F.group_norm(x, 32, weight, bias, 1e-6) * (1.0 + scale.view(B, C, 1, 1)) + shift.view(B, C, 1, 1)) + + torch.testing.assert_close(fused_out, ref_out, rtol=1e-5, atol=1e-5) + + +@pytest.mark.parametrize("spatial", [32, 64]) +def test_dit_sized_activation(spatial): + """Shapes the HunyuanImage3 DiT actually sees. + + The earlier kernel walked the spatial axis one position at a time with + ``spatial_size`` as a ``tl.constexpr``; at 64x64 that is 4096 serial steps + per pass. This test pins the realistic size so a regression to that layout + shows up as a timeout rather than silently costing throughput. + """ + B, C = 2, 256 + kw = dict(device="cuda", dtype=torch.bfloat16) + x = torch.randn(B, C, spatial, spatial, **kw) + weight = torch.randn(C, **kw) + bias = torch.randn(C, **kw) + scale = torch.randn(B, C, **kw) + shift = torch.randn(B, C, **kw) + + fused_out = fused_adaptive_group_norm_silu(x, weight, bias, scale, shift, 32, 1e-5) + ref_out = _reference(x, weight, bias, scale, shift, 32, 1e-5).to(torch.bfloat16) + + rtol, atol = _tolerance(torch.bfloat16) + torch.testing.assert_close(fused_out, ref_out, rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +def test_large_offset_small_variance(dtype): + """A large constant offset with a small variance must stay stable. + + Mirrors the fused_group_norm_silu case: ``x ~ 10000 ± 0.1`` has mean ~1e4 + and variance ~1e-2, so ``E[x^2] - E[x]^2`` cancels catastrophically in fp32 + (rounding the variance to ~8.0). The Welford reduction keeps it accurate. + fp32 tolerance is 0.1 (not 1e-6) because ``x``'s fp32 representation noise + (~0.001 ULP at 1e4) is amplified by ``rstd ~ 10`` to ~0.01-0.02 — the same + floor PyTorch's own reference hits; the old naive implementation would be + off by ~1.0 here. fp16/bf16 quantize the perturbation away, so for those + dtypes the meaningful assertion is just "no NaN". + """ + torch.manual_seed(0) + B, C, H, W = 2, 64, 16, 16 + num_groups = 32 + eps = 1e-6 + device = torch.device("cuda") + + x = torch.full((B, C, H, W), 10000.0, device=device, dtype=dtype) + x = x + torch.randn(B, C, H, W, device=device, dtype=dtype) * 0.1 + weight = torch.randn(C, device=device, dtype=dtype) + bias = torch.randn(C, device=device, dtype=dtype) + scale = torch.randn(B, C, device=device, dtype=dtype) + shift = torch.randn(B, C, device=device, dtype=dtype) + + fused_out = fused_adaptive_group_norm_silu(x, weight, bias, scale, shift, num_groups, eps) + ref_out = _reference(x, weight, bias, scale, shift, num_groups, eps).to(dtype) + + assert not torch.isnan(fused_out).any(), "fused output must not contain NaN" + + if dtype == torch.float32: + # ~0.02 is the representation-noise floor at offset 1e4; old impl ~1.0. + torch.testing.assert_close(fused_out, ref_out, rtol=0.05, atol=0.1) + # fp16/bf16: perturbation is swallowed by the dtype — no meaningful close, + # the no-NaN assertion above is the regression guard. + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/model_executor/models/common/test_fused_group_norm_silu.py b/tests/model_executor/models/common/test_fused_group_norm_silu.py new file mode 100644 index 000000000..3e9d055af --- /dev/null +++ b/tests/model_executor/models/common/test_fused_group_norm_silu.py @@ -0,0 +1,338 @@ +"""Unit tests for fused_group_norm_silu operator. + +Tests numeric correctness against PyTorch native implementation: + F.silu(F.group_norm(x, num_groups, weight, bias, eps)) +""" + +import pytest +import torch +import torch.nn.functional as F + +from vllm_omni.model_executor.models.common.ops import fused_group_norm_silu + +# Skip tests if CUDA not available +pytestmark = [ + pytest.mark.core_model, + pytest.mark.cuda, + pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required for Triton kernels"), +] + + +def _tolerance(dtype): + """Tolerance against an fp32 reference, rounded once to ``dtype``.""" + if dtype == torch.float32: + return 1e-5, 1e-6 + if dtype == torch.float16: + return 2e-3, 2e-3 + return 1e-2, 1e-2 # bfloat16 + + +@pytest.mark.parametrize("batch_size", [1, 2, 4]) +@pytest.mark.parametrize("channels", [32, 64, 128]) +@pytest.mark.parametrize("spatial_size", [(16, 16), (32, 32), (64, 64)]) +@pytest.mark.parametrize("num_groups", [8, 16, 32]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +def test_fused_group_norm_silu_correctness(batch_size, channels, spatial_size, num_groups, dtype): + """Test numeric correctness against PyTorch native ops.""" + # Skip invalid configurations + if channels % num_groups != 0: + pytest.skip(f"channels {channels} not divisible by num_groups {num_groups}") + + H, W = spatial_size + device = torch.device("cuda") + eps = 1e-6 + + # Create inputs + torch.manual_seed(42) + x = torch.randn(batch_size, channels, H, W, device=device, dtype=dtype) + weight = torch.randn(channels, device=device, dtype=dtype) + bias = torch.randn(channels, device=device, dtype=dtype) + + # Reference implementation, computed in fp32. + # + # A same-dtype reference is NOT usable here: GroupNorm's affine step + # ``(x - mean) * rstd * weight + bias`` adds two O(1) terms that can cancel + # down to ~1e-3. Rounding those terms to bf16 (ULP ~8e-3 near 1.0) destroys + # every remaining significant digit -- measured cases flip sign, e.g. a true + # +0.006136 comes out as -0.000471. The low-precision "reference" is then + # far less accurate than the kernel it is supposed to validate: the fused + # output matches the correctly-rounded fp32 result for 99.997% of elements, + # a bf16 reference for only 65.2%. + ref_out = F.silu(F.group_norm(x.float(), num_groups, weight.float(), bias.float(), eps)).to(dtype) + + # Fused implementation + fused_out = fused_group_norm_silu(x, weight, bias, num_groups, eps) + + # Check correctness + # Tolerances are set from the measured worst case across this parameter + # sweep, leaving roughly 2x headroom for each dtype. + if dtype == torch.float32: + rtol, atol = 1e-5, 1e-6 + elif dtype == torch.float16: + rtol, atol = 2e-3, 2e-3 + else: # bfloat16 + rtol, atol = 1e-2, 1e-2 + + torch.testing.assert_close( + fused_out, + ref_out, + rtol=rtol, + atol=atol, + msg=f"Mismatch for batch={batch_size}, C={channels}, HW={spatial_size}, groups={num_groups}, dtype={dtype}", + ) + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +def test_fp32_accumulation_precision(dtype): + """Test that fp32 accumulation matches PyTorch's behavior. + + GroupNorm uses fp32 for mean/variance computation regardless of input dtype. + Our kernel should match this behavior. + """ + batch_size, channels, H, W = 2, 64, 32, 32 + num_groups = 32 + eps = 1e-6 + device = torch.device("cuda") + + # Create inputs with values that expose precision differences + torch.manual_seed(123) + x = torch.randn(batch_size, channels, H, W, device=device, dtype=dtype) * 100 + weight = torch.ones(channels, device=device, dtype=dtype) + bias = torch.zeros(channels, device=device, dtype=dtype) + + # Reference: fp32 accumulation, then a single round back to ``dtype``. + # (Same discipline as the main correctness test -- a same-dtype reference + # wouldn't isolate the accumulation path this test is named after.) + ref_out = F.silu(F.group_norm(x.float(), num_groups, weight.float(), bias.float(), eps)).to(dtype) + + # Fused + fused_out = fused_group_norm_silu(x, weight, bias, num_groups, eps) + + # Should match closely even with large values + if dtype == torch.float32: + rtol, atol = 1e-5, 1e-5 + else: + rtol, atol = 1e-2, 1e-2 + + torch.testing.assert_close(fused_out, ref_out, rtol=rtol, atol=atol) + + +def test_edge_case_zero_input(): + """Test with zero input.""" + x = torch.zeros(1, 32, 16, 16, device="cuda") + weight = torch.ones(32, device="cuda") + bias = torch.zeros(32, device="cuda") + + ref_out = F.silu(F.group_norm(x, 32, weight, bias, 1e-6)) + fused_out = fused_group_norm_silu(x, weight, bias, 32, 1e-6) + + torch.testing.assert_close(fused_out, ref_out, rtol=1e-5, atol=1e-6) + + +def test_edge_case_large_eps(): + """Test with large epsilon value.""" + x = torch.randn(1, 32, 16, 16, device="cuda") + weight = torch.randn(32, device="cuda") + bias = torch.randn(32, device="cuda") + eps = 1e-2 # Larger than typical + + ref_out = F.silu(F.group_norm(x, 32, weight, bias, eps)) + fused_out = fused_group_norm_silu(x, weight, bias, 32, eps) + + torch.testing.assert_close(fused_out, ref_out, rtol=1e-4, atol=1e-5) + + +def test_output_dtype_matches_eager(): + """Output dtype must match what eager GroupNorm+SiLU would produce. + + Outside autocast that means the input dtype; inside autocast it means fp32, + because group_norm is on autocast's fp32 cast policy. HunyuanImage3's VAE + runs fp32 weights under fp16 autocast, so getting this wrong would silently + downcast the post-norm activation. + """ + for dtype in [torch.float32, torch.float16, torch.bfloat16]: + x = torch.randn(2, 64, 32, 32, device="cuda", dtype=dtype) + weight = torch.randn(64, device="cuda", dtype=dtype) + bias = torch.randn(64, device="cuda", dtype=dtype) + + eager_out = F.silu(F.group_norm(x, 32, weight, bias, 1e-6)) + out = fused_group_norm_silu(x, weight, bias, 32, 1e-6) + + assert out.dtype == eager_out.dtype, ( + f"Output dtype {out.dtype} != eager dtype {eager_out.dtype} for input dtype {dtype}" + ) + assert out.shape == x.shape, f"Output shape {out.shape} != input shape {x.shape}" + + +@pytest.mark.parametrize( + "autocast_dtype, input_dtype", + [ + (torch.float16, torch.float16), + (torch.bfloat16, torch.bfloat16), + (torch.float16, torch.float32), + (torch.bfloat16, torch.float32), + ], +) +def test_autocast_matches_eager(autocast_dtype, input_dtype): + """Inside autocast the fused op must stay a drop-in replacement. + + Mirrors the HunyuanImage3 VAE setup: fp32 module weights, activations + arriving as fp16/bf16 from an upstream conv, all under autocast. Also pins + the fp32-activation corner (some VAE stages) so the inlined + ``is_autocast_enabled -> fp32`` branch cannot silently downcast it. + """ + x = torch.randn(2, 64, 32, 32, device="cuda", dtype=input_dtype) + weight = torch.randn(64, device="cuda", dtype=torch.float32) + bias = torch.randn(64, device="cuda", dtype=torch.float32) + + with torch.autocast("cuda", dtype=autocast_dtype): + eager_out = F.silu(F.group_norm(x, 32, weight, bias, 1e-6)) + fused_out = fused_group_norm_silu(x, weight, bias, 32, 1e-6) + + assert eager_out.dtype == torch.float32, "sanity check: eager group_norm is expected to return fp32 under autocast" + assert fused_out.dtype == eager_out.dtype, ( + f"Output dtype {fused_out.dtype} != eager dtype {eager_out.dtype} " + f"under autocast({autocast_dtype}) for input {input_dtype}" + ) + torch.testing.assert_close(fused_out, eager_out, rtol=1e-5, atol=1e-5) + + +def test_invalid_channels_not_divisible(): + """Test that assertion fires when channels not divisible by num_groups.""" + x = torch.randn(1, 33, 16, 16, device="cuda") # 33 not divisible by 32 + weight = torch.randn(33, device="cuda") + bias = torch.randn(33, device="cuda") + + with pytest.raises(AssertionError, match="must be divisible by num_groups"): + fused_group_norm_silu(x, weight, bias, num_groups=32, eps=1e-6) + + +def test_invalid_weight_shape(): + """Test that assertion fires with wrong weight shape.""" + x = torch.randn(1, 64, 16, 16, device="cuda") + weight = torch.randn(32, device="cuda") # Wrong: should be 64 + bias = torch.randn(64, device="cuda") + + with pytest.raises(AssertionError, match="doesn't match channels"): + fused_group_norm_silu(x, weight, bias, num_groups=32, eps=1e-6) + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("shape", [(2, 64, 3, 16, 16), (1, 32, 1, 8, 8), (2, 64, 128)]) +def test_non_2d_spatial_matches_eager(dtype, shape): + """Spatial ranks other than 2 must work: the 3D VAE feeds (N, C, T, H, W). + + GroupNorm reduces over the whole channel group and every spatial position, + so flattening the spatial axes inside the wrapper has to be exact. + """ + torch.manual_seed(0) + C = shape[1] + x = torch.randn(*shape, device="cuda", dtype=dtype) + weight = torch.randn(C, device="cuda", dtype=dtype) + bias = torch.randn(C, device="cuda", dtype=dtype) + + fused_out = fused_group_norm_silu(x, weight, bias, num_groups=32, eps=1e-6) + ref_out = F.silu(F.group_norm(x.float(), 32, weight.float(), bias.float(), 1e-6)).to(dtype) + + assert fused_out.shape == x.shape + rtol, atol = _tolerance(dtype) + torch.testing.assert_close(fused_out, ref_out, rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +def test_permuted_input_matches_eager(dtype): + """A permuted (channel-stride-1) activation must still be correct. + + This is not a hypothetical layout: HunyuanImage3's ``UNetUp`` calls this op + on the output of ``rearrange(x, "b (h w) c -> b c h w")``. An earlier + version of the kernel indexed such inputs through their strides, which was + correct but uncoalesced enough to run at 0.44x of eager, so the wrapper now + materializes them first. + """ + torch.manual_seed(0) + B, C, H, W = 2, 64, 16, 16 + kw = dict(device="cuda", dtype=dtype) + x = torch.randn(B, H * W, C, **kw).permute(0, 2, 1).reshape(B, C, H, W) + assert not x.is_contiguous(), "test premise: input is strided" + + weight = torch.randn(C, **kw) + bias = torch.randn(C, **kw) + + fused_out = fused_group_norm_silu(x, weight, bias, num_groups=32, eps=1e-6) + ref_out = F.silu(F.group_norm(x.float(), 32, weight.float(), bias.float(), 1e-6)).to(dtype) + + rtol, atol = _tolerance(dtype) + torch.testing.assert_close(fused_out, ref_out, rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("batch_size", [1, 4]) +@pytest.mark.parametrize("channels", [64, 128]) +def test_hunyuan_vae_config(batch_size, channels): + """Test with HunyuanImage3 VAE actual configuration. + + HunyuanImage3 ResnetBlock uses: + - num_groups=32 + - eps=1e-6 + - Typical spatial sizes: 16x16 to 128x128 (after tiling) + """ + spatial_sizes = [(16, 16), (32, 32), (64, 64), (128, 128)] + + for H, W in spatial_sizes: + x = torch.randn(batch_size, channels, H, W, device="cuda", dtype=torch.float32) + weight = torch.randn(channels, device="cuda") + bias = torch.randn(channels, device="cuda") + + ref_out = F.silu(F.group_norm(x, 32, weight, bias, 1e-6)) + fused_out = fused_group_norm_silu(x, weight, bias, 32, 1e-6) + + torch.testing.assert_close( + fused_out, + ref_out, + rtol=1e-5, + atol=1e-6, + msg=f"HunyuanVAE config: batch={batch_size}, C={channels}, HW=({H},{W})", + ) + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +def test_large_offset_small_variance(dtype): + """A large constant offset with a small variance must stay stable. + + ``x ~ 10000 ± 0.1`` has mean ~1e4 and variance ~1e-2, so the naive + ``E[x^2] - E[x]^2`` subtracts two ~1e8 quantities and rounds the variance to + ~8.0 (the ULP of 1e8) — catastrophic cancellation, producing garbage + normalization. The Welford reduction keeps the variance accurate. + + Why fp32 tolerance is 0.1, not the usual 1e-6: at this offset, ``x`` is stored + in fp32 with ULP ~0.001, so ``x - mean`` carries ~0.001 of representation + noise, and ``rstd ~ 10`` amplifies it to ~0.01-0.02. Even PyTorch's own + GroupNorm reference hits this floor (the old naive implementation would be + off by ~1.0 here). fp16/bf16 quantize the 0.1 perturbation away entirely, so + for those dtypes the meaningful assertion is just "no NaN". + """ + torch.manual_seed(0) + B, C, H, W = 2, 64, 16, 16 + num_groups = 32 + eps = 1e-6 + device = torch.device("cuda") + + x = torch.full((B, C, H, W), 10000.0, device=device, dtype=dtype) + x = x + torch.randn(B, C, H, W, device=device, dtype=dtype) * 0.1 + weight = torch.randn(C, device=device, dtype=dtype) + bias = torch.randn(C, device=device, dtype=dtype) + + fused_out = fused_group_norm_silu(x, weight, bias, num_groups, eps) + # fp32 reference = PyTorch's Welford path, the ground truth. + ref_out = F.silu(F.group_norm(x.float(), num_groups, weight.float(), bias.float(), eps)).to(dtype) + + assert not torch.isnan(fused_out).any(), "fused output must not contain NaN" + + if dtype == torch.float32: + # ~0.02 is the representation-noise floor at offset 1e4; old impl ~1.0. + torch.testing.assert_close(fused_out, ref_out, rtol=0.05, atol=0.1) + # fp16/bf16: perturbation is swallowed by the dtype — no meaningful close, + # the no-NaN assertion above is the regression guard. + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/vllm_omni/diffusion/models/hunyuan_image3/__init__.py b/vllm_omni/diffusion/models/hunyuan_image3/__init__.py index 625895d99..4c93e2397 100644 --- a/vllm_omni/diffusion/models/hunyuan_image3/__init__.py +++ b/vllm_omni/diffusion/models/hunyuan_image3/__init__.py @@ -10,4 +10,8 @@ from vllm_omni.diffusion.models.hunyuan_image3.pipeline_hunyuan_image3 import ( HunyuanImage3Pipeline, ) -__all__ = ["HunyuanImage3Pipeline", "HunyuanImage3Model", "HunyuanImage3Text2ImagePipeline"] +__all__ = [ + "HunyuanImage3Pipeline", + "HunyuanImage3Model", + "HunyuanImage3Text2ImagePipeline", +] diff --git a/vllm_omni/diffusion/models/hunyuan_image3/autoencoder.py b/vllm_omni/diffusion/models/hunyuan_image3/autoencoder.py index efba2f274..1ccbafa65 100644 --- a/vllm_omni/diffusion/models/hunyuan_image3/autoencoder.py +++ b/vllm_omni/diffusion/models/hunyuan_image3/autoencoder.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import math from dataclasses import dataclass import numpy as np @@ -15,6 +14,10 @@ from diffusers.utils.torch_utils import randn_tensor from einops import rearrange from torch import Tensor, nn +# ResnetBlock is platform-dispatched in the layers package __init__: CUDA gets a +# fused GroupNorm+SiLU kernel, every other backend gets the plain PyTorch block. +from vllm_omni.diffusion.models.hunyuan_image3.layers import ResnetBlock + class DiagonalGaussianDistribution: def __init__(self, parameters: torch.Tensor, deterministic: bool = False): @@ -53,10 +56,6 @@ class DecoderOutput(BaseOutput): posterior: DiagonalGaussianDistribution | None = None -def swish(x: Tensor) -> Tensor: - return x * torch.sigmoid(x) - - def forward_with_checkpointing(module, *inputs, use_checkpointing=False): def create_custom_forward(module): def custom_forward(*inputs): @@ -70,46 +69,6 @@ def forward_with_checkpointing(module, *inputs, use_checkpointing=False): return module(*inputs) -class Conv3d(nn.Conv3d): - """ - Perform Conv3d on patches with numerical differences from nn.Conv3d within 1e-5. - Only symmetric padding is supported. - """ - - def forward(self, input): - B, C, T, H, W = input.shape - memory_count = (C * T * H * W) * 2 / 1024**3 - if memory_count > 2: - n_split = math.ceil(memory_count / 2) - assert n_split >= 2 - chunks = torch.chunk(input, chunks=n_split, dim=-3) - padded_chunks = [] - for i in range(len(chunks)): - if self.padding[0] > 0: - padded_chunk = F.pad( - chunks[i], - (0, 0, 0, 0, self.padding[0], self.padding[0]), - mode="constant" if self.padding_mode == "zeros" else self.padding_mode, - value=0, - ) - if i > 0: - padded_chunk[:, :, : self.padding[0]] = chunks[i - 1][:, :, -self.padding[0] :] - if i < len(chunks) - 1: - padded_chunk[:, :, -self.padding[0] :] = chunks[i + 1][:, :, : self.padding[0]] - else: - padded_chunk = chunks[i] - padded_chunks.append(padded_chunk) - padding_bak = self.padding - self.padding = (0, self.padding[1], self.padding[2]) - outputs = [] - for i in range(len(padded_chunks)): - outputs.append(super().forward(padded_chunks[i])) - self.padding = padding_bak - return torch.cat(outputs, dim=-3) - else: - return super().forward(input) - - class AttnBlock(nn.Module): """Attention with torch sdpa implementation.""" @@ -119,10 +78,12 @@ class AttnBlock(nn.Module): self.norm = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) - self.q = Conv3d(in_channels, in_channels, kernel_size=1) - self.k = Conv3d(in_channels, in_channels, kernel_size=1) - self.v = Conv3d(in_channels, in_channels, kernel_size=1) - self.proj_out = Conv3d(in_channels, in_channels, kernel_size=1) + # The memory concern is already handled via VAE tiling, + # so there's no need to chunk here — calling nn.Conv3d directly is fine. + self.q = nn.Conv3d(in_channels, in_channels, kernel_size=1) + self.k = nn.Conv3d(in_channels, in_channels, kernel_size=1) + self.v = nn.Conv3d(in_channels, in_channels, kernel_size=1) + self.proj_out = nn.Conv3d(in_channels, in_channels, kernel_size=1) def attention(self, h_: Tensor) -> Tensor: h_ = self.norm(h_) @@ -142,41 +103,12 @@ class AttnBlock(nn.Module): return x + self.proj_out(self.attention(x)) -class ResnetBlock(nn.Module): - def __init__(self, in_channels: int, out_channels: int): - super().__init__() - self.in_channels = in_channels - out_channels = in_channels if out_channels is None else out_channels - self.out_channels = out_channels - - self.norm1 = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) - self.conv1 = Conv3d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) - self.norm2 = nn.GroupNorm(num_groups=32, num_channels=out_channels, eps=1e-6, affine=True) - self.conv2 = Conv3d(out_channels, out_channels, kernel_size=3, stride=1, padding=1) - if self.in_channels != self.out_channels: - self.nin_shortcut = Conv3d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) - - def forward(self, x): - h = x - h = self.norm1(h) - h = swish(h) - h = self.conv1(h) - - h = self.norm2(h) - h = swish(h) - h = self.conv2(h) - - if self.in_channels != self.out_channels: - x = self.nin_shortcut(x) - return x + h - - class DownsampleDCAE(nn.Module): def __init__(self, in_channels: int, out_channels: int, add_temporal_downsample: bool = True): super().__init__() factor = 2 * 2 * 2 if add_temporal_downsample else 1 * 2 * 2 assert out_channels % factor == 0 - self.conv = Conv3d(in_channels, out_channels // factor, kernel_size=3, stride=1, padding=1) + self.conv = nn.Conv3d(in_channels, out_channels // factor, kernel_size=3, stride=1, padding=1) self.add_temporal_downsample = add_temporal_downsample self.group_size = factor * in_channels // out_channels @@ -196,7 +128,7 @@ class UpsampleDCAE(nn.Module): def __init__(self, in_channels: int, out_channels: int, add_temporal_upsample: bool = True): super().__init__() factor = 2 * 2 * 2 if add_temporal_upsample else 1 * 2 * 2 - self.conv = Conv3d(in_channels, out_channels * factor, kernel_size=3, stride=1, padding=1) + self.conv = nn.Conv3d(in_channels, out_channels * factor, kernel_size=3, stride=1, padding=1) self.add_temporal_upsample = add_temporal_upsample self.repeats = factor * out_channels // in_channels @@ -233,7 +165,7 @@ class Encoder(nn.Module): self.num_res_blocks = num_res_blocks # downsampling - self.conv_in = Conv3d(in_channels, block_out_channels[0], kernel_size=3, stride=1, padding=1) + self.conv_in = nn.Conv3d(in_channels, block_out_channels[0], kernel_size=3, stride=1, padding=1) self.down = nn.ModuleList() block_in = block_out_channels[0] @@ -265,7 +197,7 @@ class Encoder(nn.Module): # end self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True) - self.conv_out = Conv3d(block_in, 2 * z_channels, kernel_size=3, stride=1, padding=1) + self.conv_out = nn.Conv3d(block_in, 2 * z_channels, kernel_size=3, stride=1, padding=1) self.gradient_checkpointing = False @@ -291,7 +223,7 @@ class Encoder(nn.Module): group_size = self.block_out_channels[-1] // (2 * self.z_channels) shortcut = rearrange(h, "b (c r) f h w -> b c r f h w", r=group_size).mean(dim=2) h = self.norm_out(h) - h = swish(h) + h = F.silu(h) h = self.conv_out(h) h += shortcut return h @@ -321,7 +253,7 @@ class Decoder(nn.Module): # z to block_in block_in = block_out_channels[0] - self.conv_in = Conv3d(z_channels, block_in, kernel_size=3, stride=1, padding=1) + self.conv_in = nn.Conv3d(z_channels, block_in, kernel_size=3, stride=1, padding=1) # middle self.mid = nn.Module() @@ -351,7 +283,7 @@ class Decoder(nn.Module): # end self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True) - self.conv_out = Conv3d(block_in, out_channels, kernel_size=3, stride=1, padding=1) + self.conv_out = nn.Conv3d(block_in, out_channels, kernel_size=3, stride=1, padding=1) self.gradient_checkpointing = False @@ -376,7 +308,7 @@ class Decoder(nn.Module): # end h = self.norm_out(h) - h = swish(h) + h = F.silu(h) h = self.conv_out(h) return h diff --git a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py b/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py index 45a94d76d..9328abe01 100644 --- a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py +++ b/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py @@ -79,6 +79,13 @@ from vllm_omni.diffusion.forward_context import set_forward_context_denoise_step from vllm_omni.diffusion.layers.fused_moe import FusedMoE from vllm_omni.diffusion.layers.norm import RMSNorm from vllm_omni.diffusion.layers.rope import RotaryEmbedding + +# ResBlock is platform-dispatched in the layers package __init__: CUDA gets +# fused GroupNorm+SiLU and AdaGN kernels, every other backend gets the plain +# PyTorch block. UNetDown and UNetUp below instantiate whichever one this +# resolves to. +from vllm_omni.diffusion.models.hunyuan_image3.layers import ResBlock +from vllm_omni.diffusion.models.hunyuan_image3.layers.common import conv_nd, normalization from vllm_omni.diffusion.utils.kv_utils import repeat_kv from vllm_omni.model_executor.layers.timestep_embedding import timestep_embedding from vllm_omni.platforms import current_omni_platform @@ -171,44 +178,6 @@ def real_batched_index_select(t, dim, idx): return torch.stack([torch.index_select(t[i], dim - 1, idx[i]) for i in range(len(t))]) -def conv_nd(dims, *args, **kwargs): # noqa: N802 - """ - Create a 1D, 2D, or 3D convolution module. - """ - if dims == 1: - return nn.Conv1d(*args, **kwargs) - elif dims == 2: - return nn.Conv2d(*args, **kwargs) - elif dims == 3: - return nn.Conv3d(*args, **kwargs) - raise ValueError(f"unsupported dimensions: {dims}") - - -def normalization(channels, **kwargs): - """ - Make a standard normalization layer. - :param channels: number of input channels. - :return: a nn.Module for normalization. - """ - return nn.GroupNorm(32, channels, **kwargs) - - -def linear(*args, **kwargs): - """ - Create a linear module. - """ - return nn.Linear(*args, **kwargs) - - -def zero_module(module): - """ - Zero out the parameters of a module and return it. - """ - for p in module.parameters(): - p.detach().zero_() - return module - - def _to_tuple(x, dim=2): if isinstance(x, int): return (x,) * dim @@ -3282,99 +3251,6 @@ class TimestepEmbedder(nn.Module): return t_emb -class ResBlock(nn.Module): - r""" - A residual block that can optionally change the number of channels. - - Args: - in_channels (`int`): - The number of input channels. - emb_channels (`int`): - The number of timestep embedding channels. - dropout (`float`): - The rate of dropout. - out_channels (`int`, *optional*): - If specified, the number of output channels. - use_conv (`bool`, *optional*): - If True and out_channels is specified, use a spatial convolution instead of a - smaller 1x1 convolution to change the channels in the skip connection. - dims (`int`, *optional*): - Determines if the signal is 1D, 2D, or 3D. - up (`bool`, *optional*): - If True, use this block for upsampling. - down (`bool`, *optional*): - If True, use this block for downsampling. - - """ - - def __init__( - self, - in_channels, - emb_channels, - out_channels=None, - dropout=0.0, - use_conv=False, - dims=2, - up=False, - down=False, - device=None, - dtype=None, - ) -> None: - factory_kwargs = {"dtype": dtype, "device": device} - super().__init__() - self.in_channels = in_channels - self.dropout = dropout - self.out_channels = out_channels or self.in_channels - self.use_conv = use_conv - - self.in_layers = nn.Sequential( - normalization(self.in_channels, **factory_kwargs), - nn.SiLU(), - conv_nd(dims, self.in_channels, self.out_channels, 3, padding=1, **factory_kwargs), # noqa: N802 - ) - - self.updown = up or down - self.h_upd = self.x_upd = nn.Identity() - - self.emb_layers = nn.Sequential(nn.SiLU(), linear(emb_channels, 2 * self.out_channels, **factory_kwargs)) - - self.out_layers = nn.Sequential( - normalization(self.out_channels, **factory_kwargs), - nn.SiLU(), - nn.Dropout(p=dropout), - zero_module(conv_nd(dims, self.out_channels, self.out_channels, 3, padding=1, **factory_kwargs)), # noqa: N802 - ) - - if self.out_channels == self.in_channels: - self.skip_connection = nn.Identity() - elif use_conv: - self.skip_connection = conv_nd(dims, self.in_channels, self.out_channels, 3, padding=1, **factory_kwargs) # noqa: N802 - else: - self.skip_connection = conv_nd(dims, self.in_channels, self.out_channels, 1, **factory_kwargs) # noqa: N802 - - def forward(self, x, emb) -> torch.Tensor: - if self.updown: - in_rest, in_conv = self.in_layers[:-1], self.in_layers[-1] - h = in_rest(x) - h = self.h_upd(h) - x = self.x_upd(x) - h = in_conv(h) - else: - h = self.in_layers(x) - - emb_out = self.emb_layers(emb) - while len(emb_out.shape) < len(h.shape): - emb_out = emb_out[..., None] - - # Adaptive Group Normalization - out_norm, out_rest = self.out_layers[0], self.out_layers[1:] - scale, shift = torch.chunk(emb_out, 2, dim=1) - h = out_norm(h) * (1.0 + scale) + shift - h = out_rest(h) - - return self.skip_connection(x) + h - - class UNetDown(nn.Module): def __init__( self, patch_size, in_channels, emb_channels, hidden_channels, out_channels, dropout=0.0, device=None, dtype=None diff --git a/vllm_omni/diffusion/models/hunyuan_image3/layers/__init__.py b/vllm_omni/diffusion/models/hunyuan_image3/layers/__init__.py new file mode 100644 index 000000000..39081df17 --- /dev/null +++ b/vllm_omni/diffusion/models/hunyuan_image3/layers/__init__.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Platform-dispatched layer implementations for HunyuanImage3. + +The two blocks are selected by platform: + +Layout:: + native/autoencoder_blocks.py default ResnetBlock + native/transformer_blocks.py default ResBlock + nvidia/autoencoder_blocks.py CUDA ResnetBlock (fused) + nvidia/transformer_blocks.py CUDA ResBlock (fused) +""" + +from vllm_omni.platforms import current_omni_platform + +if current_omni_platform.is_cuda(): + from vllm_omni.diffusion.models.hunyuan_image3.layers.nvidia import ResBlock, ResnetBlock +else: + from vllm_omni.diffusion.models.hunyuan_image3.layers.native import ResBlock, ResnetBlock + +__all__ = ["ResBlock", "ResnetBlock"] diff --git a/vllm_omni/diffusion/models/hunyuan_image3/layers/common/__init__.py b/vllm_omni/diffusion/models/hunyuan_image3/layers/common/__init__.py new file mode 100644 index 000000000..5272e9d51 --- /dev/null +++ b/vllm_omni/diffusion/models/hunyuan_image3/layers/common/__init__.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from .utils import ( + conv_nd, + linear, + normalization, + zero_module, +) + +__all__ = ["conv_nd", "linear", "normalization", "zero_module"] diff --git a/vllm_omni/diffusion/models/hunyuan_image3/layers/common/utils.py b/vllm_omni/diffusion/models/hunyuan_image3/layers/common/utils.py new file mode 100644 index 000000000..01dd1cc4e --- /dev/null +++ b/vllm_omni/diffusion/models/hunyuan_image3/layers/common/utils.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Small platform-agnostic helpers for UNet-style blocks (adapted from diffusers). + +``conv_nd`` / ``normalization`` / ``linear`` / ``zero_module`` are used by the +``ResBlock`` / ``UNetDown`` / ``UNetUp`` implementations across HunyuanImage3's +platform-specific layer packages. They are plain ``nn.ConvNd`` / ``nn.GroupNorm`` +/ ``nn.Linear`` wrappers with no hardware-specific behaviour, so they live here +instead of being duplicated per backend. +""" + +from torch import nn + + +def conv_nd(dims, *args, **kwargs): # noqa: N802 + """ + Create a 1D, 2D, or 3D convolution module. + """ + if dims == 1: + return nn.Conv1d(*args, **kwargs) + elif dims == 2: + return nn.Conv2d(*args, **kwargs) + elif dims == 3: + return nn.Conv3d(*args, **kwargs) + raise ValueError(f"unsupported dimensions: {dims}") + + +def normalization(channels, **kwargs): + """ + Make a standard normalization layer. + :param channels: number of input channels. + :return: a nn.Module for normalization. + """ + return nn.GroupNorm(32, channels, **kwargs) + + +def linear(*args, **kwargs): + """ + Create a linear module. + """ + return nn.Linear(*args, **kwargs) + + +def zero_module(module): + """ + Zero out the parameters of a module and return it. + """ + for p in module.parameters(): + p.detach().zero_() + return module diff --git a/vllm_omni/diffusion/models/hunyuan_image3/layers/native/__init__.py b/vllm_omni/diffusion/models/hunyuan_image3/layers/native/__init__.py new file mode 100644 index 000000000..55a695af7 --- /dev/null +++ b/vllm_omni/diffusion/models/hunyuan_image3/layers/native/__init__.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Native (plain PyTorch) layer implementations.""" + +from vllm_omni.diffusion.models.hunyuan_image3.layers.native.autoencoder_blocks import ResnetBlock +from vllm_omni.diffusion.models.hunyuan_image3.layers.native.transformer_blocks import ResBlock + +__all__ = ["ResBlock", "ResnetBlock"] diff --git a/vllm_omni/diffusion/models/hunyuan_image3/layers/native/autoencoder_blocks.py b/vllm_omni/diffusion/models/hunyuan_image3/layers/native/autoencoder_blocks.py new file mode 100644 index 000000000..6e85a52f6 --- /dev/null +++ b/vllm_omni/diffusion/models/hunyuan_image3/layers/native/autoencoder_blocks.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""ResnetBlock for the HunyuanImage3 autoencoder — default fallback implementation. + +Uses native PyTorch ``nn.GroupNorm`` + ``F.silu``. Hardware-specific optimized +versions live in subdirectories (e.g., nvidia/) and are selected via __init__.py +dispatch. +""" + +import torch.nn.functional as F +from torch import nn + + +class ResnetBlock(nn.Module): + def __init__(self, in_channels: int, out_channels: int): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + + self.norm1 = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + self.conv1 = nn.Conv3d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + self.norm2 = nn.GroupNorm(num_groups=32, num_channels=out_channels, eps=1e-6, affine=True) + self.conv2 = nn.Conv3d(out_channels, out_channels, kernel_size=3, stride=1, padding=1) + if self.in_channels != self.out_channels: + self.nin_shortcut = nn.Conv3d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x): + h = x + h = self.norm1(h) + h = F.silu(h) + h = self.conv1(h) + + h = self.norm2(h) + h = F.silu(h) + h = self.conv2(h) + + if self.in_channels != self.out_channels: + x = self.nin_shortcut(x) + return x + h + + +__all__ = ["ResnetBlock"] diff --git a/vllm_omni/diffusion/models/hunyuan_image3/layers/native/transformer_blocks.py b/vllm_omni/diffusion/models/hunyuan_image3/layers/native/transformer_blocks.py new file mode 100644 index 000000000..2322fa6e3 --- /dev/null +++ b/vllm_omni/diffusion/models/hunyuan_image3/layers/native/transformer_blocks.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""DiT ``ResBlock`` for HunyuanImage3 -- default implementation. + +Plain PyTorch: ``GroupNorm -> SiLU`` runs as two ops and the adaptive +GroupNorm as ``norm(h) * (1 + scale) + shift``. Hardware-specific variants +live in sibling packages (e.g. ``nvidia/``) and are picked by the dispatch in +``hunyuan_image3/__init__.py``. + +""" + +import torch +from torch import nn + +from vllm_omni.diffusion.models.hunyuan_image3.layers.common import conv_nd, linear, normalization, zero_module + + +class ResBlock(nn.Module): + r""" + A residual block that can optionally change the number of channels. + Args: + in_channels (`int`): + The number of input channels. + emb_channels (`int`): + The number of timestep embedding channels. + dropout (`float`): + The rate of dropout. + out_channels (`int`, *optional*): + If specified, the number of output channels. + use_conv (`bool`, *optional*): + If True and out_channels is specified, use a spatial convolution instead of a + smaller 1x1 convolution to change the channels in the skip connection. + dims (`int`, *optional*): + Determines if the signal is 1D, 2D, or 3D. + up (`bool`, *optional*): + If True, use this block for upsampling. + down (`bool`, *optional*): + If True, use this block for downsampling. + """ + + def __init__( + self, + in_channels, + emb_channels, + out_channels=None, + dropout=0.0, + use_conv=False, + dims=2, + up=False, + down=False, + device=None, + dtype=None, + ) -> None: + factory_kwargs = {"dtype": dtype, "device": device} + super().__init__() + self.in_channels = in_channels + self.dropout = dropout + self.out_channels = out_channels or self.in_channels + self.use_conv = use_conv + + self.in_layers = nn.Sequential( + normalization(self.in_channels, **factory_kwargs), + nn.SiLU(), + conv_nd(dims, self.in_channels, self.out_channels, 3, padding=1, **factory_kwargs), # noqa: N802 + ) + + self.updown = up or down + self.h_upd = self.x_upd = nn.Identity() + + self.emb_layers = nn.Sequential(nn.SiLU(), linear(emb_channels, 2 * self.out_channels, **factory_kwargs)) + + self.out_layers = nn.Sequential( + normalization(self.out_channels, **factory_kwargs), + nn.SiLU(), + nn.Dropout(p=dropout), + zero_module(conv_nd(dims, self.out_channels, self.out_channels, 3, padding=1, **factory_kwargs)), # noqa: N802 + ) + + if self.out_channels == self.in_channels: + self.skip_connection = nn.Identity() + elif use_conv: + self.skip_connection = conv_nd(dims, self.in_channels, self.out_channels, 3, padding=1, **factory_kwargs) # noqa: N802 + else: + self.skip_connection = conv_nd(dims, self.in_channels, self.out_channels, 1, **factory_kwargs) # noqa: N802 + + def forward(self, x, emb) -> torch.Tensor: + if self.updown: + in_rest, in_conv = self.in_layers[:-1], self.in_layers[-1] + h = in_rest(x) + h = self.h_upd(h) + x = self.x_upd(x) + h = in_conv(h) + else: + h = self.in_layers(x) + + emb_out = self.emb_layers(emb) + while len(emb_out.shape) < len(h.shape): + emb_out = emb_out[..., None] + + # Adaptive Group Normalization + out_norm, out_rest = self.out_layers[0], self.out_layers[1:] + scale, shift = torch.chunk(emb_out, 2, dim=1) + h = out_norm(h) * (1.0 + scale) + shift + h = out_rest(h) + + return self.skip_connection(x) + h + + +__all__ = ["ResBlock"] diff --git a/vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/__init__.py b/vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/__init__.py new file mode 100644 index 000000000..78d3304a5 --- /dev/null +++ b/vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/__init__.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""NVIDIA (CUDA) implementations.""" + +from .autoencoder_blocks import ResnetBlock # noqa: E402 +from .transformer_blocks import ResBlock # noqa: E402 + +__all__ = ["ResnetBlock", "ResBlock"] diff --git a/vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/_cudnn.py b/vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/_cudnn.py new file mode 100644 index 000000000..4c541591d --- /dev/null +++ b/vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/_cudnn.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from contextlib import contextmanager + +import torch + + +@contextmanager +def cudnn_settings(benchmark: bool, deterministic: bool): + """Temporarily set cuDNN backend flags, restoring the old values on exit. + + Used to scope ``torch.backends.cudnn.benchmark``/``deterministic`` to the + few blocks whose convolutions actually benefit from autotune, instead of + setting them globally at import time (which would silently change every + other model's conv behaviour in the same process). + """ + old_benchmark = torch.backends.cudnn.benchmark + old_deterministic = torch.backends.cudnn.deterministic + + torch.backends.cudnn.benchmark = benchmark + torch.backends.cudnn.deterministic = deterministic + try: + yield + finally: + torch.backends.cudnn.benchmark = old_benchmark + torch.backends.cudnn.deterministic = old_deterministic diff --git a/vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/autoencoder_blocks.py b/vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/autoencoder_blocks.py new file mode 100644 index 000000000..c6c68957e --- /dev/null +++ b/vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/autoencoder_blocks.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""ResnetBlock for the HunyuanImage3 autoencoder — NVIDIA CUDA + Triton implementation. + +Split out from autoencoder.py because its ``GroupNorm -> SiLU`` pairs are served +by :func:`fused_group_norm_silu`, a single Triton kernel that falls back to +native ``F.silu(F.group_norm(...))`` when Triton is unavailable. +""" + +from torch import nn + +from vllm_omni.diffusion.models.hunyuan_image3.layers.nvidia._cudnn import cudnn_settings +from vllm_omni.model_executor.models.common.ops import fused_group_norm_silu + + +class ResnetBlock(nn.Module): + def __init__(self, in_channels: int, out_channels: int): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + + self.norm1 = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + self.conv1 = nn.Conv3d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + self.norm2 = nn.GroupNorm(num_groups=32, num_channels=out_channels, eps=1e-6, affine=True) + self.conv2 = nn.Conv3d(out_channels, out_channels, kernel_size=3, stride=1, padding=1) + if self.in_channels != self.out_channels: + self.nin_shortcut = nn.Conv3d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x): + # cuDNN autotune is scoped to this block rather than set at import time. + # These ResnetBlocks carry the bulk of the VAE's conv work: ~29 instances + # x 2 convs across every resolution (up to the 1024x1024 decode), versus + # the scattered convs left in autoencoder.py (AttnBlock's 1x1 at the + # lowest-res mid, down/up-samplers, conv_in/out) whose activations are + # negligible. A process-wide flag would instead silently change every + # other model's conv behaviour in the same process. + with cudnn_settings(benchmark=True, deterministic=True): + h = x + h = fused_group_norm_silu( + h, self.norm1.weight, self.norm1.bias, num_groups=self.norm1.num_groups, eps=self.norm1.eps + ) + h = self.conv1(h) + + h = fused_group_norm_silu( + h, self.norm2.weight, self.norm2.bias, num_groups=self.norm2.num_groups, eps=self.norm2.eps + ) + h = self.conv2(h) + + if self.in_channels != self.out_channels: + x = self.nin_shortcut(x) + return x + h + + +__all__ = ["ResnetBlock"] diff --git a/vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/transformer_blocks.py b/vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/transformer_blocks.py new file mode 100644 index 000000000..411dc6351 --- /dev/null +++ b/vllm_omni/diffusion/models/hunyuan_image3/layers/nvidia/transformer_blocks.py @@ -0,0 +1,138 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""``ResBlock`` for HunyuanImage3 -- NVIDIA CUDA + Triton implementation. + +Two fusions over the default block: + +* ``in_layers``: ``GroupNorm -> SiLU`` becomes one :func:`fused_group_norm_silu` + kernel instead of two ops. +* ``out_layers``: ``GroupNorm -> adaptive modulation -> SiLU`` (i.e. + ``SiLU(norm(h) * (1 + scale) + shift)``) becomes one + :func:`fused_adaptive_group_norm_silu` kernel instead of four. +""" + +import torch +from torch import nn + +from vllm_omni.diffusion.models.hunyuan_image3.layers.common import conv_nd, normalization, zero_module +from vllm_omni.diffusion.models.hunyuan_image3.layers.nvidia._cudnn import cudnn_settings +from vllm_omni.model_executor.models.common.ops import ( + fused_adaptive_group_norm_silu, + fused_group_norm_silu, +) + + +class ResBlock(nn.Module): + r""" + A residual block that can optionally change the number of channels. + Args: + in_channels (`int`): + The number of input channels. + emb_channels (`int`): + The number of timestep embedding channels. + dropout (`float`): + The rate of dropout. + out_channels (`int`, *optional*): + If specified, the number of output channels. + use_conv (`bool`, *optional*): + If True and out_channels is specified, use a spatial convolution instead of a + smaller 1x1 convolution to change the channels in the skip connection. + dims (`int`, *optional*): + Determines if the signal is 1D, 2D, or 3D. + up (`bool`, *optional*): + If True, use this block for upsampling. + down (`bool`, *optional*): + If True, use this block for downsampling. + """ + + def __init__( + self, + in_channels, + emb_channels, + out_channels=None, + dropout=0.0, + use_conv=False, + dims=2, + up=False, + down=False, + device=None, + dtype=None, + ) -> None: + factory_kwargs = {"dtype": dtype, "device": device} + super().__init__() + self.in_channels = in_channels + self.dropout = dropout + self.out_channels = out_channels or self.in_channels + self.use_conv = use_conv + + self.in_layers = nn.Sequential( + normalization(self.in_channels, **factory_kwargs), + nn.SiLU(), + conv_nd(dims, self.in_channels, self.out_channels, 3, padding=1, **factory_kwargs), # noqa: N802 + ) + + self.updown = up or down + self.h_upd = self.x_upd = nn.Identity() + + self.emb_layers = nn.Sequential(nn.SiLU(), nn.Linear(emb_channels, 2 * self.out_channels, **factory_kwargs)) + + self.out_layers = nn.Sequential( + normalization(self.out_channels, **factory_kwargs), + nn.SiLU(), + nn.Dropout(p=dropout), + zero_module(conv_nd(dims, self.out_channels, self.out_channels, 3, padding=1, **factory_kwargs)), # noqa: N802 + ) + + if self.out_channels == self.in_channels: + self.skip_connection = nn.Identity() + elif use_conv: + self.skip_connection = conv_nd(dims, self.in_channels, self.out_channels, 3, padding=1, **factory_kwargs) # noqa: N802 + else: + self.skip_connection = conv_nd(dims, self.in_channels, self.out_channels, 1, **factory_kwargs) # noqa: N802 + + def forward(self, x, emb) -> torch.Tensor: + # cuDNN autotune is scoped to this block rather than set at import time. + # The benchmark=True win is concentrated in the convolutions right here: + # this ResBlock sits inside UNetDown/UNetUp (patch_embed / final_layer), + # the heaviest conv users in the DiT. A process-wide flag would instead + # silently change every other model's conv behaviour in the same process. + with cudnn_settings(benchmark=True, deterministic=True): + # ``in_layers``/``out_layers`` stay nn.Sequential and are indexed into + # rather than unpacked into named submodules -- that is what keeps the + # state_dict keys (``in_layers.0/.2``, ``out_layers.0/.3``) identical to + # the unfused block, so checkpoints load unchanged. + in_norm, in_conv = self.in_layers[0], self.in_layers[-1] + + # GroupNorm -> SiLU, one kernel instead of two. + h = fused_group_norm_silu(x, in_norm.weight, in_norm.bias, num_groups=in_norm.num_groups, eps=in_norm.eps) + if self.updown: + h = self.h_upd(h) + x = self.x_upd(x) + h = in_conv(h) + + emb_out = self.emb_layers(emb) + while len(emb_out.shape) < len(h.shape): + emb_out = emb_out[..., None] + + # Adaptive GroupNorm -> SiLU: GroupNorm -> mul -> add -> silu, one + # kernel instead of four. The ``nn.SiLU`` at ``out_layers[1]`` is + # skipped here but stays in the Sequential, so state_dict keys are + # unchanged. + out_norm, out_rest = self.out_layers[0], self.out_layers[2:] + scale, shift = torch.chunk(emb_out, 2, dim=1) + h = fused_adaptive_group_norm_silu( + h, + out_norm.weight, + out_norm.bias, + scale, + shift, + num_groups=out_norm.num_groups, + eps=out_norm.eps, + ) + h = out_rest(h) + + return self.skip_connection(x) + h + + +__all__ = ["ResBlock"] diff --git a/vllm_omni/model_executor/models/common/ops/__init__.py b/vllm_omni/model_executor/models/common/ops/__init__.py new file mode 100644 index 000000000..d9b522bdf --- /dev/null +++ b/vllm_omni/model_executor/models/common/ops/__init__.py @@ -0,0 +1,17 @@ +"""Common operators shared across models and hardware backends. + +This module contains fused operators that can be reused across multiple models +and compiled for different hardware targets (CUDA, ROCm, etc.). +""" + +from vllm_omni.model_executor.models.common.ops.fused_adaptive_group_norm_silu import ( + fused_adaptive_group_norm_silu, +) +from vllm_omni.model_executor.models.common.ops.fused_group_norm_silu import ( + fused_group_norm_silu, +) + +__all__ = [ + "fused_group_norm_silu", + "fused_adaptive_group_norm_silu", +] diff --git a/vllm_omni/model_executor/models/common/ops/fused_adaptive_group_norm_silu.py b/vllm_omni/model_executor/models/common/ops/fused_adaptive_group_norm_silu.py new file mode 100644 index 000000000..2755c3ece --- /dev/null +++ b/vllm_omni/model_executor/models/common/ops/fused_adaptive_group_norm_silu.py @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# ruff: noqa: N803 + +"""Fused Adaptive Group Normalization (AdaGN) + SiLU operator. + +This module implements the fused AdaGN+SiLU pattern commonly used in Diffusion +Transformer ResBlocks: + + output = SiLU(GroupNorm(x, weight, bias) * (1 + scale) + shift) + +where ``scale`` and ``shift`` are per-(batch, channel) conditioning signals +derived from the timestep embedding. Eager PyTorch spends four kernels and three +full-size intermediates on this; the fused version does it in one pass. + +Falls back to native PyTorch ops when Triton is unavailable. +""" + +import torch +import torch.nn.functional as F +from vllm.triton_utils import HAS_TRITON, tl, triton + + +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE": 4096}, num_warps=16, num_stages=1), + triton.Config({"BLOCK_SIZE": 4096}, num_warps=16, num_stages=2), + triton.Config({"BLOCK_SIZE": 4096}, num_warps=16, num_stages=4), + triton.Config({"BLOCK_SIZE": 4096}, num_warps=16, num_stages=6), + ], + key=["spatial_size", "C"], +) +@triton.jit +def _adaptive_group_norm_silu_kernel( + # Input/output pointers + x_ptr, + out_ptr, + # Affine parameters + weight_ptr, + bias_ptr, + # Conditioning signals, both (B, C) contiguous + scale_ptr, + shift_ptr, + # Shape info; x is contiguous (B, C, spatial_size) + C, + spatial_size, + num_groups: tl.constexpr, + eps: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + num_stages: tl.constexpr, +): + """ + Fused AdaGN + SiLU kernel: SiLU(GroupNorm(x) * (1 + scale) + shift). + Mean and variance use a parallel Welford reduction (block-level register + reduction merged with the Chan formula), which avoids the catastrophic + cancellation of E[x^2] - E[x]^2 on large-offset inputs. x is read once for + the statistics and once for normalization (2 reads total). + One program handles each (batch, group) pair. Channels are processed serially, + while the spatial axis is vectorized for diffusion workloads. + Moments are accumulated in fp32 to match PyTorch numerics. + """ + pid = tl.program_id(0) + + group_size = C // num_groups + n_idx = pid // num_groups + g_idx = pid % num_groups + + # === Pass 1: Welford reduction over the whole group (fp32) === + # Each block is reduced in registers (block mean, then centered M2), so the + # loaded values are reused instead of re-reading memory; blocks are merged + # with the Chan et al. parallel-Welford formula. This keeps both the mean and + # the variance accurate for inputs like ``10000 +- 0.1``, where the naive + # ``E[x^2] - E[x]^2`` cancels catastrophically, and reads x only once for the + # statistics (2 reads total including normalize). + n_total = tl.zeros([1], dtype=tl.float32) + mean_total = tl.zeros([1], dtype=tl.float32) + m2_total = tl.zeros([1], dtype=tl.float32) + + for c_offset in range(group_size): + c_idx = g_idx * group_size + c_offset + base = n_idx * C * spatial_size + c_idx * spatial_size + + for s_start in tl.range(0, spatial_size, BLOCK_SIZE, num_stages=num_stages): + offsets = s_start + tl.arange(0, BLOCK_SIZE) + mask = offsets < spatial_size + + x_val = tl.load(x_ptr + base + offsets, mask=mask, other=0.0) + x_val = x_val.to(tl.float32) + + # block-level reduction in registers (no extra memory traffic) + n = tl.sum(tl.where(mask, 1.0, 0.0), axis=0) + bsum = tl.sum(x_val, axis=0) + bmean = bsum / n + bm2 = tl.sum(tl.where(mask, (x_val - bmean) * (x_val - bmean), 0.0), axis=0) + + # Chan et al. merge into the running (n, mean, m2) + delta = bmean - mean_total + new_n = n_total + n + mean_total = mean_total + delta * (n / new_n) + m2_total = m2_total + bm2 + delta * delta * (n_total * n / new_n) + n_total = new_n + + mean = mean_total + var = m2_total / n_total + rstd = 1.0 / tl.sqrt(var + eps) + + # === Pass 2: normalize, affine, then adaptive modulation === + for c_offset in range(group_size): + c_idx = g_idx * group_size + c_offset + base = n_idx * C * spatial_size + c_idx * spatial_size + + weight_val = tl.load(weight_ptr + c_idx).to(tl.float32) + bias_val = tl.load(bias_ptr + c_idx).to(tl.float32) + scale_val = tl.load(scale_ptr + n_idx * C + c_idx).to(tl.float32) + shift_val = tl.load(shift_ptr + n_idx * C + c_idx).to(tl.float32) + + for s_start in tl.range(0, spatial_size, BLOCK_SIZE, num_stages=num_stages): + offsets = s_start + tl.arange(0, BLOCK_SIZE) + mask = offsets < spatial_size + + x_val = tl.load(x_ptr + base + offsets, mask=mask, other=0.0) + x_val = x_val.to(tl.float32) + + # GroupNorm: (x - mean) * rstd * weight + bias + norm_val = (x_val - mean) * rstd * weight_val + bias_val + + # AdaGN: norm * (1 + scale) + shift + out_val = norm_val * (1.0 + scale_val) + shift_val + + # SiLU: out * sigmoid(out) + out_val = out_val * tl.sigmoid(out_val) + + # ``tl.store`` casts to the output pointer's dtype, which the + # caller picked to match eager GroupNorm's autocast behaviour. + tl.store(out_ptr + base + offsets, out_val, mask=mask) + + +def fused_adaptive_group_norm_silu( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + scale: torch.Tensor, + shift: torch.Tensor, + num_groups: int, + eps: float = 1e-6, +) -> torch.Tensor: + """ + Fused Adaptive GroupNorm + SiLU. + + Computes SiLU(GroupNorm(x) * (1 + scale) + shift) in one fused operation, avoiding intermediate tensors. + + - x: (B, C, *spatial), any spatial rank ≥ 1. + - weight, bias: per-channel parameters, shape (C,). + - scale, shift: adaptive parameters with B * C elements, broadcastable per channel. + - num_groups: GroupNorm group count. + - eps: numerical stability epsilon. + + Returns a tensor with the same shape as x, matching eager F.group_norm dtype behavior. + + Spatial dimensions are flattened during computation and restored afterward. + GroupNorm statistics cover each channel group across all spatial positions, so the result is exact. + """ + assert x.ndim >= 3, f"Expected at least 3D input (B, C, *spatial), got {x.ndim}D" + B, C = x.shape[:2] + assert C % num_groups == 0, f"num_channels ({C}) must be divisible by num_groups ({num_groups})" + assert weight.ndim == 1 and weight.size(0) == C, f"Weight shape {tuple(weight.shape)} doesn't match channels {C}" + assert bias.ndim == 1 and bias.size(0) == C, f"Bias shape {tuple(bias.shape)} doesn't match channels {C}" + assert scale.numel() == B * C, f"scale has {scale.numel()} elements, expected B*C = {B * C}" + assert shift.numel() == B * C, f"shift has {shift.numel()} elements, expected B*C = {B * C}" + + # Fallback if Triton not available (NPU, CPU, ...) + if not HAS_TRITON: + broadcast = (B, C) + (1,) * (x.ndim - 2) + normed = F.group_norm(x, num_groups, weight, bias, eps) + return F.silu(normed * (1.0 + scale.reshape(broadcast)) + shift.reshape(broadcast)) + + orig_shape = x.shape + + # The kernel indexes x as a dense (B, C, spatial) block, so normalize the + # layout here. ``reshape``/``contiguous`` are no-ops for the common case of + # a contiguous activation coming out of a conv. + x_flat = x.contiguous().reshape(B, C, -1) + spatial_size = x_flat.size(2) + + # ``reshape`` rather than ``view``: scale/shift typically arrive as halves + # of a torch.chunk along dim 1, which is not contiguous when B > 1. + scale_2d = scale.reshape(B, C).contiguous() + shift_2d = shift.reshape(B, C).contiguous() + + # Allocate with the dtype eager GroupNorm would return, so the fused path + # stays a drop-in replacement inside autocast regions. + out_dtype = x_flat.dtype + if torch.is_autocast_enabled(x_flat.device.type): + out_dtype = torch.float32 + out_flat = torch.empty_like(x_flat, dtype=out_dtype) + + # Only B*num_groups programs are launched, which is well under the SM count + # for typical diffusion batches. A memory-bound kernel can still saturate + # HBM from few CTAs, but only with enough loads in flight, so widen the CTA + # for the large activations instead of leaving it at the 4-warp default. + + # One program per (batch, group) pair. + grid = (B * num_groups,) + + _adaptive_group_norm_silu_kernel[grid]( + x_flat, + out_flat, + weight, + bias, + scale_2d, + shift_2d, + C, + spatial_size, + num_groups=num_groups, + eps=eps, + ) + + return out_flat.reshape(orig_shape) + + +__all__ = ["fused_adaptive_group_norm_silu"] diff --git a/vllm_omni/model_executor/models/common/ops/fused_group_norm_silu.py b/vllm_omni/model_executor/models/common/ops/fused_group_norm_silu.py new file mode 100644 index 000000000..21fb245c6 --- /dev/null +++ b/vllm_omni/model_executor/models/common/ops/fused_group_norm_silu.py @@ -0,0 +1,207 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# ruff: noqa: N803 + +"""Fused GroupNorm + SiLU operator. + +This operator fuses GroupNorm followed by SiLU activation into a single kernel, +reducing memory traffic and kernel launch overhead. The implementation uses +Triton for CUDA/ROCm compatibility, and falls back to native PyTorch ops when +Triton is unavailable (NPU, CPU, ...), so callers never need a platform check. + +Measured against eager ``F.silu(F.group_norm(...))`` on one L20X, bf16, 32 +groups: 1.1-1.5x at the DiT ResBlock's activation sizes, where both paths are +dominated by launch overhead, and 2.2-2.9x at the VAE's decode-resolution +activations, where the saved memory traffic is what pays. +""" + +import torch +import torch.nn.functional as F +from vllm.triton_utils import HAS_TRITON, tl, triton + + +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE": 4096}, num_warps=16, num_stages=1), + triton.Config({"BLOCK_SIZE": 4096}, num_warps=16, num_stages=2), + triton.Config({"BLOCK_SIZE": 4096}, num_warps=16, num_stages=4), + triton.Config({"BLOCK_SIZE": 4096}, num_warps=16, num_stages=6), + ], + key=["spatial_size", "C"], +) +@triton.jit +def _group_norm_silu_kernel( + # Input/Output pointers + x_ptr, + out_ptr, + # Normalization parameters + weight_ptr, + bias_ptr, + # Shape info; x is contiguous (N, C, spatial_size) + C, + spatial_size, + num_groups: tl.constexpr, + eps: tl.constexpr, + # Block sizes + BLOCK_SIZE: tl.constexpr, + num_stages: tl.constexpr, +): + """ + Fused GroupNorm + SiLU kernel. + Computes SiLU(GroupNorm(x)) in one kernel, avoiding intermediate tensors. + Mean and variance use a parallel Welford reduction (block-level register + reduction merged with the Chan formula), which avoids the catastrophic + cancellation of E[x^2] - E[x]^2 on large-offset inputs. x is read once for + the statistics and once for normalization (2 reads total). + One program handles each (batch, group) pair. Channels are processed serially, + while spatial positions are vectorized for diffusion workloads. + Moments are accumulated in fp32 to match PyTorch numerics. + """ + pid = tl.program_id(0) + + group_size = C // num_groups + n_idx = pid // num_groups + g_idx = pid % num_groups + + # === Pass 1: Welford reduction over the whole group (fp32) === + # Each block is reduced in registers (block mean, then centered M2), so the + # loaded values are reused instead of re-reading memory; blocks are merged + # with the Chan et al. parallel-Welford formula. This keeps both the mean and + # the variance accurate for inputs like ``10000 +- 0.1``, where the naive + # ``E[x^2] - E[x]^2`` cancels catastrophically, and reads x only once for the + # statistics (2 reads total including normalize). + n_total = tl.zeros([1], dtype=tl.float32) + mean_total = tl.zeros([1], dtype=tl.float32) + m2_total = tl.zeros([1], dtype=tl.float32) + + for c_offset in range(group_size): + c_idx = g_idx * group_size + c_offset + base = n_idx * C * spatial_size + c_idx * spatial_size + + for s_start in tl.range(0, spatial_size, BLOCK_SIZE, num_stages=num_stages): + offsets = s_start + tl.arange(0, BLOCK_SIZE) + mask = offsets < spatial_size + + x_val = tl.load(x_ptr + base + offsets, mask=mask, other=0.0) + x_val = x_val.to(tl.float32) + + # block-level reduction in registers (no extra memory traffic) + n = tl.sum(tl.where(mask, 1.0, 0.0), axis=0) + bsum = tl.sum(x_val, axis=0) + bmean = bsum / n + bm2 = tl.sum(tl.where(mask, (x_val - bmean) * (x_val - bmean), 0.0), axis=0) + + # Chan et al. merge into the running (n, mean, m2) + delta = bmean - mean_total + new_n = n_total + n + mean_total = mean_total + delta * (n / new_n) + m2_total = m2_total + bm2 + delta * delta * (n_total * n / new_n) + n_total = new_n + + mean = mean_total + var = m2_total / n_total + rstd = 1.0 / tl.sqrt(var + eps) + + # === Pass 2: normalize, apply affine, and SiLU === + for c_offset in range(group_size): + c_idx = g_idx * group_size + c_offset + base = n_idx * C * spatial_size + c_idx * spatial_size + + weight_val = tl.load(weight_ptr + c_idx).to(tl.float32) + bias_val = tl.load(bias_ptr + c_idx).to(tl.float32) + + for s_start in tl.range(0, spatial_size, BLOCK_SIZE, num_stages=num_stages): + offsets = s_start + tl.arange(0, BLOCK_SIZE) + mask = offsets < spatial_size + + x_val = tl.load(x_ptr + base + offsets, mask=mask, other=0.0) + x_val = x_val.to(tl.float32) + + # Normalize and apply affine + norm_val = (x_val - mean) * rstd * weight_val + bias_val + + # Apply SiLU: x * sigmoid(x) + out_val = norm_val * tl.sigmoid(norm_val) + + # ``tl.store`` casts to the output pointer's dtype, which the + # caller picked to match eager GroupNorm's autocast behaviour. + tl.store(out_ptr + base + offsets, out_val, mask=mask) + + +def fused_group_norm_silu( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + num_groups: int = 32, + eps: float = 1e-6, +) -> torch.Tensor: + """ + Fused GroupNorm + SiLU activation. + Computes SiLU(GroupNorm(x, num_groups, weight, bias, eps)) in a single Triton kernel, + avoiding intermediate tensors and reducing memory traffic and launch overhead. + + - x: (N, C, *spatial), any spatial rank. + - weight, bias: per-channel parameters, shape (C,). + - num_groups: GroupNorm group count, default 32. + - eps: numerical stability epsilon, default 1e-6. + Uses fp32 accumulation for numeric alignment with PyTorch. + Returns the same shape and eager F.group_norm-compatible dtype. + Spatial dimensions are flattened during computation and restored afterward; + this is exact since GroupNorm reduces across each channel group and all spatial positions. + Non-contiguous inputs are materialized before launch. + """ + # Fallback if Triton not available (NPU, CPU, ...) + if not HAS_TRITON: + return F.silu(F.group_norm(x, num_groups, weight, bias, eps)) + + # Validate inputs + assert x.ndim >= 3, f"Expected at least 3D input (N, C, *spatial), got {x.ndim}D" + assert x.size(1) % num_groups == 0, f"Channels {x.size(1)} must be divisible by num_groups {num_groups}" + assert weight.ndim == 1 and weight.size(0) == x.size(1), ( + f"Weight shape {weight.shape} doesn't match channels {x.size(1)}" + ) + assert bias.ndim == 1 and bias.size(0) == x.size(1), f"Bias shape {bias.shape} doesn't match channels {x.size(1)}" + + # Collapse arbitrary spatial ranks into a single axis so one kernel serves + # both the 2D DiT blocks and the 3D VAE blocks. + # + # ``contiguous()`` is not just defensive. HunyuanImage3's UNetUp feeds this + # op straight out of ``rearrange(x, "b (h w) c -> b c h w")``, i.e. a + # permuted view whose *channel* stride is 1. Indexing that layout directly + # from the kernel makes every warp's spatial load stride by C elements, and + # the resulting uncoalesced traffic turned the op into 0.44x of eager at + # (1, 4096, 64, 64). One coalesced pre-pass costs far less than that, so + # normalize the layout here and let the kernel assume a dense block. + orig_shape = x.shape + B, C = orig_shape[0], orig_shape[1] + x_flat = x.contiguous().reshape(B, C, -1) + spatial_size = x_flat.size(2) + + # Allocate output with the dtype eager GroupNorm would return, so that the + # fused path stays a drop-in replacement inside autocast regions. + out_dtype = x_flat.dtype + if torch.is_autocast_enabled(x_flat.device.type): + out_dtype = torch.float32 + out_flat = torch.empty_like(x_flat, dtype=out_dtype) + + # Only B*num_groups programs are launched, which is well under the SM count + # for typical diffusion batches. A memory-bound kernel can still saturate + # HBM from few CTAs, but only with enough loads in flight, so widen the CTA + # for the large activations instead of leaving it at the 4-warp default. + + # One program per (batch, group) pair. + grid = (B * num_groups,) + + _group_norm_silu_kernel[grid]( + x_flat, + out_flat, + weight, + bias, + C, + spatial_size, + num_groups=num_groups, + eps=eps, + ) + + return out_flat.reshape(orig_shape) From b1da47b781c02613393f319cab105e26a7d4c374 Mon Sep 17 00:00:00 2001 From: zzh <67720593+BLANKETusers@users.noreply.github.com> Date: Fri, 21 Aug 2026 00:34:24 +0800 Subject: [PATCH 091/267] Fix npu moe registration (#6350) Signed-off-by: zzh <943967662@qq.com> --- vllm_omni/platforms/npu/layers/fused_moe.py | 36 +++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/vllm_omni/platforms/npu/layers/fused_moe.py b/vllm_omni/platforms/npu/layers/fused_moe.py index e17610701..4af0e9e33 100644 --- a/vllm_omni/platforms/npu/layers/fused_moe.py +++ b/vllm_omni/platforms/npu/layers/fused_moe.py @@ -29,6 +29,40 @@ def _ensure_forward_context_attr(name: str, annotation: Any, default: Any) -> No setattr(_vllm_fc.ForwardContext, name, default) +def _ensure_ascend_moe_customop_registered() -> None: + """Register vllm-ascend's out-of-tree MoE layer implementations here. + + vllm-ascend registers its OOT layers from ``register_ascend_customop``, + which only ever runs inside their own ``NPUWorker.__init__``. The diffusion + path runs on omni's DiffusionWorker, so the OOT registry stays empty and + ``PluggableLayer.__new__`` falls back to the upstream classes. FusedMoE then + breaks outright: vllm-ascend's ``patch_fused_moe`` passes + ``n_shared_experts`` through ``routed_experts_args``, which only + ``AscendRoutedExperts`` accepts. + + Only the MoE pair is registered, not the full ``REGISTERED_ASCEND_OPS`` + table. Most of that table assumes a text-generation ``ModelConfig``: + ``AscendRotaryEmbedding.__init__`` reads ``model_config.use_mla`` and + ``hf_text_config.qk_rope_head_dim``, neither of which exists on the + diffusion config shim. ``AscendRoutedExperts``/``AscendMoERunner`` only need + ``parallel_config``, which the shim does provide. + """ + from vllm.model_executor.custom_op import CustomOp, op_registry_oot + from vllm_ascend.ops.fused_moe.fused_moe import AscendMoERunner + from vllm_ascend.ops.fused_moe.routed_experts import AscendRoutedExperts + + ascend_moe_ops: dict[str, type] = { + "MoERunner": AscendMoERunner, + "RoutedExperts": AscendRoutedExperts, + } + for name, op_cls in ascend_moe_ops.items(): + # register_oot asserts on duplicates, so skip names vllm-ascend already + # claimed. Its own registration is authoritative when it has run. + if name in op_registry_oot: + continue + CustomOp.register_oot(_decorated_op_cls=op_cls, name=name) + + def _init_mc2_group_for_diffusion( backend: str, local_rank: int, @@ -66,6 +100,8 @@ def _select_moe_comm_method(vllm_config: VllmConfig) -> MoECommType | None: def prepare_fused_moe_runtime() -> None: vllm_config = omni_get_ctx().vllm_config + _ensure_ascend_moe_customop_registered() + backend = torch.distributed.get_backend(get_world_group().device_group) local_rank = get_world_group().local_rank _init_mc2_group_for_diffusion( From c6e38860e9419b5fe8ea90b262ba9439c5d06df6 Mon Sep 17 00:00:00 2001 From: Yash Jain Date: Thu, 20 Aug 2026 17:44:03 +0100 Subject: [PATCH 092/267] [Bugfix][Core] Fix replica device split when a stage omits devices (#5445) Signed-off-by: Yash Jain --- .../test_async_omni_engine_stage_init.py | 71 +++++++++++++++++++ vllm_omni/engine/stage_init_utils.py | 19 +++-- vllm_omni/engine/stage_runtime.py | 2 +- 3 files changed, 87 insertions(+), 5 deletions(-) diff --git a/tests/engine/test_async_omni_engine_stage_init.py b/tests/engine/test_async_omni_engine_stage_init.py index 5fae5f1b2..0f7d4dab6 100644 --- a/tests/engine/test_async_omni_engine_stage_init.py +++ b/tests/engine/test_async_omni_engine_stage_init.py @@ -15,6 +15,7 @@ from vllm_omni.engine.stage_init_utils import ( ReplicaInitPlan, build_stage0_input_processor, compute_replica_layout, + split_devices_for_replicas, ) from vllm_omni.engine.stage_runtime import StageRuntime @@ -232,6 +233,76 @@ def test_compute_replica_layout_splits_diffusion_devices_by_world_size(): assert replica_devices_map == {0: ["0,1", "2,3"]} +@pytest.mark.parametrize("num_replicas", [1, 2, 4]) +def test_split_devices_for_replicas_returns_one_slot_per_replica_when_devices_unset(num_replicas): + """``devices`` is optional, and the result is indexed by replica id. + + A stage that declares ``num_replicas`` without ``devices`` lets every + replica inherit the launcher's CUDA_VISIBLE_DEVICES, so each one still + needs its own (empty) slot. + """ + assert split_devices_for_replicas(None, num_replicas, 1, 0) == [None] * num_replicas + + +def test_compute_replica_layout_covers_every_replica_when_devices_unset(): + stage_cfg = types.SimpleNamespace( + stage_id=0, + stage_type="llm", + engine_args={}, + runtime=types.SimpleNamespace(num_replicas=3), + ) + + replicas_per_stage, replica_devices_map = compute_replica_layout([stage_cfg]) + + assert replicas_per_stage == [3] + assert replica_devices_map == {0: [None, None, None]} + + +def test_build_logical_stage_init_plans_handles_stage_without_devices(monkeypatch): + """Regression: a multi-replica stage with no ``devices`` must still plan. + + ``_build_logical_stage_init_plans`` indexes ``replica_devices_map`` by + replica id, so a short list raised ``IndexError`` for every replica after + the first. + """ + import vllm_omni.engine.stage_runtime as runtime_mod + + stage_cfg = types.SimpleNamespace( + stage_id=0, + stage_type="llm", + engine_args={}, + runtime=types.SimpleNamespace(num_replicas=3), + ) + runtime = StageRuntime( + stage_configs=[stage_cfg], + model="dummy-model", + config_path="dummy-config", + stage_init_timeout=1, + diffusion_batch_size=1, + async_chunk=False, + ) + + monkeypatch.setattr( + runtime_mod, + "extract_legacy_stage_metadata", + lambda cfg: _make_llm_metadata(cfg.stage_id), + ) + monkeypatch.setattr(runtime_mod, "get_stage_connector_spec", lambda **_: {}) + monkeypatch.setattr(runtime_mod, "resolve_omni_kv_config_for_stage", lambda *_: (None, None, None)) + monkeypatch.setattr(runtime_mod, "build_engine_args_dict", lambda *_, **__: {}) + monkeypatch.setattr(runtime_mod, "build_vllm_config", lambda *_args, **_kwargs: (types.SimpleNamespace(), object)) + + replicas_per_stage, replica_devices_map = compute_replica_layout([stage_cfg]) + stage_plans = runtime._build_logical_stage_init_plans( + omni_transfer_config=None, + replicas_per_stage=replicas_per_stage, + replica_devices_map=replica_devices_map, + ) + + assert [replica.replica_id for replica in stage_plans[0].replicas] == [0, 1, 2] + assert [replica.stage_cfg.runtime.devices for replica in stage_plans[0].replicas] == [None, None, None] + + def test_collect_initialized_clients_for_cleanup_deduplicates_clients(): shared = types.SimpleNamespace(name="shared") extra = types.SimpleNamespace(name="extra") diff --git a/vllm_omni/engine/stage_init_utils.py b/vllm_omni/engine/stage_init_utils.py index 683726d28..a4c3e1650 100644 --- a/vllm_omni/engine/stage_init_utils.py +++ b/vllm_omni/engine/stage_init_utils.py @@ -567,9 +567,14 @@ def split_devices_for_replicas( num_replicas: int, tp_size: int, stage_id: int, -) -> list[str]: +) -> list[str | None]: """Split a devices string into per-replica subsets. + The result always has one entry per replica, because callers index it by + replica id. When ``devices_str`` is ``None`` the stage declares no explicit + placement, so every replica gets ``None`` and inherits the launcher's + ``CUDA_VISIBLE_DEVICES``. + When ``num_replicas`` is 1, returns ``[devices_str]`` unchanged. Otherwise, two YAML shapes are accepted: @@ -597,8 +602,14 @@ def split_devices_for_replicas( Any other length raises ``ValueError`` (the two modes are length-disjoint for ``num_replicas > 1``). """ - if num_replicas <= 1 or devices_str is None: - return [devices_str] if devices_str is not None else [devices_str] + if devices_str is None: + # No explicit placement: hand back one empty slot per replica, matching + # ``get_headless_replica_devices``. Returning a single-element list here + # made callers index past the end for every replica after the first. + return [None] * max(1, num_replicas) + + if num_replicas <= 1: + return [devices_str] device_list = [d.strip() for d in devices_str.split(",") if d.strip()] @@ -688,7 +699,7 @@ def compute_replica_layout( raise ValueError(f"num_replicas must be >= 0, got {num_replicas}") replicas_per_stage.append(num_replicas if allow_zero else max(1, num_replicas)) - replica_devices_map: dict[int, list[str]] = {} + replica_devices_map: dict[int, list[str | None]] = {} for stage_id, stage_cfg in enumerate(stage_configs): num_replicas = replicas_per_stage[stage_id] if num_replicas <= 1: diff --git a/vllm_omni/engine/stage_runtime.py b/vllm_omni/engine/stage_runtime.py index c0aafc86a..dea5242d1 100644 --- a/vllm_omni/engine/stage_runtime.py +++ b/vllm_omni/engine/stage_runtime.py @@ -337,7 +337,7 @@ class StageRuntime: self, omni_transfer_config: Any, replicas_per_stage: Sequence[int], - replica_devices_map: Mapping[int, Sequence[str]], + replica_devices_map: Mapping[int, Sequence[str | None]], ) -> list[LogicalStageInitPlan]: """Build startup plans for every logical stage and replica.""" stage_plans: list[LogicalStageInitPlan] = [] From ebb4abde0a24b2982d94c70ac1ed5e40ab3caa03 Mon Sep 17 00:00:00 2001 From: Nick Cao Date: Thu, 20 Aug 2026 12:46:32 -0400 Subject: [PATCH 093/267] [Tests] Run diffusion tiny model tests in parallel (#6339) Signed-off-by: Nick Cao --- .buildkite/cuda/test-ready.yml | 2 +- pyproject.toml | 6 +- tests/conftest.py | 1 + tests/helpers/fixtures/xdist.py | 14 +++++ tests/model_tests/conftest.py | 62 ++++++++++++------- .../diffusion/test_common_offline.py | 7 ++- .../diffusion/test_common_online.py | 4 +- 7 files changed, 68 insertions(+), 28 deletions(-) create mode 100644 tests/helpers/fixtures/xdist.py diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index 8dbe7bd71..7f7e44cd2 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -76,7 +76,7 @@ steps: - label: "Tiny Model Tests · Diffusion (base)" commands: - - timeout 15m pytest -sv tests/model_tests/diffusion -m 'core_model and cuda' --run-level "core_model" + - timeout 15m pytest -sv -n 4 tests/model_tests/diffusion -m 'core_model and cuda' --run-level "core_model" mirror_hardwares: l4_1 - label: "Engine&Model Executor Test" diff --git a/pyproject.toml b/pyproject.toml index e9bdd3b29..ea1758ada 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,10 @@ dev = [ "pytest-cov==7.1.0", "pytest-mock==3.15.1", "pytest-shard==0.1.2", + "pytest-xdist==3.8.0", "websockets==16.1", "pytest-subtests==0.15.0", + "filelock>=3.16.1,<4", "datasets==3.6.0", "mypy==1.11.1", "pre-commit==4.0.1", @@ -242,7 +244,8 @@ python_classes = ["Test*"] python_functions = ["test_*"] addopts = [ "--strict-markers", - "--strict-config" + "--strict-config", + "--dist=loadgroup" ] markers = [ # ci/cd required @@ -291,6 +294,7 @@ markers = [ "sp: Sequence parallelism tests (multi-GPU)", "slow: Slow tests (may skip in quick CI)", "benchmark: Benchmark tests", + "xdist: xdist-parallelizable tests", ] filterwarnings = [ "ignore:.*does not have '__test__' attribute.*:UserWarning", diff --git a/tests/conftest.py b/tests/conftest.py index 5c1276ec2..747b13c35 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,7 @@ pytest_plugins = ( "tests.helpers.fixtures.run_args", "tests.helpers.fixtures.runtime", "tests.helpers.fixtures.speaker_cache", + "tests.helpers.fixtures.xdist", ) diff --git a/tests/helpers/fixtures/xdist.py b/tests/helpers/fixtures/xdist.py new file mode 100644 index 000000000..1c1cb3529 --- /dev/null +++ b/tests/helpers/fixtures/xdist.py @@ -0,0 +1,14 @@ +""" +Only tests explicitly marked ``pytest.mark.xdist`` run concurrently. +Other test are pinned to a single xdist worker. +""" + +import pytest + + +def pytest_collection_modifyitems(config, items): + if not config.pluginmanager.hasplugin("xdist"): + return + for item in items: + if not item.get_closest_marker("xdist"): + item.add_marker(pytest.mark.xdist_group(name="serial")) diff --git a/tests/model_tests/conftest.py b/tests/model_tests/conftest.py index ab71a02cd..7760fa25d 100644 --- a/tests/model_tests/conftest.py +++ b/tests/model_tests/conftest.py @@ -1,12 +1,14 @@ +import json from shutil import rmtree import pytest +from filelock import ReadWriteLock from tests.model_tests.diffusion.model_settings import DIFFUSION_TEST_SETTINGS @pytest.fixture(scope="session") -def tiny_model_paths(request, run_level): +def tiny_model_paths(request, run_level, tmp_path_factory): """Build or download the tiny models for the selected tests. At core_model level, builds tiny models via the builder function. @@ -15,27 +17,43 @@ def tiny_model_paths(request, run_level): NOTE: this is session scoped to avoid churn in tiny model creation, but will ensure all the tiny models you need are created for the selected tests before it starts to execute them.""" + base_path = tmp_path_factory.getbasetemp().parent / "tiny_models" + use_lock = ReadWriteLock(base_path.with_suffix(".use.lock")) + build_lock = ReadWriteLock(base_path.with_suffix(".build.lock")) + data_file = base_path.with_suffix(".json") + model_paths = {} built_paths = [] - print("Initializing tiny models...") - for item in request.session.items: - if not hasattr(item, "callspec"): - continue - model_name = item.callspec.params.get("model_name") - if model_name is None or model_name not in DIFFUSION_TEST_SETTINGS: - continue - if model_name not in model_paths: - settings = DIFFUSION_TEST_SETTINGS[model_name] - if run_level == "core_model": - print(f"Calling tiny model builder for: {model_name}") - path = settings.builder() - built_paths.append(path) - else: - print(f"Run level is {run_level}; {model_name} will use the full model") - path = settings.model - model_paths[model_name] = path - yield model_paths - for path in built_paths: - print(f"Removing tiny model: {path}") - rmtree(path, ignore_errors=True) + with use_lock.read_lock(): + with build_lock.write_lock(): + if data_file.is_file(): + model_paths = json.loads(data_file.read_text()) + else: + print("Initializing tiny models...") + for item in request.session.items: + if not hasattr(item, "callspec"): + continue + model_name = item.callspec.params.get("model_name") + if model_name is None or model_name not in DIFFUSION_TEST_SETTINGS: + continue + if model_name not in model_paths: + settings = DIFFUSION_TEST_SETTINGS[model_name] + if run_level == "core_model": + print(f"Calling tiny model builder for: {model_name}") + path = settings.builder() + built_paths.append(path) + else: + print(f"Run level is {run_level}; {model_name} will use the full model") + path = settings.model + model_paths[model_name] = path + data_file.write_text(json.dumps(model_paths)) + + yield model_paths + + with use_lock.write_lock(): + if built_paths: + data_file.unlink(missing_ok=True) + for path in built_paths: + print(f"Removing tiny model: {path}") + rmtree(path, ignore_errors=True) diff --git a/tests/model_tests/diffusion/test_common_offline.py b/tests/model_tests/diffusion/test_common_offline.py index b0eb26c9b..884710f2e 100644 --- a/tests/model_tests/diffusion/test_common_offline.py +++ b/tests/model_tests/diffusion/test_common_offline.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + import pytest from tests.model_tests.diffusion.case_filtering import get_parametrized_options @@ -17,7 +20,7 @@ from tests.model_tests.diffusion.task_runners import ( ) # NOTE : Hardware marks are added dynamically based on test requirements -pytestmark = [pytest.mark.diffusion] +pytestmark = [pytest.mark.diffusion, pytest.mark.xdist] @pytest.mark.parametrize( @@ -50,7 +53,7 @@ def test_pipeline_on_supported_tasks( ) try: for task_type in supported_tasks: - with subtests.test(msg=task_type): + with subtests.test(msg=task_type.value): if task_type == DiffusionTasks.TEXT_TO_IMAGE: run_and_validate_text_to_image_request(omni) elif task_type == DiffusionTasks.IMAGE_TO_IMAGE: diff --git a/tests/model_tests/diffusion/test_common_online.py b/tests/model_tests/diffusion/test_common_online.py index 464adc13a..d039ce614 100644 --- a/tests/model_tests/diffusion/test_common_online.py +++ b/tests/model_tests/diffusion/test_common_online.py @@ -24,7 +24,7 @@ from tests.model_tests.diffusion.task_runners import ( ) # NOTE : Hardware marks are added dynamically based on test requirements -pytestmark = [pytest.mark.diffusion] +pytestmark = [pytest.mark.diffusion, pytest.mark.xdist] @pytest.mark.parametrize( @@ -58,7 +58,7 @@ def test_online_on_supported_tasks( log_stats=server.log_stats, ) for task_type in supported_tasks: - with subtests.test(msg=task_type): + with subtests.test(msg=task_type.value): if task_type == DiffusionTasks.TEXT_TO_IMAGE: run_and_validate_online_text_to_image_request(server, client) elif task_type == DiffusionTasks.IMAGE_TO_IMAGE: From b3270dec8d758b9801aa0d01bd7205757b65b814 Mon Sep 17 00:00:00 2001 From: MaciejBalaNV Date: Thu, 20 Aug 2026 20:40:51 +0200 Subject: [PATCH 094/267] Cosmos3 transfer fix (#5614) Signed-off-by: Maciej Bala --- .gitignore | 1 + docs/serving/videos_api.md | 20 +- recipes/cosmos3/Cosmos3-Nano.md | 29 ++- recipes/cosmos3/Cosmos3-Super.md | 17 ++ recipes/cosmos3/negative_prompt.json | 108 +++++++++++ .../models/cosmos3/test_cosmos3_pipeline.py | 177 ++++++++++++++++-- .../cosmos3/test_cosmos3_transformer.py | 41 ++++ tests/model_extras/test_model_extras.py | 1 + .../models/cosmos3/pipeline_cosmos3.py | 98 +++++++--- .../diffusion/models/cosmos3/transfer.py | 47 ++++- .../models/cosmos3/transformer_cosmos3.py | 144 +++++++++++++- vllm_omni/model_extras/cosmos3.py | 1 + 12 files changed, 629 insertions(+), 55 deletions(-) create mode 100644 recipes/cosmos3/negative_prompt.json diff --git a/.gitignore b/.gitignore index 9155a85cc..278eb50ff 100644 --- a/.gitignore +++ b/.gitignore @@ -242,6 +242,7 @@ data/ datasets/ *.csv *.json +!recipes/cosmos3/negative_prompt.json !tests/dfx/perf/tests/*.json !apps/ComfyUI-vLLM-Omni/example_workflows/*.json !tests/assets/soulxsinger/*.json diff --git a/docs/serving/videos_api.md b/docs/serving/videos_api.md index 5995e0911..7fc7d0ba8 100644 --- a/docs/serving/videos_api.md +++ b/docs/serving/videos_api.md @@ -161,11 +161,21 @@ are not implemented yet. Models may expose additional V2V controls through `extra_params`. For example, Cosmos3 supports `condition_frame_indexes_vision` and `condition_video_keep` to select which decoded reference frames are used as clean conditioning. Cosmos3 transfer mode -also accepts `edge`, `blur`, `depth`, `seg`, or `wsm` control hints plus -transfer options such as `control_path`, `control_guidance`, -`control_guidance_interval`, `num_video_frames_per_chunk`, -`num_conditional_frames`, `show_control_condition`, and `show_input`; see the -Cosmos3 recipe for complete examples. +also accepts `edge`, `blur`, `depth`, `seg`, or `wsm` control hints. Each hint +may specify its own `control_path` and `control_weight`. Request-level transfer +options include +`control_guidance`, `control_guidance_interval`, +`emphasize_control_in_prompt`, `num_video_frames_per_chunk`, +`num_conditional_frames`, `show_control_condition`, and `show_input`. Transfer +uses its transfer-specific system prompt and, by default, appends a +control-adherence directive to the positive prompt. It adds duration/FPS and +resolution metadata to both CFG branches but does not add a negative prompt +automatically. The Cosmos3 recipe includes an optional reference negative +prompt and shows how to pass it. Set `emphasize_control_in_prompt`, +`use_duration_template`, or `use_resolution_template` to `false` to disable the +corresponding addition. `negative_metadata_mode` accepts `same`, `inverse`, or +`none` and defaults to `same` for transfer. See the Cosmos3 recipe for complete +examples. HTTP redirects for `image_reference.image_url` follow vLLM's `VLLM_MEDIA_URL_ALLOW_REDIRECTS` setting. Before starting the server, set it to diff --git a/recipes/cosmos3/Cosmos3-Nano.md b/recipes/cosmos3/Cosmos3-Nano.md index b27d1b473..4aaebf0be 100644 --- a/recipes/cosmos3/Cosmos3-Nano.md +++ b/recipes/cosmos3/Cosmos3-Nano.md @@ -195,11 +195,13 @@ curl -sS -X POST http://localhost:8000/v1/videos/sync \ # Transfer V2V with a precomputed depth control video. `control_path` can point # to a local image/video; edge and blur can also be computed from `input_reference` -# by passing `"edge":true` or `"blur":true`. +# by passing `"edge":true` or `"blur":true`. The reference negative prompt is +# optional; omit the `negative_prompt` form field to use an empty negative branch. curl -sS -X POST http://localhost:8000/v1/videos/sync \ -H "Accept: video/mp4" \ -F "model=nvidia/Cosmos3-Nano" \ -F "prompt=Generate a realistic scene following the provided control video." \ + --form-string "negative_prompt=$(jq -c . recipes/cosmos3/negative_prompt.json)" \ -F "size=1280x720" -F "num_frames=121" \ -F "num_inference_steps=50" -F "seed=125" \ -F 'extra_params={"depth":{"control_path":"/path/to/depth_control.mp4"},"max_frames":121,"resolution":"720","num_video_frames_per_chunk":121}' \ @@ -322,15 +324,34 @@ vllm serve nvidia/Cosmos3-Nano-Policy-DROID \ `extra_params={"guardrails":false}` (per request) toggles safety. The per-request flag only takes effect when the server was launched **with** guardrails enabled (it cannot re-enable them on a `--no-guardrails` server). - `use_resolution_template` / `use_duration_template` are off by default and only - needed when not using upsampled prompts that already encode resolution/duration. + Outside transfer mode, `use_resolution_template` / `use_duration_template` + are off by default and only needed when not using upsampled prompts that + already encode resolution/duration. For V2V, `condition_frame_indexes_vision` selects the clean conditioned latent frame indexes (default `[0, 1]`), and `condition_video_keep` selects whether the API decodes the first or last needed reference frames (`"first"` by default). - **Transfer controls:** `extra_params` may include `edge`, `blur`, `depth`, `seg`, or `wsm`. Each hint accepts `true`, a path string, or an object such as `{"control_path": "/path/to/control.mp4"}`; `edge` also accepts - `preset_edge_threshold` and `blur` accepts `preset_blur_strength`. + `preset_edge_threshold` and `blur` accepts `preset_blur_strength`. Every hint + accepts a non-negative `control_weight`; weights are normalized across active + controls and therefore only set their relative influence. A single positive + weight always normalizes to `1.0`; use `control_guidance` to change the + absolute strength of a single control. With two or more active controls, the + per-control attention passes run replicated on every sequence-parallel + (Ulysses) rank, so Ulysses does not reduce per-rank memory or latency for + multi-control transfer requests. + Transfer always uses Cosmos3's transfer-specific system prompt. By default it + also appends a directive naming every active hint and asking the model to + follow its shape, position, and motion precisely; set the request-level + `emphasize_control_in_prompt` option to `false` for prompt ablations. Transfer + does not add a negative prompt automatically. An optional reference prompt is + provided in [`negative_prompt.json`](negative_prompt.json); compact the JSON + with `jq -c` and pass it through `negative_prompt` as shown in the transfer + example. Transfer enables duration/FPS and resolution metadata on both CFG + branches. Set `use_duration_template` or `use_resolution_template` to + `false` to disable either template. `negative_metadata_mode` accepts `same` + (the transfer default), `inverse`, or `none`. Transfer-level options include `control_guidance`, `control_guidance_interval`, `num_video_frames_per_chunk` (default `93`, `101` for WSM), `num_conditional_frames` (default `1`), diff --git a/recipes/cosmos3/Cosmos3-Super.md b/recipes/cosmos3/Cosmos3-Super.md index 54d4d7e32..c8bfb6bbd 100644 --- a/recipes/cosmos3/Cosmos3-Super.md +++ b/recipes/cosmos3/Cosmos3-Super.md @@ -125,6 +125,23 @@ curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \ under the top-level `action` field. Verified on the 64B Super under `--cfg-parallel-size 2`: async `policy` returns the predicted action (`[16, 10]`) and the rollout video reliably. +- **Transfer controls:** same semantics as Nano — `extra_params` may include + `edge`, `blur`, `depth`, `seg`, or `wsm` hints; see the Cosmos3-Nano recipe + for the request shapes and full option list. Every hint accepts a + non-negative `control_weight`; weights are normalized across active controls + and therefore only set their relative influence. A single positive weight + always normalizes to `1.0`; use `control_guidance` to change the absolute + strength of a single control. With two or more active controls, the + per-control attention passes run replicated on every sequence-parallel + (Ulysses) rank, so Ulysses does not reduce per-rank memory or latency for + multi-control transfer requests. Transfer always uses Cosmos3's + transfer-specific system prompt, appends a control-adherence directive by + default (`emphasize_control_in_prompt: false` disables it), and enables + duration/FPS and resolution metadata on both CFG branches + (`use_duration_template` / `use_resolution_template` to disable; + `negative_metadata_mode` accepts `same`, `inverse`, or `none`, defaulting to + `same`). Transfer does not add a negative prompt automatically; an optional + reference prompt is provided in [`negative_prompt.json`](negative_prompt.json). ## NPU diff --git a/recipes/cosmos3/negative_prompt.json b/recipes/cosmos3/negative_prompt.json new file mode 100644 index 000000000..9cefcb4d5 --- /dev/null +++ b/recipes/cosmos3/negative_prompt.json @@ -0,0 +1,108 @@ +{ + "subjects": [ + { + "description": "Blurry, poorly defined subjects with inconsistent shapes and unrealistic proportions.", + "appearance_details": "Distorted features, visible compression artifacts, muddy textures lacking fine detail, color bleeding between elements, and unnatural skin tones or surface textures that appear artificial or computer-generated.", + "relationship": "Subjects appear disconnected from the environment, floating or improperly grounded in the scene without proper occlusion or spatial coherence.", + "location": "Subjects are poorly placed within the frame, appearing at awkward positions that violate basic compositional rules.", + "relative_size": "Inconsistent scale relationships between subjects and the environment, with objects appearing too large or too small relative to their surroundings.", + "orientation": "Unnatural orientations that defy physics and spatial logic.", + "pose": "Stiff, mannequin-like poses with unnatural joint angles and impossible limb positions that look computer-generated.", + "action": "Incoherent motion with visible frame-to-frame discontinuities. Movement appears as a slideshow rather than smooth animation. Limbs and appendages pop between positions without interpolation.", + "state_changes": "Visual state transitions are abrupt and jarring. Colors shift without motivation. Surface textures flicker between different materials randomly. Outlines shimmer and vibrate.", + "clothing": "Clothing appears painted on with no sense of material weight or drape. Fabric textures are flat and repeat visibly.", + "expression": "Frozen, uncanny valley expressions or expressions that change abruptly without natural transition.", + "gender": "", + "age": "", + "skin_tone_and_texture": "Waxy, plastic-looking skin with visible artifacts and inconsistent texture resolution across the frame.", + "facial_features": "Asymmetric facial features, extra fingers or limbs, teeth that appear blurry or malformed.", + "number_of_subjects": 0, + "number_of_arms": 0, + "number_of_legs": 0 + }, + { + "description": "Extremely low-quality subjects with visible rendering artifacts, broken mesh geometry, and completely unrealistic proportions throughout.", + "appearance_details": "Distorted features, visible compression artifacts, muddy textures lacking fine detail, color bleeding between elements, and unnatural skin tones or surface textures that appear artificial or computer-generated.", + "relationship": "Subjects appear disconnected from the environment, floating or improperly grounded in the scene without proper occlusion or spatial coherence.", + "location": "Subjects are poorly placed within the frame, appearing at awkward positions that violate basic compositional rules.", + "relative_size": "Inconsistent scale relationships between subjects and the environment, with objects appearing too large or too small relative to their surroundings.", + "orientation": "Unnatural orientations that defy physics and spatial logic.", + "pose": "Stiff, mannequin-like poses with unnatural joint angles and impossible limb positions that look computer-generated.", + "action": "Incoherent motion with visible frame-to-frame discontinuities. Movement appears as a slideshow rather than smooth animation. Limbs and appendages pop between positions without interpolation.", + "state_changes": "Visual state transitions are abrupt and jarring. Colors shift without motivation. Surface textures flicker between different materials randomly. Outlines shimmer and vibrate.", + "clothing": "Clothing appears painted on with no sense of material weight or drape. Fabric textures are flat and repeat visibly.", + "expression": "Frozen, uncanny valley expressions or expressions that change abruptly without natural transition.", + "gender": "", + "age": "", + "skin_tone_and_texture": "Waxy, plastic-looking skin with visible artifacts and inconsistent texture resolution across the frame.", + "facial_features": "Asymmetric facial features, extra fingers or limbs, teeth that appear blurry or malformed.", + "number_of_subjects": 0, + "number_of_arms": 0, + "number_of_legs": 0 + }, + { + "description": "Poorly generated subjects exhibiting all hallmarks of failed neural rendering \u2014 flickering edges, inconsistent depth, and uncanny spatial relationships.", + "appearance_details": "Distorted features, visible compression artifacts, muddy textures lacking fine detail, color bleeding between elements, and unnatural skin tones or surface textures that appear artificial or computer-generated.", + "relationship": "Subjects appear disconnected from the environment, floating or improperly grounded in the scene without proper occlusion or spatial coherence.", + "location": "Subjects are poorly placed within the frame, appearing at awkward positions that violate basic compositional rules.", + "relative_size": "Inconsistent scale relationships between subjects and the environment, with objects appearing too large or too small relative to their surroundings.", + "orientation": "Unnatural orientations that defy physics and spatial logic.", + "pose": "Stiff, mannequin-like poses with unnatural joint angles and impossible limb positions that look computer-generated.", + "action": "Incoherent motion with visible frame-to-frame discontinuities. Movement appears as a slideshow rather than smooth animation. Limbs and appendages pop between positions without interpolation.", + "state_changes": "Visual state transitions are abrupt and jarring. Colors shift without motivation. Surface textures flicker between different materials randomly. Outlines shimmer and vibrate.", + "clothing": "Clothing appears painted on with no sense of material weight or drape. Fabric textures are flat and repeat visibly.", + "expression": "Frozen, uncanny valley expressions or expressions that change abruptly without natural transition.", + "gender": "", + "age": "", + "skin_tone_and_texture": "Waxy, plastic-looking skin with visible artifacts and inconsistent texture resolution across the frame.", + "facial_features": "Asymmetric facial features, extra fingers or limbs, teeth that appear blurry or malformed.", + "number_of_subjects": 0, + "number_of_arms": 0, + "number_of_legs": 0 + } + ], + "background_setting": "A poorly rendered, flat background with visible seams, repeated textures, and inconsistent depth cues. The environment lacks volumetric depth and appears as a painted backdrop rather than a three-dimensional space. Vegetation looks like flat cutouts with no volumetric depth. The background appears to have been composited from multiple source materials at different resolutions, creating visible seams and edge artifacts where elements meet. Textures swim and shift across surfaces in a way that breaks the illusion of solidity \u2014 patterns drift laterally rather than staying anchored to the geometry they belong to. Background elements flicker in and out of existence between frames, particularly at the edges of the field of view. The rendering resolution is visibly lower for distant elements, creating a jarring transition between near and far objects. Cloud textures repeat obviously in the sky with visible tiling. Water surfaces lack proper reflection and refraction, appearing as flat animated textures. Fog and atmospheric effects pop in and out rather than smoothly transitioning. Trees and vegetation exhibit obvious LOAD (level-of-detail) switching. Building facades have inconsistent window spacing and pattern repetition. The overall scene feels like a poorly assembled collage of individually rendered elements rather than a coherent whole.", + "lighting": { + "conditions": "Harsh, flat lighting with no natural variation. The scene appears uniformly lit as if by a single overhead fluorescent light, removing all sense of depth and atmosphere.", + "direction": "Inconsistent light sources \u2014 shadows point in multiple contradictory directions, breaking physical plausibility.", + "shadows": "Hard-edged, unrealistic shadows that pop in and out of existence between frames. Some objects cast no shadows while others have impossibly dark ones that don't animate smoothly with the object's motion. Shadow edges exhibit visible staircase aliasing artifacts. Shadow maps appear to have been rendered at extremely low resolution, creating blocky patterns. Self-shadowing on characters shows visible peter-panning artifacts where shadows detach from their source. Contact shadows between objects and the ground appear and disappear as objects move slightly. Shadow color is pure black with no ambient contribution, creating an unnaturally harsh contrast that flattens the image. Multiple shadow cascades have visible boundaries where resolution changes. The shadow rendering appears to be temporally unstable \u2014 even static objects have shadows that shimmer and crawl frame to frame, breaking the illusion of a stable light source.", + "illumination_effect": "No bounce light, no ambient occlusion, no subtle color interactions between surfaces. The scene looks like a poorly lit 3D render from the early 2000s." + }, + "aesthetics": { + "composition": "Cluttered, poorly framed composition with no clear focal point. Important elements are cut off by the frame edges. The rule of thirds is completely ignored, leading to an unbalanced and visually unpleasant arrangement.", + "color_scheme": "Oversaturated, garish colors that clash violently. Color banding is visible in gradient areas. The overall palette feels artificial and digitally processed rather than natural.", + "mood_atmosphere": "Unsettling, uncanny atmosphere that fails to evoke any intended emotional response. The scene feels lifeless and sterile despite attempting to portray dynamic action.", + "patterns": "Visible tiling artifacts in textures, moir\u00e9 patterns, and aliasing on edges." + }, + "cinematography": { + "camera_motion": "Extremely shaky, unstable camera with visible rolling shutter artifacts. The motion is jerky and discontinuous, causing motion sickness and making the scene impossible to follow.", + "framing": "Poorly framed shots that cut off important elements and include unnecessary empty space.", + "camera_angle": "Awkward, disorienting camera angles that provide no useful spatial information about the scene. The camera path exhibits visible mathematical artifacts suggesting simple interpolation between keyframes rather than natural camera operation. Camera motion is completely disconnected from the scene content \u2014 panning away from action, dollying during dialogue, and shaking during still moments. The camera appears to pass through solid objects occasionally. Zoom is applied digitally rather than optically, revealing progressively worse resolution. Camera motion exhibits non-physical acceleration profiles \u2014 instant starts and stops rather than smooth ease-in/ease-out. Rolling shutter simulation is applied inconsistently, present in some frames but not others. The camera occasionally exhibits impossible motion like teleporting between positions. Virtual camera stabilization creates an uncanny floating sensation disconnected from any physical camera rig.", + "depth_of_field": "Uniform focus throughout, creating a flat, documentary-like appearance with no cinematic depth separation.", + "focus": "Soft, out-of-focus imagery with visible chromatic aberration and lens distortion that was not corrected in post-processing.", + "lens_focal_length": "Inappropriate focal length causing barrel distortion and unnatural perspective compression." + }, + "style_medium": "Low quality compressed digital video with visible encoding artifacts", + "artistic_style": "Amateur, unpolished with inconsistent visual style", + "context": "A poorly produced video with numerous technical and artistic flaws that detract from any intended narrative or visual impact.", + "actions": [ + { + "time": "0:00-0:08", + "description": "Subjects attempt to move but their motion is jerky, temporally inconsistent, and physically implausible. Background elements flicker and shift between frames." + } + ], + "text_and_signage_elements": [], + "segments": [ + { + "segment_index": 0, + "time_range": "0:00-0:08", + "description": "A single continuous shot suffering from severe temporal inconsistencies \u2014 subjects that morph and deform between frames, backgrounds that shift and wobble, and rendering quality that fluctuates visibly over time. Motion blur is applied incorrectly, smearing in directions that don't match actual movement. Frame-to-frame coherence breaks down with individual pixels changing color randomly in flat areas. Texture detail level fluctuates between frames as if the rendering budget varied shot to shot. Color grading drifts over the duration with no creative motivation. Noise patterns change between frames in ways that draw attention rather than being invisible. Overall visual quality degrades progressively from start to finish.", + "key_changes": "No meaningful progression or narrative development. Visual quality degrades over time.", + "camera": "Unstable, poorly controlled camera work with visible mathematical interpolation artifacts." + } + ], + "transitions": [], + "temporal_caption": "The scene opens at 0.0 seconds with a poorly rendered establishing shot that immediately reveals low production quality. At 1.0 seconds, subjects begin to move but their motion is jerky and inconsistent, with limbs bending at unnatural angles and objects clipping through each other. From 2.0 to 4.0 seconds, the camera shakes violently while the scene exhibits visible compression artifacts, color banding in the sky, and flickering in the shadows. Between 4.0 and 6.0 seconds, temporal coherence breaks down as elements appear and disappear between frames, textures swim and morph unnaturally, and the lighting shifts abruptly without physical cause. In the final 2 seconds, the overall visual quality deteriorates further with increasing noise, blur, and a general loss of spatial coherence that makes the scene nearly unwatchable. Additionally, the frame rate appears inconsistent with visible judder and stuttering throughout. Color temperature shifts randomly between warm and cool tones with no motivation. The encode quality degrades in complex regions showing macro-blocking and mosquito noise around moving edges. Temporal noise patterns are spatially correlated, creating swimming artifacts on flat surfaces.", + "audio_description": "", + "physical_realism": "No adherence to physical laws. Objects defy gravity, pass through solid surfaces, and change mass and momentum without cause. Fluid dynamics, cloth simulation, and rigid body physics are all fundamentally broken. Furthermore, conservation of energy is violated as objects gain or lose kinetic energy spontaneously. Elastic collisions produce inelastic results and vice versa. Surface friction is inconsistent \u2014 objects slide on rough surfaces while sticking to smooth ones. Air resistance appears to affect only some objects while others move through the atmosphere unimpeded." + } diff --git a/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py b/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py index 796de06f1..a0eab3b5e 100644 --- a/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py +++ b/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py @@ -1019,11 +1019,19 @@ def test_transfer_config_media_helpers_and_preprocess_budget(monkeypatch: pytest cfg = transfer.resolve_transfer_config(make_sampling_params(extra_args={"edge": True})) assert cfg is not None assert list(cfg.hints) == ["edge"] + assert cfg.hints["edge"].control_weight == 1.0 + assert cfg.normalized_control_weights == [1.0] assert cfg.guidance_scale == 3.0 assert cfg.control_guidance == 1.5 assert cfg.flow_shift == 10.0 assert cfg.num_video_frames_per_chunk == 93 assert cfg.share_vision_temporal_positions is True + assert cfg.emphasize_control_in_prompt is True + prompt_ablation_cfg = transfer.resolve_transfer_config( + make_sampling_params(extra_args={"edge": True, "emphasize_control_in_prompt": False}) + ) + assert prompt_ablation_cfg is not None + assert prompt_ablation_cfg.emphasize_control_in_prompt is False # fps omitted (no fps/frame_rate on the sampling params) -> wsm preset default (10) applies. defaulted_fps_cfg = transfer.resolve_transfer_config(make_sampling_params(extra_args={"wsm": True})) assert defaulted_fps_cfg is not None @@ -1092,6 +1100,40 @@ def test_transfer_config_media_helpers_and_preprocess_budget(monkeypatch: pytest assert "preprocessed_video" not in additional +def test_transfer_control_weight_validation_and_normalization() -> None: + from vllm_omni.diffusion.models.cosmos3 import transfer + + single = transfer.resolve_transfer_config(make_sampling_params(extra_args={"edge": {"control_weight": 0.4}})) + assert single is not None + assert single.hints["edge"].control_weight == 0.4 + assert single.normalized_control_weights == [1.0] + + multiple = transfer.resolve_transfer_config( + make_sampling_params( + extra_args={ + "edge": {"control_weight": 1.0}, + "depth": {"control_weight": 3.0, "control": torch.zeros(3, 1, 8, 8, dtype=torch.uint8)}, + } + ) + ) + assert multiple is not None + assert multiple.normalized_control_weights == [0.25, 0.75] + + with pytest.raises(ValueError, match="Unsupported.*weight"): + transfer.resolve_transfer_config(make_sampling_params(extra_args={"edge": {"weight": 0.5}})) + with pytest.raises(ValueError, match="finite and non-negative"): + transfer.resolve_transfer_config(make_sampling_params(extra_args={"edge": {"control_weight": -0.5}})) + with pytest.raises(ValueError, match="positive sum"): + transfer.resolve_transfer_config( + make_sampling_params( + extra_args={ + "edge": {"control_weight": 0.0}, + "depth": {"control_weight": 0.0, "control": torch.zeros(3, 1, 8, 8, dtype=torch.uint8)}, + } + ) + ) + + def test_transfer_fps_matches_resolved_frame_rate_precedence() -> None: """When fps and frame_rate differ, transfer must pick frame_rate -- the same precedence as OmniDiffusionSamplingParams.resolved_frame_rate. Uses the real @@ -1352,6 +1394,63 @@ def test_format_and_tokenize_prompts_applies_video_templates_and_system_override ] +def test_format_and_tokenize_prompts_applies_transfer_prompt_contract(make_cosmos3_pipeline) -> None: + from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import ( + COSMOS3_TRANSFER_CONTROL_DIRECTIVE_TEMPLATE, + COSMOS3_TRANSFER_SYSTEM_PROMPT, + ) + + pipeline = make_cosmos3_pipeline() + calls = _capture_tokenize_calls(pipeline) + directive = COSMOS3_TRANSFER_CONTROL_DIRECTIVE_TEMPLATE.format(hint_names="edge") + + pipeline._format_and_tokenize_prompts( + "A robot", + "bad", + num_frames=48, + frame_rate=24, + height=720, + width=1280, + max_sequence_length=32, + sp=SimpleNamespace( + extra_args={ + "use_resolution_template": True, + "system_prompt": "request-level system prompt", + } + ), + use_system_prompt=True, + is_t2i=False, + system_prompt=COSMOS3_TRANSFER_SYSTEM_PROMPT, + prompt_suffix=directive, + use_duration_template=True, + use_resolution_template=True, + negative_metadata_mode="same", + ) + + metadata = "The video is 2.0 seconds long and is of 24 FPS. This video is of 720x1280 resolution." + assert calls[0]["text"] == f"A robot. {metadata} {directive}" + assert calls[1]["text"] == f"bad. {metadata}" + assert all(call["use_system_prompt"] is True for call in calls) + assert all(call["system_prompt"] == COSMOS3_TRANSFER_SYSTEM_PROMPT for call in calls) + + +def test_format_and_tokenize_prompts_rejects_unknown_negative_metadata_mode(make_cosmos3_pipeline) -> None: + pipeline = make_cosmos3_pipeline() + + with pytest.raises(ValueError, match="negative_metadata_mode"): + pipeline._format_and_tokenize_prompts( + "A robot", + "bad", + num_frames=48, + frame_rate=24, + height=720, + width=1280, + max_sequence_length=32, + sp=SimpleNamespace(extra_args={}), + negative_metadata_mode="unexpected", + ) + + def test_format_and_tokenize_prompts_uses_image_templates_for_t2i(make_cosmos3_pipeline) -> None: from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import COSMOS3_T2I_SYSTEM_PROMPT @@ -1740,6 +1839,7 @@ def test_diffuse_transfer_applies_control_cfg(make_cosmos3_pipeline, sequential_ control_guidance=1.5, control_guidance_interval=None, control_latents=[torch.zeros_like(latents)], + control_weights=[1.0], shared_kwargs={"video_shape": (1, 1, 1), "fps": 24.0, "noisy_frame_mask": velocity_mask}, velocity_mask=velocity_mask, condition_latents=torch.zeros_like(latents), @@ -1750,6 +1850,9 @@ def test_diffuse_transfer_applies_control_cfg(make_cosmos3_pipeline, sequential_ (2, False), (1, True), ] + assert pipeline.transformer.calls[0]["kwargs"]["control_weights"] == [1.0] + assert "control_weights" not in pipeline.transformer.calls[1]["kwargs"] + assert pipeline.transformer.calls[2]["kwargs"]["control_weights"] == [1.0] torch.testing.assert_close(result, torch.full_like(latents, 254.0)) @@ -1857,14 +1960,35 @@ def test_diffuse_transfer_interval_switches_branch_counts(make_cosmos3_pipeline, torch.testing.assert_close(result, torch.full_like(latents, 508.0)) -@pytest.mark.parametrize(("hint_key", "expected_fps"), [("edge", 8.0), ("wsm", 10.0)]) -def test_forward_transfer_uses_source_fps_except_wsm(make_cosmos3_pipeline, hint_key: str, expected_fps: float) -> None: +@pytest.mark.parametrize( + ("hint_key", "emphasize_control", "expected_fps", "negative_prompt"), + [ + ("edge", None, 8.0, None), + ("wsm", None, 10.0, ""), + ("edge", False, 8.0, "custom negative"), + ], +) +def test_forward_transfer_uses_transfer_prompt_contract_and_source_fps_except_wsm( + make_cosmos3_pipeline, + hint_key: str, + emphasize_control: bool | None, + expected_fps: float, + negative_prompt: str | None, +) -> None: + from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import ( + COSMOS3_TRANSFER_CONTROL_DIRECTIVE_TEMPLATE, + COSMOS3_TRANSFER_SYSTEM_PROMPT, + ) + pipeline = make_cosmos3_pipeline() captured: dict[str, Any] = {} def fake_format(prompt, negative_prompt, num_frames, frame_rate, height, width, *args, **kwargs): - del prompt, negative_prompt, num_frames, height, width, args, kwargs + del num_frames, height, width, args + captured["prompt"] = prompt + captured["negative_prompt"] = negative_prompt captured["format_frame_rate"] = frame_rate + captured["format_kwargs"] = kwargs return _ids(2), _mask(), _ids(1), _mask() def fake_encode(video: torch.Tensor) -> torch.Tensor: @@ -1901,33 +2025,48 @@ def test_forward_transfer_uses_source_fps_except_wsm(make_cosmos3_pipeline, hint pipeline._decode_latents = lambda latents: torch.zeros(1, 3, 5, 16, 16, device="meta") control = torch.zeros(3, 5, 16, 16, dtype=torch.uint8) + extra_args = { + hint_key: {"control": control}, + "max_frames": 5, + "num_video_frames_per_chunk": 5, + "show_control_condition": True, + } + if emphasize_control is not None: + extra_args["emphasize_control_in_prompt"] = emphasize_control + prompt_data = { + "prompt": "transfer", + "modalities": ["video"], + "additional_information": { + "preprocessed_transfer_video": torch.zeros(1, 3, 5, 16, 16), + "transfer_input_fps": 8.0, + }, + } + if negative_prompt is not None: + prompt_data["negative_prompt"] = negative_prompt request = SimpleNamespace( - prompts=[ - { - "prompt": "transfer", - "modalities": ["video"], - "additional_information": { - "preprocessed_transfer_video": torch.zeros(1, 3, 5, 16, 16), - "transfer_input_fps": 8.0, - }, - } - ], + prompts=[prompt_data], sampling_params=make_sampling_params( height=16, width=16, # fps omitted (no frame_rate) -> non-wsm uses the source video fps (8), wsm uses its preset (10). - extra_args={ - hint_key: {"control": control}, - "max_frames": 5, - "num_video_frames_per_chunk": 5, - "show_control_condition": True, - }, + extra_args=extra_args, ), ) output = pipeline.forward(request) assert captured["format_frame_rate"] == expected_fps + expected_negative_prompt = "" if negative_prompt is None else negative_prompt + assert captured["negative_prompt"] == expected_negative_prompt + assert captured["format_kwargs"]["use_system_prompt"] is True + assert captured["format_kwargs"]["system_prompt"] == COSMOS3_TRANSFER_SYSTEM_PROMPT + assert captured["format_kwargs"]["use_duration_template"] is True + assert captured["format_kwargs"]["use_resolution_template"] is True + assert captured["format_kwargs"]["negative_metadata_mode"] == "same" + expected_suffix = ( + None if emphasize_control is False else COSMOS3_TRANSFER_CONTROL_DIRECTIVE_TEMPLATE.format(hint_names=hint_key) + ) + assert captured["format_kwargs"]["prompt_suffix"] == expected_suffix assert captured["shared_kwargs"]["fps"] == expected_fps assert captured["flow_shifts"] == [10.0] # Transfer applies the V2V flow shift when building its timestep schedule. diff --git a/tests/diffusion/models/cosmos3/test_cosmos3_transformer.py b/tests/diffusion/models/cosmos3/test_cosmos3_transformer.py index 5dd8a875a..2bfa7b558 100644 --- a/tests/diffusion/models/cosmos3/test_cosmos3_transformer.py +++ b/tests/diffusion/models/cosmos3/test_cosmos3_transformer.py @@ -504,6 +504,35 @@ def test_model_cpu_offload_moves_reasoner_and_generator_between_cpu_and_device( assert generator_param.device == accelerator_device +def test_multi_control_attention_weights_target_outputs() -> None: + from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3CrossAttention + + class ControlValueAttention: + def __call__(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor: + del k + # Each per-control K/V layout is [text, control_i, target]. + return v[:, 1:2].expand(q.shape[0], q.shape[1], -1, -1) + + attention = SimpleNamespace(multi_control_attn=ControlValueAttention()) + q = torch.zeros(1, 3, 1, 1) + k = torch.zeros_like(q) + v = torch.tensor([[[[10.0]], [[20.0]], [[30.0]]]]) + text_kv = torch.zeros(1, 1, 1, 1) + + output = Cosmos3CrossAttention._forward_multi_control( + attention, + q, + k, + v, + text_kv, + text_kv, + control_token_sizes=(1, 1), + control_weights=(0.25, 0.75), + ) + + torch.testing.assert_close(output, torch.tensor([[[10.0], [20.0], [17.5]]])) + + def test_forward_accepts_transfer_control_latents(monkeypatch: pytest.MonkeyPatch) -> None: from vllm_omni.diffusion.models.cosmos3 import transformer_cosmos3 @@ -521,9 +550,21 @@ def test_forward_accepts_transfer_control_latents(monkeypatch: pytest.MonkeyPatc video_shape=(1, 2, 2), fps=24.0, control_latents=[torch.ones_like(hidden_states), torch.full_like(hidden_states, 2.0)], + control_weights=[1.0, 3.0], ) assert tuple(output.shape) == tuple(hidden_states.shape) + with pytest.raises(ValueError, match="control_weights length"): + model( + hidden_states=hidden_states, + timestep=torch.tensor([1.0]), + text_ids=torch.tensor([[1, 2]], dtype=torch.long), + text_mask=torch.ones(1, 2, dtype=torch.long), + video_shape=(1, 2, 2), + fps=24.0, + control_latents=[torch.ones_like(hidden_states), torch.full_like(hidden_states, 2.0)], + control_weights=[1.0], + ) with pytest.raises(ValueError, match="control latent shape"): model( hidden_states=hidden_states, diff --git a/tests/model_extras/test_model_extras.py b/tests/model_extras/test_model_extras.py index 573f0bd79..b7f2decf9 100644 --- a/tests/model_extras/test_model_extras.py +++ b/tests/model_extras/test_model_extras.py @@ -151,6 +151,7 @@ def test_cosmos3_extra_registry_declares_request_and_response_params(pipeline_na "max_sequence_length", "use_resolution_template", "use_duration_template", + "negative_metadata_mode", "use_system_prompt", "system_prompt", "negative_prompt", diff --git a/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py b/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py index 97cf92e26..2304c0212 100644 --- a/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py +++ b/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py @@ -163,6 +163,14 @@ COSMOS3_INVERSE_IMAGE_RESOLUTION_TEMPLATE = "This image is not of {height}x{widt # NOTE: Intentional typo in "give" instead of "given" to match training setup. COSMOS3_SYSTEM_PROMPT = "You are a helpful assistant who will generate videos from a give prompt." COSMOS3_T2I_SYSTEM_PROMPT = "You are a helpful assistant who will generate images from a give prompt." +COSMOS3_TRANSFER_SYSTEM_PROMPT = ( + "You are a helpful assistant that generates images or videos following the user's instructions" + " and control signals (edge maps, blur, depth, or segmentation)." +) +COSMOS3_TRANSFER_CONTROL_DIRECTIVE_TEMPLATE = ( + "Follow the {hint_names} control video precisely: shape, contour, silhouette, position, and motion of every " + "visible structure must align with the {hint_names} signal at every frame." +) COSMOS3_T2V_DEFAULT_HEIGHT = 720 COSMOS3_T2V_DEFAULT_WIDTH = 1280 @@ -2068,6 +2076,11 @@ class Cosmos3OmniDiffusersPipeline( sp: OmniDiffusionSamplingParams, use_system_prompt: bool = False, is_t2i: bool = False, + system_prompt: str | None = None, + prompt_suffix: str | None = None, + use_duration_template: bool | None = None, + use_resolution_template: bool | None = None, + negative_metadata_mode: str | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """Format prompts with metadata templates and tokenize. @@ -2075,14 +2088,30 @@ class Cosmos3OmniDiffusersPipeline( For T2I (``is_t2i=True``) the duration template is suppressed (no FPS or duration concept for a single image) and the image-flavored - resolution template is used. + resolution template is used. ``prompt_suffix`` is appended only to the + positive prompt, after metadata formatting. ``negative_metadata_mode`` + controls whether the negative branch receives no metadata, the same + metadata as the positive branch, or inverse metadata. """ # Route cosmos3-specific controls through ``_get_sp_param`` so they # are picked up from ``extra_args`` (OpenAI endpoint path) as well # as from direct attributes. - use_duration_template = bool(self._get_sp_param(sp, "use_duration_template", False)) and not is_t2i + if use_duration_template is None: + use_duration_template = bool(self._get_sp_param(sp, "use_duration_template", False)) + if use_resolution_template is None: + use_resolution_template = bool(self._get_sp_param(sp, "use_resolution_template", False)) + if negative_metadata_mode is None: + negative_metadata_mode = str(self._get_sp_param(sp, "negative_metadata_mode", "inverse")) + negative_metadata_mode = negative_metadata_mode.strip().lower() + if negative_metadata_mode not in {"none", "same", "inverse"}: + raise ValueError( + "Cosmos3 negative_metadata_mode must be one of 'none', 'same', or 'inverse'; " + f"got {negative_metadata_mode!r}." + ) + + use_duration_template = use_duration_template and not is_t2i dur_tmpl = COSMOS3_DURATION_TEMPLATE if use_duration_template else None - if bool(self._get_sp_param(sp, "use_resolution_template", False)): + if use_resolution_template: res_tmpl = COSMOS3_IMAGE_RESOLUTION_TEMPLATE if is_t2i else COSMOS3_RESOLUTION_TEMPLATE else: res_tmpl = None @@ -2106,37 +2135,44 @@ class Cosmos3OmniDiffusersPipeline( duration_template=dur_tmpl, resolution_template=res_tmpl, ) + if prompt_suffix: + prompt = f"{prompt.rstrip()} {prompt_suffix.lstrip()}".strip() if _is_rank_zero(): logger.info("Final prompt: '%s'", prompt) - # Negative prompt: inverse templates ("not {duration}...", "not {height}x{width}..."). - # Applied whenever the matching positive template is enabled; an empty - # negative_prompt yields output that starts with the template, not a dot. - inv_dur = COSMOS3_INVERSE_DURATION_TEMPLATE if dur_tmpl else None - if res_tmpl is None: - inv_res = None - elif is_t2i: - inv_res = COSMOS3_INVERSE_IMAGE_RESOLUTION_TEMPLATE + if negative_metadata_mode == "none": + negative_dur_tmpl = None + negative_res_tmpl = None + elif negative_metadata_mode == "same": + negative_dur_tmpl = dur_tmpl + negative_res_tmpl = res_tmpl else: - inv_res = COSMOS3_INVERSE_RESOLUTION_TEMPLATE + negative_dur_tmpl = COSMOS3_INVERSE_DURATION_TEMPLATE if dur_tmpl else None + if res_tmpl is None: + negative_res_tmpl = None + elif is_t2i: + negative_res_tmpl = COSMOS3_INVERSE_IMAGE_RESOLUTION_TEMPLATE + else: + negative_res_tmpl = COSMOS3_INVERSE_RESOLUTION_TEMPLATE negative_prompt = self._apply_metadata_templates( negative_prompt, num_frames, frame_rate, height, width, - duration_template=inv_dur, - resolution_template=inv_res, - force_duration_template=True, + duration_template=negative_dur_tmpl, + resolution_template=negative_res_tmpl, + force_duration_template=negative_metadata_mode == "inverse", ) - default_sys_prompt = COSMOS3_T2I_SYSTEM_PROMPT if is_t2i else COSMOS3_SYSTEM_PROMPT - sys_prompt = self._get_sp_param(sp, "system_prompt", default_sys_prompt) or default_sys_prompt + if system_prompt is None: + default_sys_prompt = COSMOS3_T2I_SYSTEM_PROMPT if is_t2i else COSMOS3_SYSTEM_PROMPT + system_prompt = self._get_sp_param(sp, "system_prompt", default_sys_prompt) or default_sys_prompt cond_ids, cond_mask = self._tokenize_prompt( - prompt, max_sequence_length, use_system_prompt, system_prompt=sys_prompt + prompt, max_sequence_length, use_system_prompt, system_prompt=system_prompt ) uncond_ids, uncond_mask = self._tokenize_prompt( - negative_prompt, max_sequence_length, use_system_prompt, system_prompt=sys_prompt + negative_prompt, max_sequence_length, use_system_prompt, system_prompt=system_prompt ) return cond_ids, cond_mask, uncond_ids, uncond_mask @@ -2890,6 +2926,7 @@ class Cosmos3OmniDiffusersPipeline( control_latents: list[torch.Tensor], shared_kwargs: dict[str, Any], *, + control_weights: list[float] | None = None, velocity_mask: torch.Tensor, condition_latents: torch.Tensor, guidance_interval: tuple[float, float] | None = None, @@ -2925,6 +2962,7 @@ class Cosmos3OmniDiffusersPipeline( text_ids=cond_ids, text_mask=cond_mask, control_latents=control_latents, + control_weights=control_weights, _cosmos3_cache_key="transfer_cond_full", **shared_kwargs, ) @@ -2946,6 +2984,7 @@ class Cosmos3OmniDiffusersPipeline( text_ids=uncond_ids, text_mask=uncond_mask, control_latents=control_latents, + control_weights=control_weights, _cosmos3_cache_key="transfer_uncond_full", **shared_kwargs, ), @@ -2993,6 +3032,7 @@ class Cosmos3OmniDiffusersPipeline( text_ids=uncond_ids, text_mask=uncond_mask, control_latents=control_latents, + control_weights=control_weights, _cosmos3_cache_key="transfer_uncond_full", **shared_kwargs, ), @@ -3113,7 +3153,10 @@ class Cosmos3OmniDiffusersPipeline( self._get_sp_param(sp, "max_sequence_length", COSMOS3_DEFAULT_MAX_SEQUENCE_LENGTH) or COSMOS3_DEFAULT_MAX_SEQUENCE_LENGTH ) - use_system_prompt = bool(self._get_sp_param(sp, "use_system_prompt", False)) + transfer_prompt_suffix = None + if transfer_config.emphasize_control_in_prompt: + hint_names = ", ".join(hint.key for hint in transfer_config.ordered_hints) + transfer_prompt_suffix = COSMOS3_TRANSFER_CONTROL_DIRECTIVE_TEMPLATE.format(hint_names=hint_names) self._guidance_scale = guidance_scale self._num_timesteps = num_inference_steps @@ -3124,6 +3167,13 @@ class Cosmos3OmniDiffusersPipeline( if generator is None: generator = torch.Generator(device=self.device).manual_seed(seed) + for ignored_key in ("system_prompt", "use_system_prompt"): + if self._get_sp_param(sp, ignored_key, None) is not None: + logger.warning_once( + "Cosmos3 transfer ignores the '%s' request parameter and always " + "tokenizes both CFG branches with the transfer-specific system prompt.", + ignored_key, + ) cond_ids, cond_mask, uncond_ids, uncond_mask = self._format_and_tokenize_prompts( prompt, negative_prompt, @@ -3133,8 +3183,13 @@ class Cosmos3OmniDiffusersPipeline( width, max_sequence_length, sp, - use_system_prompt, + use_system_prompt=True, is_t2i=False, + system_prompt=COSMOS3_TRANSFER_SYSTEM_PROMPT, + prompt_suffix=transfer_prompt_suffix, + use_duration_template=bool(self._get_sp_param(sp, "use_duration_template", True)), + use_resolution_template=bool(self._get_sp_param(sp, "use_resolution_template", True)), + negative_metadata_mode=str(self._get_sp_param(sp, "negative_metadata_mode", "same")), ) output_chunks: list[torch.Tensor] = [] @@ -3227,6 +3282,7 @@ class Cosmos3OmniDiffusersPipeline( control_guidance=transfer_config.control_guidance, control_guidance_interval=transfer_config.control_guidance_interval, control_latents=control_latents, + control_weights=transfer_config.normalized_control_weights, shared_kwargs=shared_kwargs, velocity_mask=velocity_mask, condition_latents=condition_latents, diff --git a/vllm_omni/diffusion/models/cosmos3/transfer.py b/vllm_omni/diffusion/models/cosmos3/transfer.py index 42cbea559..9c8a52f3f 100644 --- a/vllm_omni/diffusion/models/cosmos3/transfer.py +++ b/vllm_omni/diffusion/models/cosmos3/transfer.py @@ -14,6 +14,7 @@ generation, or action generation. from __future__ import annotations import importlib +import math from collections.abc import Mapping from dataclasses import dataclass, field from pathlib import Path @@ -25,6 +26,14 @@ import torch import torch.nn.functional as F TRANSFER_HINT_KEYS: tuple[str, ...] = ("edge", "blur", "depth", "seg", "wsm") +_TRANSFER_HINT_COMMON_FIELDS = frozenset({"control_path", "control", "control_weight"}) +_TRANSFER_HINT_FIELDS: dict[str, frozenset[str]] = { + "edge": _TRANSFER_HINT_COMMON_FIELDS | {"preset_edge_threshold"}, + "blur": _TRANSFER_HINT_COMMON_FIELDS | {"preset_blur_strength"}, + "depth": _TRANSFER_HINT_COMMON_FIELDS, + "seg": _TRANSFER_HINT_COMMON_FIELDS, + "wsm": _TRANSFER_HINT_COMMON_FIELDS, +} TRANSFER_SAMPLE_DEFAULTS: dict[str, Any] = { "num_video_frames_per_chunk": 93, "num_conditional_frames": 1, @@ -33,6 +42,7 @@ TRANSFER_SAMPLE_DEFAULTS: dict[str, Any] = { "show_input": False, "num_first_chunk_conditional_frames": 0, "share_vision_temporal_positions": True, + "emphasize_control_in_prompt": True, } TRANSFER_DEFAULTS: dict[str, dict[str, Any]] = { "edge": {"guidance_scale": 3.0, "control_guidance": 1.5, "flow_shift": 10.0}, @@ -85,6 +95,7 @@ class Cosmos3TransferHint: key: str control_path: str | None = None control: Any | None = None + control_weight: float = 1.0 preset_edge_threshold: str = "medium" preset_blur_strength: str = "medium" @@ -103,6 +114,7 @@ class Cosmos3TransferConfig: show_input: bool = False num_first_chunk_conditional_frames: int = 0 share_vision_temporal_positions: bool = True + emphasize_control_in_prompt: bool = True num_frames: int | None = None fps: float | None = None @@ -110,6 +122,16 @@ class Cosmos3TransferConfig: def ordered_hints(self) -> list[Cosmos3TransferHint]: return [self.hints[key] for key in TRANSFER_HINT_KEYS if key in self.hints] + @property + def normalized_control_weights(self) -> list[float]: + weights = [hint.control_weight for hint in self.ordered_hints] + if any(not math.isfinite(weight) or weight < 0.0 for weight in weights): + raise ValueError(f"Cosmos3 transfer control_weight values must be finite and non-negative, got {weights}.") + total = sum(weights) + if total <= 0.0: + raise ValueError("Cosmos3 transfer control_weight values must have a positive sum.") + return [weight / total for weight in weights] + def _extra_args(sp: Any) -> Mapping[str, Any]: extra = getattr(sp, "extra_args", None) @@ -184,7 +206,7 @@ def _as_interval(value: Any) -> tuple[float, float] | None: return None if isinstance(value, str): value = [item.strip() for item in value.split(",") if item.strip()] - if not isinstance(value, (list, tuple)) or len(value) != 2: + if not isinstance(value, list | tuple) or len(value) != 2: raise ValueError("Cosmos3 transfer control_guidance_interval must contain exactly two values.") lo, hi = float(value[0]), float(value[1]) if lo > hi: @@ -220,10 +242,27 @@ def resolve_transfer_config(sp: Any, prompt_data: Any = None) -> Cosmos3Transfer raise TypeError( f"Cosmos3 transfer hint '{key}' must be an object, path string, or true; got {type(raw)!r}." ) + unknown_fields = set(raw) - _TRANSFER_HINT_FIELDS[key] + if unknown_fields: + raise ValueError( + f"Unsupported Cosmos3 transfer hint '{key}' fields: {sorted(unknown_fields)}. " + f"Supported fields are: {sorted(_TRANSFER_HINT_FIELDS[key])}." + ) + try: + control_weight = float(raw.get("control_weight", 1.0)) + except (TypeError, ValueError) as exc: + raise TypeError( + f"Cosmos3 transfer hint '{key}' control_weight must be a finite non-negative number." + ) from exc + if not math.isfinite(control_weight) or control_weight < 0.0: + raise ValueError( + f"Cosmos3 transfer hint '{key}' control_weight must be finite and non-negative, got {control_weight!r}." + ) hints[key] = Cosmos3TransferHint( key=key, control_path=str(raw["control_path"]) if raw.get("control_path") is not None else None, control=raw.get("control"), + control_weight=control_weight, preset_edge_threshold=str(raw.get("preset_edge_threshold") or "medium").lower(), preset_blur_strength=str(raw.get("preset_blur_strength") or "medium").lower(), ) @@ -239,6 +278,7 @@ def resolve_transfer_config(sp: Any, prompt_data: Any = None) -> Cosmos3Transfer "show_input", "num_first_chunk_conditional_frames", "share_vision_temporal_positions", + "emphasize_control_in_prompt", ) if any(_is_user_field(extra, sp, prompt_data, key) for key in transfer_only): raise ValueError("Cosmos3 transfer options were provided, but no transfer hint was selected.") @@ -293,6 +333,10 @@ def resolve_transfer_config(sp: Any, prompt_data: Any = None) -> Cosmos3Transfer _param(extra, sp, prompt_data, "share_vision_temporal_positions", None), bool(TRANSFER_SAMPLE_DEFAULTS["share_vision_temporal_positions"]), ), + emphasize_control_in_prompt=_as_bool( + _param(extra, sp, prompt_data, "emphasize_control_in_prompt", None), + bool(TRANSFER_SAMPLE_DEFAULTS["emphasize_control_in_prompt"]), + ), num_frames=( int(_param(extra, sp, prompt_data, "num_frames")) if _param(extra, sp, prompt_data, "num_frames", None) is not None @@ -328,6 +372,7 @@ def resolve_transfer_config(sp: Any, prompt_data: Any = None) -> Cosmos3Transfer raise ValueError(f"Unsupported Cosmos3 edge preset: {hint.preset_edge_threshold!r}.") if hint.key == "blur" and hint.preset_blur_strength not in BLUR_DOWNUP_PRESETS: raise ValueError(f"Unsupported Cosmos3 blur preset: {hint.preset_blur_strength!r}.") + _ = config.normalized_control_weights return config diff --git a/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py b/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py index cea5a32ff..c946eb4ca 100644 --- a/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py +++ b/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py @@ -651,6 +651,19 @@ class Cosmos3CrossAttention(nn.Module): softmax_scale=1.0 / (self.head_dim**0.5), num_kv_heads=self.num_kv_heads, ) + # Multi-control attention operates on one full [control_i, target] + # sequence at a time. Keep those sequences replicated when Ulysses is + # active; sharding the concatenated [control_1, ..., control_N, target] + # layout would split the per-control ranges across ranks. + self.multi_control_attn = FrameworkAttention( + num_heads=self.num_heads, + head_size=self.head_dim, + causal=False, + softmax_scale=1.0 / (self.head_dim**0.5), + num_kv_heads=self.num_kv_heads, + prefix=f"{prefix}.multi_control_attn", + skip_sequence_parallel=True, + ) # TODO(follow-up): collapse _forward_local and _forward_sp into a single # joint-based path when NoParallelAttention can process joint_key/value. @@ -701,6 +714,61 @@ class Cosmos3CrossAttention(nn.Module): out = self.attn(q, k, v, attn_metadata) return out.reshape(B, S_gen, -1) + def _forward_multi_control( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + k_und: torch.Tensor, + v_und: torch.Tensor, + control_token_sizes: tuple[int, ...], + control_weights: tuple[float, ...], + ) -> torch.Tensor: + """Run independent attention per control and weight target outputs. + + For control ``i``, both its queries and the target queries attend to + ``[text, control_i, target]``. Each control keeps the output from its + own pass, while target outputs are combined using normalized weights. + """ + if len(control_token_sizes) != len(control_weights): + raise ValueError( + "Cosmos3 control token sizes and weights must have the same length: " + f"sizes={len(control_token_sizes)}, weights={len(control_weights)}." + ) + control_tokens = sum(control_token_sizes) + if control_tokens <= 0 or control_tokens >= q.shape[1]: + raise ValueError( + "Cosmos3 multi-control attention requires non-empty control and target token ranges, " + f"got control_tokens={control_tokens}, total_tokens={q.shape[1]}." + ) + + q_target = q[:, control_tokens:] + k_target = k[:, control_tokens:] + v_target = v[:, control_tokens:] + control_outputs: list[torch.Tensor] = [] + target_output: torch.Tensor | None = None + start = 0 + for size, weight in zip(control_token_sizes, control_weights, strict=True): + if size <= 0: + raise ValueError(f"Cosmos3 control token sizes must be positive, got {control_token_sizes}.") + end = start + size + q_control = q[:, start:end] + k_control = k[:, start:end] + v_control = v[:, start:end] + q_pair = torch.cat([q_control, q_target], dim=1) + k_pair = torch.cat([k_und, k_control, k_target], dim=1) + v_pair = torch.cat([v_und, v_control, v_target], dim=1) + pair_output = self.multi_control_attn(q_pair, k_pair, v_pair) + control_outputs.append(pair_output[:, :size]) + weighted_target = pair_output[:, size:] * weight + target_output = weighted_target if target_output is None else target_output + weighted_target + start = end + + if start != control_tokens or target_output is None: + raise RuntimeError("Cosmos3 multi-control attention failed to cover all control token ranges.") + output = torch.cat([*control_outputs, target_output], dim=1) + return output.reshape(q.shape[0], q.shape[1], -1) + # -- Public forward: routes to the appropriate path ---------------------- def forward( @@ -710,6 +778,8 @@ class Cosmos3CrossAttention(nn.Module): v_und: torch.Tensor, freqs_cos: torch.Tensor, freqs_sin: torch.Tensor, + control_token_sizes: tuple[int, ...] | None = None, + control_weights: tuple[float, ...] | None = None, ) -> torch.Tensor: """ Args: @@ -733,7 +803,19 @@ class Cosmos3CrossAttention(nn.Module): # Qwen3-style RoPE q, k = _apply_rotary_pos_emb(q, k, freqs_cos, freqs_sin) - if _is_sp_active(): + if control_token_sizes is not None or control_weights is not None: + if control_token_sizes is None or control_weights is None: + raise ValueError("Cosmos3 multi-control attention requires both control token sizes and weights.") + out = self._forward_multi_control( + q, + k, + v, + k_und, + v_und, + control_token_sizes, + control_weights, + ) + elif _is_sp_active(): out = self._forward_sp(q, k, v, k_und, v_und) else: out = self._forward_local(q, k, v, k_und, v_und) @@ -847,6 +929,8 @@ class Cosmos3GenDecoderLayer(nn.Module): freqs_sin: torch.Tensor | None = None, cached_kv: list[tuple[torch.Tensor, torch.Tensor]] | None = None, freqs_gen: tuple[torch.Tensor, torch.Tensor] | None = None, + control_token_sizes: tuple[int, ...] | None = None, + control_weights: tuple[float, ...] | None = None, ) -> torch.Tensor: if cached_kv is not None: if self.layer_idx is None: @@ -861,7 +945,13 @@ class Cosmos3GenDecoderLayer(nn.Module): hidden_states = self.input_layernorm(hidden_states) hidden_states = self.cross_attention( - hidden_states, k_und=k_und, v_und=v_und, freqs_cos=freqs_cos, freqs_sin=freqs_sin + hidden_states, + k_und=k_und, + v_und=v_und, + freqs_cos=freqs_cos, + freqs_sin=freqs_sin, + control_token_sizes=control_token_sizes, + control_weights=control_weights, ) hidden_states = residual + hidden_states @@ -1579,6 +1669,7 @@ class Cosmos3VFMTransformer(nn.Module): sound_latents: torch.Tensor | None = None, noisy_frame_mask: torch.Tensor | None = None, control_latents: list[torch.Tensor] | tuple[torch.Tensor, ...] | torch.Tensor | None = None, + control_weights: list[float] | tuple[float, ...] | torch.Tensor | None = None, transfer_share_vision_temporal_positions: bool = True, **kwargs, ) -> torch.Tensor | tuple[torch.Tensor, ...]: @@ -1601,6 +1692,8 @@ class Cosmos3VFMTransformer(nn.Module): frames are noisy, as in T2I/T2V. control_latents: Optional transfer-control latents. Controls are clean vision context and are packed before the noisy target. + control_weights: Optional non-negative relative weights for + transfer controls. Values are normalized to sum to one. Returns: [B, C, t, h, w] velocity prediction, or @@ -1629,6 +1722,31 @@ class Cosmos3VFMTransformer(nn.Module): else: control_latent_list = list(control_latents) has_control = len(control_latent_list) > 0 + if control_weights is None: + normalized_control_weights = ( + [1.0 / len(control_latent_list)] * len(control_latent_list) if has_control else [] + ) + else: + if isinstance(control_weights, torch.Tensor): + raw_control_weights = control_weights.detach().cpu().flatten().tolist() + else: + raw_control_weights = list(control_weights) + if len(raw_control_weights) != len(control_latent_list): + raise ValueError( + "Cosmos3 transfer control_weights length must match control_latents: " + f"weights={len(raw_control_weights)}, controls={len(control_latent_list)}." + ) + normalized_control_weights = [float(weight) for weight in raw_control_weights] + if any(not math.isfinite(weight) or weight < 0.0 for weight in normalized_control_weights): + raise ValueError( + "Cosmos3 transfer control_weights must be finite and non-negative, " + f"got {normalized_control_weights}." + ) + control_weight_sum = sum(normalized_control_weights) + if control_weight_sum <= 0.0: + raise ValueError("Cosmos3 transfer control_weights must have a positive sum.") + normalized_control_weights = [weight / control_weight_sum for weight in normalized_control_weights] + use_multi_control_attention = len(control_latent_list) > 1 if has_control and (has_action or has_sound): raise ValueError("Cosmos3 transfer control latents cannot be combined with action or sound latents.") if has_action and not self.action_gen: @@ -1646,6 +1764,13 @@ class Cosmos3VFMTransformer(nn.Module): # Query Ulysses state at runtime ulysses_size, _, _ = _get_ulysses_state() + if use_multi_control_attention and ulysses_size > 1: + logger.warning_once( + "Cosmos3 multi-control attention requires full [control_i | target] " + "sequences, so the GEN pathway runs replicated on every Ulysses rank. " + "Sequence parallelism will not reduce per-rank memory or latency for " + "multi-control transfer requests." + ) # Pack action/sound tokens (no learned weights) up front so the UND # cache sizing knows their token lengths. The modality projections are @@ -1722,6 +1847,9 @@ class Cosmos3VFMTransformer(nn.Module): ) hidden_controls.append(hidden_control) s_control += hidden_control.shape[1] + control_token_sizes = tuple(control.shape[1] for control in hidden_controls) + multi_control_weights = tuple(normalized_control_weights) if use_multi_control_attention else None + multi_control_token_sizes = control_token_sizes if use_multi_control_attention else None hidden_action = None hidden_sound = None if action_tokens is not None: @@ -1788,10 +1916,11 @@ class Cosmos3VFMTransformer(nn.Module): has_action=has_action, has_sound=has_sound, has_control=has_control, - ulysses_size=ulysses_size, + ulysses_size=1 if use_multi_control_attention else ulysses_size, ) freqs_cos, freqs_sin = self.cached_freqs_gen - hidden_gen, freqs_cos, freqs_sin = self.gen_sp_prepare(hidden_gen, freqs_cos, freqs_sin) + if not use_multi_control_attention: + hidden_gen, freqs_cos, freqs_sin = self.gen_sp_prepare(hidden_gen, freqs_cos, freqs_sin) freqs_gen = (freqs_cos, freqs_sin) if len(self.gen_layers) == len(self.cached_kv): @@ -1802,6 +1931,8 @@ class Cosmos3VFMTransformer(nn.Module): v_und=v_und, freqs_cos=freqs_cos, freqs_sin=freqs_sin, + control_token_sizes=multi_control_token_sizes, + control_weights=multi_control_weights, ) # Cache-dit's block wrapper may return a tuple; unwrap it. if isinstance(hidden_gen, tuple): @@ -1813,11 +1944,14 @@ class Cosmos3VFMTransformer(nn.Module): hidden_gen, cached_kv=self.cached_kv, freqs_gen=freqs_gen, + control_token_sizes=multi_control_token_sizes, + control_weights=multi_control_weights, ) if isinstance(hidden_gen, tuple): hidden_gen = hidden_gen[0] - hidden_gen = self.gen_sp_gather(hidden_gen) + if not use_multi_control_attention: + hidden_gen = self.gen_sp_gather(hidden_gen) # Final norm and project back to latent space hidden_gen = self.norm_moe_gen(hidden_gen) diff --git a/vllm_omni/model_extras/cosmos3.py b/vllm_omni/model_extras/cosmos3.py index d0ffa5182..01cecf94f 100644 --- a/vllm_omni/model_extras/cosmos3.py +++ b/vllm_omni/model_extras/cosmos3.py @@ -9,6 +9,7 @@ COSMOS3_EXTRA_BODY_PARAMS = frozenset( "max_sequence_length", "use_resolution_template", "use_duration_template", + "negative_metadata_mode", "use_system_prompt", "system_prompt", "negative_prompt", From b2011a3edda1dbea8057067957a27559114a15b6 Mon Sep 17 00:00:00 2001 From: Alicia <115451386+congw729@users.noreply.github.com> Date: Fri, 21 Aug 2026 02:56:23 +0800 Subject: [PATCH 095/267] [CI/Build] Update perf baselines base on 8/1-8/7 7 days avg (#6201) Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- .../dfx/perf/tests/test_bagel_vllm_omni.json | 16 +- .../perf/tests/test_cosmos3_vllm_omni.json | 22 +- .../perf/tests/test_hunyuan_image3_it2i.json | 4 +- tests/dfx/perf/tests/test_minicpmo_4_5.json | 30 +-- .../test_minicpmo_4_5_duplex_seed_tts.json | 10 +- .../perf/tests/test_minicpmo_4_5_ready.json | 20 +- .../tests/test_qwen3_omni_async_chunk.json | 72 ++----- .../tests/test_qwen3_omni_multi_replicas.json | 22 +- .../tests/test_qwen3_omni_no_async_chunk.json | 46 ++--- .../perf/tests/test_qwen3_omni_vllm_text.json | 32 +-- .../test_qwen_image_edit_2511_vllm_omni.json | 40 ++-- .../tests/test_qwen_image_edit_vllm_omni.json | 190 ------------------ .../test_qwen_image_layered_vllm_omni.json | 4 +- .../perf/tests/test_qwen_image_vllm_omni.json | 20 +- 14 files changed, 134 insertions(+), 394 deletions(-) delete mode 100644 tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json diff --git a/tests/dfx/perf/tests/test_bagel_vllm_omni.json b/tests/dfx/perf/tests/test_bagel_vllm_omni.json index 9dda3148d..1189ddd9d 100644 --- a/tests/dfx/perf/tests/test_bagel_vllm_omni.json +++ b/tests/dfx/perf/tests/test_bagel_vllm_omni.json @@ -43,8 +43,8 @@ "H100": { "throughput_qps": 0.495, "latency_mean": 2.0238, - "peak_memory_mb_max": 29393.4286, - "peak_memory_mb_mean": 29392.7143 + "peak_memory_mb_max": 29392.0, + "peak_memory_mb_mean": 29392.0 } } } @@ -93,10 +93,10 @@ "num-input-images": 1, "baseline": { "H100": { - "throughput_qps": 0.3451, - "latency_mean": 2.9031, - "peak_memory_mb_max": 30480.2857, - "peak_memory_mb_mean": 30479.2857 + "throughput_qps": 0.3499, + "latency_mean": 2.8701, + "peak_memory_mb_max": 30466.0, + "peak_memory_mb_mean": 30465.0 } } } @@ -196,8 +196,8 @@ "num-input-images": 1, "baseline": { "H100": { - "throughput_qps": 0.149, - "latency_mean": 6.7157, + "throughput_qps": 0.1533, + "latency_mean": 6.5243, "peak_memory_mb_max": 30574.0, "peak_memory_mb_mean": 30574.0 } diff --git a/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json b/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json index c4d2305e6..bcd649de1 100644 --- a/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json +++ b/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json @@ -49,8 +49,8 @@ }, "baseline": { "H100": { - "throughput_qps": 1.107, - "latency_mean": 0.8929, + "throughput_qps": 1.1958, + "latency_mean": 0.843, "peak_memory_mb_mean": 80000.0 } } @@ -118,9 +118,9 @@ }, "baseline": { "H100": { - "throughput_qps": 0.027, - "latency_mean": 36.6824, - "peak_memory_mb_mean": 30155.4 + "throughput_qps": 0.0307, + "latency_mean": 32.7665, + "peak_memory_mb_mean": 28762.4762 } } } @@ -186,9 +186,9 @@ }, "baseline": { "H100": { - "throughput_qps": 0.027, - "latency_mean": 36.968, - "peak_memory_mb_mean": 30443.6 + "throughput_qps": 0.0305, + "latency_mean": 33.1009, + "peak_memory_mb_mean": 29441.1429 } } } @@ -258,9 +258,9 @@ }, "baseline": { "H100": { - "throughput_qps": 0.027, - "latency_mean": 37.2117, - "peak_memory_mb_mean": 30135.6 + "throughput_qps": 0.0299, + "latency_mean": 33.925, + "peak_memory_mb_mean": 29443.1429 } } } diff --git a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json index c4ef34b96..098dcd5a0 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json +++ b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json @@ -137,9 +137,7 @@ "H200": { "throughput_qps": 0.1874, "latency_p99": 101.852, - "latency_mean": 79.0385, - "peak_memory_mb_max": 70000, - "peak_memory_mb_mean": 70000 + "latency_mean": 79.0385 } } } diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5.json b/tests/dfx/perf/tests/test_minicpmo_4_5.json index 9296cf5b3..cac1880c5 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5.json @@ -56,29 +56,29 @@ "baseline": { "H100": { "request_throughput": [ - 0.7803, - 1.1008, - 1.2027 + 0.9731, + 1.1986, + 1.686 ], "mean_ttft_ms": [ - 166.9764, - 307.7133, - 323.584 + 152.5665, + 270.2559, + 301.455 ], "mean_e2el_ms": [ - 1361.1682, - 3800.6332, - 6961.8636 + 1027.2953, + 3306.3735, + 4686.5366 ], "mean_audio_ttfp_ms": [ - 635.8099, - 1615.6591, - 2888.9831 + 458.1553, + 2030.2335, + 2068.6233 ], "mean_audio_rtf": [ - 0.3151, - 0.8568, - 1.5425 + 0.2357, + 0.791, + 1.0324 ] }, "A3": { diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json b/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json index 6ef27680c..2038e45e0 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json @@ -47,19 +47,19 @@ "baseline": { "H100": { "request_throughput": [ - 0.1594 + 0.3038 ], "mean_ttft_ms": [ - 274.164 + 139.997 ], "mean_e2el_ms": [ - 6250.7696 + 3269.8807 ], "mean_audio_ttfp_ms": [ - 994.422 + 456.073 ], "mean_audio_rtf": [ - 0.46163 + 0.2254 ] }, "A3": { diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json b/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json index 15d52e6ad..26dc54aab 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json @@ -54,24 +54,24 @@ "baseline": { "H100": { "request_throughput": [ - 0.7803, - 1.1008 + 0.9731, + 1.1986 ], "mean_ttft_ms": [ - 166.9764, - 307.7133 + 152.5665, + 270.2559 ], "mean_e2el_ms": [ - 1361.1682, - 3800.6332 + 1027.2953, + 3306.3735 ], "mean_audio_ttfp_ms": [ - 635.8099, - 1615.6591 + 458.1553, + 2030.2335 ], "mean_audio_rtf": [ - 0.3151, - 0.8568 + 0.2357, + 0.791 ] } } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index a73abc5fb..666e993bb 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -45,26 +45,19 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "H100": { - "mean_ttft_ms": [ - 96.4061, - 140.8147, - 271.9025, - 362.2916, - 507.776 - ], "mean_e2el_ms": [ - 18507.4582, - 28365.4956, + 11701.3818, + 27510.3956, 31907.9613, - 48161.5829, - 72630.8601 + 42173.9832, + 63970.0751 ], "mean_audio_rtf": [ 0.1175, - 0.18, - 0.2359, - 0.3278, - 0.6296 + 0.1647, + 0.2003, + 0.2726, + 0.4195 ] } } @@ -94,14 +87,11 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "H100": { - "mean_ttft_ms": [ - 354.0951 - ], "mean_e2el_ms": [ - 3952.5615 + 3646.797 ], "mean_audio_rtf": [ - 0.117 + 0.1098 ] } } @@ -133,14 +123,11 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "H100": { - "mean_ttft_ms": [ - 259.6317 - ], "mean_e2el_ms": [ - 4253.3009 + 4059.0878 ], "mean_audio_rtf": [ - 0.1448 + 0.1335 ] } } @@ -174,14 +161,11 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "H100": { - "mean_ttft_ms": [ - 457.4453 - ], "mean_e2el_ms": [ - 5912.5623 + 5639.5769 ], "mean_audio_rtf": [ - 0.1967 + 0.1798 ] } } @@ -215,19 +199,12 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "H100": { - "mean_ttft_ms": [ - 90.8033, - 202.1779, - 326.5864, - 544.8914, - 961.3929 - ], "mean_e2el_ms": [ - 4517.3411, - 7857.9711, + 4354.8945, + 7672.4248, 9861.1687, - 13283.3363, - 19541.0736 + 13086.8865, + 18716.709 ] } } @@ -262,11 +239,8 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "H100": { - "mean_ttft_ms": [ - 318.048 - ], "mean_e2el_ms": [ - 807.6114 + 756.8415 ] } } @@ -303,11 +277,8 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "H100": { - "mean_ttft_ms": [ - 186.168 - ], "mean_e2el_ms": [ - 726.1137 + 694.9777 ] } } @@ -346,9 +317,6 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "H100": { - "mean_ttft_ms": [ - 375.5282 - ], "mean_e2el_ms": [ 1006.1663 ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json index a3d8aaea8..55ccf14ba 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json @@ -36,10 +36,9 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "H100": { - "mean_ttft_ms": [130.5751, 278.6469, 324.7249, 911.6085], - "mean_tpot_ms": [7.9854, 11.1209, 16.1507, 20.7485], - "mean_audio_ttfp_ms": [486.486, 954.1791, 1289.6514, 2196.8275], - "mean_audio_rtf": [0.1896, 0.3962, 0.4098, 0.7322] + "mean_tpot_ms": [7.6955, 10.6457, 12.7363, 15.1608], + "mean_audio_ttfp_ms": [325.1624, 602.5192, 699.3891, 1024.088], + "mean_audio_rtf": [0.1689, 0.2088, 0.2397, 0.2791] } } }, @@ -64,9 +63,8 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "H100": { - "mean_ttft_ms": [340.5122], - "mean_audio_ttfp_ms": [463.0752], - "mean_audio_rtf": [0.1153] + "mean_audio_ttfp_ms": [436.4663], + "mean_audio_rtf": [0.1069] } } }, @@ -93,9 +91,8 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "H100": { - "mean_ttft_ms": [236.926], - "mean_audio_ttfp_ms": [395.2182], - "mean_audio_rtf": [0.1259] + "mean_audio_ttfp_ms": [376.4911], + "mean_audio_rtf": [0.1145] } } }, @@ -124,9 +121,8 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "H100": { - "mean_ttft_ms": [470.6126], - "mean_audio_ttfp_ms": [665.5474], - "mean_audio_rtf": [0.166] + "mean_audio_ttfp_ms": [638.2227], + "mean_audio_rtf": [0.1438] } } } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json index 38dbfb2e3..26e4b6370 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json @@ -45,26 +45,19 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "H100": { - "mean_ttft_ms": [ - 115.8348, - 173.1823, - 446.672, - 464.5985, - 3420.0634 - ], "mean_e2el_ms": [ - 25070.9523, - 31992.0646, - 37619.7669, - 50393.8257, - 136619.2743 + 15793.1209, + 30658.6632, + 37463.4943, + 47990.5839, + 64980.2199 ], "mean_audio_rtf": [ 0.169, - 0.2111, - 0.2475, - 0.3551, - 0.9266 + 0.2041, + 0.2382, + 0.3177, + 0.4595 ] } } @@ -94,14 +87,11 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "H100": { - "mean_ttft_ms": [ - 343.1377 - ], "mean_e2el_ms": [ - 4811.1123 + 4556.4347 ], "mean_audio_rtf": [ - 0.1463 + 0.138 ] } } @@ -133,14 +123,11 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "H100": { - "mean_ttft_ms": [ - 245.5446 - ], "mean_e2el_ms": [ - 5019.1256 + 4768.0972 ], "mean_audio_rtf": [ - 0.1713 + 0.1609 ] } } @@ -174,14 +161,11 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "H100": { - "mean_ttft_ms": [ - 506.1407 - ], "mean_e2el_ms": [ - 6814.9656 + 6395.3517 ], "mean_audio_rtf": [ - 0.2213 + 0.2048 ] } } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json index 8775ee381..f24bd957f 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json @@ -52,19 +52,12 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "H100": { - "mean_ttft_ms": [ - 69.8081, - 173.0651, - 271.8108, - 413.4079, - 497.2546 - ], "mean_e2el_ms": [ - 4720.8225, - 7937.2532, - 9962.3566, - 13724.5479, - 18324.091 + 4619.0068, + 7868.3523, + 9760.6175, + 13422.3491, + 18007.6918 ] } } @@ -100,11 +93,8 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "H100": { - "mean_ttft_ms": [ - 25.9477 - ], "mean_e2el_ms": [ - 517.3155 + 491.2534 ] } } @@ -142,11 +132,8 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "H100": { - "mean_ttft_ms": [ - 26.7056 - ], "mean_e2el_ms": [ - 551.308 + 524.5321 ] } } @@ -186,11 +173,8 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "H100": { - "mean_ttft_ms": [ - 27.6906 - ], "mean_e2el_ms": [ - 596.5284 + 559.6674 ] } } diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json index fd8e437fd..34d275ac9 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json @@ -35,10 +35,10 @@ "enable-negative-prompt": true, "baseline": { "H100": { - "throughput_qps": 0.0721, - "latency_mean": 13.8398, - "peak_memory_mb_max": 57610.0, - "peak_memory_mb_mean": 57610.0 + "throughput_qps": 0.0684, + "latency_mean": 14.6249, + "peak_memory_mb_max": 57632.0, + "peak_memory_mb_mean": 57632.0 } } }, @@ -55,10 +55,10 @@ "enable-negative-prompt": true, "baseline": { "H100": { - "throughput_qps": 0.0186, - "latency_mean": 53.6942, - "peak_memory_mb_max": 67790.0, - "peak_memory_mb_mean": 67790.0 + "throughput_qps": 0.0177, + "latency_mean": 56.5285, + "peak_memory_mb_max": 67840.0, + "peak_memory_mb_mean": 67840.0 } } } @@ -105,10 +105,10 @@ "enable-negative-prompt": true, "baseline": { "H100": { - "throughput_qps": 0.0538, - "latency_mean": 18.5591, - "peak_memory_mb_max": 56449.4286, - "peak_memory_mb_mean": 56449.4286 + "throughput_qps": 0.0515, + "latency_mean": 19.4345, + "peak_memory_mb_max": 56348.0, + "peak_memory_mb_mean": 56348.0 } } } @@ -164,10 +164,10 @@ "enable-negative-prompt": true, "baseline": { "H100": { - "throughput_qps": 0.2776, - "latency_mean": 3.5938, - "peak_memory_mb_max": 58113.7143, - "peak_memory_mb_mean": 58075.1429 + "throughput_qps": 0.2479, + "latency_mean": 4.0389, + "peak_memory_mb_max": 57750.0, + "peak_memory_mb_mean": 57750.0 } } }, @@ -184,10 +184,10 @@ "enable-negative-prompt": true, "baseline": { "H100": { - "throughput_qps": 0.1051, - "latency_mean": 9.4918, - "peak_memory_mb_max": 68107.4286, - "peak_memory_mb_mean": 68107.4286 + "throughput_qps": 0.0948, + "latency_mean": 10.5462, + "peak_memory_mb_max": 67764.0, + "peak_memory_mb_mean": 67764.0 } } } diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json deleted file mode 100644 index 62ee70aa2..000000000 --- a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json +++ /dev/null @@ -1,190 +0,0 @@ -[ - { - "test_name": "test_qwen_image_edit_single_device", - "mark": [ - { - "hardware_marks": { - "res": { - "cuda": "H100" - }, - "num_cards": 1 - } - }, - "local_model", - "diffusion" - ], - "description": "Single-device baseline", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Edit", - "serve_args": { - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "512x512_steps20_i2i", - "dataset": "random", - "task": "i2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "H100": { - "throughput_qps": 0.1129, - "latency_mean": 8.8352, - "peak_memory_mb_max": 58572.0, - "peak_memory_mb_mean": 58572.0 - } - } - }, - { - "name": "1536x1536_steps35_i2i", - "dataset": "random", - "task": "i2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "H100": { - "throughput_qps": 0.024, - "latency_mean": 41.5686, - "peak_memory_mb_max": 66368.0, - "peak_memory_mb_mean": 66368.0 - } - } - } - ] - }, - { - "test_name": "test_qwen_image_edit_ulysses2_cfg2_vae_patch4", - "mark": [ - { - "hardware_marks": { - "res": { - "cuda": "H100" - }, - "num_cards": 4 - } - }, - "local_model", - "diffusion" - ], - "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Edit", - "serve_args": { - "ulysses-degree": 2, - "cfg-parallel-size": 2, - "vae-patch-parallel-size": 4, - "vae-use-tiling": true, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "1536x1536_steps35_i2i", - "dataset": "random", - "task": "i2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "H100": { - "throughput_qps": 0.0672, - "latency_mean": 14.8517, - "peak_memory_mb_max": 57076.0, - "peak_memory_mb_mean": 57076.0 - } - } - } - ] - }, - { - "test_name": "test_qwen_image_edit_ulysses2_cfg2_cache_dit", - "mark": [ - { - "hardware_marks": { - "res": { - "cuda": "H100" - }, - "num_cards": 4 - } - }, - "local_model", - "diffusion" - ], - "description": "Ulysses SP=2 + CFG=2 + CacheDiT", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Edit", - "serve_args": { - "ulysses-degree": 2, - "cfg-parallel-size": 2, - "cache-backend": "cache_dit", - "cache-config": { - "Fn_compute_blocks": 1, - "Bn_compute_blocks": 0, - "max_warmup_steps": 4, - "residual_diff_threshold": 0.24, - "max_continuous_cached_steps": 3, - "enable_taylorseer": false, - "taylorseer_order": 1, - "scm_steps_mask_policy": null, - "scm_steps_policy": "dynamic" - }, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "512x512_steps20_i2i", - "dataset": "random", - "task": "i2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "H100": { - "throughput_qps": 0.2879, - "latency_mean": 3.4701, - "peak_memory_mb_max": 58758.0, - "peak_memory_mb_mean": 58758.0 - } - } - }, - { - "name": "1536x1536_steps35_i2i", - "dataset": "random", - "task": "i2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "H100": { - "throughput_qps": 0.1113, - "latency_mean": 8.9656, - "peak_memory_mb_max": 67424.0, - "peak_memory_mb_mean": 67424.0 - } - } - } - ] - } -] diff --git a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json index b5fa2815b..bc0f5c923 100644 --- a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json @@ -55,8 +55,8 @@ "H100": { "throughput_qps": 0.0417, "latency_mean": 23.914, - "peak_memory_mb_max": 63225.1429, - "peak_memory_mb_mean": 63225.1429 + "peak_memory_mb_max": 62912.0, + "peak_memory_mb_mean": 62912.0 } } } diff --git a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json index ebf9ccc29..31a4ef8cd 100644 --- a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json @@ -36,7 +36,7 @@ "H100": { "throughput_qps": 0.4153, "latency_mean": 2.4065, - "peak_memory_mb_mean": 55121.4286 + "peak_memory_mb_mean": 55062.0 } } }, @@ -54,7 +54,7 @@ "H100": { "throughput_qps": 0.0423, "latency_mean": 23.5991, - "peak_memory_mb_mean": 65118.0 + "peak_memory_mb_mean": 64546.0 } } } @@ -98,7 +98,7 @@ "H100": { "throughput_qps": 0.4175, "latency_mean": 2.3953, - "peak_memory_mb_mean": 55088.5714 + "peak_memory_mb_mean": 54952.0 } } }, @@ -116,7 +116,7 @@ "H100": { "throughput_qps": 0.0423, "latency_mean": 23.5768, - "peak_memory_mb_mean": 66012.2857 + "peak_memory_mb_mean": 64500.0 } } } @@ -163,7 +163,7 @@ "H100": { "throughput_qps": 0.1214, "latency_mean": 8.2211, - "peak_memory_mb_mean": 54501.1429 + "peak_memory_mb_mean": 54474.0 } } } @@ -220,7 +220,7 @@ "H100": { "throughput_qps": 0.559, "latency_mean": 1.794, - "peak_memory_mb_mean": 55290.5714 + "peak_memory_mb_mean": 55174.0 } } }, @@ -238,7 +238,7 @@ "H100": { "throughput_qps": 0.1918, "latency_mean": 5.2022, - "peak_memory_mb_mean": 64838.8571 + "peak_memory_mb_mean": 64750.0 } } } @@ -284,9 +284,9 @@ "enable-negative-prompt": true, "baseline": { "H100": { - "throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159], - "latency_mean": [2.4215, 2.992, 5.9293, 9.656], - "peak_memory_mb_mean": [55147.4286, 55148.0, 55150.4643, 55152.3821] + "throughput_qps": [0.3937, 0.6606, 0.6188, 0.7752], + "latency_mean": [2.5466, 3.0235, 6.3948, 10.1944], + "peak_memory_mb_mean": [55149.0, 55150.0, 55153.1458, 55156.325] } } } From 13c2de684c3fca3882e508b7ed5a1add1e405504 Mon Sep 17 00:00:00 2001 From: MaciejBalaNV Date: Thu, 20 Aug 2026 21:04:49 +0200 Subject: [PATCH 096/267] Cosmos3 logging and prompt improvements (#6325) Signed-off-by: Maciej Bala Co-authored-by: Roger Wang --- .../models/cosmos3/test_cosmos3_pipeline.py | 141 ++++++++++++++- vllm_omni/diffusion/models/cosmos3/action.py | 34 +--- .../models/cosmos3/pipeline_cosmos3.py | 167 +++++++++++++++++- vllm_omni/diffusion/models/cosmos3/utils.py | 31 ++++ 4 files changed, 335 insertions(+), 38 deletions(-) diff --git a/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py b/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py index a0eab3b5e..f2527bd99 100644 --- a/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py +++ b/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py @@ -1095,6 +1095,7 @@ def test_transfer_config_media_helpers_and_preprocess_budget(monkeypatch: pytest ) additional = preprocess(request).prompt["additional_information"] assert (request.sampling_params.height, request.sampling_params.width) == (192, 320) + assert request.sampling_params.extra_args["_cosmos3_transfer_requested_size"] == {"width": 32, "height": 16} assert tuple(additional["preprocessed_transfer_video"].shape) == (1, 3, 4, 192, 320) assert additional["transfer_input_fps"] == 12.5 assert "preprocessed_video" not in additional @@ -1520,6 +1521,140 @@ def test_format_and_tokenize_prompts_rewrites_json_object_metadata(make_cosmos3_ ) +def test_format_and_tokenize_prompts_transfer_aspect_ratio_override(make_cosmos3_pipeline) -> None: + import json + + pipeline = make_cosmos3_pipeline() + calls = _capture_tokenize_calls(pipeline) + + pipeline._format_and_tokenize_prompts( + '{"caption": "A robot", "aspect_ratio": "4,3"}', + "", + num_frames=48, + frame_rate=24, + height=480, + width=832, + max_sequence_length=32, + sp=SimpleNamespace(extra_args={"aspect_ratio": "4:3"}), + is_t2i=False, + aspect_ratio_override="16,9", + ) + + assert json.loads(calls[0]["text"])["aspect_ratio"] == "16,9" + + +@pytest.mark.parametrize("rank_zero", [True, False]) +def test_format_and_tokenize_prompts_corrects_conflicting_aspect_ratio_metadata_on_every_rank( + make_cosmos3_pipeline, + monkeypatch: pytest.MonkeyPatch, + mocker, + rank_zero: bool, +) -> None: + import json + + from vllm_omni.diffusion.models.cosmos3 import pipeline_cosmos3 + + pipeline = make_cosmos3_pipeline() + calls = _capture_tokenize_calls(pipeline) + monkeypatch.setattr(pipeline_cosmos3, "_is_rank_zero", lambda: rank_zero) + warning = mocker.patch.object(pipeline_cosmos3.logger, "warning") + + pipeline._format_and_tokenize_prompts( + '{"caption": "A robot", "aspect_ratio": "4,3"}', + "", + num_frames=48, + frame_rate=24, + height=720, + width=1280, + max_sequence_length=32, + sp=SimpleNamespace(extra_args={}), + is_t2i=False, + ) + + assert json.loads(calls[0]["text"])["aspect_ratio"] == "16,9" + if rank_zero: + rendered_warning = warning.call_args.args[0] % warning.call_args.args[1:] + assert "JSON prompt aspect_ratio='4,3' conflicts with the generated 1280x720 canvas" in rendered_warning + else: + warning.assert_not_called() + + +@pytest.mark.parametrize( + ("width", "height", "aspect_ratio"), + [ + (1104, 816, "4,3"), + (832, 468, "16,9"), + ], +) +def test_format_and_tokenize_prompts_keeps_nearest_canonical_aspect_ratio( + make_cosmos3_pipeline, + mocker, + width: int, + height: int, + aspect_ratio: str, +) -> None: + import json + + from vllm_omni.diffusion.models.cosmos3 import pipeline_cosmos3 + + pipeline = make_cosmos3_pipeline() + calls = _capture_tokenize_calls(pipeline) + warning = mocker.patch.object(pipeline_cosmos3.logger, "warning") + + pipeline._format_and_tokenize_prompts( + json.dumps({"caption": "A robot", "aspect_ratio": aspect_ratio}), + "", + num_frames=48, + frame_rate=24, + height=height, + width=width, + max_sequence_length=32, + sp=SimpleNamespace(extra_args={}), + is_t2i=False, + ) + + assert json.loads(calls[0]["text"])["aspect_ratio"] == aspect_ratio + warning.assert_not_called() + + +def test_transfer_bucket_selection_warns_on_size_and_aspect_ratio_conflicts( + make_cosmos3_pipeline, + mocker, +) -> None: + from vllm_omni.diffusion.models.cosmos3 import pipeline_cosmos3 + + pipeline = make_cosmos3_pipeline() + warning = mocker.patch.object(pipeline_cosmos3.logger, "warning") + sp = make_sampling_params( + width=832, + height=480, + extra_args={ + "resolution": "480", + "aspect_ratio": "4:3", + "_cosmos3_transfer_requested_size": {"width": 1280, "height": 720}, + }, + ) + + height, width, aspect_ratio = pipeline._transfer_bucket_size(sp, (480, 832)) + assert (height, width, aspect_ratio) == (480, 832, "16,9") + + pipeline._warn_transfer_bucket_conflicts( + sp, + "A transfer prompt", + source_hw=(480, 832), + height=height, + width=width, + aspect_ratio=aspect_ratio, + ) + + rendered_warnings = [call.args[0] % call.args[1:] for call in warning.call_args_list] + assert any("ignores requested size=1280x720 (WxH)" in message for message in rendered_warnings) + assert any( + "requested aspect_ratio='4:3' conflicts with the control-selected 16,9 bucket" in message + for message in rendered_warnings + ) + + def test_format_and_tokenize_prompts_removes_video_metadata_from_t2i_json(make_cosmos3_pipeline) -> None: import json @@ -1989,6 +2124,7 @@ def test_forward_transfer_uses_transfer_prompt_contract_and_source_fps_except_ws captured["negative_prompt"] = negative_prompt captured["format_frame_rate"] = frame_rate captured["format_kwargs"] = kwargs + captured["format_aspect_ratio"] = kwargs["aspect_ratio_override"] return _ids(2), _mask(), _ids(1), _mask() def fake_encode(video: torch.Tensor) -> torch.Tensor: @@ -2011,7 +2147,7 @@ def test_forward_transfer_uses_transfer_prompt_contract_and_source_fps_except_ws captured["shared_kwargs"] = kwargs["shared_kwargs"] return kwargs["latents"] - pipeline._transfer_bucket_size = lambda sp, source_hw: (16, 16) + pipeline._transfer_bucket_size = lambda sp, source_hw: (16, 16, "1,1") pipeline._format_and_tokenize_prompts = fake_format pipeline._encode_video_tensor = fake_encode pipeline._prepare_transfer_latents = fake_prepare @@ -2056,6 +2192,7 @@ def test_forward_transfer_uses_transfer_prompt_contract_and_source_fps_except_ws output = pipeline.forward(request) assert captured["format_frame_rate"] == expected_fps + assert captured["format_aspect_ratio"] == "1,1" expected_negative_prompt = "" if negative_prompt is None else negative_prompt assert captured["negative_prompt"] == expected_negative_prompt assert captured["format_kwargs"]["use_system_prompt"] is True @@ -2082,7 +2219,7 @@ def test_forward_transfer_runs_multichunk_overlap_path( pipeline = make_cosmos3_pipeline() captured: dict[str, Any] = {"targets": [], "conditional_frames": []} - pipeline._transfer_bucket_size = lambda sp, source_hw: (16, 16) + pipeline._transfer_bucket_size = lambda sp, source_hw: (16, 16, "1,1") pipeline._format_and_tokenize_prompts = lambda *args, **kwargs: (_ids(2), _mask(), _ids(1), _mask()) pipeline._set_flow_shift = lambda target, **_kwargs: captured.setdefault("flow_shifts", []).append(target) diff --git a/vllm_omni/diffusion/models/cosmos3/action.py b/vllm_omni/diffusion/models/cosmos3/action.py index c5fe41a91..dc755b39c 100644 --- a/vllm_omni/diffusion/models/cosmos3/action.py +++ b/vllm_omni/diffusion/models/cosmos3/action.py @@ -15,6 +15,8 @@ from typing import Any import numpy as np import torch +from .utils import VIDEO_RES_SIZE_INFO + ACTION_MODE_POLICY = "policy" ACTION_MODE_FORWARD_DYNAMICS = "forward_dynamics" ACTION_MODE_INVERSE_DYNAMICS = "inverse_dynamics" @@ -46,38 +48,6 @@ EMBODIMENT_TO_DOMAIN_ID: dict[str, int] = { } -VIDEO_RES_SIZE_INFO: dict[str, dict[str, tuple[int, int]]] = { - "256": { - "1,1": (256, 256), - "4,3": (320, 256), - "3,4": (256, 320), - "16,9": (320, 192), - "9,16": (192, 320), - }, - "480": { - "1,1": (640, 640), - "4,3": (736, 544), - "3,4": (544, 736), - "16,9": (832, 480), - "9,16": (480, 832), - }, - "704": { - "1,1": (960, 960), - "4,3": (1088, 832), - "3,4": (832, 1088), - "16,9": (1280, 704), - "9,16": (704, 1280), - }, - "720": { - "1,1": (960, 960), - "4,3": (1104, 832), - "3,4": (832, 1104), - "16,9": (1280, 720), - "9,16": (720, 1280), - }, -} - - def normalize_action_mode(mode: Any) -> str | None: if mode is None: return None diff --git a/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py b/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py index 2304c0212..7a924dd52 100644 --- a/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py +++ b/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py @@ -126,6 +126,7 @@ from .utils import ( ROBOLAB_DEFAULT_RAW_ACTION_DIM, ROBOLAB_DEFAULT_RESOLUTION, ROBOLAB_MIDTRAIN_RAW_ACTION_DIM, + VIDEO_RES_SIZE_INFO, RoboLabActionPostprocessInputs, RoboLabPolicyInputs, build_abs_pose_from_components, @@ -199,6 +200,7 @@ COSMOS3_DISTILLED_CHECKPOINT_SCHEDULER_CLASS = "FlowMatchEulerDiscreteScheduler" # are tokenized to their natural length (no padding); this only bounds the # UND pathway / GEN cross-attention cost for pathologically long prompts. COSMOS3_DEFAULT_MAX_SEQUENCE_LENGTH = 4096 +COSMOS3_TRANSFER_REQUESTED_SIZE_KEY = "_cosmos3_transfer_requested_size" def _ceil_video_num_frames(num_frames: int, temporal_compression_factor: int) -> int: @@ -243,6 +245,50 @@ def _format_json_object_prompt( return json.dumps(prompt_obj) +def _json_object_aspect_ratio(prompt: str) -> Any | None: + """Return an aspect-ratio field from a JSON-object prompt, if present.""" + try: + prompt_obj = json.loads(prompt) + except (json.JSONDecodeError, TypeError, ValueError): + return None + if not isinstance(prompt_obj, dict): + return None + return prompt_obj.get("aspect_ratio") + + +def _normalize_aspect_ratio(value: Any) -> str | None: + """Normalize supported ratio spellings for comparison and prompt metadata.""" + if value is None: + return None + parts = str(value).strip().replace(":", ",").split(",") + if len(parts) != 2: + return None + try: + width, height = (int(part.strip()) for part in parts) + except ValueError: + return None + if width <= 0 or height <= 0: + return None + divisor = math.gcd(width, height) + return f"{width // divisor},{height // divisor}" + + +def _aspect_ratio_for_dimensions(width: int, height: int) -> str: + """Return the nearest supported Cosmos aspect-ratio label.""" + if width <= 0 or height <= 0: + raise ValueError(f"Cosmos3 canvas dimensions must be positive, got {width}x{height}.") + canvas_ratio = width / height + + def ratio_value(label: str) -> float: + ratio_width, ratio_height = (int(part) for part in label.split(",")) + return ratio_width / ratio_height + + return min( + (label for sizes in VIDEO_RES_SIZE_INFO.values() for label in sizes), + key=lambda label: abs(canvas_ratio - ratio_value(label)), + ) + + def resolve_cosmos3_transformer_cls(model_config: Any) -> type[Cosmos3VFMTransformer]: """Select the Cosmos3 transformer implementation from transformer/config.json.""" backbone_type = _tf_config_get(model_config, "backbone_type", None) @@ -500,6 +546,17 @@ def get_cosmos3_pre_process_func(od_config: OmniDiffusionConfig): # Resolve missing H/W. if transfer_requested: + requested_width = request.sampling_params.width + requested_height = request.sampling_params.height + if ( + requested_width is not None + and requested_height is not None + and COSMOS3_TRANSFER_REQUESTED_SIZE_KEY not in extra + ): + extra[COSMOS3_TRANSFER_REQUESTED_SIZE_KEY] = { + "width": int(requested_width), + "height": int(requested_height), + } _set_transfer_size_from_image(request, image) elif request.sampling_params.height is None or request.sampling_params.width is None: if action_mode is not None: @@ -2081,6 +2138,7 @@ class Cosmos3OmniDiffusersPipeline( use_duration_template: bool | None = None, use_resolution_template: bool | None = None, negative_metadata_mode: str | None = None, + aspect_ratio_override: str | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """Format prompts with metadata templates and tokenize. @@ -2115,13 +2173,40 @@ class Cosmos3OmniDiffusersPipeline( res_tmpl = COSMOS3_IMAGE_RESOLUTION_TEMPLATE if is_t2i else COSMOS3_RESOLUTION_TEMPLATE else: res_tmpl = None + requested_aspect_ratio = self._get_sp_param(sp, "aspect_ratio", None) + prompt_aspect_ratio = _json_object_aspect_ratio(prompt) + metadata_aspect_ratio = requested_aspect_ratio if requested_aspect_ratio is not None else prompt_aspect_ratio + canvas_aspect_ratio = aspect_ratio_override or _aspect_ratio_for_dimensions(width, height) + normalized_metadata_ratio = _normalize_aspect_ratio(metadata_aspect_ratio) + has_aspect_ratio_conflict = ( + aspect_ratio_override is None + and metadata_aspect_ratio is not None + and normalized_metadata_ratio != canvas_aspect_ratio + ) + if has_aspect_ratio_conflict: + aspect_ratio_source = ( + "requested aspect_ratio" if requested_aspect_ratio is not None else "JSON prompt aspect_ratio" + ) + if _is_rank_zero(): + logger.warning( + "Cosmos3 %s=%r conflicts with the generated %dx%d canvas (WxH); using %s for prompt metadata.", + aspect_ratio_source, + metadata_aspect_ratio, + width, + height, + canvas_aspect_ratio, + ) + metadata_aspect_ratio = canvas_aspect_ratio + elif aspect_ratio_override is not None: + metadata_aspect_ratio = aspect_ratio_override + json_prompt = _format_json_object_prompt( prompt, num_frames=num_frames, frame_rate=frame_rate, height=height, width=width, - aspect_ratio=self._get_sp_param(sp, "aspect_ratio", None), + aspect_ratio=metadata_aspect_ratio, ) if json_prompt is not None: prompt = json_prompt @@ -2893,11 +2978,76 @@ class Cosmos3OmniDiffusersPipeline( self, sp: OmniDiffusionSamplingParams, source_hw: tuple[int, int] | None, - ) -> tuple[int, int]: + ) -> tuple[int, int, str]: resolution = self._get_sp_param(sp, "resolution", self._get_sp_param(sp, "image_size", 720)) source_h, source_w = source_hw or (COSMOS3_T2V_DEFAULT_HEIGHT, COSMOS3_T2V_DEFAULT_WIDTH) target_w, target_h = find_closest_target_size(int(source_h), int(source_w), resolution) - return int(target_h), int(target_w) + aspect_ratio = next( + ratio for ratio, size in VIDEO_RES_SIZE_INFO[str(resolution)].items() if size == (target_w, target_h) + ) + return int(target_h), int(target_w), aspect_ratio + + def _warn_transfer_bucket_conflicts( + self, + sp: OmniDiffusionSamplingParams, + prompt: str, + *, + source_hw: tuple[int, int] | None, + height: int, + width: int, + aspect_ratio: str, + ) -> None: + """Explain transfer's bucket selection when it overrides request metadata.""" + if not _is_rank_zero(): + return + + requested_size_data = self._get_sp_param(sp, COSMOS3_TRANSFER_REQUESTED_SIZE_KEY, None) + if isinstance(requested_size_data, Mapping): + requested_width = requested_size_data.get("width") + requested_height = requested_size_data.get("height") + else: + requested_width = getattr(sp, "width", None) + requested_height = getattr(sp, "height", None) + if requested_width is not None and requested_height is not None: + try: + requested_size = (int(requested_width), int(requested_height)) + except (TypeError, ValueError): + requested_size = None + if requested_size is not None and requested_size != (width, height): + source_description = ( + f"control input {source_hw[1]}x{source_hw[0]} (WxH)" + if source_hw is not None + else "the default transfer geometry" + ) + resolution = self._get_sp_param(sp, "resolution", self._get_sp_param(sp, "image_size", 720)) + logger.warning( + "Cosmos3 transfer ignores requested size=%dx%d (WxH); %s selected the %s resolution=%s " + "bucket, %dx%d (WxH). Transfer supports canonical Cosmos buckets, not arbitrary output sizes.", + *requested_size, + source_description, + aspect_ratio, + resolution, + width, + height, + ) + + request_aspect_ratio = self._get_sp_param(sp, "aspect_ratio", None) + aspect_ratio_source = "requested aspect_ratio" + if request_aspect_ratio is None: + request_aspect_ratio = _json_object_aspect_ratio(prompt) + aspect_ratio_source = "JSON prompt aspect_ratio" + normalized_requested_ratio = _normalize_aspect_ratio(request_aspect_ratio) + if normalized_requested_ratio is not None and normalized_requested_ratio != aspect_ratio: + logger.warning( + "Cosmos3 transfer %s=%r conflicts with the control-selected %s bucket, %dx%d (WxH); " + "using %s for prompt metadata.", + aspect_ratio_source, + request_aspect_ratio, + aspect_ratio, + width, + height, + aspect_ratio, + ) @staticmethod def _first_transfer_control_hw(transfer_config: Cosmos3TransferConfig) -> tuple[int, int] | None: @@ -3083,7 +3233,15 @@ class Cosmos3OmniDiffusersPipeline( source_hw = (int(input_frames.shape[-2]), int(input_frames.shape[-1])) else: source_hw = self._first_transfer_control_hw(transfer_config) - height, width = self._transfer_bucket_size(sp, source_hw) + height, width, transfer_aspect_ratio = self._transfer_bucket_size(sp, source_hw) + self._warn_transfer_bucket_conflicts( + sp, + prompt, + source_hw=source_hw, + height=height, + width=width, + aspect_ratio=transfer_aspect_ratio, + ) if input_frames is not None: if tuple(input_frames.shape[-2:]) != (height, width): @@ -3190,6 +3348,7 @@ class Cosmos3OmniDiffusersPipeline( use_duration_template=bool(self._get_sp_param(sp, "use_duration_template", True)), use_resolution_template=bool(self._get_sp_param(sp, "use_resolution_template", True)), negative_metadata_mode=str(self._get_sp_param(sp, "negative_metadata_mode", "same")), + aspect_ratio_override=transfer_aspect_ratio, ) output_chunks: list[torch.Tensor] = [] diff --git a/vllm_omni/diffusion/models/cosmos3/utils.py b/vllm_omni/diffusion/models/cosmos3/utils.py index bfa3fd4db..76df64b3b 100644 --- a/vllm_omni/diffusion/models/cosmos3/utils.py +++ b/vllm_omni/diffusion/models/cosmos3/utils.py @@ -27,6 +27,37 @@ COSMOS3_DEFAULT_CONDITION_VIDEO_KEEP = "first" # pixel-frame budgets without instantiating the VAE. COSMOS3_VAE_TEMPORAL_COMPRESSION = 4 +VIDEO_RES_SIZE_INFO: dict[str, dict[str, tuple[int, int]]] = { + "256": { + "1,1": (256, 256), + "4,3": (320, 256), + "3,4": (256, 320), + "16,9": (320, 192), + "9,16": (192, 320), + }, + "480": { + "1,1": (640, 640), + "4,3": (736, 544), + "3,4": (544, 736), + "16,9": (832, 480), + "9,16": (480, 832), + }, + "704": { + "1,1": (960, 960), + "4,3": (1088, 832), + "3,4": (832, 1088), + "16,9": (1280, 704), + "9,16": (704, 1280), + }, + "720": { + "1,1": (960, 960), + "4,3": (1104, 832), + "3,4": (832, 1104), + "16,9": (1280, 720), + "9,16": (720, 1280), + }, +} + ROBOLAB_DEFAULT_CONDITIONING_FPS = 15.0 ROBOLAB_DEFAULT_ACTION_CHUNK_SIZE = 32 ROBOLAB_DEFAULT_IMAGE_HEIGHT = 540 From 37c11e6e4f60f9edf19e103d7278391def9caa28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E9=91=AB?= <77673148+l-wave@users.noreply.github.com> Date: Fri, 21 Aug 2026 03:22:40 +0800 Subject: [PATCH 097/267] [Perf][Qwen3-TTS] Fuse QKV and gate_up projections in code predictor (#5791) Signed-off-by: sheenlin Signed-off-by: Gao Han Co-authored-by: sheenlin Co-authored-by: yurain Co-authored-by: Gao Han --- .../models/moss_tts/test_moss_fused_load.py | 120 +++++ .../qwen3_tts/test_code_predictor_dtype.py | 478 ++++++++++++++++++ tests/platforms/npu/test_310p_patches.py | 102 ++++ .../models/common/qwen3_code_predictor.py | 223 +++++++- .../moss_tts/modeling_moss_tts_local.py | 18 + .../moss_tts/modeling_moss_tts_talker.py | 16 +- .../qwen3_omni/qwen3_omni_moe_talker.py | 6 +- .../qwen3_tts/configuration_qwen3_tts.py | 17 +- .../platforms/npu/_310p/patch/qwen3_tts.py | 33 +- 9 files changed, 972 insertions(+), 41 deletions(-) create mode 100644 tests/model_executor/models/moss_tts/test_moss_fused_load.py diff --git a/tests/model_executor/models/moss_tts/test_moss_fused_load.py b/tests/model_executor/models/moss_tts/test_moss_fused_load.py new file mode 100644 index 000000000..60039c693 --- /dev/null +++ b/tests/model_executor/models/moss_tts/test_moss_fused_load.py @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Regression tests for MOSS-TTS-Realtime fused code-predictor loading. + +The MOSS-TTS-Realtime depth transformer shares ``CodePredictorBaseModel``, whose +q/k/v and gate/up projections are fused. Before the fix, the talker loaded those +shards with a per-tensor ``default_weight_loader``, which silently dropped every +q/k/v/gate/up shard and left the fused params randomly initialized -- no +exception, just a corrupted model. + +These tests assert the property that failure mode violates: after +``load_weights``, every fused parameter is both reported loaded and numerically +equal to the concatenation of the shards that fed it. +""" + +import pytest +import torch + +from vllm_omni.model_executor.models.moss_tts.configuration_moss_tts import ( + MossTTSLocalTransformerConfig, +) +from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_local import ( + MossTTSRealtimeLocalTransformer, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _tiny_cfg(): + return MossTTSLocalTransformerConfig( + head_dim=8, + hidden_size=32, + num_hidden_layers=2, + num_attention_heads=4, + num_key_value_heads=2, + intermediate_size=64, + max_position_embeddings=8, + ) + + +def _hf_shards(cfg, *, drop: set[str] | None = None): + """Synthetic HF-style checkpoint tensors for the depth transformer.""" + drop = drop or set() + h, inter = cfg.hidden_size, cfg.intermediate_size + q_out = cfg.num_attention_heads * cfg.head_dim + kv_out = cfg.num_key_value_heads * cfg.head_dim + out = {} + for layer in range(cfg.num_hidden_layers): + p = f"model.layers.{layer}" + for nm, rows in (("q_proj", q_out), ("k_proj", kv_out), ("v_proj", kv_out)): + key = f"{p}.self_attn.{nm}.weight" + if nm not in drop: + out[key] = torch.randn(rows, h) + out[f"{p}.self_attn.o_proj.weight"] = torch.randn(h, q_out) + for nm in ("gate_proj", "up_proj"): + key = f"{p}.mlp.{nm}.weight" + if nm not in drop: + out[key] = torch.randn(inter, h) + out[f"{p}.mlp.down_proj.weight"] = torch.randn(h, inter) + out[f"{p}.input_layernorm.weight"] = torch.randn(h) + out[f"{p}.post_attention_layernorm.weight"] = torch.randn(h) + for nm in ("q_norm", "k_norm"): + out[f"{p}.self_attn.{nm}.weight"] = torch.randn(cfg.head_dim) + out["model.norm.weight"] = torch.randn(h) + return out + + +def test_every_fused_param_is_loaded_and_correctly_packed(): + cfg = _tiny_cfg() + lt = MossTTSRealtimeLocalTransformer(cfg) + shards = _hf_shards(cfg) + + loaded = lt.load_weights(iter([(k, v) for k, v in shards.items()])) + + params = dict(lt.named_parameters()) + fused = [n for n in params if ".self_attn.qkv_proj." in n or ".mlp.gate_up_proj." in n] + assert fused, "model declares no fused params -- test would be vacuous" + + # 1. Reported as loaded (this is what the old path silently failed). + unreported = [n for n in fused if n not in loaded] + assert not unreported, f"fused params not reported loaded: {unreported}" + + # 2. Numerically equal to the concatenation of their shards, in order. + for layer in range(cfg.num_hidden_layers): + p = f"model.layers.{layer}" + want_qkv = torch.cat([shards[f"{p}.self_attn.{n}.weight"] for n in ("q_proj", "k_proj", "v_proj")], dim=0) + got_qkv = params[f"{p}.self_attn.qkv_proj.weight"].detach() + assert torch.equal(got_qkv, want_qkv), f"layer {layer}: qkv packing mismatch" + + want_gu = torch.cat([shards[f"{p}.mlp.{n}.weight"] for n in ("gate_proj", "up_proj")], dim=0) + got_gu = params[f"{p}.mlp.gate_up_proj.weight"].detach() + assert torch.equal(got_gu, want_gu), f"layer {layer}: gate_up packing mismatch" + + +def test_incomplete_shard_group_raises_instead_of_corrupting(): + """A checkpoint missing v_proj must fail loudly, not load a random qkv.""" + cfg = _tiny_cfg() + lt = MossTTSRealtimeLocalTransformer(cfg) + shards = _hf_shards(cfg, drop={"v_proj"}) + + with pytest.raises(RuntimeError, match="incomplete fused shards"): + lt.load_weights(iter([(k, v) for k, v in shards.items()])) + + +def test_absent_fused_group_raises(): + """No q/k/v at all must also raise, via the missing-fused-param guard.""" + cfg = _tiny_cfg() + lt = MossTTSRealtimeLocalTransformer(cfg) + shards = _hf_shards(cfg, drop={"q_proj", "k_proj", "v_proj"}) + + with pytest.raises(RuntimeError, match="missing fused parameters"): + lt.load_weights(iter([(k, v) for k, v in shards.items()])) + + +def test_returned_names_keep_the_model_prefix(): + """Returned names must be ``model.*`` for talker re-prefixing.""" + cfg = _tiny_cfg() + lt = MossTTSRealtimeLocalTransformer(cfg) + loaded = lt.load_weights(iter([(k, v) for k, v in _hf_shards(cfg).items()])) + assert loaded, "nothing reported loaded" + assert all(n.startswith("model.") for n in loaded), sorted(n for n in loaded if not n.startswith("model.")) diff --git a/tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py b/tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py index 20f8eb2f0..619f2169a 100644 --- a/tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py +++ b/tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py @@ -697,3 +697,481 @@ class TestGumbelMaxSampling: empirical = torch.bincount(samples, minlength=vocab).float() / n_draws # L1 distance under 0.03 with 20k draws is comfortable for vocab=16. assert (empirical - target).abs().sum().item() < 0.05 + + +class TestCodePredictorFusedProjections: + """Reference tests for the fused qkv_proj / gate_up_proj paths. + + These CPU tests confirm that the packed projections implement the same + linear algebra as the unfused reference under this setup. GPU executions + may still diverge at the bit level when kernel choices change. + """ + + def test_fused_qkv_matches_separate(self, loaded_target_classes) -> None: + torch.manual_seed(0) + hidden = 32 + num_heads, num_kv_heads, head_dim = 4, 2, 8 + q_size = num_heads * head_dim + kv_size = num_kv_heads * head_dim + x = torch.randn(2, 5, hidden) + + q = torch.nn.Linear(hidden, q_size, bias=False) + k = torch.nn.Linear(hidden, kv_size, bias=False) + v = torch.nn.Linear(hidden, kv_size, bias=False) + fused = torch.nn.Linear(hidden, q_size + 2 * kv_size, bias=False) + with torch.no_grad(): + fused.weight.copy_(torch.cat([q.weight, k.weight, v.weight], dim=0)) + + ref_q, ref_k, ref_v = q(x), k(x), v(x) + out_q, out_k, out_v = fused(x).split([q_size, kv_size, kv_size], dim=-1) + assert torch.equal(ref_q, out_q) + assert torch.equal(ref_k, out_k) + assert torch.equal(ref_v, out_v) + + def test_attention_split_redensifies_kv_slices(self, loaded_target_classes) -> None: + cp_config, _ = _make_tiny_config(loaded_target_classes) + common_mod = sys.modules["vllm_omni.model_executor.models.common.qwen3_code_predictor"] + attn = common_mod.CodePredictorAttention(cp_config) + + qkv = torch.randn(2, 5, attn._q_size + 2 * attn._kv_size) + q_ref, k_ref, v_ref = qkv.split([attn._q_size, attn._kv_size, attn._kv_size], dim=-1) + q_raw, k_raw, v_raw = attn._split_qkv(qkv) + + assert torch.equal(q_raw, q_ref) + assert torch.equal(k_raw, k_ref) + assert torch.equal(v_raw, v_ref) + assert k_raw.is_contiguous() + assert v_raw.is_contiguous() + + def test_fused_gate_up_matches_separate(self, loaded_target_classes) -> None: + torch.manual_seed(1) + hidden, intermediate = 32, 64 + x = torch.randn(2, 5, hidden) + + g = torch.nn.Linear(hidden, intermediate, bias=False) + u = torch.nn.Linear(hidden, intermediate, bias=False) + fused = torch.nn.Linear(hidden, 2 * intermediate, bias=False) + with torch.no_grad(): + fused.weight.copy_(torch.cat([g.weight, u.weight], dim=0)) + + ref_g, ref_u = g(x), u(x) + out_g, out_u = fused(x).split([intermediate, intermediate], dim=-1) + assert torch.equal(ref_g, out_g) + assert torch.equal(ref_u, out_u) + + def test_load_weights_repacks_hf_shards(self, mocker: MockerFixture, loaded_target_classes) -> None: + """``load_weights`` must transparently re-pack HF q/k/v and gate/up + shards into the fused ``qkv_proj`` and ``gate_up_proj`` weights.""" + _, _, _, code_predictor_model, _ = loaded_target_classes + cp_config, _ = _make_tiny_config(loaded_target_classes) + model = code_predictor_model(cp_config, embedding_dim=cp_config.hidden_size) + + # The test harness mocks ``default_weight_loader`` as a no-op; install a + # real copier so this test verifies both the returned names and the + # packed parameter values/order. + common_mod = sys.modules["vllm_omni.model_executor.models.common.qwen3_code_predictor"] + mocker.patch.object( + common_mod, + "default_weight_loader", + lambda param, weight: param.data.copy_(weight), + ) + + # Build a synthetic "HF checkpoint" that exposes separate q/k/v + # and gate/up shards, mirroring what HuggingFace ships. + param_dict = dict(model.named_parameters(remove_duplicate=False)) + weights: list[tuple[str, torch.Tensor]] = [] + expected_fused: dict[str, torch.Tensor] = {} + torch.manual_seed(42) + for name, param in param_dict.items(): + if name.endswith(".self_attn.qkv_proj.weight"): + prefix = name[: -len(".qkv_proj.weight")] + num_heads = cp_config.num_attention_heads + num_kv_heads = cp_config.num_key_value_heads + head_dim = cp_config.head_dim + q_size = num_heads * head_dim + kv_size = num_kv_heads * head_dim + hidden = cp_config.hidden_size + q_w = torch.randn(q_size, hidden) + k_w = torch.randn(kv_size, hidden) + v_w = torch.randn(kv_size, hidden) + weights.append((f"{prefix}.q_proj.weight", q_w)) + weights.append((f"{prefix}.k_proj.weight", k_w)) + weights.append((f"{prefix}.v_proj.weight", v_w)) + expected_fused[name] = torch.cat([q_w, k_w, v_w], dim=0) + elif name.endswith(".mlp.gate_up_proj.weight"): + prefix = name[: -len(".gate_up_proj.weight")] + inter = cp_config.intermediate_size + hidden = cp_config.hidden_size + gate_w = torch.randn(inter, hidden) + up_w = torch.randn(inter, hidden) + weights.append((f"{prefix}.gate_proj.weight", gate_w)) + weights.append((f"{prefix}.up_proj.weight", up_w)) + expected_fused[name] = torch.cat([gate_w, up_w], dim=0) + else: + # Pass-through parameters (norms, embeddings, o_proj, down_proj). + weights.append((name, torch.randn_like(param))) + + loaded = model.load_weights(weights) + after = dict(model.named_parameters(remove_duplicate=False)) + assert expected_fused, "test config must contain at least one fused projection layer" + for fname, expected in expected_fused.items(): + assert fname in loaded, f"{fname} was not assembled by load_weights" + assert torch.equal(after[fname], expected), f"{fname} does not match the packed shard order" + + def test_load_weights_repacks_bias_shards(self, mocker: MockerFixture, loaded_target_classes) -> None: + """When ``attention_bias=True``, the q/k/v ``.bias`` shards must be + concatenated into ``qkv_proj.bias`` instead of being dropped (which + would leave the fused bias randomly initialized).""" + _, _, _, code_predictor_model, _ = loaded_target_classes + cp_config, _ = _make_tiny_config(loaded_target_classes) + cp_config.attention_bias = True + model = code_predictor_model(cp_config, embedding_dim=cp_config.hidden_size) + + # The test harness mocks ``default_weight_loader`` as a no-op; install a + # real copier so we can assert the packed values land on the parameter. + common_mod = sys.modules["vllm_omni.model_executor.models.common.qwen3_code_predictor"] + mocker.patch.object( + common_mod, + "default_weight_loader", + lambda param, weight: param.data.copy_(weight), + ) + + param_dict = dict(model.named_parameters(remove_duplicate=False)) + weights: list[tuple[str, torch.Tensor]] = [] + expected_bias: dict[str, torch.Tensor] = {} + torch.manual_seed(11) + for name, param in param_dict.items(): + if name.endswith(".self_attn.qkv_proj.weight"): + prefix = name[: -len(".qkv_proj.weight")] + q_size = cp_config.num_attention_heads * cp_config.head_dim + kv_size = cp_config.num_key_value_heads * cp_config.head_dim + hidden = cp_config.hidden_size + q_w, k_w, v_w = (torch.randn(s, hidden) for s in (q_size, kv_size, kv_size)) + q_b, k_b, v_b = (torch.randn(s) for s in (q_size, kv_size, kv_size)) + weights.append((f"{prefix}.q_proj.weight", q_w)) + weights.append((f"{prefix}.k_proj.weight", k_w)) + weights.append((f"{prefix}.v_proj.weight", v_w)) + weights.append((f"{prefix}.q_proj.bias", q_b)) + weights.append((f"{prefix}.k_proj.bias", k_b)) + weights.append((f"{prefix}.v_proj.bias", v_b)) + expected_bias[f"{prefix}.qkv_proj.bias"] = torch.cat([q_b, k_b, v_b], dim=0) + elif name.endswith(".mlp.gate_up_proj.weight"): + prefix = name[: -len(".gate_up_proj.weight")] + inter, hidden = cp_config.intermediate_size, cp_config.hidden_size + weights.append((f"{prefix}.gate_proj.weight", torch.randn(inter, hidden))) + weights.append((f"{prefix}.up_proj.weight", torch.randn(inter, hidden))) + elif name.endswith(".qkv_proj.bias"): + # Fused target is assembled from the q/k/v bias shards above; a + # real HF checkpoint never ships a qkv_proj.bias tensor directly. + continue + else: + weights.append((name, torch.randn_like(param))) + + loaded = model.load_weights(weights) + after = dict(model.named_parameters()) + assert expected_bias, "test config must contain at least one qkv layer" + for bname, exp in expected_bias.items(): + assert bname in loaded, f"{bname} bias was not assembled by load_weights" + assert torch.equal(after[bname], exp), f"{bname} does not match the packed q/k/v bias" + + def test_wrapper_load_weights_handles_model_prefixed_shards( + self, + mocker: MockerFixture, + loaded_target_classes, + ) -> None: + _, _, code_predictor_wrapper, _, _ = loaded_target_classes + cp_config, talker_config = _make_tiny_config(loaded_target_classes) + vllm_config = _make_vllm_config(mocker) + predictor = code_predictor_wrapper( + vllm_config=vllm_config, + config=cp_config, + talker_config=talker_config, + ) + + common_mod = sys.modules["vllm_omni.model_executor.models.common.qwen3_code_predictor"] + mocker.patch.object( + common_mod, + "default_weight_loader", + lambda param, weight: param.data.copy_(weight), + ) + + param_dict = dict(predictor.named_parameters(remove_duplicate=False)) + weights: list[tuple[str, torch.Tensor]] = [] + expected_fused: dict[str, torch.Tensor] = {} + torch.manual_seed(7) + for name, param in param_dict.items(): + if name.endswith(".self_attn.qkv_proj.weight"): + prefix = name[: -len(".qkv_proj.weight")] + q_size = cp_config.num_attention_heads * cp_config.head_dim + kv_size = cp_config.num_key_value_heads * cp_config.head_dim + hidden = cp_config.hidden_size + q_w = torch.randn(q_size, hidden) + k_w = torch.randn(kv_size, hidden) + v_w = torch.randn(kv_size, hidden) + weights.append((f"{prefix}.q_proj.weight", q_w)) + weights.append((f"{prefix}.k_proj.weight", k_w)) + weights.append((f"{prefix}.v_proj.weight", v_w)) + expected_fused[name] = torch.cat([q_w, k_w, v_w], dim=0) + elif name.endswith(".mlp.gate_up_proj.weight"): + prefix = name[: -len(".gate_up_proj.weight")] + inter = cp_config.intermediate_size + hidden = cp_config.hidden_size + gate_w = torch.randn(inter, hidden) + up_w = torch.randn(inter, hidden) + weights.append((f"{prefix}.gate_proj.weight", gate_w)) + weights.append((f"{prefix}.up_proj.weight", up_w)) + expected_fused[name] = torch.cat([gate_w, up_w], dim=0) + else: + weights.append((name, torch.randn_like(param))) + + loaded = predictor.load_weights(weights) + after = dict(predictor.named_parameters(remove_duplicate=False)) + assert expected_fused, "test config must contain at least one fused projection layer" + for fname, expected in expected_fused.items(): + assert fname in loaded, f"{fname} was not loaded through the outer wrapper" + assert torch.equal(after[fname], expected), f"{fname} does not match the packed shard order" + + def test_wrapper_load_weights_handles_legacy_unprefixed_model_shards( + self, + mocker: MockerFixture, + loaded_target_classes, + ) -> None: + """Qwen3-Omni checkpoints may expose code_predictor body tensors as + ``layers.*`` / ``codec_embedding.*`` instead of ``model.*``. The + wrapper must still route those shards through the fused inner loader.""" + _, _, code_predictor_wrapper, _, _ = loaded_target_classes + cp_config, talker_config = _make_tiny_config(loaded_target_classes) + vllm_config = _make_vllm_config(mocker) + predictor = code_predictor_wrapper( + vllm_config=vllm_config, + config=cp_config, + talker_config=talker_config, + ) + + common_mod = sys.modules["vllm_omni.model_executor.models.common.qwen3_code_predictor"] + mocker.patch.object( + common_mod, + "default_weight_loader", + lambda param, weight: param.data.copy_(weight), + ) + + param_dict = dict(predictor.named_parameters(remove_duplicate=False)) + weights: list[tuple[str, torch.Tensor]] = [] + expected_fused: dict[str, torch.Tensor] = {} + torch.manual_seed(17) + for name, param in param_dict.items(): + if name.endswith(".self_attn.qkv_proj.weight"): + prefix = name[: -len(".qkv_proj.weight")] + legacy_prefix = prefix.removeprefix("model.") + q_size = cp_config.num_attention_heads * cp_config.head_dim + kv_size = cp_config.num_key_value_heads * cp_config.head_dim + hidden = cp_config.hidden_size + q_w = torch.randn(q_size, hidden) + k_w = torch.randn(kv_size, hidden) + v_w = torch.randn(kv_size, hidden) + weights.append((f"{legacy_prefix}.q_proj.weight", q_w)) + weights.append((f"{legacy_prefix}.k_proj.weight", k_w)) + weights.append((f"{legacy_prefix}.v_proj.weight", v_w)) + expected_fused[name] = torch.cat([q_w, k_w, v_w], dim=0) + elif name.endswith(".mlp.gate_up_proj.weight"): + prefix = name[: -len(".gate_up_proj.weight")] + legacy_prefix = prefix.removeprefix("model.") + inter = cp_config.intermediate_size + hidden = cp_config.hidden_size + gate_w = torch.randn(inter, hidden) + up_w = torch.randn(inter, hidden) + weights.append((f"{legacy_prefix}.gate_proj.weight", gate_w)) + weights.append((f"{legacy_prefix}.up_proj.weight", up_w)) + expected_fused[name] = torch.cat([gate_w, up_w], dim=0) + elif name.startswith("model."): + weights.append((name.removeprefix("model."), torch.randn_like(param))) + else: + weights.append((name, torch.randn_like(param))) + + loaded = predictor.load_weights(weights) + after = dict(predictor.named_parameters(remove_duplicate=False)) + assert expected_fused, "test config must contain at least one fused projection layer" + for fname, expected in expected_fused.items(): + assert fname in loaded, f"{fname} was not loaded from legacy unprefixed shards" + assert torch.equal(after[fname], expected), f"{fname} does not match the packed shard order" + + def test_wrapper_load_weights_handles_nested_code_predictor_prefixed_shards( + self, + mocker: MockerFixture, + loaded_target_classes, + ) -> None: + """Realtime Qwen3-Omni loaders may still prepend ``talker.code_predictor.`` + before the inner ``model.*`` body; the wrapper must normalize that layer + too before delegating to the fused inner loader.""" + _, _, code_predictor_wrapper, _, _ = loaded_target_classes + cp_config, talker_config = _make_tiny_config(loaded_target_classes) + vllm_config = _make_vllm_config(mocker) + predictor = code_predictor_wrapper( + vllm_config=vllm_config, + config=cp_config, + talker_config=talker_config, + prefix="talker.code_predictor", + ) + + common_mod = sys.modules["vllm_omni.model_executor.models.common.qwen3_code_predictor"] + mocker.patch.object( + common_mod, + "default_weight_loader", + lambda param, weight: param.data.copy_(weight), + ) + + param_dict = dict(predictor.named_parameters(remove_duplicate=False)) + weights: list[tuple[str, torch.Tensor]] = [] + expected_fused: dict[str, torch.Tensor] = {} + torch.manual_seed(23) + wrapper_prefix = "talker.code_predictor." + for name, param in param_dict.items(): + wrapped_name = f"{wrapper_prefix}{name}" + if name.endswith(".self_attn.qkv_proj.weight"): + prefix = wrapped_name[: -len(".qkv_proj.weight")] + q_size = cp_config.num_attention_heads * cp_config.head_dim + kv_size = cp_config.num_key_value_heads * cp_config.head_dim + hidden = cp_config.hidden_size + q_w = torch.randn(q_size, hidden) + k_w = torch.randn(kv_size, hidden) + v_w = torch.randn(kv_size, hidden) + weights.append((f"{prefix}.q_proj.weight", q_w)) + weights.append((f"{prefix}.k_proj.weight", k_w)) + weights.append((f"{prefix}.v_proj.weight", v_w)) + expected_fused[name] = torch.cat([q_w, k_w, v_w], dim=0) + elif name.endswith(".mlp.gate_up_proj.weight"): + prefix = wrapped_name[: -len(".gate_up_proj.weight")] + inter = cp_config.intermediate_size + hidden = cp_config.hidden_size + gate_w = torch.randn(inter, hidden) + up_w = torch.randn(inter, hidden) + weights.append((f"{prefix}.gate_proj.weight", gate_w)) + weights.append((f"{prefix}.up_proj.weight", up_w)) + expected_fused[name] = torch.cat([gate_w, up_w], dim=0) + else: + weights.append((wrapped_name, torch.randn_like(param))) + + loaded = predictor.load_weights(weights) + after = dict(predictor.named_parameters(remove_duplicate=False)) + assert expected_fused, "test config must contain at least one fused projection layer" + for fname, expected in expected_fused.items(): + assert fname in loaded, f"{fname} was not loaded from nested code_predictor shards" + assert torch.equal(after[fname], expected), f"{fname} does not match the packed shard order" + + def test_wrapper_load_weights_accepts_incremental_non_layer_shard( + self, + mocker: MockerFixture, + loaded_target_classes, + ) -> None: + """Qwen3-Omni splits code-predictor weights across checkpoint shards. + + Its first shard contains the transformer layers, while the next shard + re-enters the wrapper with only codec embeddings and LM heads. Loading + that second shard must not require every fused layer to appear again. + """ + _, _, code_predictor_wrapper, _, _ = loaded_target_classes + cp_config, talker_config = _make_tiny_config(loaded_target_classes) + vllm_config = _make_vllm_config(mocker) + predictor = code_predictor_wrapper( + vllm_config=vllm_config, + config=cp_config, + talker_config=talker_config, + ) + + common_mod = sys.modules["vllm_omni.model_executor.models.common.qwen3_code_predictor"] + mocker.patch.object( + common_mod, + "default_weight_loader", + lambda param, weight: param.data.copy_(weight), + ) + + first_shard: list[tuple[str, torch.Tensor]] = [] + second_shard: list[tuple[str, torch.Tensor]] = [] + for name, param in predictor.named_parameters(remove_duplicate=False): + if name.startswith("model.codec_embedding.") or name.startswith("lm_head."): + second_shard.append((name, torch.randn_like(param))) + elif name.endswith(".self_attn.qkv_proj.weight"): + prefix = name[: -len(".qkv_proj.weight")] + q_size = cp_config.num_attention_heads * cp_config.head_dim + kv_size = cp_config.num_key_value_heads * cp_config.head_dim + hidden = cp_config.hidden_size + first_shard.extend( + [ + (f"{prefix}.q_proj.weight", torch.randn(q_size, hidden)), + (f"{prefix}.k_proj.weight", torch.randn(kv_size, hidden)), + (f"{prefix}.v_proj.weight", torch.randn(kv_size, hidden)), + ] + ) + elif name.endswith(".mlp.gate_up_proj.weight"): + prefix = name[: -len(".gate_up_proj.weight")] + inter = cp_config.intermediate_size + hidden = cp_config.hidden_size + first_shard.extend( + [ + (f"{prefix}.gate_proj.weight", torch.randn(inter, hidden)), + (f"{prefix}.up_proj.weight", torch.randn(inter, hidden)), + ] + ) + else: + first_shard.append((name, torch.randn_like(param))) + + assert first_shard and second_shard + common_mod.CodePredictorWrapper.load_weights(predictor, first_shard) + loaded_second = common_mod.CodePredictorWrapper.load_weights(predictor, second_shard) + + assert {name for name, _ in second_shard} <= loaded_second + + def test_load_weights_raises_on_incomplete_shards(self, loaded_target_classes) -> None: + """Missing one of q/k/v (or gate/up) must surface a clear error. + + ``CodePredictorBaseModel`` is the inner transformer (its parameter + paths look like ``layers..self_attn.qkv_proj.weight`` -- without + the ``model.`` prefix that the outer ``CodePredictorWrapper`` strips + before forwarding). The test mimics that contract. + """ + _, _, _, code_predictor_model, _ = loaded_target_classes + cp_config, _ = _make_tiny_config(loaded_target_classes) + model = code_predictor_model(cp_config, embedding_dim=cp_config.hidden_size) + + # Find any actual self_attn layer prefix, to make the test robust + # to renaming of the inner model's parameter scheme. + attn_prefix = next( + ( + name[: -len(".qkv_proj.weight")] + for name in dict(model.named_parameters()).keys() + if name.endswith(".self_attn.qkv_proj.weight") + ), + None, + ) + assert attn_prefix is not None, "test config must contain at least one self_attn layer" + + num_heads = cp_config.num_attention_heads + num_kv_heads = cp_config.num_key_value_heads + head_dim = cp_config.head_dim + hidden = cp_config.hidden_size + + # Provide only q + k (no v). The fused param exists in the model, + # so load_weights must refuse to silently leave it uninitialized. + bad_weights: list[tuple[str, torch.Tensor]] = [ + (f"{attn_prefix}.q_proj.weight", torch.randn(num_heads * head_dim, hidden)), + (f"{attn_prefix}.k_proj.weight", torch.randn(num_kv_heads * head_dim, hidden)), + # v_proj missing on purpose. + ] + with pytest.raises(RuntimeError, match="incomplete fused shards"): + model.load_weights(bad_weights) + + def test_load_weights_raises_when_fused_shards_absent(self, loaded_target_classes) -> None: + """Entirely absent q/k/v or gate/up groups must not leave fused params + randomly initialized.""" + _, _, _, code_predictor_model, _ = loaded_target_classes + cp_config, _ = _make_tiny_config(loaded_target_classes) + model = code_predictor_model(cp_config, embedding_dim=cp_config.hidden_size) + + weights = [ + (name, torch.randn_like(param)) + for name, param in model.named_parameters(remove_duplicate=False) + if not (name.endswith(".qkv_proj.weight") or name.endswith(".gate_up_proj.weight")) + ] + + with pytest.raises(RuntimeError, match="missing fused parameters"): + model.load_weights(weights) diff --git a/tests/platforms/npu/test_310p_patches.py b/tests/platforms/npu/test_310p_patches.py index 99dbb4650..08cbb2adc 100644 --- a/tests/platforms/npu/test_310p_patches.py +++ b/tests/platforms/npu/test_310p_patches.py @@ -414,6 +414,108 @@ def test_qwen3_tts_code_predictor_forward_uses_projected_embedding_and_sampling( assert sample_calls == [{0: generator}] +def test_310p_attention_forward_runs_fused_qkv_runtime(monkeypatch: pytest.MonkeyPatch) -> None: + """Exercise the mocked 310P fused-QKV path at runtime on CPU. + + The reviewer explicitly asked for more than source inspection: this test now + instantiates the patched attention, runs ``prepare_qkv_weights()``, verifies + the packed weight/bias cache, and drives the mocked 310P flash-attention op. + """ + module, _ = _load_qwen3_tts_patch(monkeypatch) + + trans_calls = [] + rotary_calls = [] + flash_calls = [] + + def maybe_trans_nz(weight: torch.Tensor) -> torch.Tensor: + trans_calls.append(weight.detach().clone()) + return weight + 1.0 + + def npu_rotary_mul(hidden_states: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + rotary_calls.append((hidden_states.detach().clone(), cos.detach().clone(), sin.detach().clone())) + return hidden_states + cos - sin + + def npu_flash_attention(**kwargs) -> None: + flash_calls.append( + {key: value.detach().clone() if torch.is_tensor(value) else value for key, value in kwargs.items()} + ) + kwargs["out"].copy_((kwargs["query"] + kwargs["key"] + kwargs["value"]).to(kwargs["out"].dtype)) + + monkeypatch.setattr(module, "maybe_trans_nz", maybe_trans_nz) + monkeypatch.setattr(module, "aligned_16", lambda tensor: tensor) + monkeypatch.setattr(module.torch_npu, "npu_rotary_mul", npu_rotary_mul, raising=False) + monkeypatch.setattr(module.torch_npu, "_npu_flash_attention", npu_flash_attention, raising=False) + + config = SimpleNamespace( + num_attention_heads=2, + num_key_value_heads=2, + head_dim=2, + hidden_size=4, + num_code_groups=2, + attention_bias=True, + rms_norm_eps=1e-6, + ) + attention = module._Qwen3CodePredictorAttention310P(config) + attention.num_heads = 2 + attention.num_kv_heads = 2 + attention.head_dim = 2 + attention.scaling = 0.5 + attention._q_size = attention.num_heads * attention.head_dim + attention._kv_size = attention.num_kv_heads * attention.head_dim + attention.q_norm = torch.nn.Identity() + attention.k_norm = torch.nn.Identity() + attention.o_proj = torch.nn.Identity() + attention.qkv_proj = torch.nn.Linear(4, attention._q_size + 2 * attention._kv_size, bias=True) + with torch.no_grad(): + attention.qkv_proj.weight.copy_(torch.arange(48, dtype=torch.float32).reshape(12, 4) / 10) + attention.qkv_proj.bias.copy_(torch.arange(12, dtype=torch.float32) / 10) + attention._split_qkv = lambda qkv: qkv.split([attention._q_size, attention._kv_size, attention._kv_size], dim=-1) + + attention.prepare_qkv_weights() + + assert len(trans_calls) == 1 + assert trans_calls[0].is_contiguous() + expected_fused_weight = attention.qkv_proj.weight.detach().clone().contiguous() + 1.0 + torch.testing.assert_close(attention._fused_qkv_weight, expected_fused_weight) + assert attention._fused_qkv_bias is attention.qkv_proj.bias + + hidden_states = (torch.arange(12, dtype=torch.float32).reshape(1, 3, 4) - 3) / 10 + cos = torch.full((1, 3, 2), 0.25, dtype=torch.float32) + sin = torch.full((1, 3, 2), -0.5, dtype=torch.float32) + attention_mask = torch.arange(9, dtype=torch.float32).reshape(3, 3) + + output = attention(hidden_states, (cos, sin), attention_mask=attention_mask) + + expected_qkv = torch.nn.functional.linear(hidden_states, expected_fused_weight, attention.qkv_proj.bias) + expected_q_raw, expected_k_raw, expected_v_raw = expected_qkv.split( + [attention._q_size, attention._kv_size, attention._kv_size], + dim=-1, + ) + expected_q = expected_q_raw.view(1, 3, attention.num_heads, attention.head_dim).transpose(1, 2) + expected_k = expected_k_raw.view(1, 3, attention.num_kv_heads, attention.head_dim).transpose(1, 2) + expected_v = expected_v_raw.view(1, 3, attention.num_kv_heads, attention.head_dim).transpose(1, 2) + expected_cos = cos.unsqueeze(1) + expected_sin = sin.unsqueeze(1) + expected_q = expected_q + expected_cos - expected_sin + expected_k = expected_k + expected_cos - expected_sin + expected_q_f = expected_q.transpose(1, 2).reshape(3, attention.num_heads, attention.head_dim) + expected_k_f = expected_k.transpose(1, 2).reshape(3, attention.num_kv_heads, attention.head_dim) + expected_v_f = expected_v.transpose(1, 2).reshape(3, attention.num_kv_heads, attention.head_dim) + expected_flash = (expected_q_f + expected_k_f + expected_v_f).to(torch.float16).to(torch.float32) + + assert len(rotary_calls) == 2 + assert len(flash_calls) == 1 + torch.testing.assert_close(flash_calls[0]["query"], expected_q_f) + torch.testing.assert_close(flash_calls[0]["key"], expected_k_f) + torch.testing.assert_close(flash_calls[0]["value"], expected_v_f) + torch.testing.assert_close(flash_calls[0]["mask"], attention_mask) + torch.testing.assert_close(flash_calls[0]["seq_len"], torch.tensor([3], dtype=torch.int32)) + assert flash_calls[0]["scale_value"] == pytest.approx(0.5) + assert flash_calls[0]["num_heads"] == 2 + assert flash_calls[0]["num_kv_heads"] == 2 + torch.testing.assert_close(output, expected_flash.reshape(1, 3, 4)) + + def test_qwen3_tts_talker_patch_uses_fp16_runtime_dtype(monkeypatch: pytest.MonkeyPatch) -> None: module, _ = _load_qwen3_tts_patch(monkeypatch) talker = module._Qwen3TTSTalker310P(vllm_config=object()) diff --git a/vllm_omni/model_executor/models/common/qwen3_code_predictor.py b/vllm_omni/model_executor/models/common/qwen3_code_predictor.py index 8c9c134cf..3a47e2912 100644 --- a/vllm_omni/model_executor/models/common/qwen3_code_predictor.py +++ b/vllm_omni/model_executor/models/common/qwen3_code_predictor.py @@ -3,7 +3,8 @@ Shared by Qwen3-Omni and Qwen3-TTS talker models. * SDPA attention (F.scaled_dot_product_attention) with native GQA support -* HF-compatible CPU/CUDA numerics with NPU-only fused norm/RoPE fast paths +* HF-compatible CPU/CUDA numerics with packed qkv/gate_up projections + and NPU-only fused norm/RoPE fast paths * Per-call embedding buffer to avoid cross-request aliasing * Pre-allocated position_ids (read-only, safe to persist) * torch.compile (epilogue_fusion=False) on inner transformer by default @@ -45,8 +46,9 @@ if current_omni_platform.is_npu(): # HuggingFace reference on CPU/CUDA. vLLM's fused kernels (RMSNorm, # QKVParallel, get_rope) introduce small precision differences that compound # across the autoregressive steps of the code predictor, causing severe audio -# quality degradation. The Ascend fused norm/RoPE kernels below are dispatched -# by the current device platform. +# quality degradation. Packing qkv/gate_up into plain ``nn.Linear`` keeps the +# same algebra but can still change GPU bit-level accumulation. The Ascend +# fused norm/RoPE kernels below are dispatched by the current device platform. # # See: https://github.com/vllm-project/vllm-omni/issues/2274 @@ -175,11 +177,23 @@ class CodePredictorAttention(nn.Module): self.scaling = self.head_dim**-0.5 self.max_seq = int(config.num_code_groups) + 1 - # Separate q/k/v projections matching HF (no fused packing) + # Fused QKV projection. + # + # The HF reference uses three separate ``nn.Linear`` modules for + # q/k/v. Packing those weights row-wise into one ``nn.Linear`` keeps + # the same matmul at the algebra level; see the module-level + # HF-numerics note for the GPU bit-level drift caveat. The goal here + # is narrower: reduce eager host launch overhead while keeping the + # downstream q_norm / k_norm / SDPA math and HF checkpoint loading + # contract intact. bias = getattr(config, "attention_bias", False) - self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=bias) - self.k_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=bias) - self.v_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=bias) + self._q_size = self.num_heads * self.head_dim + self._kv_size = self.num_kv_heads * self.head_dim + self.qkv_proj = nn.Linear( + self.hidden_size, + self._q_size + 2 * self._kv_size, + bias=bias, + ) self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) self.q_norm = _RMSNorm(self.head_dim, eps=config.rms_norm_eps) self.k_norm = _RMSNorm(self.head_dim, eps=config.rms_norm_eps) @@ -197,6 +211,15 @@ class CodePredictorAttention(nn.Module): ) self.register_buffer("_fusion_causal_mask", fusion_mask, persistent=False) + def _split_qkv(self, qkv: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + q_raw = qkv[..., : self._q_size] + kv_raw = qkv[..., self._q_size :] + # ``split`` would leave the non-leading k/v slices strided like the + # packed qkv buffer. Materialize them once here so SDPA sees the same + # dense k/v layout as the unfused projection path. + k_raw, v_raw = (t.contiguous() for t in kv_raw.split(self._kv_size, dim=-1)) + return q_raw, k_raw, v_raw + def _forward_npu_attention( self, q: torch.Tensor, @@ -257,9 +280,14 @@ class CodePredictorAttention(nn.Module): hidden_shape_q = (bsz, seq_len, self.num_heads, self.head_dim) hidden_shape_kv = (bsz, seq_len, self.num_kv_heads, self.head_dim) - q = self.q_norm(self.q_proj(hidden_states).view(hidden_shape_q)).transpose(1, 2) - k = self.k_norm(self.k_proj(hidden_states).view(hidden_shape_kv)).transpose(1, 2) - v = self.v_proj(hidden_states).view(hidden_shape_kv).transpose(1, 2) + # Single fused matmul. ``q`` is immediately consumed by RMSNorm; for + # ``k/v`` we re-densify the packed tail once so SDPA does not inherit + # the packed-buffer stride pattern from the fused projection. + qkv = self.qkv_proj(hidden_states) + q_raw, k_raw, v_raw = self._split_qkv(qkv) + q = self.q_norm(q_raw.view(hidden_shape_q)).transpose(1, 2) + k = self.k_norm(k_raw.view(hidden_shape_kv)).transpose(1, 2) + v = v_raw.view(hidden_shape_kv).transpose(1, 2) cos, sin = position_embeddings # cos/sin are [batch, seq_len, head_dim], need unsqueeze at dim=1 for heads @@ -295,12 +323,22 @@ class CodePredictorMLP(nn.Module): def __init__(self, config, *, prefix: str = "") -> None: super().__init__() - self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False) - self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False) + # Fused gate_up projection. Same packing idea as ``qkv_proj`` above: + # one matmul instead of two, with the same algebraic result as + # separate gate/up linears. See the module-level HF-numerics note for + # the GPU bit-level drift caveat. + self._intermediate_size = int(config.intermediate_size) + self.gate_up_proj = nn.Linear( + config.hidden_size, + 2 * config.intermediate_size, + bias=False, + ) self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - return self.down_proj(F.silu(self.gate_proj(hidden_states)) * self.up_proj(hidden_states)) + gate_up = self.gate_up_proj(hidden_states) + gate, up = gate_up.split([self._intermediate_size, self._intermediate_size], dim=-1) + return self.down_proj(F.silu(gate) * up) # =================================================================== @@ -411,18 +449,140 @@ class CodePredictorBaseModel(nn.Module): hidden_states = self.norm(hidden_states) return hidden_states.to(input_dtype) + # Mapping from HF checkpoint shard name to the (fused parameter, + # slice index) pair used to assemble the fused tensor. See + # ``load_weights`` below. + _FUSED_QKV_SHARDS: tuple[tuple[str, str, int], ...] = ( + ("q_proj", "qkv_proj", 0), + ("k_proj", "qkv_proj", 1), + ("v_proj", "qkv_proj", 2), + ) + _FUSED_GATE_UP_SHARDS: tuple[tuple[str, str, int], ...] = ( + ("gate_proj", "gate_up_proj", 0), + ("up_proj", "gate_up_proj", 1), + ) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Load weights, transparently re-packing HF q/k/v and gate/up shards + into the fused ``qkv_proj`` and ``gate_up_proj`` parameters. + + HF checkpoints ship q_proj / k_proj / v_proj (and gate_proj / + up_proj) as separate tensors. Our model defines a single + ``qkv_proj`` (``gate_up_proj``) whose weight is the row-wise + concatenation of those shards in q/k/v (gate/up) order. We + buffer the shards per layer prefix and write the fused parameter + in one go once all shards have arrived. + + Both ``.weight`` and ``.bias`` shards are packed this way, so a + code-predictor config with ``attention_bias=True`` is handled + correctly: the q/k/v bias vectors are concatenated into + ``qkv_proj.bias`` instead of being dropped (which would leave the + fused bias randomly initialized). ``gate_up_proj`` has no bias, + so only its ``.weight`` shards are ever routed. + """ params_dict = dict(self.named_parameters(remove_duplicate=False)) loaded_params: set[str] = set() + fused_layer_prefixes = { + name.split(marker, 1)[0] + for name in params_dict + for marker in (".self_attn.qkv_proj.", ".mlp.gate_up_proj.") + if marker in name + } + touched_fused_layers: set[str] = set() + + # layer_prefix -> {fused_name: {kind: {slice_idx: shard_tensor}}} + # where ``kind`` is "weight" or "bias". + pending: dict[str, dict[str, dict[str, dict[int, torch.Tensor]]]] = {} + + def _try_finalize_fused( + layer_prefix: str, + fused_name: str, + kind: str, + num_shards: int, + ) -> None: + bucket = pending.get(layer_prefix, {}).get(fused_name, {}).get(kind) + if bucket is None or len(bucket) < num_shards: + return + target_param_name = f"{layer_prefix}.{fused_name}.{kind}" + target_param = params_dict.get(target_param_name) + if target_param is None: + return + ordered = [bucket[i] for i in range(num_shards)] + fused_weight = torch.cat(ordered, dim=0) + weight_loader = getattr(target_param, "weight_loader", default_weight_loader) + weight_loader(target_param, fused_weight) + loaded_params.add(target_param_name) + del pending[layer_prefix][fused_name][kind] + if not pending[layer_prefix][fused_name]: + del pending[layer_prefix][fused_name] + if not pending[layer_prefix]: + del pending[layer_prefix] + + def _route_shard( + name: str, + loaded_weight: torch.Tensor, + shard_table: tuple[tuple[str, str, int], ...], + parent_attr: str, + ) -> bool: + """Stash a HF ``.weight``/``.bias`` shard if its fused counterpart + exists in this module; return True iff the shard was consumed.""" + for shard_name, fused_name, slice_idx in shard_table: + for kind in ("weight", "bias"): + suffix = f".{parent_attr}.{shard_name}.{kind}" + if not name.endswith(suffix): + continue + layer_prefix = name[: -len(f".{shard_name}.{kind}")] # ...self_attn or ...mlp + fused_param_name = f"{layer_prefix}.{fused_name}.{kind}" + if fused_param_name not in params_dict: + # e.g. a gate/up ``.bias`` shard when the fused + # projection has no bias: nothing to pack into. + return False + pending.setdefault(layer_prefix, {}).setdefault(fused_name, {}).setdefault(kind, {})[slice_idx] = ( + loaded_weight + ) + _try_finalize_fused(layer_prefix, fused_name, kind, num_shards=len(shard_table)) + return True + return False + for name, loaded_weight in weights: if "rotary_emb.inv_freq" in name: continue + touched_fused_layers.update( + layer_prefix for layer_prefix in fused_layer_prefixes if name.startswith(f"{layer_prefix}.") + ) + if _route_shard(name, loaded_weight, self._FUSED_QKV_SHARDS, parent_attr="self_attn"): + continue + if _route_shard(name, loaded_weight, self._FUSED_GATE_UP_SHARDS, parent_attr="mlp"): + continue param = params_dict.get(name) if param is None: continue weight_loader = getattr(param, "weight_loader", default_weight_loader) weight_loader(param, loaded_weight) loaded_params.add(name) + + if pending: + unfinished = { + lp: {fn: list(kinds.keys()) for fn, kinds in buckets.items()} for lp, buckets in pending.items() + } + raise RuntimeError( + f"CodePredictor load_weights: incomplete fused shards for layers {unfinished}; " + f"check that the checkpoint contains all q/k/v and gate/up tensors." + ) + + missing_fused = sorted( + name + for name in params_dict + if (".self_attn.qkv_proj." in name or ".mlp.gate_up_proj." in name) + and name not in loaded_params + and any(name.startswith(f"{layer_prefix}.") for layer_prefix in touched_fused_layers) + ) + if missing_fused: + raise RuntimeError( + f"CodePredictor load_weights: missing fused parameters {missing_fused}; " + f"check that the checkpoint contains the q/k/v and gate/up shards, " + f"or the already-fused qkv_proj/gate_up_proj tensors." + ) return loaded_params @@ -997,18 +1157,47 @@ class CodePredictorWrapper(nn.Module): logger.info("Prepared NPU code predictor weights: linear=%d", linear_count) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - """Load weights directly (no fused projection remapping needed).""" + """Load wrapper weights, delegating inner-model tensors to the fused loader. + + Qwen3-TTS checkpoints use the current ``model.*`` wrapper layout, while + some Qwen3-Omni artifacts expose the shared body directly as + ``layers.*`` / ``codec_embedding.*``. Normalize both forms before + calling ``CodePredictorBaseModel.load_weights`` so its q/k/v and gate/up + repacking guards see the HF shards instead of treating them as unrelated + wrapper-level tensors. + """ loaded: set[str] = set() model_weights: list[tuple[str, torch.Tensor]] = [] other_weights: list[tuple[str, torch.Tensor]] = [] + inner_model_prefixes = ("codec_embedding.", "layers.", "norm.", "rotary_emb.") + wrapper_prefixes = tuple( + dict.fromkeys( + prefix + for prefix in ( + f"{self.prefix}.", + f"{self.prefix.rsplit('.', 1)[-1]}." if self.prefix else "", + "code_predictor.", + ) + if prefix + ) + ) for name, w in weights: if "rotary_emb.inv_freq" in name: continue - if name.startswith("model."): - model_weights.append((name[len("model.") :], w)) + + normalized_name = name + for wrapper_prefix in wrapper_prefixes: + if normalized_name.startswith(wrapper_prefix): + normalized_name = normalized_name[len(wrapper_prefix) :] + break + + if normalized_name.startswith("model."): + model_weights.append((normalized_name[len("model.") :], w)) + elif normalized_name.startswith(inner_model_prefixes): + model_weights.append((normalized_name, w)) else: - other_weights.append((name, w)) + other_weights.append((normalized_name, w)) loaded_model = self.model.load_weights(model_weights) loaded |= {f"model.{n}" for n in loaded_model} diff --git a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py index 31a9fced1..c9ce11a93 100644 --- a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py +++ b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py @@ -23,6 +23,8 @@ previous KV-cache loop -- causal attention, only the last position is read. from __future__ import annotations +from collections.abc import Iterable + import torch import torch.nn as nn import torch.nn.functional as F @@ -61,6 +63,22 @@ class MossTTSRealtimeLocalTransformer(nn.Module): prefix="model", ) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Delegate ``model.*`` tensors to ``CodePredictorBaseModel.load_weights``. + + The shared body fuses ``q/k/v_proj`` into ``qkv_proj`` and + ``gate/up_proj`` into ``gate_up_proj``; its loader re-packs the HF + shards and raises on incomplete or missing fused parameters. Going + through it (instead of ``default_weight_loader`` per tensor) keeps + those guarantees for the MossTTSRealtime checkpoint too. + """ + model_weights: list[tuple[str, torch.Tensor]] = [] + for name, tensor in weights: + if name.startswith("model."): + model_weights.append((name[len("model.") :], tensor)) + loaded = self.model.load_weights(iter(model_weights)) + return {f"model.{n}" for n in loaded} + @torch.no_grad() def generate_frame( self, diff --git a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py index 3f9ebf42f..0495d79d8 100644 --- a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py +++ b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py @@ -1179,7 +1179,13 @@ class MossTTSRealtimeTalkerForGeneration(nn.Module): # ``gate_proj``/``up_proj`` → ``gate_up_proj``. ``default_weight_loader`` # alone leaves those fused params un-initialised, which silently turns # the backbone into a slightly-corrupted model that never emits EOS. + # The local depth transformer (shared ``CodePredictorBaseModel``) has + # the exact same hazard: its ``qkv_proj``/``gate_up_proj`` are fused, + # so its shards must go through ``CodePredictorBaseModel.load_weights`` + # (via ``MossTTSRealtimeLocalTransformer.load_weights``), which re-packs + # them and raises on incomplete/missing fused parameters. backbone_weights: list[tuple[str, torch.Tensor]] = [] + local_weights: list[tuple[str, torch.Tensor]] = [] skipped: list[str] = [] for name, tensor in weights: if name.startswith("language_model."): @@ -1194,8 +1200,9 @@ class MossTTSRealtimeTalkerForGeneration(nn.Module): continue # non-persistent buffer, recomputed at runtime if sub.startswith("embed_tokens."): sub = "codec_embedding." + sub[len("embed_tokens.") :] - tgt = "local_transformer.model." + sub - elif name.startswith("local_transformer.local_lm_heads."): + local_weights.append((f"model.{sub}", tensor)) + continue + if name.startswith("local_transformer.local_lm_heads."): tgt = "local_lm_heads." + name[len("local_transformer.local_lm_heads.") :] else: tgt = name @@ -1210,6 +1217,11 @@ class MossTTSRealtimeTalkerForGeneration(nn.Module): for n in backbone_loaded: loaded.add(f"model.{n}") + # Delegate depth-transformer weights to the shared fused loader. + local_loaded = self.local_transformer.load_weights(iter(local_weights)) + for n in local_loaded: + loaded.add(f"local_transformer.{n}") + # Build stacked embedding cache for vectorised decode ops. # embed_tokens[0] is the text embedding; [1..n_vq] are audio codebooks. self._stacked_audio_emb_w = torch.stack( diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py index 4b27e4ba0..46401fb13 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py @@ -58,8 +58,10 @@ class Qwen3OmniMoeTalkerForConditionalGeneration( "talker.model.": "language_model.model.", # Codec head remains separate (outputs audio codes, not text) "talker.codec_head.": "codec_head.", - # Code predictor: Now matches HF structure exactly (has .model sub-module) - # e.g., "talker.code_predictor.model.codec_embedding.0" → "code_predictor.model.codec_embedding.0" + # Code predictor: keep the subtree under ``code_predictor.``. + # ``CodePredictorWrapper.load_weights`` accepts both current + # ``model.*`` body names and legacy body-direct names such as + # ``layers.*`` / ``codec_embedding.*``. "talker.code_predictor.": "code_predictor.", # Projection layers "talker.text_projection.": "text_projection.", diff --git a/vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py b/vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py index 28ca4f362..36bf663dd 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py +++ b/vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py @@ -173,16 +173,13 @@ class Qwen3TTSTalkerCodePredictorConfig(PretrainedConfig): model_type = "qwen3_tts_talker_code_predictor" keys_to_ignore_at_inference = ["past_key_values"] - # Default tensor parallel plan for base model `Qwen3TTSTalkerCodePredictor` - base_model_tp_plan = { - "layers.*.self_attn.q_proj": "colwise", - "layers.*.self_attn.k_proj": "colwise", - "layers.*.self_attn.v_proj": "colwise", - "layers.*.self_attn.o_proj": "rowwise", - "layers.*.mlp.gate_proj": "colwise", - "layers.*.mlp.up_proj": "colwise", - "layers.*.mlp.down_proj": "rowwise", - } + # The code predictor currently runs TP=1. Do not advertise a tensor + # parallel plan for the packed qkv/gate_up projections until the fused + # loader and ``_split_qkv`` path are taught TP-aware packing/sharding: + # a generic colwise split of the plain ``nn.Linear`` would split the + # packed [q, k, v] layout across ranks instead of preserving each rank's + # local q/k/v groups. + base_model_tp_plan = {} base_model_pp_plan = { "embed_tokens": (["input_ids"], ["inputs_embeds"]), "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), diff --git a/vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py b/vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py index 713b67f81..dd03ba7d2 100644 --- a/vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py +++ b/vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py @@ -148,6 +148,15 @@ class _Qwen3CodePredictorAttention310P(qwen3_code_predictor.CodePredictorAttenti def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) + config = args[0] if args else kwargs.get("config") + if config is not None: + self.num_heads = getattr(self, "num_heads", config.num_attention_heads) + self.num_kv_heads = getattr(self, "num_kv_heads", config.num_key_value_heads) + self.head_dim = getattr( + self, + "head_dim", + getattr(config, "head_dim", config.hidden_size // config.num_attention_heads), + ) self._buffers.pop("_fusion_causal_mask", None) self._q_size = self.num_heads * self.head_dim self._kv_size = self.num_kv_heads * self.head_dim @@ -157,11 +166,9 @@ class _Qwen3CodePredictorAttention310P(qwen3_code_predictor.CodePredictorAttenti def prepare_qkv_weights(self) -> None: # Pack QKV once so each graph replay uses one matmul and consumes the # weight directly in the 310P matmul layout. - self._fused_qkv_weight = maybe_trans_nz( - torch.cat((self.q_proj.weight, self.k_proj.weight, self.v_proj.weight), dim=0).contiguous() - ) - if self.q_proj.bias is not None: - self._fused_qkv_bias = torch.cat((self.q_proj.bias, self.k_proj.bias, self.v_proj.bias), dim=0) + self._fused_qkv_weight = maybe_trans_nz(self.qkv_proj.weight.contiguous()) + if self.qkv_proj.bias is not None: + self._fused_qkv_bias = self.qkv_proj.bias def forward( self, @@ -170,11 +177,17 @@ class _Qwen3CodePredictorAttention310P(qwen3_code_predictor.CodePredictorAttenti attention_mask: torch.Tensor | None = None, ) -> torch.Tensor: bsz, seq_len, _ = hidden_states.shape + hidden_shape_q = (bsz, seq_len, self.num_heads, self.head_dim) + hidden_shape_kv = (bsz, seq_len, self.num_kv_heads, self.head_dim) + + # ``prepare_qkv_weights`` packs qkv_proj into the 310P matmul layout; + # split it back out the same way before re-applying the 310P + # flash-attention path. qkv = F.linear(hidden_states, self._fused_qkv_weight, self._fused_qkv_bias) - q, k, v = qkv.split((self._q_size, self._kv_size, self._kv_size), dim=-1) - q = self.q_norm(q.view(bsz, seq_len, self.num_heads, self.head_dim)).transpose(1, 2) - k = self.k_norm(k.view(bsz, seq_len, self.num_kv_heads, self.head_dim)).transpose(1, 2) - v = v.view(bsz, seq_len, self.num_kv_heads, self.head_dim).transpose(1, 2) + q_raw, k_raw, v_raw = self._split_qkv(qkv) + q = self.q_norm(q_raw.view(hidden_shape_q)).transpose(1, 2) + k = self.k_norm(k_raw.view(hidden_shape_kv)).transpose(1, 2) + v = v_raw.view(hidden_shape_kv).transpose(1, 2) cos, sin = position_embeddings cos = cos.unsqueeze(1) @@ -304,7 +317,7 @@ class _Qwen3TTSTalkerCodePredictor310P( for layer in self.model.layers: attention = layer.self_attn attention.prepare_qkv_weights() - qkv_projections.update((attention.q_proj, attention.k_proj, attention.v_proj)) + qkv_projections.add(attention.qkv_proj) for module in self.modules(): if isinstance(module, nn.Linear) and module not in qkv_projections: From 5730af0a95299387c233e12dfbdbe728aace0e58 Mon Sep 17 00:00:00 2001 From: Ian Eaves Date: Thu, 20 Aug 2026 15:05:05 -0500 Subject: [PATCH 098/267] [Bugfix] Preserve prompt token usage details (#5181) Signed-off-by: Ian Eaves --- .buildkite/cuda/test-merge.yml | 2 + .buildkite/cuda/test-ready.yml | 2 + tests/e2e/online_serving/test_qwen3_omni.py | 18 +- .../openai_api/test_serving_chat_metrics.py | 68 ++++++ .../test_serving_chat_sampling_params.py | 160 ++++++------ .../entrypoints/test_stream_finish_reason.py | 228 +++++------------- tests/helpers/runtime.py | 3 + tests/helpers/serving_chat.py | 160 ++++++++++++ vllm_omni/entrypoints/openai/serving_chat.py | 34 ++- 9 files changed, 418 insertions(+), 257 deletions(-) create mode 100644 tests/helpers/serving_chat.py diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index 69913bf55..6a402254b 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -119,6 +119,8 @@ steps: source_file_dependencies: - tests/e2e/offline_inference/test_qwen3_omni.py - tests/e2e/online_serving/test_qwen3_omni.py + - tests/helpers/runtime.py + - vllm_omni/entrypoints/openai/serving_chat.py - vllm_omni/model_executor/models/qwen3_omni/ - vllm_omni/model_executor/models/common/qwen3_code_predictor.py - vllm_omni/model_executor/models/common/snake_activation.py diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index 7f7e44cd2..ec3d7485a 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -168,6 +168,8 @@ steps: - label: "Omni · Qwen3-Omni Test" source_file_dependencies: - tests/e2e/online_serving/test_qwen3_omni.py + - tests/helpers/runtime.py + - vllm_omni/entrypoints/openai/serving_chat.py - vllm_omni/model_executor/models/qwen3_omni/ - vllm_omni/model_executor/models/common/qwen3_code_predictor.py - vllm_omni/model_executor/models/common/snake_activation.py diff --git a/tests/e2e/online_serving/test_qwen3_omni.py b/tests/e2e/online_serving/test_qwen3_omni.py index a54d2fe8a..036740ead 100644 --- a/tests/e2e/online_serving/test_qwen3_omni.py +++ b/tests/e2e/online_serving/test_qwen3_omni.py @@ -149,10 +149,19 @@ def _run_prefix_cache_check(openai_client, request_config: dict): 2. The number of cached tokens is divisible by the block size used in test_params, because currently upstream vLLM does not cache partial blocks. + 3. The first request also reports prompt token details; details must be + present whenever --enable-prompt-tokens-details is on, even when zero + tokens are cached. (An exact cached_tokens == 0 check would be + order-dependent here: the module-scoped server may have already cached + the shared system-prompt blocks from earlier tests.) + + Returns the (first, second) responses for additional per-test assertions. """ - openai_client.send_omni_request(request_config, request_num=1)[0] + first_response = openai_client.send_omni_request(request_config, request_num=1)[0] cached_response = openai_client.send_omni_request(request_config, request_num=1)[0] + assert first_response.cached_tokens is not None + # Ensure that we have a prefix cache hit on the second request and that only the last # partial block is uncached (since currently we don't cache partial blocks). num_cached_tokens = cached_response.cached_tokens @@ -163,6 +172,8 @@ def _run_prefix_cache_check(openai_client, request_config: dict): assert num_cached_tokens % BLOCK_SIZE == 0 assert (num_cached_tokens + num_uncached_tokens) == num_prompt_tokens + return first_response, cached_response + @pytest.mark.advanced_model @pytest.mark.core_model @@ -195,7 +206,10 @@ def test_thinker_prefix_caching_text_output(omni_server, openai_client) -> None: "stream": False, "modalities": ["text"], } - _run_prefix_cache_check(openai_client, request_config) + first_response, _ = _run_prefix_cache_check(openai_client, request_config) + + assert first_response.multimodal_tokens is not None + assert first_response.multimodal_tokens.get("image", 0) > 0 @pytest.mark.advanced_model diff --git a/tests/entrypoints/openai_api/test_serving_chat_metrics.py b/tests/entrypoints/openai_api/test_serving_chat_metrics.py index 0647c40a3..2fdf9f646 100644 --- a/tests/entrypoints/openai_api/test_serving_chat_metrics.py +++ b/tests/entrypoints/openai_api/test_serving_chat_metrics.py @@ -2,9 +2,18 @@ """Unit tests for OmniChatCompletionResponse/StreamResponse metrics field.""" from types import SimpleNamespace +from unittest.mock import MagicMock import pytest +from tests.helpers.serving_chat import ( + build_serving_chat, + collect_stream, + make_request, + make_text_omni_output, + parse_sse_chunks, +) + pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -54,6 +63,65 @@ def test_omni_chat_completion_stream_response_metrics(): assert response.metrics == {"stage_latency": 0.5} +@pytest.mark.asyncio +async def test_stream_prompt_token_details_include_zero_cached_and_multimodal_tokens(): + serving_chat = build_serving_chat() + serving_chat.enable_prompt_tokens_details = True + request = make_request(modalities=["text"], include_usage=True) + + async def result_generator(): + yield make_text_omni_output( + token_ids=[10], + finish_reason="stop", + num_cached_tokens=0, + ) + + raw_lines = await collect_stream( + serving_chat.chat_completion_stream_generator( + request=request, + result_generator=result_generator(), + request_id="test-req", + model_name="test-model", + conversation=[], + tokenizer=MagicMock(), + request_metadata=MagicMock(), + mm_token_counts={"image": 7}, + ) + ) + + chunks = parse_sse_chunks(raw_lines) + usage = next(chunk["usage"] for chunk in chunks if chunk.get("usage")) + + assert usage["prompt_tokens_details"] == { + "cached_tokens": 0, + "multimodal_tokens": {"image": 7}, + } + + +def test_non_stream_prompt_token_details_include_zero_cached_and_multimodal_tokens(): + serving_chat = build_serving_chat() + serving_chat.enable_prompt_tokens_details = True + request = make_request(modalities=["text"], stream=False) + omni_output = make_text_omni_output( + token_ids=[10], + finish_reason="stop", + num_cached_tokens=0, + ) + + _, usage, _, _, _ = serving_chat._create_text_choice( + request=request, + omni_outputs=omni_output, + tokenizer=MagicMock(), + conversation=[], + role="assistant", + mm_token_counts={"audio": 5}, + ) + + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.cached_tokens == 0 + assert usage.prompt_tokens_details.multimodal_tokens == {"audio": 5} + + def test_create_image_choice_exposes_diffusion_metrics(): """Ensure image chat content exposes profiler metrics for clients.""" from PIL import Image diff --git a/tests/entrypoints/openai_api/test_serving_chat_sampling_params.py b/tests/entrypoints/openai_api/test_serving_chat_sampling_params.py index e7f014a3b..f81e61cf1 100644 --- a/tests/entrypoints/openai_api/test_serving_chat_sampling_params.py +++ b/tests/entrypoints/openai_api/test_serving_chat_sampling_params.py @@ -14,6 +14,8 @@ from pytest_mock import MockerFixture from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.sampling_params import SamplingParams +from tests.helpers.serving_chat import build_serving_chat + pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -77,18 +79,8 @@ def mock_engine_client( return engine_client -@pytest.fixture -def serving_chat(mock_engine_client): - """Create OmniOpenAIServingChat instance with mocked dependencies.""" - from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat - - # Create instance without calling __init__ - instance = object.__new__(OmniOpenAIServingChat) - instance.engine_client = mock_engine_client - return instance - - -def test_serving_boundary_normalizes_declared_root_and_nested_extras(serving_chat): +def test_serving_boundary_normalizes_declared_root_and_nested_extras(mock_engine_client): + serving_chat = build_serving_chat(engine_client=mock_engine_client) serving_chat._diffusion_extra_body_params = frozenset({"cfg_text_scale", "negative_prompt"}) request = ChatCompletionRequest( model="test", @@ -118,7 +110,8 @@ def test_serving_boundary_normalizes_declared_root_and_nested_extras(serving_cha assert diffusion_request_args["modalities"] == ["image"] -def test_unknown_root_extra_does_not_claim_canonical_extra(serving_chat): +def test_unknown_root_extra_does_not_claim_canonical_extra(mock_engine_client): + serving_chat = build_serving_chat(engine_client=mock_engine_client) serving_chat._diffusion_extra_body_params = frozenset() request = ChatCompletionRequest( model="test", @@ -135,7 +128,8 @@ def test_unknown_root_extra_does_not_claim_canonical_extra(serving_chat): assert "pipeline_option" not in diffusion_request_args -def test_serving_boundary_rejects_invalid_quality(serving_chat): +def test_serving_boundary_rejects_invalid_quality(mock_engine_client): + serving_chat = build_serving_chat(engine_client=mock_engine_client) serving_chat._diffusion_extra_body_params = frozenset() request = ChatCompletionRequest( model="test", @@ -148,7 +142,8 @@ def test_serving_boundary_rejects_invalid_quality(serving_chat): serving_chat._normalize_diffusion_request_args(request) -def test_unregistered_cfg_scale_aliases_common_true_cfg_scale(serving_chat): +def test_unregistered_cfg_scale_aliases_common_true_cfg_scale(mock_engine_client): + serving_chat = build_serving_chat(engine_client=mock_engine_client) serving_chat._diffusion_extra_body_params = frozenset() request = ChatCompletionRequest( model="test", @@ -170,7 +165,8 @@ def test_unregistered_cfg_scale_aliases_common_true_cfg_scale(serving_chat): ], ids=["registered-model-extra", "unregistered-common-alias"], ) -def test_cfg_scale_owner_conflicts_are_rejected(serving_chat, registered, request_kwargs): +def test_cfg_scale_owner_conflicts_are_rejected(mock_engine_client, registered, request_kwargs): + serving_chat = build_serving_chat(engine_client=mock_engine_client) serving_chat._diffusion_extra_body_params = frozenset(registered) request = ChatCompletionRequest(model="test", messages=[], **request_kwargs) @@ -180,10 +176,11 @@ def test_cfg_scale_owner_conflicts_are_rejected(serving_chat, registered, reques @pytest.mark.parametrize("diffusion_mode", [True, False], ids=["pure", "mixed"]) def test_duplicate_extras_return_the_same_bad_request_before_dispatch( - serving_chat, + mock_engine_client, mocker: MockerFixture, diffusion_mode: bool, ): + serving_chat = build_serving_chat(engine_client=mock_engine_client) serving_chat._diffusion_mode = diffusion_mode serving_chat._diffusion_extra_body_params = frozenset({"sample_solver"}) serving_chat.engine_client.stage_configs = [SimpleNamespace(stage_type="diffusion")] @@ -211,9 +208,10 @@ def test_duplicate_extras_return_the_same_bad_request_before_dispatch( pure_dispatch.assert_not_awaited() -def test_pure_consumer_preserves_defaults_and_separate_cfg_owners(serving_chat, mocker: MockerFixture): +def test_pure_consumer_preserves_defaults_and_separate_cfg_owners(mock_engine_client, mocker: MockerFixture): from vllm_omni.inputs.data import OmniDiffusionSamplingParams + serving_chat = build_serving_chat(engine_client=mock_engine_client) captured: dict[str, object] = {} async def generate(**kwargs): @@ -261,19 +259,20 @@ def test_pure_consumer_preserves_defaults_and_separate_cfg_owners(serving_chat, assert response.error.message == "No output generated from AsyncOmni" -def test_mixed_consumer_keeps_root_common_args_with_nested_extras(serving_chat, mocker: MockerFixture): +def test_mixed_consumer_keeps_root_common_args_with_nested_extras(mock_engine_client, mocker: MockerFixture): from vllm_omni.inputs.data import OmniDiffusionSamplingParams - serving_chat.engine_client.stage_configs = [ + mock_engine_client.stage_configs = [ SimpleNamespace(stage_type="llm", is_comprehension=True), SimpleNamespace(stage_type="diffusion", is_comprehension=False), ] - serving_chat.engine_client.default_sampling_params_list = [ + mock_engine_client.default_sampling_params_list = [ SamplingParams(), OmniDiffusionSamplingParams(), ] - serving_chat.engine_client.output_modalities = ["image"] - serving_chat.engine_client.errored = False + mock_engine_client.output_modalities = ["image"] + mock_engine_client.errored = False + mock_engine_client.renderer = SimpleNamespace(get_tokenizer=lambda: object()) captured: dict[str, object] = {} @@ -285,21 +284,15 @@ def test_mixed_consumer_keeps_root_common_args_with_nested_extras(serving_chat, captured.update(kwargs) return results() - serving_chat.engine_client.generate = generate - serving_chat.__dict__.update( - _diffusion_mode=False, - _diffusion_extra_body_params=frozenset(), + mock_engine_client.generate = generate + serving_chat = build_serving_chat( + engine_client=mock_engine_client, models=SimpleNamespace(model_name=lambda _: "test"), - renderer=SimpleNamespace(get_tokenizer=lambda: object()), online_renderer=SimpleNamespace(validate_chat_template=lambda **_: None), - parser_cls=None, - use_harmony=False, - enable_auto_tools=False, - exclude_tools_when_tool_choice_none=False, trust_request_chat_template=True, - chat_template=None, - chat_template_content_format="auto", ) + serving_chat._diffusion_mode = False + serving_chat._diffusion_extra_body_params = frozenset() mocker.patch.multiple( serving_chat, _check_model=mocker.AsyncMock(return_value=None), @@ -386,8 +379,9 @@ def test_openai_sampling_fields_contains_expected_fields(): # ============================================================================= -def test_preserves_yaml_defaults_when_no_request_params(serving_chat, mock_request): +def test_preserves_yaml_defaults_when_no_request_params(mock_engine_client, mock_request): """Test that YAML defaults are preserved when request has no params.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) result = serving_chat._build_sampling_params_list_from_request(mock_request) assert len(result) == 2 @@ -400,8 +394,9 @@ def test_preserves_yaml_defaults_when_no_request_params(serving_chat, mock_reque assert comprehension_params.repetition_penalty == 1.05 # YAML custom param preserved -def test_request_temperature_overrides_yaml_default(serving_chat, mock_request): +def test_request_temperature_overrides_yaml_default(mock_engine_client, mock_request): """Test that request temperature overrides YAML default.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) mock_request.temperature = 0.8 mock_request.model_fields_set = {"temperature"} @@ -413,8 +408,9 @@ def test_request_temperature_overrides_yaml_default(serving_chat, mock_request): assert comprehension_params.top_k == 1 # YAML custom param preserved -def test_request_top_p_overrides_yaml_default(serving_chat, mock_request): +def test_request_top_p_overrides_yaml_default(mock_engine_client, mock_request): """Test that request top_p overrides YAML default.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) mock_request.top_p = 0.95 mock_request.model_fields_set = {"top_p"} @@ -425,8 +421,9 @@ def test_request_top_p_overrides_yaml_default(serving_chat, mock_request): assert comprehension_params.temperature == 0.4 # Preserved from YAML -def test_request_max_tokens_overrides_yaml_default(serving_chat, mock_request): +def test_request_max_tokens_overrides_yaml_default(mock_engine_client, mock_request): """Test that request max_tokens overrides YAML default.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) mock_request.max_tokens = 100 mock_request.model_fields_set = {"max_tokens"} @@ -435,15 +432,17 @@ def test_request_max_tokens_overrides_yaml_default(serving_chat, mock_request): assert result[0].max_tokens == 100 -def test_max_tokens_uses_yaml_default_when_not_specified(serving_chat, mock_request): +def test_max_tokens_uses_yaml_default_when_not_specified(mock_engine_client, mock_request): """Test that max_tokens falls back to YAML default when not in request.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) result = serving_chat._build_sampling_params_list_from_request(mock_request) assert result[0].max_tokens == 4353 -def test_request_seed_overrides_yaml_default(serving_chat, mock_request): +def test_request_seed_overrides_yaml_default(mock_engine_client, mock_request): """Test that request seed overrides YAML default.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) mock_request.seed = 123 mock_request.model_fields_set = {"seed"} @@ -454,8 +453,9 @@ def test_request_seed_overrides_yaml_default(serving_chat, mock_request): assert comprehension_params.temperature == 0.4 # Preserved from YAML -def test_request_frequency_penalty_overrides(serving_chat, mock_request): +def test_request_frequency_penalty_overrides(mock_engine_client, mock_request): """Test that request frequency_penalty is applied.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) mock_request.frequency_penalty = 0.5 mock_request.model_fields_set = {"frequency_penalty"} @@ -464,8 +464,9 @@ def test_request_frequency_penalty_overrides(serving_chat, mock_request): assert result[0].frequency_penalty == 0.5 -def test_request_presence_penalty_overrides(serving_chat, mock_request): +def test_request_presence_penalty_overrides(mock_engine_client, mock_request): """Test that request presence_penalty is applied.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) mock_request.presence_penalty = 0.3 mock_request.model_fields_set = {"presence_penalty"} @@ -474,8 +475,9 @@ def test_request_presence_penalty_overrides(serving_chat, mock_request): assert result[0].presence_penalty == 0.3 -def test_non_comprehension_stages_use_cloned_defaults(serving_chat, mock_request): +def test_non_comprehension_stages_use_cloned_defaults(mock_engine_client, mock_request): """Test that non-comprehension stages always use cloned YAML defaults.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) mock_request.max_tokens = 50 mock_request.temperature = 0.1 @@ -488,8 +490,9 @@ def test_non_comprehension_stages_use_cloned_defaults(serving_chat, mock_request assert other_params.seed == 42 # YAML default -def test_multiple_params_override_together(serving_chat, mock_request): +def test_multiple_params_override_together(mock_engine_client, mock_request): """Test that multiple request params can override together.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) mock_request.max_tokens = 200 mock_request.temperature = 0.7 mock_request.top_p = 0.85 @@ -514,15 +517,17 @@ def test_multiple_params_override_together(serving_chat, mock_request): # ============================================================================= -def test_apply_request_overrides_clones_params(serving_chat, mock_request, default_comprehension_params): +def test_apply_request_overrides_clones_params(mock_engine_client, mock_request, default_comprehension_params): """Test that _apply_request_overrides returns a cloned object.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) result = serving_chat._apply_request_overrides(default_comprehension_params, mock_request) assert result is not default_comprehension_params # Different object -def test_apply_request_overrides_preserves_defaults(serving_chat, mock_request, default_comprehension_params): +def test_apply_request_overrides_preserves_defaults(mock_engine_client, mock_request, default_comprehension_params): """Test that _apply_request_overrides preserves defaults when request has None.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) result = serving_chat._apply_request_overrides(default_comprehension_params, mock_request) assert result.temperature == 0.4 @@ -531,8 +536,9 @@ def test_apply_request_overrides_preserves_defaults(serving_chat, mock_request, assert result.top_k == 1 # YAML custom param -def test_apply_request_overrides_applies_values(serving_chat, mock_request, default_comprehension_params): +def test_apply_request_overrides_applies_values(mock_engine_client, mock_request, default_comprehension_params): """Test that _apply_request_overrides applies non-None request values.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) mock_request.temperature = 0.8 mock_request.seed = 123 mock_request.model_fields_set = {"temperature", "seed"} @@ -550,8 +556,9 @@ def test_apply_request_overrides_applies_values(serving_chat, mock_request, defa # ============================================================================= -def test_apply_overrides_empty_stop_list_preserves_default(serving_chat, mocker): +def test_apply_overrides_empty_stop_list_preserves_default(mock_engine_client, mocker): """Test that request.stop=[] does NOT override YAML default stop words.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) default_params = SamplingParams(temperature=0.5, stop=["<|im_end|>"]) request = mocker.MagicMock() request.temperature = None @@ -573,8 +580,9 @@ def test_apply_overrides_empty_stop_list_preserves_default(serving_chat, mocker) assert result.stop == ["<|im_end|>"] # YAML default preserved -def test_apply_overrides_nonempty_stop_list_overrides_default(serving_chat, mocker): +def test_apply_overrides_nonempty_stop_list_overrides_default(mock_engine_client, mocker): """Test that request.stop=["\\n"] overrides YAML default stop words.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) default_params = SamplingParams(temperature=0.5, stop=["<|im_end|>"]) request = mocker.MagicMock() request.temperature = None @@ -596,8 +604,9 @@ def test_apply_overrides_nonempty_stop_list_overrides_default(serving_chat, mock assert result.stop == ["\n"] # Overridden by request -def test_apply_overrides_empty_stop_token_ids_preserves_default(serving_chat, mocker): +def test_apply_overrides_empty_stop_token_ids_preserves_default(mock_engine_client, mocker): """Test that request.stop_token_ids=[] does NOT override YAML default.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) default_params = SamplingParams(temperature=0.5, stop_token_ids=[2, 3]) request = mocker.MagicMock() request.temperature = None @@ -617,8 +626,9 @@ def test_apply_overrides_empty_stop_token_ids_preserves_default(serving_chat, mo assert result.stop_token_ids == [2, 3] # YAML default preserved -def test_apply_overrides_nonempty_stop_token_ids_overrides_default(serving_chat, mocker): +def test_apply_overrides_nonempty_stop_token_ids_overrides_default(mock_engine_client, mocker): """Test that request.stop_token_ids=[100] overrides YAML default.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) default_params = SamplingParams(temperature=0.5, stop_token_ids=[2, 3]) request = mocker.MagicMock() request.temperature = None @@ -640,8 +650,9 @@ def test_apply_overrides_nonempty_stop_token_ids_overrides_default(serving_chat, assert result.stop_token_ids == [100] # Overridden by request -def test_apply_overrides_mixed_empty_and_nonempty_lists(serving_chat, mocker): +def test_apply_overrides_mixed_empty_and_nonempty_lists(mock_engine_client, mocker): """Test mixing empty and non-empty list fields with scalar fields.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) default_params = SamplingParams( temperature=0.4, stop=["<|end|>"], @@ -669,8 +680,9 @@ def test_apply_overrides_mixed_empty_and_nonempty_lists(serving_chat, mocker): assert result.stop_token_ids == [100, 200] # Non-empty list DID override -def test_apply_overrides_none_scalar_still_preserves_default(serving_chat, mocker): +def test_apply_overrides_none_scalar_still_preserves_default(mock_engine_client, mocker): """Regression: ensure None scalar values still don't override defaults.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) default_params = SamplingParams(temperature=0.5, max_tokens=100, seed=42) request = mocker.MagicMock() request.temperature = None @@ -694,8 +706,9 @@ def test_apply_overrides_none_scalar_still_preserves_default(serving_chat, mocke assert result.seed == 42 -def test_apply_overrides_both_lists_empty_preserves_defaults(serving_chat, mocker): +def test_apply_overrides_both_lists_empty_preserves_defaults(mock_engine_client, mocker): """Test that both stop=[] and stop_token_ids=[] preserve YAML defaults.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) default_params = SamplingParams( temperature=0.5, stop=["<|end|>", "\\n"], @@ -722,9 +735,10 @@ def test_apply_overrides_both_lists_empty_preserves_defaults(serving_chat, mocke assert result.stop_token_ids == [2, 32000] -def test_build_sampling_params_list_empty_stop_preserves_yaml(serving_chat, mock_request): +def test_build_sampling_params_list_empty_stop_preserves_yaml(mock_engine_client, mock_request): """Test that empty stop list in request preserves YAML defaults via _build_sampling_params_list_from_request.""" + serving_chat = build_serving_chat(engine_client=mock_engine_client) mock_request.stop = [] mock_request.stop_token_ids = [] @@ -736,21 +750,18 @@ def test_build_sampling_params_list_empty_stop_preserves_yaml(serving_chat, mock assert comprehension_params.stop_token_ids == [] -def test_to_sampling_params_list_pads_missing_tail_stage_with_defaults(): +def test_to_sampling_params_list_pads_missing_tail_stage_with_defaults(mocker: MockerFixture): """AURA callers may pass 3 semantic model params for a 4-stage engine pipeline.""" - from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat - - instance = object.__new__(OmniOpenAIServingChat) default_params = [ SamplingParams(max_tokens=10), SamplingParams(max_tokens=20), SamplingParams(max_tokens=30), SamplingParams(max_tokens=40), ] - instance.engine_client = SimpleNamespace( - stage_configs=[SimpleNamespace(stage_type="llm") for _ in range(4)], - default_sampling_params_list=default_params, - ) + engine_client = mocker.MagicMock() + engine_client.stage_configs = [SimpleNamespace(stage_type="llm") for _ in range(4)] + engine_client.default_sampling_params_list = default_params + instance = build_serving_chat(engine_client=engine_client) result = instance._to_sampling_params_list( [ @@ -772,44 +783,35 @@ def test_to_sampling_params_list_pads_missing_tail_stage_with_defaults(): def test_get_comprehension_stage_index_finds_first_stage(mock_engine_client): """Test finding comprehension stage when it's at index 0.""" - from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat - - instance = object.__new__(OmniOpenAIServingChat) - instance.engine_client = mock_engine_client + instance = build_serving_chat(engine_client=mock_engine_client) assert instance._get_comprehension_stage_index() == 0 def test_get_comprehension_stage_index_finds_second_stage(mocker: MockerFixture): """Test finding comprehension stage when it's at index 1.""" - from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat - - instance = object.__new__(OmniOpenAIServingChat) - other = mocker.MagicMock() other.is_comprehension = False comprehension = mocker.MagicMock() comprehension.is_comprehension = True - instance.engine_client = mocker.MagicMock() - instance.engine_client.stage_configs = [other, comprehension] + engine_client = mocker.MagicMock() + engine_client.stage_configs = [other, comprehension] + instance = build_serving_chat(engine_client=engine_client) assert instance._get_comprehension_stage_index() == 1 def test_get_comprehension_stage_index_raises_when_not_found(mocker: MockerFixture): """Test that ValueError is raised when no comprehension stage exists.""" - from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat - - instance = object.__new__(OmniOpenAIServingChat) - stage1 = mocker.MagicMock() stage1.is_comprehension = False stage2 = mocker.MagicMock() stage2.is_comprehension = False - instance.engine_client = mocker.MagicMock() - instance.engine_client.stage_configs = [stage1, stage2] + engine_client = mocker.MagicMock() + engine_client.stage_configs = [stage1, stage2] + instance = build_serving_chat(engine_client=engine_client) with pytest.raises(ValueError, match="No comprehension stage"): instance._get_comprehension_stage_index() diff --git a/tests/entrypoints/test_stream_finish_reason.py b/tests/entrypoints/test_stream_finish_reason.py index b02936838..97ed2370b 100644 --- a/tests/entrypoints/test_stream_finish_reason.py +++ b/tests/entrypoints/test_stream_finish_reason.py @@ -16,19 +16,20 @@ Key invariants tested: - voice/speaker parameter compatibility in chat completions """ -import json import time from unittest.mock import MagicMock import pytest -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, - ChatCompletionResponseStreamChoice, -) -from vllm.entrypoints.openai.engine.protocol import DeltaMessage -from vllm.entrypoints.openai.models.serving import OpenAIServingModels +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.outputs import CompletionOutput, RequestOutput +from tests.helpers.serving_chat import ( + build_serving_chat, + collect_stream, + make_request, + make_text_omni_output, + parse_sse_chunks, +) from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat from vllm_omni.outputs import OmniRequestOutput @@ -40,43 +41,6 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] # --------------------------------------------------------------------------- -def _make_text_omni_output( - request_id: str = "test-req", - text: str = "hello", - token_ids: list[int] | None = None, - finish_reason: str | None = None, - index: int = 0, - num_prompt_tokens: int = 3, -) -> OmniRequestOutput: - """Build an OmniRequestOutput wrapping a text RequestOutput.""" - if token_ids is None: - token_ids = [10, 11, 12] - res = RequestOutput( - request_id=request_id, - prompt="test", - prompt_token_ids=list(range(num_prompt_tokens)), - prompt_logprobs=None, - outputs=[ - CompletionOutput( - index=index, - text=text, - token_ids=token_ids, - cumulative_logprob=0.0, - logprobs=None, - finish_reason=finish_reason, - stop_reason=None, - ) - ], - finished=finish_reason is not None, - ) - return OmniRequestOutput.from_stage_output( - res, - request_id=request_id, - final_output_type="text", - finished=finish_reason is not None, - ) - - def _make_audio_omni_output( request_id: str = "test-req", index: int = 0, @@ -118,82 +82,6 @@ def _make_audio_omni_output( ) -def _mock_audio_choices(index: int = 0, role: str = "assistant"): - return [ - ChatCompletionResponseStreamChoice( - index=index, - delta=DeltaMessage(role=role, content="dGVzdA=="), - logprobs=None, - finish_reason="stop", - ) - ] - - -def _build_serving_chat(): - """Create a minimal OmniOpenAIServingChat for testing.""" - mock_engine = MagicMock() - mock_engine.errored = False - - models = OpenAIServingModels( - engine_client=mock_engine, - base_model_paths=[], - ) - mock_render = MagicMock() - - instance = OmniOpenAIServingChat( - engine_client=mock_engine, - models=models, - response_role="assistant", - online_renderer=mock_render, - request_logger=None, - chat_template=None, - chat_template_content_format="auto", - ) - instance._create_audio_choice = MagicMock( - side_effect=lambda omni_res, role, request, stream=False: _mock_audio_choices( - index=omni_res.outputs[0].index, - role=role, - ) - ) - return instance - - -def _make_request(modalities: list[str], n: int = 1) -> ChatCompletionRequest: - req = ChatCompletionRequest( - model="test-model", - messages=[{"role": "user", "content": "hello"}], - n=n, - stream=True, - ) - req.modalities = modalities # type: ignore[attr-defined] - return req - - -def _parse_sse_chunks(lines: list[str]) -> list[dict]: - """Parse SSE lines into JSON dicts.""" - prefix = "data: " - chunks = [] - for line in lines: - line = line.strip() - if not line.startswith(prefix): - continue - payload = line[len(prefix) :].strip() - if payload == "[DONE]": - continue - try: - chunks.append(json.loads(payload)) - except json.JSONDecodeError: - pass - return chunks - - -async def _collect_stream(gen): - result = [] - async for item in gen: - result.append(item) - return result - - # --------------------------------------------------------------------------- # Tests: finish_reason correctness # --------------------------------------------------------------------------- @@ -202,14 +90,14 @@ async def _collect_stream(gen): @pytest.mark.asyncio async def test_single_modality_text_only_one_stop(): """Text-only streaming: exactly one chunk has finish_reason='stop'.""" - serving_chat = _build_serving_chat() - request = _make_request(modalities=["text"]) + serving_chat = build_serving_chat() + request = make_request(modalities=["text"]) async def result_generator(): - yield _make_text_omni_output(text="he", token_ids=[10, 11], finish_reason=None) - yield _make_text_omni_output(text="llo", token_ids=[12], finish_reason="stop") + yield make_text_omni_output(text="he", token_ids=[10, 11], finish_reason=None) + yield make_text_omni_output(text="llo", token_ids=[12], finish_reason="stop") - raw_lines = await _collect_stream( + raw_lines = await collect_stream( serving_chat.chat_completion_stream_generator( request=request, result_generator=result_generator(), @@ -221,7 +109,7 @@ async def test_single_modality_text_only_one_stop(): ) ) - chunks = _parse_sse_chunks(raw_lines) + chunks = parse_sse_chunks(raw_lines) finish_reasons = [c["choices"][0]["finish_reason"] for c in chunks if c.get("choices")] assert finish_reasons[-1] == "stop" @@ -233,15 +121,15 @@ async def test_single_modality_text_only_one_stop(): @pytest.mark.asyncio async def test_multi_modal_text_audio_only_last_stop(): """text+audio: text finish sends finish_reason=null, audio sends stop.""" - serving_chat = _build_serving_chat() - request = _make_request(modalities=["text", "audio"]) + serving_chat = build_serving_chat() + request = make_request(modalities=["text", "audio"]) async def result_generator(): - yield _make_text_omni_output(text="he", token_ids=[10, 11], finish_reason=None) - yield _make_text_omni_output(text="llo", token_ids=[12], finish_reason="stop") + yield make_text_omni_output(text="he", token_ids=[10, 11], finish_reason=None) + yield make_text_omni_output(text="llo", token_ids=[12], finish_reason="stop") yield _make_audio_omni_output() - raw_lines = await _collect_stream( + raw_lines = await collect_stream( serving_chat.chat_completion_stream_generator( request=request, result_generator=result_generator(), @@ -253,7 +141,7 @@ async def test_multi_modal_text_audio_only_last_stop(): ) ) - chunks = _parse_sse_chunks(raw_lines) + chunks = parse_sse_chunks(raw_lines) finish_reasons = [ch["finish_reason"] for c in chunks for ch in c.get("choices", [])] assert finish_reasons.count("stop") == 1 @@ -269,18 +157,18 @@ async def test_multi_modal_text_audio_only_last_stop(): @pytest.mark.asyncio async def test_multi_modal_n2_independent_per_choice(): """n=2 with text+audio: each choice gets exactly one stop, at the end.""" - serving_chat = _build_serving_chat() - request = _make_request(modalities=["text", "audio"], n=2) + serving_chat = build_serving_chat() + request = make_request(modalities=["text", "audio"], n=2) async def result_generator(): - yield _make_text_omni_output(text="A", token_ids=[10], finish_reason=None, index=0) - yield _make_text_omni_output(text="B", token_ids=[20], finish_reason=None, index=1) - yield _make_text_omni_output(text="", token_ids=[11], finish_reason="stop", index=0) - yield _make_text_omni_output(text="", token_ids=[21], finish_reason="stop", index=1) + yield make_text_omni_output(text="A", token_ids=[10], finish_reason=None, index=0) + yield make_text_omni_output(text="B", token_ids=[20], finish_reason=None, index=1) + yield make_text_omni_output(text="", token_ids=[11], finish_reason="stop", index=0) + yield make_text_omni_output(text="", token_ids=[21], finish_reason="stop", index=1) yield _make_audio_omni_output(index=0) yield _make_audio_omni_output(index=1) - raw_lines = await _collect_stream( + raw_lines = await collect_stream( serving_chat.chat_completion_stream_generator( request=request, result_generator=result_generator(), @@ -292,7 +180,7 @@ async def test_multi_modal_n2_independent_per_choice(): ) ) - chunks = _parse_sse_chunks(raw_lines) + chunks = parse_sse_chunks(raw_lines) per_choice: dict[int, list] = {} for c in chunks: for ch in c.get("choices", []): @@ -306,13 +194,13 @@ async def test_multi_modal_n2_independent_per_choice(): @pytest.mark.asyncio async def test_single_modality_audio_only_one_stop(): """Audio-only streaming: the audio chunk carries finish_reason='stop'.""" - serving_chat = _build_serving_chat() - request = _make_request(modalities=["audio"]) + serving_chat = build_serving_chat() + request = make_request(modalities=["audio"]) async def result_generator(): yield _make_audio_omni_output() - raw_lines = await _collect_stream( + raw_lines = await collect_stream( serving_chat.chat_completion_stream_generator( request=request, result_generator=result_generator(), @@ -324,7 +212,7 @@ async def test_single_modality_audio_only_one_stop(): ) ) - chunks = _parse_sse_chunks(raw_lines) + chunks = parse_sse_chunks(raw_lines) finish_reasons = [ch["finish_reason"] for c in chunks for ch in c.get("choices", [])] assert finish_reasons.count("stop") == 1 @@ -349,8 +237,8 @@ async def test_streaming_audio_metrics_resolve_replica_id( import vllm_omni.entrypoints.openai.serving_chat as serving_chat_mod from vllm_omni.entrypoints.client_request_state import ClientRequestState - serving_chat = _build_serving_chat() - request = _make_request(modalities=["audio"]) + serving_chat = build_serving_chat() + request = make_request(modalities=["audio"]) req_state = ClientRequestState( request_id="internal-req", external_request_id="test-req", @@ -377,7 +265,7 @@ async def test_streaming_audio_metrics_resolve_replica_id( async def result_generator(): yield _make_audio_omni_output(stage_id=2, replica_id=output_replica_id, audio_samples=2400) - await _collect_stream( + await collect_stream( serving_chat.chat_completion_stream_generator( request=request, result_generator=result_generator(), @@ -408,15 +296,15 @@ async def test_streaming_audio_metrics_resolve_replica_id( async def test_declared_modality_not_produced_emits_fallback_stop(): """If request.modalities declares ["text","audio"] but engine only produces text, a fallback stop chunk is emitted at stream end.""" - serving_chat = _build_serving_chat() - request = _make_request(modalities=["text", "audio"]) + serving_chat = build_serving_chat() + request = make_request(modalities=["text", "audio"]) async def result_generator(): # Engine only produces text, no audio output at all - yield _make_text_omni_output(text="hi", token_ids=[10], finish_reason=None) - yield _make_text_omni_output(text="!", token_ids=[11], finish_reason="stop") + yield make_text_omni_output(text="hi", token_ids=[10], finish_reason=None) + yield make_text_omni_output(text="!", token_ids=[11], finish_reason="stop") - raw_lines = await _collect_stream( + raw_lines = await collect_stream( serving_chat.chat_completion_stream_generator( request=request, result_generator=result_generator(), @@ -428,7 +316,7 @@ async def test_declared_modality_not_produced_emits_fallback_stop(): ) ) - chunks = _parse_sse_chunks(raw_lines) + chunks = parse_sse_chunks(raw_lines) finish_reasons = [ch["finish_reason"] for c in chunks for ch in c.get("choices", [])] # Text finish is suppressed (audio not seen yet), but fallback stop @@ -441,15 +329,15 @@ async def test_declared_modality_not_produced_emits_fallback_stop(): async def test_declared_modality_not_produced_text_finish_suppressed(): """When text finishes but audio (declared in modalities) never appears, the text finish chunk has finish_reason=null (suppressed).""" - serving_chat = _build_serving_chat() - request = _make_request(modalities=["text", "audio"]) + serving_chat = build_serving_chat() + request = make_request(modalities=["text", "audio"]) async def result_generator(): - yield _make_text_omni_output(text="hi", token_ids=[10], finish_reason=None) - yield _make_text_omni_output(text="!", token_ids=[11], finish_reason="stop") + yield make_text_omni_output(text="hi", token_ids=[10], finish_reason=None) + yield make_text_omni_output(text="!", token_ids=[11], finish_reason="stop") # No audio output — stream ends - raw_lines = await _collect_stream( + raw_lines = await collect_stream( serving_chat.chat_completion_stream_generator( request=request, result_generator=result_generator(), @@ -461,7 +349,7 @@ async def test_declared_modality_not_produced_text_finish_suppressed(): ) ) - chunks = _parse_sse_chunks(raw_lines) + chunks = parse_sse_chunks(raw_lines) # Find the text finish chunk (content "!") for c in chunks: @@ -485,8 +373,8 @@ async def test_audio_chunk_without_waveform_keeps_stream_alive(): yields (field, value) tuples, so the loop raised ``AttributeError: 'tuple' object has no attribute 'finish_reason'``. """ - serving_chat = _build_serving_chat() - request = _make_request(modalities=["text", "audio"]) + serving_chat = build_serving_chat() + request = make_request(modalities=["text", "audio"]) empty_audio = _make_audio_omni_output() empty_audio.outputs[0].multimodal_output = {"audio": []} @@ -497,11 +385,11 @@ async def test_audio_chunk_without_waveform_keeps_stream_alive(): serving_chat._create_audio_choice = create_audio_choice async def result_generator(): - yield _make_text_omni_output(text="hi", token_ids=[10], finish_reason=None) - yield _make_text_omni_output(text="!", token_ids=[11], finish_reason="stop") + yield make_text_omni_output(text="hi", token_ids=[10], finish_reason=None) + yield make_text_omni_output(text="!", token_ids=[11], finish_reason="stop") yield empty_audio - raw_lines = await _collect_stream( + raw_lines = await collect_stream( serving_chat.chat_completion_stream_generator( request=request, result_generator=result_generator(), @@ -514,7 +402,7 @@ async def test_audio_chunk_without_waveform_keeps_stream_alive(): ) assert not any("Error in chat completion stream generator" in line for line in raw_lines) - chunks = _parse_sse_chunks(raw_lines) + chunks = parse_sse_chunks(raw_lines) finish_reasons = [ch["finish_reason"] for c in chunks for ch in c.get("choices", [])] assert finish_reasons.count("stop") == 1, f"Expected 1 stop, got {finish_reasons}" assert finish_reasons[-1] == "stop" @@ -525,18 +413,18 @@ async def test_audio_choice_error_response_is_not_iterated_as_choices(): """Any ErrorResponse from the audio path is skipped, not iterated.""" from vllm.entrypoints.openai.engine.protocol import ErrorResponse - serving_chat = _build_serving_chat() - request = _make_request(modalities=["text", "audio"]) + serving_chat = build_serving_chat() + request = make_request(modalities=["text", "audio"]) serving_chat._create_audio_choice = MagicMock( side_effect=lambda omni_res, role, request, stream=False: serving_chat._create_error_response("boom") ) async def result_generator(): - yield _make_text_omni_output(text="hi", token_ids=[10], finish_reason="stop") + yield make_text_omni_output(text="hi", token_ids=[10], finish_reason="stop") yield _make_audio_omni_output() - raw_lines = await _collect_stream( + raw_lines = await collect_stream( serving_chat.chat_completion_stream_generator( request=request, result_generator=result_generator(), @@ -550,7 +438,7 @@ async def test_audio_choice_error_response_is_not_iterated_as_choices(): assert isinstance(serving_chat._create_error_response("boom"), ErrorResponse) assert not any("AttributeError" in line for line in raw_lines) - chunks = _parse_sse_chunks(raw_lines) + chunks = parse_sse_chunks(raw_lines) finish_reasons = [ch["finish_reason"] for c in chunks for ch in c.get("choices", [])] assert finish_reasons.count("stop") == 1, f"Expected 1 stop, got {finish_reasons}" diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index 56f7aaccc..51df9ef84 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -770,6 +770,7 @@ class OmniResponse: success: bool = False prompt_tokens: int | None = None cached_tokens: int | None = None + multimodal_tokens: dict[str, int] | None = None logprobs: list | None = None #: HTTP status + error text for the error-handling path (e.g. validator #: rejections); populated when the OpenAI client raises an APIError. @@ -1089,6 +1090,7 @@ class OpenAIClientHandler: result.prompt_tokens = chunk.usage.prompt_tokens if details := getattr(chunk.usage, "prompt_tokens_details", None): result.cached_tokens = details.cached_tokens + result.multimodal_tokens = getattr(details, "multimodal_tokens", None) if audio_data: merged_seg = _merge_base64_audio_to_segment(audio_data) @@ -1121,6 +1123,7 @@ class OpenAIClientHandler: result.prompt_tokens = usage.prompt_tokens if details := getattr(usage, "prompt_tokens_details", None): result.cached_tokens = details.cached_tokens + result.multimodal_tokens = getattr(details, "multimodal_tokens", None) if audio_data: result.audio_bytes = base64.b64decode(audio_data) result.text_content = text_content diff --git a/tests/helpers/serving_chat.py b/tests/helpers/serving_chat.py new file mode 100644 index 000000000..f1f2710b5 --- /dev/null +++ b/tests/helpers/serving_chat.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Shared scaffolding for OmniOpenAIServingChat unit tests. + +Provides factories for a minimal serving-chat instance, chat requests, and +OmniRequestOutput objects, plus SSE stream collection/parsing utilities. +""" + +import json +from unittest.mock import MagicMock + +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ChatCompletionResponseStreamChoice, +) +from vllm.entrypoints.openai.engine.protocol import DeltaMessage +from vllm.entrypoints.openai.models.serving import OpenAIServingModels +from vllm.outputs import CompletionOutput, RequestOutput + +from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat +from vllm_omni.outputs import OmniRequestOutput + + +def make_text_omni_output( + request_id: str = "test-req", + text: str = "hello", + token_ids: list[int] | None = None, + finish_reason: str | None = None, + index: int = 0, + num_prompt_tokens: int = 3, + num_cached_tokens: int | None = None, +) -> OmniRequestOutput: + """Build an OmniRequestOutput wrapping a text RequestOutput.""" + if token_ids is None: + token_ids = [10, 11, 12] + res = RequestOutput( + request_id=request_id, + prompt="test", + prompt_token_ids=list(range(num_prompt_tokens)), + prompt_logprobs=None, + outputs=[ + CompletionOutput( + index=index, + text=text, + token_ids=token_ids, + cumulative_logprob=0.0, + logprobs=None, + finish_reason=finish_reason, + stop_reason=None, + ) + ], + finished=finish_reason is not None, + num_cached_tokens=num_cached_tokens, + ) + return OmniRequestOutput.from_stage_output( + res, + request_id=request_id, + final_output_type="text", + finished=finish_reason is not None, + ) + + +def mock_audio_choices(index: int = 0, role: str = "assistant"): + return [ + ChatCompletionResponseStreamChoice( + index=index, + delta=DeltaMessage(role=role, content="dGVzdA=="), + logprobs=None, + finish_reason="stop", + ) + ] + + +def build_serving_chat( + *, + engine_client=None, + models=None, + online_renderer=None, + response_role: str = "assistant", + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + **init_overrides, +) -> OmniOpenAIServingChat: + """Create a real OmniOpenAIServingChat via its actual __init__. + + engine_client/models/online_renderer default to minimal mocks; pass your + own to wire in test-specific behavior (e.g. engine_client.stage_configs). + Any other OmniOpenAIServingChat.__init__ keyword — enable_auto_tools, + enable_prompt_tokens_details, trust_request_chat_template, etc. — can be + passed through **init_overrides instead of set on the instance after + construction. + """ + if engine_client is None: + engine_client = MagicMock() + engine_client.errored = False + if models is None: + models = OpenAIServingModels( + engine_client=engine_client, + base_model_paths=[], + ) + if online_renderer is None: + online_renderer = MagicMock() + + instance = OmniOpenAIServingChat( + engine_client=engine_client, + models=models, + response_role=response_role, + online_renderer=online_renderer, + request_logger=request_logger, + chat_template=chat_template, + chat_template_content_format=chat_template_content_format, + **init_overrides, + ) + instance._create_audio_choice = MagicMock( + side_effect=lambda omni_res, role, request, stream=False: mock_audio_choices( + index=omni_res.outputs[0].index, + role=role, + ) + ) + return instance + + +def make_request( + modalities: list[str], + n: int = 1, + *, + stream: bool = True, + include_usage: bool = False, +) -> ChatCompletionRequest: + req = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hello"}], + n=n, + stream=stream, + stream_options={"include_usage": True} if include_usage else None, + ) + req.modalities = modalities # type: ignore[attr-defined] + return req + + +def parse_sse_chunks(lines: list[str]) -> list[dict]: + """Parse SSE lines into JSON dicts.""" + prefix = "data: " + chunks = [] + for line in lines: + line = line.strip() + if not line.startswith(prefix): + continue + payload = line[len(prefix) :].strip() + if payload == "[DONE]": + continue + chunks.append(json.loads(payload)) + return chunks + + +async def collect_stream(gen): + result = [] + async for item in gen: + result.append(item) + return result diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index 752d93577..2bcbecc33 100755 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -58,7 +58,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionResponseStreamChoice, ChatMessage, ) -from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat +from vllm.entrypoints.openai.chat_completion.serving import ( + OpenAIServingChat, + _get_mm_token_counts, + _make_prompt_tokens_details, +) from vllm.entrypoints.openai.engine.protocol import ( DeltaFunctionCall, DeltaMessage, @@ -67,7 +71,6 @@ from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse, FunctionCall, FunctionDefinition, - PromptTokenUsageInfo, RequestResponseMetadata, ToolCall, UsageInfo, @@ -730,8 +733,11 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): # Schedule the request and get the result generator. generators: list[AsyncGenerator[RequestOutput, None]] = [] + mm_token_counts: dict[str, int] | None = None try: for i, engine_prompt in enumerate(engine_prompts): + if self.enable_prompt_tokens_details: + mm_token_counts = _get_mm_token_counts(engine_prompt) if hasattr(request, "sampling_params_list"): sampling_params_list = self._to_sampling_params_list(request.sampling_params_list) else: @@ -814,6 +820,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): tokenizer, request_metadata, reasoning_parser, + mm_token_counts=mm_token_counts, raw_request=raw_request, ) @@ -827,6 +834,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): tokenizer, request_metadata, reasoning_parser, + mm_token_counts=mm_token_counts, ) except ValueError as e: return self.create_error_response(e) @@ -1387,6 +1395,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): tokenizer: AnyTokenizer, request_metadata: RequestResponseMetadata, reasoning_parser: ReasoningParser | None = None, + mm_token_counts: dict[str, int] | None = None, raw_request: Request | None = None, ): created_time = int(time.time()) @@ -1407,6 +1416,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): stop_reason_emitted: list[bool] = [False] * num_choices num_prompt_tokens = 0 num_cached_tokens = None + num_cache_creation_tokens = None if self.use_harmony: harmony_parsers = [get_streamable_parser_for_assistant() for _ in range(num_choices)] harmony_tools_streamed = [False] * num_choices @@ -1503,6 +1513,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): num_prompt_tokens += len(res.encoder_prompt_token_ids) num_cached_tokens = res.num_cached_tokens + num_cache_creation_tokens = res.num_cache_creation_tokens # Send first response for each choice with role # NOTE: num_choices defaults to 1 so this usually executes once per request for i in range(num_choices): @@ -2234,8 +2245,12 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): completion_tokens=completion_tokens, total_tokens=num_prompt_tokens + completion_tokens, ) - if self.enable_prompt_tokens_details and num_cached_tokens: - final_usage.prompt_tokens_details = PromptTokenUsageInfo(cached_tokens=num_cached_tokens) + final_usage.prompt_tokens_details = _make_prompt_tokens_details( + self.enable_prompt_tokens_details, + num_cached_tokens, + num_cache_creation_tokens, + mm_token_counts, + ) final_usage_chunk = OmniChatCompletionStreamResponse( id=request_id, @@ -2307,6 +2322,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, reasoning_parser: ReasoningParser | None = None, + mm_token_counts: dict[str, int] | None = None, ) -> ErrorResponse | OmniChatCompletionResponse: created_time = int(time.time()) final_res: RequestOutput | None = None @@ -2361,6 +2377,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): conversation, role, reasoning_parser, + mm_token_counts=mm_token_counts, ) final_res = omni_outputs else: @@ -2462,6 +2479,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): conversation: list[ConversationMessage], role: str, reasoning_parser: ReasoningParser | None = None, + mm_token_counts: dict[str, int] | None = None, ): final_res = omni_outputs if self.tool_call_id_type == "kimi_k2": @@ -2698,8 +2716,12 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): completion_tokens=num_generated_tokens, total_tokens=num_prompt_tokens + num_generated_tokens, ) - if self.enable_prompt_tokens_details and final_res.num_cached_tokens: - usage.prompt_tokens_details = PromptTokenUsageInfo(cached_tokens=final_res.num_cached_tokens) + usage.prompt_tokens_details = _make_prompt_tokens_details( + self.enable_prompt_tokens_details, + final_res.num_cached_tokens, + final_res.num_cache_creation_tokens, + mm_token_counts, + ) prompt_logprobs = clamp_prompt_logprobs(final_res.prompt_logprobs) prompt_token_ids = final_res.prompt_token_ids if request.return_token_ids else None From 73b623f2f7db092053c1c86fe796bed89eb3dc71 Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Fri, 21 Aug 2026 08:50:59 +0800 Subject: [PATCH 099/267] [Core] Add Host Weight Runtime foundation (#6419) Signed-off-by: Hongsheng Liu --- docs/.nav.yml | 2 + docs/design/feature/host_weight_runtime.md | 302 +++++ docs/design/index.md | 14 +- docs/design/module/host_weight_runtime.md | 264 ++++ tests/host_weight_runtime/test_contracts.py | 232 ++++ .../test_filesystem_store.py | 1106 +++++++++++++++ tests/host_weight_runtime/test_runtime.py | 364 +++++ vllm_omni/host_weight_runtime/__init__.py | 139 ++ vllm_omni/host_weight_runtime/config.py | 183 +++ vllm_omni/host_weight_runtime/errors.py | 106 ++ .../filesystem/__init__.py | 11 + .../host_weight_runtime/filesystem/locks.py | 115 ++ .../host_weight_runtime/filesystem/store.py | 1189 +++++++++++++++++ .../host_weight_runtime/filesystem/writer.py | 425 ++++++ vllm_omni/host_weight_runtime/identity.py | 407 ++++++ vllm_omni/host_weight_runtime/inspection.py | 80 ++ vllm_omni/host_weight_runtime/lease.py | 143 ++ vllm_omni/host_weight_runtime/manifest.py | 353 +++++ vllm_omni/host_weight_runtime/outcomes.py | 119 ++ vllm_omni/host_weight_runtime/protocols.py | 228 ++++ vllm_omni/host_weight_runtime/runtime.py | 332 +++++ 21 files changed, 6108 insertions(+), 6 deletions(-) create mode 100644 docs/design/feature/host_weight_runtime.md create mode 100644 docs/design/module/host_weight_runtime.md create mode 100644 tests/host_weight_runtime/test_contracts.py create mode 100644 tests/host_weight_runtime/test_filesystem_store.py create mode 100644 tests/host_weight_runtime/test_runtime.py create mode 100644 vllm_omni/host_weight_runtime/__init__.py create mode 100644 vllm_omni/host_weight_runtime/config.py create mode 100644 vllm_omni/host_weight_runtime/errors.py create mode 100644 vllm_omni/host_weight_runtime/filesystem/__init__.py create mode 100644 vllm_omni/host_weight_runtime/filesystem/locks.py create mode 100644 vllm_omni/host_weight_runtime/filesystem/store.py create mode 100644 vllm_omni/host_weight_runtime/filesystem/writer.py create mode 100644 vllm_omni/host_weight_runtime/identity.py create mode 100644 vllm_omni/host_weight_runtime/inspection.py create mode 100644 vllm_omni/host_weight_runtime/lease.py create mode 100644 vllm_omni/host_weight_runtime/manifest.py create mode 100644 vllm_omni/host_weight_runtime/outcomes.py create mode 100644 vllm_omni/host_weight_runtime/protocols.py create mode 100644 vllm_omni/host_weight_runtime/runtime.py diff --git a/docs/.nav.yml b/docs/.nav.yml index a2c93d534..ded87fe9c 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -113,6 +113,7 @@ nav: - Feature Design: - Runtime and Stage Execution: - Disaggregated Inference: design/feature/disaggregated_inference.md + - Host Weight Runtime: design/feature/host_weight_runtime.md - Async Chunk: design/feature/async_chunk.md - Async Diffusion Output: design/feature/async_diffusion_output.md - Async Omni Output Materialization: design/feature/omni_async_output_materialization.md @@ -169,6 +170,7 @@ nav: - Offloader: design/module/diffusion/offloader.md - Execution Platforms: design/module/execution_platforms.md - Cache Management: design/module/cache_management.md + - Host Weight Runtime: design/module/host_weight_runtime.md - Quantization: design/module/quantization.md - Observability: design/module/observability.md - Profiling: design/module/profiling.md diff --git a/docs/design/feature/host_weight_runtime.md b/docs/design/feature/host_weight_runtime.md new file mode 100644 index 000000000..d26aa6202 --- /dev/null +++ b/docs/design/feature/host_weight_runtime.md @@ -0,0 +1,302 @@ +# Host Weight Runtime + +The Host Weight Runtime is a loader-facing feature for reusing immutable, +runtime-ready model weights across workers on one host. It lets a model +integration replace repeated checkpoint loading and transformation with an +exact lookup of a previously published host representation, while keeping the +canonical model source authoritative. + +This feature is shared infrastructure. Diffusion, autoregressive, speech, and +future model integrations may depend on the top-level +`vllm_omni.host_weight_runtime` package without depending on one another's +loader or execution stack. The internal component and filesystem contracts are +specified in the [Host Weight Runtime module design](../module/host_weight_runtime.md). + +## Status + +The first implementation provides contracts and a CPU local-filesystem store. +It does not enable cached loading for a model by itself. A consumer must still +provide an exact identity, a representation producer, and a model restorer. + +V1 includes: + +- exact runtime-weight identity and immutable manifests; +- coordinated local lookup and one-producer publication; +- descriptor-backed safetensors mmap leases; +- preferred and required resolution policy; +- validation, deny, quarantine, cleanup, and capacity controls; and +- typed reports for every terminal resolution outcome. + +V1 does not include: + +- a public CLI or default-on model integration; +- a generic BF16, FP8, quantized, or model-specific producer; +- CUDA registration, pinned staging, H2D scheduling, or GPU kernels; +- a remote artifact provider or cross-node coordination; +- post-load publication (V1 accepts only pre-load-safe producers); +- automatic eviction; or +- a change to DLO AllGather or no-AllGather behavior. + +## Motivation and use cases + +Model loading can create the same final host representation repeatedly. This +is especially expensive when loading performs checkpoint decoding, tensor +renaming, TP slicing, quantization, packing, or scale construction before GPU +transfer. Independent workers may then retain private CPU copies of identical +runtime weights. + +The Host Weight Runtime is useful when: + +- multiple same-node workers request semantically identical final weights; +- constructing those weights is materially more expensive than validating and + mapping a local artifact; +- the runtime layout can be identified exactly and reproduced deterministically; +- the workers can share immutable file-backed pages through the OS page cache; + and +- canonical loading remains available when policy permits fallback. + +Typical consumers include same-node diffusion replicas, DLO host-weight +sources, and future transformed or quantized model loaders. Request routing and +replica orchestration remain outside this feature. + +This feature is not zero-copy GPU execution. A lease exposes stable CPU tensor +views and mapped ranges. A separate transport still decides whether to use +registered mmap, private pinned staging, synchronous copies, or asynchronous +H2D transfer. + +## Resolution behavior + +The loader resolves the immutable canonical source and computes the exact +requested identity before asking the runtime for a representation. + +```mermaid +flowchart TD + L["Loader resolves canonical source"] --> I["Compute exact WeightArtifactIdentity"] + I --> M{"Runtime mode"} + M -->|"disabled"| C["Canonical loader"] + M -->|"preferred or required"| A{"Validated local artifact?"} + A -->|"yes"| R["Return HostWeightLease"] + A -->|"no"| X["Remote exact artifact (future)"] + X --> P{"Registered producer available?"} + P -->|"yes"| B["Build, validate, and publish atomically"] + B --> R + P -->|"no or recoverable failure"| F{"Policy permits fallback?"} + F -->|"preferred"| C + F -->|"required"| E["Fail startup"] + P -->|"nonretryable failure"| E +``` + +The modes are: + +| Mode | Observable behavior | +| --- | --- | +| `disabled` | Use canonical loading directly without probing storage, identity, credentials, or topology. | +| `preferred` | Return an exact lease when available; otherwise use canonical fallback only for a miss, invalid cache entry, or typed retryable failure. | +| `required` | Fail when an exact lease cannot be acquired. | + +An unsupported capability, semantic identity collision, producer failure, or +publication failure is nonretryable and remains visible even in preferred +mode. Storage policy must never disguise a semantic or configuration error as +a cache miss. + +## Loader and restoration sequence + +The model integration owns the end-to-end loading transaction. The runtime +does not construct a model or invoke the canonical loader. + +```mermaid +sequenceDiagram + participant L as Loader adapter + participant R as HostWeightRuntime + participant S as HostWeightStore + participant P as WeightProducer + participant X as WeightRestorer + participant T as GPU transport + + L->>L: Resolve canonical revision and exact identity + L->>R: resolve(identity, producer) + R->>S: lookup(exact identity) + alt validated hit + S-->>R: HostWeightLease + else miss with allowed producer + R->>S: get_or_build(identity, producer) + S->>P: produce(store-scoped writer) + P-->>S: final-layout tensors and metadata + S-->>R: validated HostWeightLease + else policy permits canonical fallback + R-->>L: CANONICAL_FALLBACK + L->>L: Run canonical loader + else required or nonretryable failure + R-->>L: FAILED report + end + R-->>L: lease and terminal report + L->>X: plan_restore(model, lease) + X-->>L: validation-only restore plan + L->>X: commit() once + L->>T: consume restored CPU tensors + T-->>L: transfer teardown complete + L->>L: close lease +``` + +`plan_restore()` must not mutate the model or lease. `commit() -> None` is the +sole one-shot model mutation. If planning fails, canonical fallback may reuse +the untouched model. If commit begins and fails, the partially hydrated model +must be discarded and canonical fallback must construct a fresh model. + +## Exact representation identity + +The store performs exact lookup. It never silently converts one representation +into another or substitutes the first backing that responds. + +Identity includes: + +- immutable model revision and source fingerprint; +- model component and ownership boundary; +- representation name, dtype, and format metadata; +- final tensor layout and semantic parallel coordinates; +- static adaptation identity; and +- producer implementation, manifest, and restorer schema versions. + +The requested representation is selected by the loader. The store chooses +where to obtain that exact representation: validated local artifact, future +remote materialization, or a registered producer. + +### Parallelism + +Parallel coordinates are included only when they change bytes, shape, layout, +or component ownership. + +| Dimension | Identity and sharing rule | +| --- | --- | +| DP | Exclude DP rank for replicated weights so same-node replicas can acquire the same artifact. Include a coordinate only if DP changes weight ownership. | +| TP | Include TP size and rank when each rank owns different tensor slices or packed layouts. Each distinct TP shard is a separate exact artifact. | +| SP | Include SP size and backend when they change the weight layout. Exclude SP rank when all SP ranks consume identical bytes. | +| PP | Encode component ownership or stage-local layout when PP changes which weights belong to the consumer. | +| EP | Encode expert ownership and layout whenever ranks own different expert weights. | + +The runtime does not infer these rules from process-group topology. The model +adapter is responsible for constructing a correct identity. + +### Quantization and adaptations + +Runtime BF16, FP8, packed quantized weights, and future formats are distinct +representations. Conversion is performed by an explicitly versioned producer, +never by cache coercion. A producer must publish the final layout consumed by +its matching restorer. + +Dynamic LoRA overlays are not part of a reusable base-weight artifact. A +statically merged adapter is cacheable only as a separate identity containing +the adapter fingerprint and merge semantics. + +## Host sharing and GPU transport + +Every process receives its own virtual mappings and `HostWeightLease`, but +workers mapping the same local artifact can share physical file-backed pages +through the OS page cache. This avoids independent private tensor allocations; +it does not guarantee that process PSS reports exactly one model copy. + +The ownership boundary is: + +- the store owns immutable artifact files, publication, and lifecycle; +- the kernel owns page-cache residency and placement; +- the lease owns process-local mappings and the shared artifact lock; and +- transport owns page registration, page locking, private staging, H2D copies, + streams, device buffers, and lease release ordering. + +DLO AllGather and no-AllGather remain transport and execution choices. A DLO +integration may consume a lease, but the Host Weight Runtime does not choose a +parallel collective or orchestrate DP requests. See the +[DLO feature design](offloader/distributed_layerwise_offload.md). + +## Locality and NUMA policy + +The V1 backend is node-local. It accepts an allowlist of known local disk +filesystems plus tmpfs/ramfs, records the detected kernel filesystem type, and +rejects known remote or unknown filesystems. NFS, CIFS, Lustre, Ceph, and other +cross-node mounts cannot silently satisfy the local backend because their +page-cache and advisory-lock behavior does not provide the required node-local +contract. + +A NUMA-specific root creates a separate storage domain but does not by itself +place page-cache pages on that NUMA node. A topology-aware integration must +also pin workers, establish local first touch or prefault, perform transport +registration in the same domain, and collect residency evidence. + +Tmpfs artifacts consume host memory and may consume swap. They must be +accounted as memory-backed storage rather than ordinary disk capacity. + +## Concurrency and timeouts + +One process per exact identity owns a build; other workers wait and then acquire +leases for the published artifact. Publication is invisible until all payloads +and metadata are validated, hashed, fsynced, and atomically renamed. + +`coordination_timeout_seconds` bounds filesystem lock acquisition. It does not +cancel synchronous validation, a producer that has already started, or atomic +publication. A hung in-process producer therefore blocks its owning process and +must be handled by external process supervision. Enforceable producer +cancellation requires a future process-isolated producer contract. + +Leases keep a shared artifact lock for their lifetime. A forked child may close +its inherited descriptors but cannot unlock the parent process's lease. Cleanup +uses a nonblocking exclusive lock and reports an active lease instead of +removing live mappings. + +## Failure and fallback contract + +| Condition | Preferred mode | Required mode | +| --- | --- | --- | +| Exact local hit | Return lease | Return lease | +| Local miss | Try later backing or canonical fallback | Try later backing or fail | +| Invalid or denied artifact | Quarantine/rebuild when possible; otherwise canonical fallback | Quarantine/rebuild or fail | +| Retryable lock, domain, or capacity failure | Canonical fallback | Fail | +| Unsupported producer or backend | Fail | Fail | +| Identity collision | Fail | Fail | +| Producer or publication failure | Fail | Fail | +| Restore planning failure | Canonical fallback may reuse untouched model | Fail | +| Restore commit failure | Discard model; fallback requires a fresh instance | Fail | + +Cache failures should normally reduce performance rather than availability, +but only when the typed failure explicitly permits fallback. Canonical source, +authentication, configuration, and semantic errors remain visible. + +## Consumer integration requirements + +A consumer PR must: + +1. Resolve an immutable canonical model source before cache lookup. +2. Construct the exact final representation identity, including relevant + parallel, quantization, and adaptation semantics. +3. Register a deterministic producer for that identity or support lookup-only + operation. +4. Provide a validation-only restorer with a one-shot commit. +5. Keep GPU transport outside the producer, store, and restorer contracts. +6. Handle preferred fallback and required failure without reusing a model after + a failed restore commit. +7. Retain the lease until every transport operation that may access mapped + memory has completed. +8. Emit and test the terminal resolution report. + +Consumer validation must prove: + +- output parity with canonical loading; +- exact warm-hit work avoidance; +- correct TP/SP/PP/EP ownership and layout identity; +- correct BF16, FP8, quantized, or adaptation semantics; +- shared-backing evidence when host-memory savings are claimed; +- clean unmapping, unregistration, and artifact-lock release; and +- startup, host-memory, and transport effects for the claimed deployment. + +## Rollout + +The rollout is intentionally staged: + +1. Land the neutral contracts and local filesystem store. +2. Add one model-specific producer/restorer integration with parity evidence. +3. Connect eligible DLO or other transport paths without moving transport + ownership into the runtime. +4. Add additional final-layout and quantized producers independently. +5. Introduce remote materialization only through the same local lease contract. + +The architectural decisions and deferred work are tracked in +[RFC #6414](https://github.com/vllm-project/vllm-omni/issues/6414). diff --git a/docs/design/index.md b/docs/design/index.md index acac90458..066d257d1 100644 --- a/docs/design/index.md +++ b/docs/design/index.md @@ -18,6 +18,7 @@ implementation contract; it is not, by itself, a general support claim. ### Runtime and stage execution - [Disaggregated Inference](feature/disaggregated_inference.md) +- [Host Weight Runtime](feature/host_weight_runtime.md) - [Async Chunk](feature/async_chunk.md) - [Async Diffusion Output](feature/async_diffusion_output.md) - [Async Omni Output Materialization](feature/omni_async_output_materialization.md) @@ -88,14 +89,15 @@ The design contracts separate selection mechanics from backend algorithms: - [Model Integration](module/model_integration.md) - [Autoregressive Runtime](module/ar_runtime.md) - Diffusion - - [Overview](module/diffusion/index.md) - - [Runtime](module/diffusion/diffusion_runtime.md) - - [Model Integration](module/diffusion/diffusion_model_integration.md) - - [Continuous Batching](module/diffusion/continuous_batching.md) - - [Parallelism](module/diffusion/parallelism.md) - - [Offloader](module/diffusion/offloader.md) + - [Overview](module/diffusion/index.md) + - [Runtime](module/diffusion/diffusion_runtime.md) + - [Model Integration](module/diffusion/diffusion_model_integration.md) + - [Continuous Batching](module/diffusion/continuous_batching.md) + - [Parallelism](module/diffusion/parallelism.md) + - [Offloader](module/diffusion/offloader.md) - [Execution Platforms](module/execution_platforms.md) - [Cache Management](module/cache_management.md) +- [Host Weight Runtime](module/host_weight_runtime.md) - [Quantization](module/quantization.md) - [Observability](module/observability.md) - [Profiling](module/profiling.md) diff --git a/docs/design/module/host_weight_runtime.md b/docs/design/module/host_weight_runtime.md new file mode 100644 index 000000000..441d2d07e --- /dev/null +++ b/docs/design/module/host_weight_runtime.md @@ -0,0 +1,264 @@ +--- +title: Host Weight Runtime +kind: module +status: draft +owners: + - "@hsliuustc0106" +primary_code_paths: + - vllm_omni/host_weight_runtime/** +related_code_paths: + - vllm_omni/diffusion/model_loader/** + - vllm_omni/diffusion/offloader/** +depends_on: + - model_integration.md + - quantization.md +validation_paths: + - tests/host_weight_runtime/** +upstream_refs: + - https://github.com/vllm-project/vllm-omni/issues/6414 +last_reviewed: 2026-08-21 +--- + +# Host Weight Runtime + +The Host Weight Runtime is a loader-adjacent, transport-independent foundation +for reusing immutable, runtime-ready model weights. It separates the semantic +question—"are these exactly the weights this loader requested?"—from the +physical question—"where is the validated host representation stored?" + +The runtime can save repeated checkpoint loading, transformation, and peak host +memory when several workers need the same final representation. It also creates +a second, versioned path for reconstructing a valid model, so a cache miss or +recoverable store failure normally falls back to the canonical loader. + +This module does not enable host-weight caching for any model by itself. A model +integration must still provide an exact identity, a producer, and a restorer. +For feature behavior and consumer integration requirements, see the +[Host Weight Runtime feature design](../feature/host_weight_runtime.md). + +## Component boundary + +```mermaid +flowchart LR + LA["Model loader adapter"] -->|"exact identity + policy"| RT["HostWeightRuntime"] + RT -->|"lookup / get_or_build"| ST["HostWeightStore"] + PR["WeightProducer"] -->|"runtime-ready tensors"| ST + ST -->|"validated mmap views"| LE["HostWeightLease"] + LE --> RE["WeightRestorer"] + RE --> MO["Restored host model"] + MO --> TR["DLO or another GPU transport"] + + CS["Canonical model source"] --> PR + LA -. "fallback" .-> CS +``` + +The contracts have deliberately narrow ownership: + +- `HostWeightRuntime` applies resolution and fallback policy and emits one + terminal report. It does not own a model or call the canonical loader. +- `HostWeightStore` owns artifact lookup, coordinated construction, + publication, validation, quarantine, and lifecycle. +- `HostWeightLease` owns process-local tensor views, mapped-file resources, and + the shared artifact lock that stabilizes their lifetime. +- `WeightProducer` creates one declared final-layout representation. It writes + only through a store-scoped writer and cannot publish paths directly. +- `WeightRestorer` validates a lease-to-model plan without mutation. Its + one-shot `commit() -> None` is the sole model-mutating phase. Concrete + restorers remain with the model or quantization integration. +- GPU transport owns page registration, private staging, H2D scheduling, and + lease release. DLO must not parse manifests or manage store artifacts. + +## Exact semantic identity + +Storage lookup is exact; the store never coerces one representation into +another. `WeightArtifactIdentity` includes: + +- immutable model revision and source-file fingerprint; +- component and ownership boundary; +- runtime representation, dtype, and representation metadata; +- final tensor layout and semantic parallel coordinates; +- static adaptation identity; and +- producer version, implementation fingerprint, manifest schema, and restorer + schema. + +The identity excludes transfer policy such as registered mmap versus private +pinned staging. Those paths move the same host representation and must not +produce duplicate artifacts. + +Parallel coordinates follow semantics rather than process topology: + +- TP size and rank are included when they change tensor contents or shape. +- DP rank is excluded for replicated weights. +- SP size and backend are included when they affect layout. SP rank is excluded + only when all SP ranks consume an identical representation. +- PP, EP, or future parallel modes must be added by the model adapter whenever + they change component ownership or tensor layout. + +Dynamic LoRA overlays are not cached as base weights. A statically merged +adapter is a distinct artifact identity with its own fingerprint. + +## Deterministic resolution + +The loader first resolves the canonical source and computes the exact requested +identity. The runtime then follows this order: + +```text +validated local artifact + -> exact remote artifact materialized locally (future) + -> registered local producer + -> canonical loader fallback, if policy permits +``` + +An already published local artifact wins because it avoids network transfer and +is already in node-accessible storage. "Wins" means the highest-priority valid +representation, not whichever backing responds first. + +Configuration modes are: + +- `disabled`: return the canonical-direct outcome without probing a root, + credentials, topology, or identity; +- `preferred`: recoverable store failures may return canonical fallback; and +- `required`: fail when no exact lease can be acquired. + +Failure-to-action policy is centralized in `HostWeightRuntime`. A clean miss or +invalid cache artifact may fall back in preferred mode. A typed failure may do +so only when it is marked retryable; unsupported capabilities, identity +collisions, producer failures, and publication failures remain visible as +startup failures. + +`coordination_timeout_seconds` bounds acquisition of lookup and build locks; it +is not a hard wall-clock deadline for in-process validation, production, or +atomic publication. Once this V1 implementation becomes the producer, the +synchronous producer and publication run to completion. A hung producer blocks +its owning process and requires external process supervision; enforceable +producer cancellation requires a future process-isolated producer contract. A +waiter timing out never cancels another process's valid build. + +Remote providers are represented by protocols but are explicitly unsupported +in the first implementation. V1 also invokes only `PRE_LOAD_SAFE` producers; +enabling post-load publication is rejected during configuration until a +separate post-load entry point and model-lifetime contract exist. + +## Restoration transaction boundary + +`WeightRestorer.plan_restore()` is validation-only: it must not modify the model +or lease. The returned plan may be committed exactly once, `commit()` returns +`None`, and commit is the sole phase allowed to mutate the model. Registration, +staging, and GPU transfer remain transport responsibilities rather than restore +side effects. + +If planning fails, preferred-mode canonical fallback may reuse the untouched +model. If commit begins and then fails, the model is considered partially +hydrated and must be discarded; canonical fallback must construct a fresh model +instance instead of reusing it. + +## Filesystem domain and artifact lifecycle + +A storage domain describes physical locality and policy; it is not part of +semantic identity. The first implementation supports node- or NUMA-scoped +roots on verified local disk filesystems or tmpfs. The detected kernel +filesystem type is preserved in `domain.json` and inspection output. Known +remote filesystems and unknown filesystem types are rejected rather than being +silently treated as local disk, because cross-node page cache and advisory-lock +semantics do not satisfy this backend's contract. A NUMA root alone does not +guarantee page placement; CPU pinning, local first-touch/prefault, and residency +evidence belong to a later topology-aware integration. + +```text +/ +├── domain.json +├── domain-policy.json +├── locks/ +│ ├── .build.lock +│ └── .artifact.lock +├── artifacts// +│ ├── READY.json +│ ├── manifest.json +│ └── *.safetensors +├── tmp/ +├── quarantine/ +└── deny/ +``` + +Lock files live outside replaceable artifact directories. Every cooperative +mutation follows one lock order: build lock, then exclusive artifact lock. +Lookup acquires a shared artifact lock, validates `READY` and the manifest, +opens payloads with non-following descriptor-based access, creates mmap-backed +views, and transfers that shared lock to the lease. + +Lease acquisition rechecks the producer and restorer schemas against the exact +identity. Malformed safetensors payloads are reported as typed invalid +artifacts and denied rather than escaping the resolution policy as raw parser +exceptions. + +Each `FileLock` records its creator PID. A forked child closes an inherited lock +descriptor without issuing `LOCK_UN`, because `flock` state belongs to the +shared open-file description and unlocking in the child would also release the +parent's active lease lock. + +Publication follows this lifecycle: + +```text +ABSENT -> invisible temporary build -> fully hashed and fsynced + -> atomic rename -> READY + +READY -> denied -> quarantined -> explicit removal +``` + +`READY.json` is written last in the temporary directory. Readers therefore see +either a complete validated artifact or a miss, never an in-progress build. +Failed process-local builds are left invisible and a later build owner removes +their stale temporary directories. Explicit cleanup is nonblocking and reports +an active builder or lease instead of unlinking live mappings. + +## Validation and capacity + +Every publication computes full file and tensor hashes. Local lookup supports: + +- `manifest_and_metadata`, which validates identity, immutable publication + metadata, file descriptors, sizes, tensor keys, shapes, dtypes, and strides; + and +- `full_checksum`, which additionally reads and hashes every payload file. + +Filesystem verity is modeled but unsupported. Invalid artifacts receive an +external deny marker before their shared lookup lock is released; a build owner +then quarantines the immutable directory under the exclusive lock. +Lookup checks the marker both before validation and after constructing the +lease so a weaker overlapping lookup cannot return a lease after a stronger +validation has denied the artifact. + +`inspect_domain()` and `inspect_artifact()` return typed, read-only snapshots. +Inspection validation does not create a deny marker. Reported lock activity is +point-in-time evidence only; kernel locks are not a persistent owner registry. + +Capacity policy covers all store-owned bytes, including ready, temporary, and +quarantined data. The local writer preflights `max_artifact_bytes`, +`max_store_bytes`, and `min_free_bytes`, and preallocates payloads where the +filesystem supports it. `ENOSPC` remains a normal typed store failure because +concurrent preflight is inherently racy. Automatic eviction and strict +concurrent reservations are not implemented. + +For tmpfs, artifact bytes are host-memory consumption and may consume swap; +they must not be reported operationally as ordinary disk capacity. The store +owns files and reports capacity. The kernel controls page-cache residency, and +transport controls registration or page locking. + +## First implementation scope + +The foundation implements a CPU local filesystem store and +`SINGLE_PROCESS` production for one exact artifact. It intentionally adds no: + +- public CLI or DLO configuration; +- BF16, FP8, or model-specific producer/restorer; +- change to DLO AllGather or no-AllGather behavior; +- CUDA registration, pinned staging, or H2D scheduling; +- remote provider, group-cohort producer, artifact bundle, eviction, or + cross-node coordination. + +Consumer PRs must prove ordinary-loader parity, warm-hit work avoidance, +parallel-layout correctness, clean teardown, shared-backing evidence, and +representation-specific startup and memory behavior. Generic foundation tests +remain CPU-only and make no production performance claim. + +The complete rationale and staged rollout are tracked in +[RFC #6414](https://github.com/vllm-project/vllm-omni/issues/6414). diff --git a/tests/host_weight_runtime/test_contracts.py b/tests/host_weight_runtime/test_contracts.py new file mode 100644 index 000000000..d831863da --- /dev/null +++ b/tests/host_weight_runtime/test_contracts.py @@ -0,0 +1,232 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Contract tests for semantic identity and immutable artifact schemas.""" + +from __future__ import annotations + +import ast +import math +from pathlib import Path +from typing import get_type_hints + +import pytest + +from vllm_omni.host_weight_runtime import ( + AdaptationIdentity, + CanonicalJson, + ComponentIdentity, + FileManifestEntry, + IdentityValidationError, + ManifestValidationError, + ProducerIdentity, + ProductionPolicy, + PublicationMarker, + RuntimeWeightLayout, + TensorKind, + TensorManifestEntry, + WeightArtifactIdentity, + WeightManifest, + WeightRepresentation, + WeightRestorePlan, + WeightRestorer, + WeightSourceIdentity, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_SHA_A = "a" * 64 +_SHA_B = "b" * 64 +_SHA_C = "c" * 64 + + +def _identity( + *, + tp_size: int = 1, + tp_rank: int = 0, + sp_size: int = 1, + adaptation: AdaptationIdentity | None = None, + source_metadata: CanonicalJson | None = None, +) -> WeightArtifactIdentity: + return WeightArtifactIdentity( + schema_version=1, + source=WeightSourceIdentity( + model_id="test-org/test-model", + revision="0123456789abcdef", + fingerprint="source-files-sha256", + metadata=source_metadata or CanonicalJson.empty(), + ), + component=ComponentIdentity(name="transformer", ownership="diffusion.dit"), + representation=WeightRepresentation(name="runtime-bf16", dtype="torch.bfloat16"), + layout=RuntimeWeightLayout( + name="final-module-layout", + tensor_parallel_size=tp_size, + tensor_parallel_rank=tp_rank, + sequence_parallel_size=sp_size, + sequence_parallel_backend="ulysses" if sp_size > 1 else "none", + ), + adaptation=adaptation or AdaptationIdentity(), + producer=ProducerIdentity( + producer_id="test.final-layout", + version="1", + implementation_fingerprint="implementation-sha256", + manifest_schema="test-producer-v1", + restorer_schema="test-restorer-v1", + ), + ) + + +def test_canonical_metadata_is_a_stable_immutable_snapshot() -> None: + items: list[object] = [1, {"value": 2}] + mutable = {"z": items, "a": True} + snapshot = CanonicalJson.from_value(mutable) + items.append(3) + + assert snapshot.encoded == b'{"a":true,"z":[1,{"value":2}]}' + assert snapshot.to_value() == {"a": True, "z": [1, {"value": 2}]} + assert CanonicalJson.from_value({"a": True, "z": [1, {"value": 2}]}) == snapshot + + +@pytest.mark.parametrize("value", [math.inf, math.nan, object(), {1: "non-string-key"}]) +def test_canonical_metadata_rejects_process_specific_or_non_json_values(value: object) -> None: + with pytest.raises(IdentityValidationError): + CanonicalJson.from_value(value) + + +@pytest.mark.parametrize("encoded", [b"not-json", b'{"b":1, "a":2}', b'{"value":NaN}']) +def test_canonical_metadata_cannot_bypass_stable_encoding(encoded: bytes) -> None: + with pytest.raises(IdentityValidationError): + CanonicalJson(encoded) + + +def test_identity_round_trip_and_hash_are_deterministic() -> None: + first = _identity(source_metadata=CanonicalJson.from_value({"files": ["b", "a"], "size": 3})) + second = _identity(source_metadata=CanonicalJson.from_value({"size": 3, "files": ["b", "a"]})) + + assert first == second + assert first.key == second.key + assert WeightArtifactIdentity.from_dict(first.to_dict()) == first + assert len(first.key) == 64 + + +def test_identity_separates_semantic_parallel_layout_and_static_adaptation() -> None: + base = _identity() + tp0 = _identity(tp_size=2, tp_rank=0) + tp1 = _identity(tp_size=2, tp_rank=1) + sp2 = _identity(sp_size=2) + merged_lora = _identity(adaptation=AdaptationIdentity(kind="merged-lora", fingerprint="adapter-sha256")) + + assert len({item.key for item in (base, tp0, tp1, sp2, merged_lora)}) == 5 + assert "data_parallel" not in base.layout.to_dict() + assert "sequence_parallel_rank" not in sp2.layout.to_dict() + + +def test_identity_schema_rejects_unknown_fields() -> None: + document = _identity().to_dict() + document["transfer_mode"] = "registered" + + with pytest.raises(IdentityValidationError, match="unexpected"): + WeightArtifactIdentity.from_dict(document) + + +def test_identity_schema_does_not_coerce_field_types() -> None: + document = _identity().to_dict() + document["schema_version"] = "1" + + with pytest.raises(IdentityValidationError, match="must be an integer"): + WeightArtifactIdentity.from_dict(document) + + +def test_restoration_contract_has_a_validation_only_one_shot_commit_boundary() -> None: + assert get_type_hints(WeightRestorePlan.commit)["return"] is type(None) + assert "one-shot" in (WeightRestorePlan.__doc__ or "") + assert "without mutating" in (WeightRestorer.__doc__ or "") + + +def test_post_load_publication_policy_is_explicitly_unsupported() -> None: + with pytest.raises(ValueError, match="post-load publication is not implemented"): + ProductionPolicy(allow_post_load_publish=True) + + +def test_manifest_and_publication_marker_round_trip_exactly() -> None: + identity = _identity() + manifest = WeightManifest( + schema_version=1, + artifact_key=identity.key, + identity=identity, + producer_schema="test-producer-v1", + restorer_schema="test-restorer-v1", + tensors=( + TensorManifestEntry( + name="layer.weight", + kind=TensorKind.PARAMETER, + role="weight", + file_name="weights.safetensors", + storage_key="layer.weight", + shape=(2, 4), + dtype="torch.bfloat16", + stride=(4, 1), + nbytes=16, + sha256=_SHA_A, + ), + ), + files=(FileManifestEntry(file_name="weights.safetensors", size=128, sha256=_SHA_B),), + artifact_content_sha256=_SHA_C, + format_metadata=CanonicalJson.from_value({"quantization": "none"}), + ) + marker = PublicationMarker( + schema_version=1, + publication_id="publication-id", + manifest_sha256=_SHA_A, + artifact_content_sha256=_SHA_C, + integrity_mode="full_checksum", + ) + + assert WeightManifest.from_dict(manifest.to_dict()) == manifest + assert PublicationMarker.from_dict(marker.to_dict()) == marker + + document = manifest.to_dict() + document["unknown"] = True + with pytest.raises(ManifestValidationError, match="unexpected"): + WeightManifest.from_dict(document) + + malformed = manifest.to_dict() + tensors = malformed["tensors"] + assert isinstance(tensors, list) and isinstance(tensors[0], dict) + tensors[0]["nbytes"] = "16" + with pytest.raises(ManifestValidationError, match="must be an integer"): + WeightManifest.from_dict(malformed) + + +@pytest.mark.parametrize("file_name", ["", ".", "..", "../weights.safetensors", "nested/weights.safetensors"]) +def test_manifest_rejects_unsafe_payload_names(file_name: str) -> None: + with pytest.raises(ManifestValidationError, match="safe relative basename"): + FileManifestEntry(file_name=file_name, size=1, sha256=_SHA_A) + + +def test_package_source_keeps_the_neutral_dependency_boundary() -> None: + package_root = Path(__file__).resolve().parents[2] / "vllm_omni" / "host_weight_runtime" + assert package_root.is_dir() + forbidden = ( + "vllm_omni.diffusion", + "vllm_omni.distributed", + "torch.cuda", + "torch.distributed", + ) + violations: list[str] = [] + for source_path in package_root.rglob("*.py"): + tree = ast.parse(source_path.read_text(encoding="utf-8"), filename=str(source_path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + violations.extend( + f"{source_path.name}:{node.lineno}:{alias.name}" + for alias in node.names + if alias.name.startswith(forbidden) + ) + elif isinstance(node, ast.ImportFrom) and node.level == 0: + module = node.module or "" + if module.startswith(forbidden) or ( + module == "torch" and any(alias.name in {"cuda", "distributed"} for alias in node.names) + ): + violations.append(f"{source_path.name}:{node.lineno}:{module}") + + assert violations == [] diff --git a/tests/host_weight_runtime/test_filesystem_store.py b/tests/host_weight_runtime/test_filesystem_store.py new file mode 100644 index 000000000..763d2622a --- /dev/null +++ b/tests/host_weight_runtime/test_filesystem_store.py @@ -0,0 +1,1106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Lifecycle and concurrency tests for the local filesystem store.""" + +from __future__ import annotations + +import contextlib +import errno +import hashlib +import json +import multiprocessing as mp +import os +import shutil +import threading +import time +import warnings +from collections.abc import Sequence +from dataclasses import replace +from multiprocessing.process import BaseProcess +from pathlib import Path + +import pytest +import torch + +from vllm_omni.host_weight_runtime import ( + AdaptationIdentity, + ArtifactInventoryState, + BuildRequest, + CapacityPolicy, + ComponentIdentity, + CoordinationScope, + FailureCode, + HostWeightError, + HostWeightLease, + IntegrityPolicy, + ProducerIdentity, + ProductionMetadata, + ProductionSourceMode, + ResolutionStage, + RuntimeWeightLayout, + StorageClass, + StorageDomainPolicy, + StorageScope, + StoreResult, + StoreStatus, + TensorKind, + TensorWriteSpec, + ValidationLevel, + WeightArtifactIdentity, + WeightProductionSpec, + WeightRepresentation, + WeightSourceIdentity, +) +from vllm_omni.host_weight_runtime.filesystem import ( + FilesystemHostWeightStore, + detect_filesystem_type, + detect_storage_class, +) +from vllm_omni.host_weight_runtime.filesystem import store as filesystem_store_module +from vllm_omni.host_weight_runtime.filesystem.locks import FileLock + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +# Spawning imports the complete vLLM package and exceeded 10 seconds on a loaded +# CI worker. Keep process-bootstrap bounds separate from the short store +# deadlines exercised by these tests. +_PROCESS_START_TIMEOUT_SECONDS = 60.0 +_PROCESS_COMPLETION_TIMEOUT_SECONDS = 60.0 +_PROCESS_CLEANUP_TIMEOUT_SECONDS = 5.0 + + +def _identity(*, tp_rank: int = 0) -> WeightArtifactIdentity: + return WeightArtifactIdentity( + schema_version=1, + source=WeightSourceIdentity("test-org/test-model", "0123456789abcdef", "source-files-sha256"), + component=ComponentIdentity("transformer", "diffusion.dit"), + representation=WeightRepresentation("runtime-bf16", "torch.bfloat16"), + layout=RuntimeWeightLayout( + "final-module-layout", + tensor_parallel_size=2, + tensor_parallel_rank=tp_rank, + ), + adaptation=AdaptationIdentity(), + producer=ProducerIdentity( + "test.final-layout", + "1", + "implementation-sha256", + "test-producer-v1", + "test-restorer-v1", + ), + ) + + +def _weight(offset: int = 0) -> torch.Tensor: + return torch.arange(offset, offset + 12, dtype=torch.float32).to(torch.bfloat16).reshape(4, 3) + + +def _specs() -> tuple[TensorWriteSpec, ...]: + return ( + TensorWriteSpec("layer.weight", (4, 3), torch.bfloat16), + TensorWriteSpec("layer.scale", (2,), torch.float32, kind=TensorKind.BUFFER, role="scale"), + ) + + +def _open_fd_targets() -> tuple[str, ...]: + targets = [] + for path in Path("/proc/self/fd").iterdir(): + with contextlib.suppress(OSError): + targets.append(os.readlink(path)) + return tuple(targets) + + +class FakeProducer: + def __init__( + self, + identity: WeightArtifactIdentity, + *, + counter_path: Path | None = None, + delay_seconds: float = 0.0, + started: object | None = None, + release: object | None = None, + write_mode: str = "rows", + weight_offset: int = 0, + coordination_scope: CoordinationScope = CoordinationScope.SINGLE_PROCESS, + ) -> None: + self._spec = WeightProductionSpec( + producer_id="test.final-layout", + outputs=(identity,), + source_mode=ProductionSourceMode.FINALIZED_MODEL, + coordination_scope=coordination_scope, + ) + self.counter_path = counter_path + self.delay_seconds = delay_seconds + self.started = started + self.release = release + self.write_mode = write_mode + self.weight_offset = weight_offset + + @property + def spec(self) -> WeightProductionSpec: + return self._spec + + def produce(self, writer: object) -> ProductionMetadata: + if self.counter_path is not None: + fd = os.open(self.counter_path, os.O_CREAT | os.O_APPEND | os.O_WRONLY | os.O_CLOEXEC, 0o600) + try: + os.write(fd, f"{os.getpid()}\n".encode()) + os.fsync(fd) + finally: + os.close(fd) + if self.started is not None: + self.started.set() # type: ignore[attr-defined] + if self.release is not None and not self.release.wait(timeout=10): # type: ignore[attr-defined] + raise TimeoutError("test producer was not released") + if self.delay_seconds: + time.sleep(self.delay_seconds) + + with writer.open_tensor_file("weights.safetensors", _specs()) as output: # type: ignore[attr-defined] + if self.write_mode == "rows": + output.write_rows("layer.weight", 0, _weight(self.weight_offset)[:2]) + output.write_rows("layer.weight", 2, _weight(self.weight_offset)[2:]) + elif self.write_mode == "complete": + output.write_tensor("layer.weight", _weight(self.weight_offset)) + elif self.write_mode == "incomplete": + output.write_rows("layer.weight", 0, _weight(self.weight_offset)[:2]) + elif self.write_mode == "overlap": + output.write_rows("layer.weight", 0, _weight(self.weight_offset)[:3]) + output.write_rows("layer.weight", 2, _weight(self.weight_offset)[2:]) + else: # pragma: no cover - test helper guard + raise AssertionError(f"unknown write mode {self.write_mode}") + output.write_tensor("layer.scale", torch.tensor([1.0, 2.0])) + return ProductionMetadata("test-producer-v1", "test-restorer-v1") + + +class CrashProducer(FakeProducer): + def produce(self, writer: object) -> ProductionMetadata: + output = writer.open_tensor_file("weights.safetensors", _specs()) # type: ignore[attr-defined] + output.write_rows("layer.weight", 0, _weight()[:2]) + os._exit(17) + + +class NonCooperativeStore(FilesystemHostWeightStore): + """Test double representing a publisher that does not join the build lock.""" + + def _build_lock_path(self, key: str) -> Path: + return self.locks_dir / f"{key}.noncooperative-build.lock" + + def _remove_stale_temps_locked(self, _key: str) -> None: + return + + +class InitialLookupBarrierStore(FilesystemHostWeightStore): + """Make both test workers observe the initial miss before either builds.""" + + def __init__( + self, + domain: StorageDomainPolicy, + capacity: CapacityPolicy, + integrity: IntegrityPolicy, + *, + initial_lookup_barrier: object, + ) -> None: + super().__init__(domain, capacity, integrity) + self._initial_lookup_barrier = initial_lookup_barrier + self._lookup_count = 0 + + def lookup( + self, + identity: WeightArtifactIdentity, + *, + validation: ValidationLevel, + deadline: float | None = None, + ) -> StoreResult: + result = super().lookup(identity, validation=validation, deadline=deadline) + self._lookup_count += 1 + if self._lookup_count == 1: + self._initial_lookup_barrier.wait(timeout=10) # type: ignore[attr-defined] + return result + + +class PausingOpenLeaseStore(FilesystemHostWeightStore): + """Pause after opening a lease so another lookup can publish a deny marker.""" + + def __init__( + self, + domain: StorageDomainPolicy, + capacity: CapacityPolicy, + integrity: IntegrityPolicy, + *, + lease_opened: threading.Event, + resume_lookup: threading.Event, + ) -> None: + super().__init__(domain, capacity, integrity) + self._lease_opened = lease_opened + self._resume_lookup = resume_lookup + + def _open_lease( + self, + identity: WeightArtifactIdentity, + entry_path: Path, + artifact_lock: FileLock, + validation: ValidationLevel, + ) -> HostWeightLease: + lease = super()._open_lease(identity, entry_path, artifact_lock, validation) + self._lease_opened.set() + if not self._resume_lookup.wait(5): + lease.close() + raise TimeoutError("test did not resume paused lookup") + return lease + + +def _make_store( + root: Path, + *, + capacity: CapacityPolicy | None = None, + integrity: IntegrityPolicy | None = None, +) -> FilesystemHostWeightStore: + root.mkdir(parents=True, exist_ok=True, mode=0o700) + domain = StorageDomainPolicy(root=root, storage_class=detect_storage_class(root)) + return FilesystemHostWeightStore(domain, capacity or CapacityPolicy(), integrity or IntegrityPolicy()) + + +def _build_process(root: str, counter: str, initial_lookup_barrier: object, result_queue: object) -> None: + try: + identity = _identity() + root_path = Path(root) + root_path.mkdir(parents=True, exist_ok=True, mode=0o700) + store = InitialLookupBarrierStore( + StorageDomainPolicy(root=root_path, storage_class=detect_storage_class(root_path)), + CapacityPolicy(), + IntegrityPolicy(), + initial_lookup_barrier=initial_lookup_barrier, + ) + result = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity, counter_path=Path(counter), delay_seconds=0.25), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 15, + ) + fingerprint = result.lease.provenance.backing_fingerprint if result.lease is not None else None + if result.lease is not None: + result.lease.close() + result_queue.put((result.status.value, fingerprint, None)) # type: ignore[attr-defined] + except BaseException as exc: + result_queue.put(("error", None, repr(exc))) # type: ignore[attr-defined] + + +def _controlled_build_process(root: str, started: object, release: object, result_queue: object) -> None: + try: + identity = _identity() + store = _make_store(Path(root)) + result = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity, started=started, release=release), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 15, + ) + if result.lease is not None: + result.lease.close() + result_queue.put((result.status.value, None)) # type: ignore[attr-defined] + except BaseException as exc: + result_queue.put(("error", repr(exc))) # type: ignore[attr-defined] + + +def _crash_build_process(root: str) -> None: + identity = _identity() + store = _make_store(Path(root)) + store.get_or_build( + BuildRequest(identity), + CrashProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 15, + ) + + +def _join_processes( + processes: Sequence[BaseProcess], + timeout: float = _PROCESS_COMPLETION_TIMEOUT_SECONDS, +) -> None: + deadline = time.monotonic() + timeout + for process in processes: + process.join(max(0.0, deadline - time.monotonic())) + timed_out = [process for process in processes if process.is_alive()] + if not timed_out: + return + timed_out_workers = ", ".join(f"{process.name}(pid={process.pid})" for process in timed_out) + for process in timed_out: + process.terminate() + for process in timed_out: + process.join(_PROCESS_CLEANUP_TIMEOUT_SECONDS) + survivors = [process for process in timed_out if process.is_alive()] + for process in survivors: + process.kill() + process.join(_PROCESS_CLEANUP_TIMEOUT_SECONDS) + unstoppable = [process for process in survivors if process.is_alive()] + assert not unstoppable, "timed-out test worker processes could not be stopped" + pytest.fail(f"test worker processes exceeded the {timeout:g}-second group deadline: {timed_out_workers}") + + +def test_build_lookup_share_one_published_backing_and_lease_blocks_cleanup(tmp_path: Path) -> None: + identity = _identity() + store = _make_store(tmp_path / "store") + + built = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert built.status is StoreStatus.BUILT + assert built.lease is not None + assert torch.equal(built.lease.tensors["layer.weight"], _weight()) + assert torch.equal(built.lease.tensors["layer.scale"], torch.tensor([1.0, 2.0])) + assert built.lease.mapped_regions + assert all(region.address > 0 and region.length > 0 and region.read_only for region in built.lease.mapped_regions) + with pytest.raises(TypeError, match="process-local"): + built.lease.__reduce_ex__(4) + + second = store.lookup(identity, validation=ValidationLevel.FULL_CHECKSUM) + assert second.status is StoreStatus.HIT + assert second.lease is not None + assert second.lease.provenance.publication_id == built.lease.provenance.publication_id + assert second.lease.provenance.backing_fingerprint == built.lease.provenance.backing_fingerprint + payload = store.artifacts_dir / identity.key / "weights.safetensors" + assert any(str(payload) in target for target in _open_fd_targets()) + + artifact = store.inspect_artifact(identity, validation=ValidationLevel.FULL_CHECKSUM) + assert artifact.status is StoreStatus.HIT + assert artifact.manifest is not None and artifact.manifest.identity == identity + assert artifact.publication_id == built.lease.provenance.publication_id + assert artifact.artifact_lock_active + domain = store.inspect_domain() + assert domain.domain_uuid == built.lease.provenance.domain_uuid + assert domain.filesystem_type == detect_filesystem_type(store.root) + assert any( + item.artifact_key == identity.key and item.state is ArtifactInventoryState.READY and item.size_bytes > 0 + for item in domain.inventory + ) + + active = store.cleanup(identity) + assert active is not None and active.code is FailureCode.ACTIVE_LEASE + second.lease.close() + built.lease.close() + built.lease.close() + assert not built.lease.tensors + assert not any(str(payload) in target for target in _open_fd_targets()) + + assert store.cleanup(identity) is None + assert store.lookup(identity, validation=ValidationLevel.FULL_CHECKSUM).status is StoreStatus.MISS + + +def test_concurrent_lease_close_waits_for_resource_teardown(tmp_path: Path) -> None: + identity = _identity() + store = _make_store(tmp_path / "store") + built = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert built.lease is not None + lease = built.lease + teardown_started = threading.Event() + allow_teardown = threading.Event() + second_started = threading.Event() + second_returned = threading.Event() + errors: list[BaseException] = [] + + def blocking_closer() -> None: + teardown_started.set() + if not allow_teardown.wait(5): + raise TimeoutError("test did not release lease teardown") + + def close_lease(*, started: threading.Event | None = None, returned: threading.Event | None = None) -> None: + if started is not None: + started.set() + try: + lease.close() + except BaseException as exc: + errors.append(exc) + finally: + if returned is not None: + returned.set() + + lease._resource_closers = (*lease._resource_closers, blocking_closer) + first = threading.Thread(target=close_lease) + second = threading.Thread(target=close_lease, kwargs={"started": second_started, "returned": second_returned}) + first.start() + assert teardown_started.wait(5) + second.start() + assert second_started.wait(5) + assert not second_returned.wait(0.2) + + allow_teardown.set() + first.join(5) + second.join(5) + assert not first.is_alive() and not second.is_alive() + assert second_returned.is_set() + assert not errors + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="fork-specific flock ownership regression") +def test_forked_child_close_does_not_release_parent_lease_lock(tmp_path: Path) -> None: + identity = _identity() + store = _make_store(tmp_path / "store") + built = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert built.lease is not None + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + child_pid = os.fork() + if child_pid == 0: # pragma: no cover - assertions run in the parent + try: + built.lease.close() + except BaseException: + os._exit(1) + os._exit(0) + _, child_status = os.waitpid(child_pid, 0) + assert os.waitstatus_to_exitcode(child_status) == 0 + + active = store.cleanup(identity) + assert active is not None and active.code is FailureCode.ACTIVE_LEASE + built.lease.close() + assert store.cleanup(identity) is None + + +@pytest.mark.parametrize("write_mode", ["incomplete", "overlap"]) +def test_failed_streaming_writes_never_publish_partial_artifacts(tmp_path: Path, write_mode: str) -> None: + identity = _identity() + store = _make_store(tmp_path / "store") + + result = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity, write_mode=write_mode), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + + assert result.status is StoreStatus.FAILED + assert result.failure is not None and result.failure.code is FailureCode.PRODUCER_FAILED + assert not list(store.tmp_dir.iterdir()) + assert not list(store.artifacts_dir.iterdir()) + + +@pytest.mark.parametrize( + "dtype", + [ + dtype + for name in ("float8_e4m3fn", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz") + if (dtype := getattr(torch, name, None)) is not None + ], +) +def test_available_float8_formats_round_trip_through_streaming_store(tmp_path: Path, dtype: torch.dtype) -> None: + identity = replace( + _identity(), + representation=WeightRepresentation(name="runtime-fp8", dtype=str(dtype)), + ) + expected = torch.zeros((2, 4), dtype=dtype) + + class Float8Producer: + spec = WeightProductionSpec( + producer_id="test.final-layout", + outputs=(identity,), + source_mode=ProductionSourceMode.FINALIZED_MODEL, + ) + + def produce(self, writer: object) -> ProductionMetadata: + spec = TensorWriteSpec("weight", tuple(expected.shape), dtype) + with writer.open_tensor_file("weights.safetensors", (spec,)) as output: # type: ignore[attr-defined] + output.write_tensor("weight", expected) + return ProductionMetadata("test-producer-v1", "test-restorer-v1") + + store = _make_store(tmp_path / "store") + result = store.get_or_build( + BuildRequest(identity), + Float8Producer(), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + + assert result.status is StoreStatus.BUILT + assert result.lease is not None + actual = result.lease.tensors["weight"] + assert actual.dtype is dtype + assert torch.equal(actual.view(torch.uint8), expected.view(torch.uint8)) + result.lease.close() + + +def test_lookup_treats_orphan_deny_marker_as_miss(tmp_path: Path) -> None: + identity = _identity() + store = _make_store(tmp_path / "store") + deny = store.deny_dir / f"{identity.key}.json" + deny.write_text("{}", encoding="utf-8") + + result = store.lookup(identity, validation=ValidationLevel.FULL_CHECKSUM) + + assert result.status is StoreStatus.MISS + + +def test_overlapping_weaker_lookup_cannot_return_after_artifact_is_denied(tmp_path: Path) -> None: + identity = _identity() + root = tmp_path / "store" + store = _make_store(root) + built = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert built.lease is not None + built.lease.close() + + artifact_dir = store.artifacts_dir / identity.key + payload = artifact_dir / "weights.safetensors" + os.chmod(artifact_dir, 0o755) + os.chmod(payload, 0o644) + with payload.open("r+b") as handle: + handle.seek(-1, os.SEEK_END) + original = handle.read(1) + handle.seek(-1, os.SEEK_END) + handle.write(bytes([original[0] ^ 0xFF])) + handle.flush() + os.fsync(handle.fileno()) + os.chmod(payload, 0o444) + os.chmod(artifact_dir, 0o555) + + lease_opened = threading.Event() + resume_lookup = threading.Event() + overlapping = PausingOpenLeaseStore( + StorageDomainPolicy(root=root, storage_class=detect_storage_class(root)), + CapacityPolicy(), + IntegrityPolicy(), + lease_opened=lease_opened, + resume_lookup=resume_lookup, + ) + weak_results: list[StoreResult] = [] + + def weak_lookup() -> None: + weak_results.append(overlapping.lookup(identity, validation=ValidationLevel.MANIFEST_AND_METADATA)) + + thread = threading.Thread(target=weak_lookup) + thread.start() + try: + assert lease_opened.wait(5) + strong = store.lookup(identity, validation=ValidationLevel.FULL_CHECKSUM) + assert strong.status is StoreStatus.INVALID + assert (store.deny_dir / f"{identity.key}.json").is_file() + finally: + resume_lookup.set() + thread.join(5) + + assert not thread.is_alive() + assert len(weak_results) == 1 + weak = weak_results[0] + assert weak.status is StoreStatus.INVALID + assert weak.failure is not None and weak.failure.code is FailureCode.ARTIFACT_DENIED + + +def test_full_checksum_denies_quarantines_and_rebuilds_corrupt_payload(tmp_path: Path) -> None: + identity = _identity() + store = _make_store(tmp_path / "store") + built = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert built.lease is not None + built.lease.close() + + artifact_dir = store.artifacts_dir / identity.key + payload = artifact_dir / "weights.safetensors" + os.chmod(artifact_dir, 0o755) + os.chmod(payload, 0o644) + with payload.open("r+b") as handle: + handle.seek(-1, os.SEEK_END) + original = handle.read(1) + handle.seek(-1, os.SEEK_END) + handle.write(bytes([original[0] ^ 0xFF])) + handle.flush() + os.fsync(handle.fileno()) + os.chmod(payload, 0o444) + os.chmod(artifact_dir, 0o555) + + metadata_only = store.lookup(identity, validation=ValidationLevel.MANIFEST_AND_METADATA) + assert metadata_only.status is StoreStatus.HIT + assert metadata_only.lease is not None + metadata_only.lease.close() + + inspected = store.inspect_artifact(identity, validation=ValidationLevel.FULL_CHECKSUM) + assert inspected.status is StoreStatus.INVALID + assert inspected.failure is not None and inspected.failure.code is FailureCode.CHECKSUM_MISMATCH + assert not (store.deny_dir / f"{identity.key}.json").exists() + + invalid = store.lookup(identity, validation=ValidationLevel.FULL_CHECKSUM) + assert invalid.status is StoreStatus.INVALID + assert invalid.failure is not None and invalid.failure.code is FailureCode.CHECKSUM_MISMATCH + assert (store.deny_dir / f"{identity.key}.json").is_file() + + rebuilt = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert rebuilt.status is StoreStatus.BUILT + assert rebuilt.lease is not None + assert torch.equal(rebuilt.lease.tensors["layer.weight"], _weight()) + rebuilt.lease.close() + assert not (store.deny_dir / f"{identity.key}.json").exists() + assert len(list(store.quarantine_dir.iterdir())) == 1 + + +def test_malformed_safetensors_is_a_typed_invalid_artifact(tmp_path: Path) -> None: + identity = _identity() + store = _make_store(tmp_path / "store") + built = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert built.lease is not None + built.lease.close() + + artifact_dir = store.artifacts_dir / identity.key + payload = artifact_dir / "weights.safetensors" + os.chmod(artifact_dir, 0o755) + os.chmod(payload, 0o644) + with payload.open("r+b") as handle: + handle.write(b"\xff" * 8) + handle.flush() + os.fsync(handle.fileno()) + os.chmod(payload, 0o444) + os.chmod(artifact_dir, 0o555) + + result = store.lookup(identity, validation=ValidationLevel.MANIFEST_AND_METADATA) + + assert result.status is StoreStatus.INVALID + assert result.failure is not None and result.failure.code is FailureCode.MANIFEST_CORRUPT + assert (store.deny_dir / f"{identity.key}.json").is_file() + + +@pytest.mark.parametrize("schema_field", ["producer_schema", "restorer_schema"]) +def test_lookup_rejects_manifest_schema_that_differs_from_identity( + tmp_path: Path, + schema_field: str, +) -> None: + identity = _identity() + store = _make_store(tmp_path / "store") + built = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert built.lease is not None + built.lease.close() + + artifact_dir = store.artifacts_dir / identity.key + manifest_path = artifact_dir / "manifest.json" + marker_path = artifact_dir / "READY.json" + manifest_document = json.loads(manifest_path.read_bytes()) + marker_document = json.loads(marker_path.read_bytes()) + manifest_document[schema_field] = "incompatible-schema" + manifest_bytes = json.dumps( + manifest_document, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + marker_document["manifest_sha256"] = hashlib.sha256(manifest_bytes).hexdigest() + marker_bytes = json.dumps( + marker_document, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + os.chmod(artifact_dir, 0o755) + for path, contents in ((manifest_path, manifest_bytes), (marker_path, marker_bytes)): + os.chmod(path, 0o644) + path.write_bytes(contents) + os.chmod(path, 0o444) + os.chmod(artifact_dir, 0o555) + + result = store.lookup(identity, validation=ValidationLevel.MANIFEST_AND_METADATA) + + assert result.status is StoreStatus.INVALID + assert result.failure is not None and result.failure.code is FailureCode.MANIFEST_INCOMPATIBLE + assert (store.deny_dir / f"{identity.key}.json").is_file() + + +def test_noncooperative_conflicting_publication_is_reported_without_replacement(tmp_path: Path) -> None: + identity = _identity() + root = tmp_path / "store" + store = _make_store(root) + started = threading.Event() + release = threading.Event() + thread_result: list[StoreResult] = [] + + def build_expected_content() -> None: + thread_result.append( + store.get_or_build( + BuildRequest(identity), + FakeProducer(identity, started=started, release=release), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + ) + + builder = threading.Thread(target=build_expected_content) + builder.start() + try: + assert started.wait(timeout=5) + competing = NonCooperativeStore(store.domain_policy, store.capacity_policy, store.integrity_policy) + published = competing.get_or_build( + BuildRequest(identity), + FakeProducer(identity, weight_offset=100), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert published.status is StoreStatus.BUILT + assert published.lease is not None + published.lease.close() + finally: + release.set() + builder.join(timeout=10) + assert not builder.is_alive() + assert len(thread_result) == 1 + collision = thread_result[0] + assert collision.status is StoreStatus.FAILED + assert collision.failure is not None and collision.failure.code is FailureCode.IDENTITY_COLLISION + + winner = store.lookup(identity, validation=ValidationLevel.FULL_CHECKSUM) + assert winner.status is StoreStatus.HIT + assert winner.lease is not None + assert torch.equal(winner.lease.tensors["layer.weight"], _weight(100)) + winner.lease.close() + assert len(list(store.quarantine_dir.iterdir())) == 1 + + +def test_payload_symlink_is_rejected_without_following_it(tmp_path: Path) -> None: + identity = _identity() + store = _make_store(tmp_path / "store") + built = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert built.lease is not None + built.lease.close() + + artifact_dir = store.artifacts_dir / identity.key + payload = artifact_dir / "weights.safetensors" + outside = tmp_path / "outside.safetensors" + os.chmod(artifact_dir, 0o755) + payload.rename(outside) + payload.symlink_to(outside) + os.chmod(artifact_dir, 0o555) + + result = store.lookup(identity, validation=ValidationLevel.MANIFEST_AND_METADATA) + assert result.status is StoreStatus.INVALID + assert result.failure is not None and result.failure.code is FailureCode.MANIFEST_CORRUPT + + +def test_coordination_deadline_does_not_claim_to_cancel_an_active_producer(tmp_path: Path) -> None: + identity = _identity() + store = _make_store(tmp_path / "store") + started = time.monotonic() + + result = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity, delay_seconds=0.15), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 0.02, + ) + + assert time.monotonic() - started >= 0.1 + assert result.status is StoreStatus.BUILT + assert result.lease is not None + result.lease.close() + + +@pytest.mark.parallel +def test_multiprocess_contention_runs_one_builder_and_joins_same_artifact(tmp_path: Path) -> None: + ctx = mp.get_context("spawn") + root = tmp_path / "store" + _make_store(root) + counter = tmp_path / "producer-count" + initial_lookup_barrier = ctx.Barrier(2) + result_queue = ctx.Queue() + processes = [ + ctx.Process( + target=_build_process, + args=(str(root), str(counter), initial_lookup_barrier, result_queue), + ) + for _ in range(2) + ] + for process in processes: + process.start() + _join_processes(processes) + + results = [result_queue.get(timeout=3) for _ in processes] + assert all(process.exitcode == 0 for process in processes) + assert sorted(item[0] for item in results) == [StoreStatus.BUILT.value, StoreStatus.JOINED.value] + assert all(item[2] is None for item in results) + assert len({item[1] for item in results}) == 1 + assert counter.read_text(encoding="utf-8").count("\n") == 1 + + +@pytest.mark.parallel +def test_reader_sees_miss_during_build_and_waiter_has_bounded_timeout(tmp_path: Path) -> None: + ctx = mp.get_context("spawn") + root = tmp_path / "store" + store = _make_store(root) + identity = _identity() + started = ctx.Event() + release = ctx.Event() + result_queue = ctx.Queue() + process = ctx.Process( + target=_controlled_build_process, + args=(str(root), started, release, result_queue), + ) + process.start() + try: + assert started.wait(timeout=_PROCESS_START_TIMEOUT_SECONDS) + assert store.lookup(identity, validation=ValidationLevel.FULL_CHECKSUM).status is StoreStatus.MISS + timed_out = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 0.1, + ) + assert timed_out.status is StoreStatus.TIMEOUT + assert timed_out.failure is not None and timed_out.failure.code is FailureCode.ACTIVE_BUILD_TIMEOUT + finally: + release.set() + _join_processes([process]) + + status, error = result_queue.get(timeout=3) + assert process.exitcode == 0 + assert (status, error) == (StoreStatus.BUILT.value, None) + + +@pytest.mark.parallel +def test_crashed_builder_leaves_only_invisible_temp_and_next_builder_recovers(tmp_path: Path) -> None: + ctx = mp.get_context("spawn") + root = tmp_path / "store" + store = _make_store(root) + process = ctx.Process(target=_crash_build_process, args=(str(root),)) + process.start() + _join_processes([process]) + + assert process.exitcode == 17 + assert list(store.tmp_dir.iterdir()) + assert not list(store.artifacts_dir.iterdir()) + + identity = _identity() + recovered = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert recovered.status is StoreStatus.BUILT + assert recovered.lease is not None + recovered.lease.close() + assert not list(store.tmp_dir.iterdir()) + + +def test_capacity_and_enospc_failures_are_typed_and_clean_staging( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + identity = _identity() + limited = _make_store(tmp_path / "limited", capacity=CapacityPolicy(max_artifact_bytes=64)) + too_large = limited.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert too_large.status is StoreStatus.FAILED + assert too_large.failure is not None and too_large.failure.code is FailureCode.ARTIFACT_TOO_LARGE + assert not list(limited.tmp_dir.iterdir()) + + no_space = _make_store(tmp_path / "enospc") + + def raise_enospc(_fd: int, _offset: int, _length: int) -> None: + raise OSError(errno.ENOSPC, "injected no space") + + monkeypatch.setattr(os, "posix_fallocate", raise_enospc) + failed = no_space.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert failed.status is StoreStatus.FAILED + assert failed.failure is not None and failed.failure.code is FailureCode.ENOSPC + assert not list(no_space.tmp_dir.iterdir()) + assert not list(no_space.artifacts_dir.iterdir()) + + +@pytest.mark.parametrize( + ("failure_errno", "expected_stage", "expected_code"), + [ + (errno.ENOSPC, ResolutionStage.CAPACITY, FailureCode.ENOSPC), + (errno.EDQUOT, ResolutionStage.CAPACITY, FailureCode.ENOSPC), + (errno.EACCES, ResolutionStage.DOMAIN, FailureCode.DOMAIN_UNAVAILABLE), + ], +) +def test_staging_directory_failures_are_typed_and_clean( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure_errno: int, + expected_stage: ResolutionStage, + expected_code: FailureCode, +) -> None: + identity = _identity() + store = _make_store(tmp_path / "store") + original_mkdir = Path.mkdir + + def fail_staging_mkdir( + path: Path, + mode: int = 0o777, + parents: bool = False, + exist_ok: bool = False, + ) -> None: + if path.parent == store.tmp_dir: + raise OSError(failure_errno, "injected staging directory failure") + original_mkdir(path, mode=mode, parents=parents, exist_ok=exist_ok) + + monkeypatch.setattr(Path, "mkdir", fail_staging_mkdir) + result = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + + assert result.status is StoreStatus.FAILED + assert result.failure is not None + assert result.failure.stage is expected_stage + assert result.failure.code is expected_code + assert not list(store.tmp_dir.iterdir()) + assert not list(store.artifacts_dir.iterdir()) + + +def test_store_and_free_space_capacity_limits_fail_before_publication(tmp_path: Path) -> None: + identity = _identity() + store_limited = _make_store(tmp_path / "store-limit", capacity=CapacityPolicy(max_store_bytes=1)) + store_result = store_limited.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert store_result.status is StoreStatus.FAILED + assert store_result.failure is not None and store_result.failure.code is FailureCode.STORE_LIMIT_EXCEEDED + assert not list(store_limited.artifacts_dir.iterdir()) + + free_root = tmp_path / "free-limit" + free_root.mkdir(mode=0o700) + unavailable_reserve = shutil.disk_usage(free_root).free + 1 + free_limited = _make_store(free_root, capacity=CapacityPolicy(min_free_bytes=unavailable_reserve)) + free_result = free_limited.get_or_build( + BuildRequest(identity), + FakeProducer(identity), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert free_result.status is StoreStatus.FAILED + assert free_result.failure is not None and free_result.failure.code is FailureCode.INSUFFICIENT_CAPACITY + assert not list(free_limited.artifacts_dir.iterdir()) + + +def test_domain_capacity_policy_is_authoritative(tmp_path: Path) -> None: + root = tmp_path / "store" + _make_store(root, capacity=CapacityPolicy(max_store_bytes=4096)) + + with pytest.raises(HostWeightError, match="capacity policy differs"): + _make_store(root, capacity=CapacityPolicy(max_store_bytes=8192)) + + +@pytest.mark.parametrize("filesystem_type", ["nfs4", "cifs", "lustre", "ceph", "unknownfs"]) +def test_remote_and_unknown_filesystems_cannot_back_the_local_store( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + filesystem_type: str, +) -> None: + monkeypatch.setattr(filesystem_store_module, "detect_filesystem_type", lambda _path: filesystem_type) + root = tmp_path / "store" + + with pytest.raises(ValueError, match="not node-local|not recognized"): + detect_storage_class(root) + with pytest.raises(HostWeightError) as error: + FilesystemHostWeightStore( + StorageDomainPolicy(root=root, storage_class=StorageClass.DISK), + CapacityPolicy(), + IntegrityPolicy(), + ) + + assert error.value.failure.code is FailureCode.DOMAIN_POLICY_MISMATCH + assert not root.exists() + + +def test_trusted_store_rejects_group_or_world_writable_root(tmp_path: Path) -> None: + root = tmp_path / "untrusted" + root.mkdir(mode=0o700) + root.chmod(0o777) + + with pytest.raises(HostWeightError) as error: + FilesystemHostWeightStore( + StorageDomainPolicy(root=root, storage_class=detect_storage_class(root)), + CapacityPolicy(), + IntegrityPolicy(), + ) + + assert error.value.failure.code is FailureCode.DOMAIN_UNAVAILABLE + + +def test_unimplemented_coordination_and_fs_verity_return_typed_failures(tmp_path: Path) -> None: + identity = _identity() + store = _make_store(tmp_path / "store") + + unsupported_producer = store.get_or_build( + BuildRequest(identity), + FakeProducer(identity, coordination_scope=CoordinationScope.GROUP_COHORT), + validation=ValidationLevel.FULL_CHECKSUM, + deadline=time.monotonic() + 10, + ) + assert unsupported_producer.status is StoreStatus.FAILED + assert unsupported_producer.failure is not None + assert unsupported_producer.failure.code is FailureCode.PRODUCER_UNSUPPORTED + + fs_verity = store.lookup(identity, validation=ValidationLevel.FS_VERITY) + assert fs_verity.status is StoreStatus.FAILED + assert fs_verity.failure is not None and fs_verity.failure.code is FailureCode.UNSUPPORTED + + +def test_unimplemented_numa_enforcement_and_background_scrub_fail_explicitly(tmp_path: Path) -> None: + numa_root = tmp_path / "numa" + numa_root.mkdir(mode=0o700) + with pytest.raises(HostWeightError) as numa_error: + FilesystemHostWeightStore( + StorageDomainPolicy( + root=numa_root, + scope=StorageScope.NUMA, + storage_class=detect_storage_class(numa_root), + numa_prefault=True, + ), + CapacityPolicy(), + IntegrityPolicy(), + ) + assert numa_error.value.failure.code is FailureCode.UNSUPPORTED + + with pytest.raises(HostWeightError) as scrub_error: + _make_store(tmp_path / "scrub", integrity=IntegrityPolicy(background_scrub=True)) + assert scrub_error.value.failure.code is FailureCode.UNSUPPORTED diff --git a/tests/host_weight_runtime/test_runtime.py b/tests/host_weight_runtime/test_runtime.py new file mode 100644 index 000000000..afd7c943a --- /dev/null +++ b/tests/host_weight_runtime/test_runtime.py @@ -0,0 +1,364 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Resolution-policy tests for the loader-adjacent Host Weight Runtime.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import torch + +from vllm_omni.host_weight_runtime import ( + AdaptationIdentity, + CanonicalJson, + ComponentIdentity, + FailureCode, + HostWeightFailure, + HostWeightRuntime, + HostWeightRuntimeConfig, + IntegrityPolicy, + LookupPhase, + ProducerIdentity, + ProductionMetadata, + ProductionSourceMode, + RemoteImportPolicy, + RemoteOnMiss, + ResolutionOutcome, + ResolutionReport, + ResolutionStage, + RuntimeMode, + RuntimeWeightLayout, + StorageDomainPolicy, + StoreResult, + StoreStatus, + TensorWriteSpec, + ValidationLevel, + WeightArtifactIdentity, + WeightProductionSpec, + WeightRepresentation, + WeightSourceIdentity, +) +from vllm_omni.host_weight_runtime.filesystem import detect_storage_class + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _identity() -> WeightArtifactIdentity: + return WeightArtifactIdentity( + schema_version=1, + source=WeightSourceIdentity("test-org/test-model", "0123456789abcdef", "source-files-sha256"), + component=ComponentIdentity("transformer", "diffusion.dit"), + representation=WeightRepresentation("runtime-bf16", "torch.bfloat16"), + layout=RuntimeWeightLayout("final-module-layout"), + adaptation=AdaptationIdentity(), + producer=ProducerIdentity( + "test.final-layout", + "1", + "implementation-sha256", + "test-producer-v1", + "test-restorer-v1", + ), + ) + + +class CountingProducer: + def __init__( + self, + identity: WeightArtifactIdentity, + *, + lookup_phase: LookupPhase = LookupPhase.PRE_LOAD_SAFE, + ) -> None: + self.calls = 0 + self._spec = WeightProductionSpec( + producer_id="test.final-layout", + outputs=(identity,), + source_mode=ProductionSourceMode.FINALIZED_MODEL, + lookup_phase=lookup_phase, + ) + + @property + def spec(self) -> WeightProductionSpec: + return self._spec + + def produce(self, writer: object) -> ProductionMetadata: + self.calls += 1 + spec = TensorWriteSpec("weight", (2, 2), torch.bfloat16) + with writer.open_tensor_file("weights.safetensors", (spec,)) as output: # type: ignore[attr-defined] + output.write_tensor("weight", torch.arange(4, dtype=torch.float32).to(torch.bfloat16).reshape(2, 2)) + return ProductionMetadata( + producer_schema="test-producer-v1", + restorer_schema="test-restorer-v1", + format_metadata=CanonicalJson.from_value({"source": "test"}), + ) + + +def _domain(root: Path) -> StorageDomainPolicy: + return StorageDomainPolicy(root=root, storage_class=detect_storage_class(root.parent)) + + +def test_disabled_mode_does_not_probe_or_create_configured_root(tmp_path: Path) -> None: + root = tmp_path / "must-not-exist" + runtime = HostWeightRuntime.from_config(HostWeightRuntimeConfig(mode=RuntimeMode.DISABLED, domain=_domain(root))) + + resolution = runtime.resolve() + + assert resolution.report.outcome is ResolutionOutcome.CANONICAL_DIRECT + assert resolution.report.attempts == () + assert resolution.lease is None + assert not root.exists() + + +@pytest.mark.parametrize( + ("mode", "expected"), + [ + (RuntimeMode.PREFERRED, ResolutionOutcome.CANONICAL_FALLBACK), + (RuntimeMode.REQUIRED, ResolutionOutcome.FAILED), + ], +) +def test_local_miss_uses_configured_fallback_policy( + tmp_path: Path, + mode: RuntimeMode, + expected: ResolutionOutcome, +) -> None: + runtime = HostWeightRuntime.from_config(HostWeightRuntimeConfig(mode=mode, domain=_domain(tmp_path / mode.value))) + + resolution = runtime.resolve(_identity()) + + assert resolution.report.outcome is expected + assert resolution.lease is None + assert [attempt.result.value for attempt in resolution.report.attempts] == ["miss", "skipped"] + expected_action = "canonical_fallback" if mode is RuntimeMode.PREFERRED else "fail_startup" + assert [attempt.action.value for attempt in resolution.report.attempts] == [expected_action, expected_action] + + +def test_local_production_then_exact_warm_hit_emits_one_terminal_report_each(tmp_path: Path) -> None: + reports: list[ResolutionReport] = [] + identity = _identity() + producer = CountingProducer(identity) + runtime = HostWeightRuntime.from_config( + HostWeightRuntimeConfig(mode=RuntimeMode.PREFERRED, domain=_domain(tmp_path / "store")), + observer=reports.append, + ) + + cold = runtime.resolve(identity, producer=producer) + assert cold.report.outcome is ResolutionOutcome.LOCAL_PRODUCTION + assert cold.lease is not None + assert torch.equal( + cold.lease.tensors["weight"], + torch.arange(4, dtype=torch.float32).to(torch.bfloat16).reshape(2, 2), + ) + cold.lease.close() + assert cold.report.attempts[0].action.value == "try_producer" + + warm = runtime.resolve(identity) + assert warm.report.outcome is ResolutionOutcome.LOCAL_HIT + assert warm.lease is not None + warm.lease.close() + + assert producer.calls == 1 + assert reports == [cold.report, warm.report] + assert all(report.resolution_id for report in reports) + + +@pytest.mark.parametrize( + ("mode", "expected"), + [ + (RuntimeMode.PREFERRED, ResolutionOutcome.FAILED), + (RuntimeMode.REQUIRED, ResolutionOutcome.FAILED), + ], +) +def test_remote_required_is_explicitly_unsupported_and_skips_local_production( + tmp_path: Path, + mode: RuntimeMode, + expected: ResolutionOutcome, +) -> None: + identity = _identity() + producer = CountingProducer(identity) + runtime = HostWeightRuntime.from_config( + HostWeightRuntimeConfig( + mode=mode, + domain=_domain(tmp_path / mode.value), + remote=RemoteImportPolicy( + on_local_miss=RemoteOnMiss.REQUIRE, + providers=("future-mooncake",), + ), + ) + ) + + resolution = runtime.resolve(identity, producer=producer) + + assert resolution.report.outcome is expected + assert producer.calls == 0 + assert [attempt.stage.value for attempt in resolution.report.attempts] == ["lookup", "remote"] + assert resolution.report.attempts[-1].failure is not None + assert resolution.report.attempts[-1].failure.code.value == "unsupported" + assert resolution.report.attempts[-1].action.value == "fail_startup" + + +def test_observer_failure_cannot_change_resolution_outcome(tmp_path: Path) -> None: + def broken_observer(_report: object) -> None: + raise RuntimeError("metrics backend unavailable") + + runtime = HostWeightRuntime.from_config( + HostWeightRuntimeConfig(mode=RuntimeMode.PREFERRED, domain=_domain(tmp_path / "store")), + observer=broken_observer, + ) + + resolution = runtime.resolve(_identity()) + + assert resolution.report.outcome is ResolutionOutcome.CANONICAL_FALLBACK + + +def test_preload_resolution_does_not_run_postload_only_producer(tmp_path: Path) -> None: + identity = _identity() + producer = CountingProducer(identity, lookup_phase=LookupPhase.POST_LOAD_ONLY) + runtime = HostWeightRuntime.from_config( + HostWeightRuntimeConfig(mode=RuntimeMode.PREFERRED, domain=_domain(tmp_path / "store")) + ) + + resolution = runtime.resolve(identity, producer=producer) + + assert resolution.report.outcome is ResolutionOutcome.CANONICAL_FALLBACK + assert producer.calls == 0 + assert resolution.report.attempts[0].action.value == "canonical_fallback" + + +@pytest.mark.parametrize( + ("mode", "expected"), + [ + (RuntimeMode.PREFERRED, ResolutionOutcome.CANONICAL_FALLBACK), + (RuntimeMode.REQUIRED, ResolutionOutcome.FAILED), + ], +) +def test_retryable_store_initialization_failure_obeys_runtime_mode( + tmp_path: Path, + mode: RuntimeMode, + expected: ResolutionOutcome, +) -> None: + root = tmp_path / "not-a-directory" + root.write_text("occupied", encoding="utf-8") + runtime = HostWeightRuntime.from_config(HostWeightRuntimeConfig(mode=mode, domain=_domain(root))) + + resolution = runtime.resolve(_identity()) + + assert resolution.report.outcome is expected + assert resolution.lease is None + assert len(resolution.report.attempts) == 1 + assert resolution.report.attempts[0].failure is not None + assert resolution.report.attempts[0].failure.code.value == "domain_unavailable" + assert resolution.report.attempts[0].failure.retryable + + +@pytest.mark.parametrize( + ("stage", "code"), + [ + (ResolutionStage.PRODUCTION, FailureCode.PRODUCER_UNSUPPORTED), + (ResolutionStage.VALIDATION, FailureCode.IDENTITY_COLLISION), + (ResolutionStage.LIFECYCLE, FailureCode.PUBLICATION_FAILED), + ], +) +def test_nonretryable_post_initialization_failure_cannot_fall_back_in_preferred_mode( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + stage: ResolutionStage, + code: FailureCode, +) -> None: + identity = _identity() + runtime = HostWeightRuntime.from_config( + HostWeightRuntimeConfig(mode=RuntimeMode.PREFERRED, domain=_domain(tmp_path / code.value)) + ) + assert runtime.store is not None + + def fail_production( + _request: object, + _producer: object, + *, + validation: object, + deadline: float, + ) -> StoreResult: + del validation, deadline + return StoreResult( + StoreStatus.FAILED, + failure=HostWeightFailure( + stage=stage, + code=code, + retryable=False, + message="injected nonretryable production failure", + details=CanonicalJson.empty(), + ), + ) + + monkeypatch.setattr(runtime.store, "get_or_build", fail_production) + resolution = runtime.resolve(identity, producer=CountingProducer(identity)) + + assert resolution.report.outcome is ResolutionOutcome.FAILED + assert resolution.report.attempts[-1].action.value == "fail_startup" + assert resolution.report.attempts[-1].failure is not None + assert resolution.report.attempts[-1].failure.code is code + + +def test_retryable_post_initialization_failure_can_fall_back_in_preferred_mode( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + identity = _identity() + runtime = HostWeightRuntime.from_config( + HostWeightRuntimeConfig(mode=RuntimeMode.PREFERRED, domain=_domain(tmp_path / "retryable")) + ) + assert runtime.store is not None + + def fail_production( + _request: object, + _producer: object, + *, + validation: object, + deadline: float, + ) -> StoreResult: + del validation, deadline + return StoreResult( + StoreStatus.FAILED, + failure=HostWeightFailure( + stage=ResolutionStage.CAPACITY, + code=FailureCode.ENOSPC, + retryable=True, + message="injected recoverable capacity failure", + details=CanonicalJson.empty(), + ), + ) + + monkeypatch.setattr(runtime.store, "get_or_build", fail_production) + resolution = runtime.resolve(identity, producer=CountingProducer(identity)) + + assert resolution.report.outcome is ResolutionOutcome.CANONICAL_FALLBACK + assert resolution.report.attempts[-1].action.value == "canonical_fallback" + + +def test_nonretryable_store_configuration_failure_remains_visible_in_preferred_mode(tmp_path: Path) -> None: + root = tmp_path / "untrusted" + root.mkdir(mode=0o700) + root.chmod(0o777) + runtime = HostWeightRuntime.from_config(HostWeightRuntimeConfig(mode=RuntimeMode.PREFERRED, domain=_domain(root))) + + resolution = runtime.resolve(_identity()) + + assert resolution.report.outcome is ResolutionOutcome.FAILED + assert resolution.report.attempts[0].action.value == "fail_startup" + assert resolution.report.attempts[0].failure is not None + assert not resolution.report.attempts[0].failure.retryable + + +def test_unsupported_integrity_policy_fails_instead_of_disguising_configuration_error(tmp_path: Path) -> None: + runtime = HostWeightRuntime.from_config( + HostWeightRuntimeConfig( + mode=RuntimeMode.PREFERRED, + domain=_domain(tmp_path / "store"), + integrity=IntegrityPolicy(local_lookup=ValidationLevel.FS_VERITY), + ) + ) + + resolution = runtime.resolve(_identity()) + + assert resolution.report.outcome is ResolutionOutcome.FAILED + assert resolution.report.attempts[0].failure is not None + assert resolution.report.attempts[0].failure.code.value == "unsupported" diff --git a/vllm_omni/host_weight_runtime/__init__.py b/vllm_omni/host_weight_runtime/__init__.py new file mode 100644 index 000000000..c7bbf51dc --- /dev/null +++ b/vllm_omni/host_weight_runtime/__init__.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Neutral contracts for immutable runtime-ready host weight artifacts.""" + +from .config import ( + CapacityPolicy, + HostWeightRuntimeConfig, + IntegrityPolicy, + ProductionPolicy, + RemoteImportPolicy, + RemoteOnMiss, + RuntimeMode, + StorageClass, + StorageDomainPolicy, + StorageScope, + ValidationLevel, + WaitPolicy, +) +from .errors import FailureCode, HostWeightError, HostWeightFailure, ResolutionAction, ResolutionStage +from .identity import ( + AdaptationIdentity, + CanonicalJson, + ComponentIdentity, + IdentityValidationError, + ProducerIdentity, + RuntimeWeightLayout, + WeightArtifactIdentity, + WeightRepresentation, + WeightSourceIdentity, +) +from .inspection import ( + ArtifactInspection, + ArtifactInventoryEntry, + ArtifactInventoryState, + DomainInspection, + HostWeightInspector, +) +from .lease import HostWeightLease, LeaseProvenance, MappedHostRegion +from .manifest import ( + FileManifestEntry, + ManifestValidationError, + PublicationMarker, + TensorKind, + TensorManifestEntry, + WeightManifest, +) +from .outcomes import ( + AttemptResult, + HostWeightResolution, + ResolutionAttempt, + ResolutionOutcome, + ResolutionReport, + StoreResult, + StoreStatus, +) +from .protocols import ( + ArtifactWriter, + BuildRequest, + CoordinationScope, + HostWeightStore, + LookupPhase, + ProductionMetadata, + ProductionSourceMode, + RemoteArtifactDescriptor, + RemoteArtifactProvider, + TensorFileWriter, + TensorWriteSpec, + WeightProducer, + WeightProductionSpec, + WeightRestorePlan, + WeightRestorer, +) +from .runtime import HostWeightRuntime + +__all__ = [ + "AdaptationIdentity", + "ArtifactInspection", + "ArtifactInventoryEntry", + "ArtifactInventoryState", + "ArtifactWriter", + "AttemptResult", + "BuildRequest", + "CanonicalJson", + "CapacityPolicy", + "ComponentIdentity", + "CoordinationScope", + "DomainInspection", + "FailureCode", + "FileManifestEntry", + "HostWeightError", + "HostWeightFailure", + "HostWeightLease", + "HostWeightResolution", + "HostWeightInspector", + "HostWeightRuntime", + "HostWeightRuntimeConfig", + "HostWeightStore", + "IdentityValidationError", + "IntegrityPolicy", + "LeaseProvenance", + "LookupPhase", + "ManifestValidationError", + "MappedHostRegion", + "ProducerIdentity", + "ProductionMetadata", + "ProductionPolicy", + "ProductionSourceMode", + "PublicationMarker", + "RemoteArtifactDescriptor", + "RemoteArtifactProvider", + "RemoteImportPolicy", + "RemoteOnMiss", + "ResolutionAction", + "ResolutionAttempt", + "ResolutionOutcome", + "ResolutionReport", + "ResolutionStage", + "RuntimeMode", + "RuntimeWeightLayout", + "StorageClass", + "StorageDomainPolicy", + "StorageScope", + "StoreResult", + "StoreStatus", + "TensorFileWriter", + "TensorKind", + "TensorManifestEntry", + "TensorWriteSpec", + "ValidationLevel", + "WaitPolicy", + "WeightArtifactIdentity", + "WeightManifest", + "WeightProducer", + "WeightProductionSpec", + "WeightRepresentation", + "WeightRestorePlan", + "WeightRestorer", + "WeightSourceIdentity", +] diff --git a/vllm_omni/host_weight_runtime/config.py b/vllm_omni/host_weight_runtime/config.py new file mode 100644 index 000000000..f2422899e --- /dev/null +++ b/vllm_omni/host_weight_runtime/config.py @@ -0,0 +1,183 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Typed policy for the loader-adjacent Host Weight Runtime.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path + + +class RuntimeMode(str, Enum): + DISABLED = "disabled" + PREFERRED = "preferred" + REQUIRED = "required" + + +class StorageScope(str, Enum): + NODE = "node" + NUMA = "numa" + + +class StorageClass(str, Enum): + DISK = "disk" + TMPFS = "tmpfs" + + +class ValidationLevel(str, Enum): + MANIFEST_AND_METADATA = "manifest_and_metadata" + FULL_CHECKSUM = "full_checksum" + FS_VERITY = "fs_verity" + + +class RemoteOnMiss(str, Enum): + DISABLED = "disabled" + TRY = "try" + REQUIRE = "require" + + +@dataclass(frozen=True) +class StorageDomainPolicy: + root: Path + scope: StorageScope = StorageScope.NODE + domain_id: str = "node" + storage_class: StorageClass = StorageClass.DISK + numa_prefault: bool = False + verify_numa_residency: bool = False + + def __post_init__(self) -> None: + if not isinstance(self.root, Path) or not str(self.root): + raise ValueError("host weight store root must not be empty") + if not isinstance(self.scope, StorageScope) or not isinstance(self.storage_class, StorageClass): + raise ValueError("storage scope and class must use their typed enum values") + if not isinstance(self.domain_id, str) or not self.domain_id: + raise ValueError("host weight store domain_id must not be empty") + if not isinstance(self.numa_prefault, bool) or not isinstance(self.verify_numa_residency, bool): + raise ValueError("NUMA placement options must be booleans") + if self.scope is StorageScope.NODE and (self.numa_prefault or self.verify_numa_residency): + raise ValueError("NUMA placement options require scope='numa'") + + +@dataclass(frozen=True) +class CapacityPolicy: + max_artifact_bytes: int | None = None + max_store_bytes: int | None = None + min_free_bytes: int = 0 + eviction: str = "none" + + def __post_init__(self) -> None: + for name, value in ( + ("max_artifact_bytes", self.max_artifact_bytes), + ("max_store_bytes", self.max_store_bytes), + ): + if value is not None and (isinstance(value, bool) or not isinstance(value, int) or value <= 0): + raise ValueError(f"{name} must be positive when configured") + if isinstance(self.min_free_bytes, bool) or not isinstance(self.min_free_bytes, int) or self.min_free_bytes < 0: + raise ValueError("min_free_bytes must not be negative") + if not isinstance(self.eviction, str) or self.eviction != "none": + raise ValueError("automatic host weight eviction is not implemented") + + +@dataclass(frozen=True) +class IntegrityPolicy: + local_lookup: ValidationLevel = ValidationLevel.MANIFEST_AND_METADATA + require_trusted_permissions: bool = True + background_scrub: bool = False + + def __post_init__(self) -> None: + if not isinstance(self.local_lookup, ValidationLevel): + raise ValueError("local lookup validation must use ValidationLevel") + if not isinstance(self.require_trusted_permissions, bool) or not isinstance(self.background_scrub, bool): + raise ValueError("integrity switches must be booleans") + + +@dataclass(frozen=True) +class RemoteImportPolicy: + on_local_miss: RemoteOnMiss = RemoteOnMiss.DISABLED + providers: tuple[str, ...] = () + + def __post_init__(self) -> None: + if not isinstance(self.on_local_miss, RemoteOnMiss): + raise ValueError("remote on-miss policy must use RemoteOnMiss") + if not isinstance(self.providers, tuple): + raise ValueError("remote providers must be an immutable tuple") + if self.on_local_miss is not RemoteOnMiss.DISABLED and not self.providers: + raise ValueError("remote providers are required when remote import is enabled") + if any(not isinstance(provider, str) or not provider for provider in self.providers): + raise ValueError("remote provider identifiers must not be empty") + + +@dataclass(frozen=True) +class ProductionPolicy: + allow_local_build: bool = True + allow_post_load_publish: bool = False + + def __post_init__(self) -> None: + if not isinstance(self.allow_local_build, bool) or not isinstance(self.allow_post_load_publish, bool): + raise ValueError("production switches must be booleans") + if self.allow_post_load_publish: + raise ValueError("post-load publication is not implemented") + + +@dataclass(frozen=True) +class WaitPolicy: + coordination_timeout_seconds: float = 600.0 + + def __post_init__(self) -> None: + timeout = self.coordination_timeout_seconds + if ( + isinstance(timeout, bool) + or not isinstance(timeout, int | float) + or not math.isfinite(timeout) + or timeout <= 0 + ): + raise ValueError("coordination_timeout_seconds must be positive") + + +@dataclass(frozen=True) +class HostWeightRuntimeConfig: + mode: RuntimeMode = RuntimeMode.DISABLED + domain: StorageDomainPolicy | None = None + remote: RemoteImportPolicy = field(default_factory=RemoteImportPolicy) + production: ProductionPolicy = field(default_factory=ProductionPolicy) + capacity: CapacityPolicy = field(default_factory=CapacityPolicy) + wait: WaitPolicy = field(default_factory=WaitPolicy) + integrity: IntegrityPolicy = field(default_factory=IntegrityPolicy) + schema_version: int = 1 + + def __post_init__(self) -> None: + if isinstance(self.schema_version, bool) or self.schema_version != 1: + raise ValueError("only Host Weight Runtime config schema_version=1 is supported") + if not isinstance(self.mode, RuntimeMode): + raise ValueError("runtime mode must use RuntimeMode") + if self.domain is not None and not isinstance(self.domain, StorageDomainPolicy): + raise ValueError("runtime domain must use StorageDomainPolicy") + for name, value, expected in ( + ("remote", self.remote, RemoteImportPolicy), + ("production", self.production, ProductionPolicy), + ("capacity", self.capacity, CapacityPolicy), + ("wait", self.wait, WaitPolicy), + ("integrity", self.integrity, IntegrityPolicy), + ): + if not isinstance(value, expected): + raise ValueError(f"runtime {name} policy has the wrong type") + if self.mode is not RuntimeMode.DISABLED and self.domain is None: + raise ValueError("enabled Host Weight Runtime requires a storage domain") + + +__all__ = [ + "CapacityPolicy", + "HostWeightRuntimeConfig", + "IntegrityPolicy", + "ProductionPolicy", + "RemoteImportPolicy", + "RemoteOnMiss", + "RuntimeMode", + "StorageClass", + "StorageDomainPolicy", + "StorageScope", + "ValidationLevel", + "WaitPolicy", +] diff --git a/vllm_omni/host_weight_runtime/errors.py b/vllm_omni/host_weight_runtime/errors.py new file mode 100644 index 000000000..d3ab4f3c2 --- /dev/null +++ b/vllm_omni/host_weight_runtime/errors.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Stable Host Weight Runtime failures and actions.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + +from .identity import CanonicalJson + + +class ResolutionStage(str, Enum): + CONFIGURATION = "configuration" + DOMAIN = "domain" + LOOKUP = "lookup" + VALIDATION = "validation" + REMOTE = "remote" + PRODUCTION = "production" + CAPACITY = "capacity" + RESTORATION = "restoration" + LIFECYCLE = "lifecycle" + CANONICAL_LOADING = "canonical_loading" + + +class FailureCode(str, Enum): + INVALID_CONFIG = "invalid_config" + COHORT_POLICY_MISMATCH = "cohort_policy_mismatch" + DOMAIN_POLICY_MISMATCH = "domain_policy_mismatch" + DOMAIN_UNAVAILABLE = "domain_unavailable" + STORAGE_CLASS_MISMATCH = "storage_class_mismatch" + NUMA_PLACEMENT_INVALID = "numa_placement_invalid" + LOCAL_MISS = "local_miss" + ARTIFACT_DENIED = "artifact_denied" + ACTIVE_BUILD_TIMEOUT = "active_build_timeout" + MANIFEST_INCOMPATIBLE = "manifest_incompatible" + MANIFEST_CORRUPT = "manifest_corrupt" + CHECKSUM_MISMATCH = "checksum_mismatch" + IDENTITY_COLLISION = "identity_collision" + REMOTE_MISS = "remote_miss" + REMOTE_UNAVAILABLE = "remote_unavailable" + REMOTE_AUTH_FAILED = "remote_auth_failed" + REMOTE_TIMEOUT = "remote_timeout" + REMOTE_ARTIFACT_INVALID = "remote_artifact_invalid" + PRODUCER_UNAVAILABLE = "producer_unavailable" + PRODUCER_UNSUPPORTED = "producer_unsupported" + PRODUCER_FAILED = "producer_failed" + PRODUCTION_TIMEOUT = "production_timeout" + ARTIFACT_TOO_LARGE = "artifact_too_large" + STORE_LIMIT_EXCEEDED = "store_limit_exceeded" + INSUFFICIENT_CAPACITY = "insufficient_capacity" + ENOSPC = "enospc" + RESTORER_UNAVAILABLE = "restorer_unavailable" + RESTORE_PREFLIGHT_FAILED = "restore_preflight_failed" + RESTORE_COMMIT_FAILED = "restore_commit_failed" + ACTIVE_LEASE = "active_lease" + PUBLICATION_FAILED = "publication_failed" + QUARANTINE_FAILED = "quarantine_failed" + CANONICAL_SOURCE_FAILED = "canonical_source_failed" + UNSUPPORTED = "unsupported" + + +class ResolutionAction(str, Enum): + RETURN_LEASE = "return_lease" + TRY_REMOTE = "try_remote" + TRY_PRODUCER = "try_producer" + QUARANTINE_THEN_PRODUCE = "quarantine_then_produce" + CANONICAL_FALLBACK = "canonical_fallback" + FAIL_STARTUP = "fail_startup" + SKIP = "skip" + + +@dataclass(frozen=True) +class HostWeightFailure: + stage: ResolutionStage + code: FailureCode + retryable: bool + message: str + details: CanonicalJson = field(default_factory=CanonicalJson.empty) + + def __post_init__(self) -> None: + if not isinstance(self.stage, ResolutionStage) or not isinstance(self.code, FailureCode): + raise ValueError("host weight failure stage and code must use their typed enums") + if not isinstance(self.retryable, bool): + raise ValueError("host weight failure retryable must be a boolean") + if not isinstance(self.message, str) or not self.message: + raise ValueError("host weight failure requires a message") + if not isinstance(self.details, CanonicalJson): + raise ValueError("host weight failure details must use CanonicalJson") + + +class HostWeightError(RuntimeError): + """Internal exception carrying a stable, policy-neutral failure.""" + + def __init__(self, failure: HostWeightFailure): + super().__init__(failure.message) + self.failure = failure + + +__all__ = [ + "FailureCode", + "HostWeightError", + "HostWeightFailure", + "ResolutionAction", + "ResolutionStage", +] diff --git a/vllm_omni/host_weight_runtime/filesystem/__init__.py b/vllm_omni/host_weight_runtime/filesystem/__init__.py new file mode 100644 index 000000000..2083c1e85 --- /dev/null +++ b/vllm_omni/host_weight_runtime/filesystem/__init__.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Local filesystem Host Weight Store implementation.""" + +from .store import FilesystemHostWeightStore, detect_filesystem_type, detect_storage_class + +__all__ = [ + "FilesystemHostWeightStore", + "detect_filesystem_type", + "detect_storage_class", +] diff --git a/vllm_omni/host_weight_runtime/filesystem/locks.py b/vllm_omni/host_weight_runtime/filesystem/locks.py new file mode 100644 index 000000000..70a400dac --- /dev/null +++ b/vllm_omni/host_weight_runtime/filesystem/locks.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Stable POSIX advisory locks used by the filesystem store.""" + +from __future__ import annotations + +import errno +import fcntl +import os +import time +from pathlib import Path + + +class FileLockTimeoutError(TimeoutError): + pass + + +class FileLock: + """One held flock whose inode is stable outside artifact directories.""" + + def __init__(self, path: Path, *, exclusive: bool, deadline: float | None, nonblocking: bool = False) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + self.path = path + self._fd = os.open(path, os.O_CREAT | os.O_RDWR | os.O_CLOEXEC | os.O_NOFOLLOW, 0o600) + self._creator_pid = os.getpid() + self._closed = False + operation = fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH + if nonblocking: + operation |= fcntl.LOCK_NB + try: + if nonblocking: + self._acquire_once(operation) + elif deadline is None: + self._acquire_blocking(operation) + else: + self._acquire_until(operation | fcntl.LOCK_NB, deadline) + except BaseException: + os.close(self._fd) + self._closed = True + raise + + def _acquire_once(self, operation: int) -> None: + try: + fcntl.flock(self._fd, operation) + except OSError as exc: + if exc.errno in (errno.EACCES, errno.EAGAIN): + raise FileLockTimeoutError(f"lock is active: {self.path}") from exc + raise + + def _acquire_blocking(self, operation: int) -> None: + while True: + try: + fcntl.flock(self._fd, operation) + return + except OSError as exc: + if exc.errno != errno.EINTR: + raise + + def _acquire_until(self, operation: int, deadline: float) -> None: + while True: + try: + fcntl.flock(self._fd, operation) + return + except OSError as exc: + if exc.errno == errno.EINTR: + continue + if exc.errno not in (errno.EACCES, errno.EAGAIN): + raise + remaining = deadline - time.monotonic() + if remaining <= 0: + raise FileLockTimeoutError(f"timed out waiting for {self.path}") from exc + time.sleep(min(0.05, remaining)) + + def close(self) -> None: + if self._closed: + return + self._closed = True + if os.getpid() != self._creator_pid: + # flock state belongs to the inherited open-file description. A + # child must only drop its descriptor reference; LOCK_UN would + # release the creator process's active lock as well. + os.close(self._fd) + return + try: + fcntl.flock(self._fd, fcntl.LOCK_UN) + finally: + os.close(self._fd) + + def __enter__(self) -> FileLock: + return self + + def __exit__(self, exc_type: object, exc: object, _traceback: object) -> None: + self.close() + + +def lock_is_active(path: Path) -> bool: + """Return a point-in-time contention snapshot without creating a lock file.""" + try: + fd = os.open(path, os.O_RDWR | os.O_CLOEXEC | os.O_NOFOLLOW) + except FileNotFoundError: + return False + try: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as exc: + if exc.errno in (errno.EACCES, errno.EAGAIN): + return True + raise + fcntl.flock(fd, fcntl.LOCK_UN) + return False + finally: + os.close(fd) + + +__all__ = ["FileLock", "FileLockTimeoutError", "lock_is_active"] diff --git a/vllm_omni/host_weight_runtime/filesystem/store.py b/vllm_omni/host_weight_runtime/filesystem/store.py new file mode 100644 index 000000000..9c81d2972 --- /dev/null +++ b/vllm_omni/host_weight_runtime/filesystem/store.py @@ -0,0 +1,1189 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Crash-safe node-local filesystem implementation of HostWeightStore.""" + +from __future__ import annotations + +import contextlib +import errno +import hashlib +import json +import os +import shutil +import stat +import time +import uuid +from collections import defaultdict +from collections.abc import Callable +from pathlib import Path +from typing import Protocol + +import regex as re +import torch +from safetensors import SafetensorError, safe_open + +from ..config import CapacityPolicy, IntegrityPolicy, StorageClass, StorageDomainPolicy, ValidationLevel +from ..errors import FailureCode, HostWeightError, HostWeightFailure, ResolutionStage +from ..identity import CanonicalJson, WeightArtifactIdentity, canonical_json +from ..inspection import ( + ArtifactInspection, + ArtifactInventoryEntry, + ArtifactInventoryState, + DomainInspection, +) +from ..lease import HostWeightLease, LeaseProvenance, MappedHostRegion +from ..manifest import ManifestValidationError, PublicationMarker, TensorManifestEntry, WeightManifest +from ..outcomes import StoreResult, StoreStatus +from ..protocols import BuildRequest, CoordinationScope, WeightProducer +from .locks import FileLock, FileLockTimeoutError, lock_is_active +from .writer import FilesystemArtifactWriter + +_DOMAIN_FILE = "domain.json" +_DOMAIN_POLICY_FILE = "domain-policy.json" +_MANIFEST_FILE = "manifest.json" +_READY_FILE = "READY.json" +_MAX_METADATA_BYTES = 64 * 1024**2 +_FILE_HASH_CHUNK_BYTES = 8 * 1024**2 +_ARTIFACT_KEY_RE = re.compile(r"[0-9a-f]{64}") +_LOCAL_DISK_FILESYSTEM_TYPES = frozenset( + { + "aufs", + "bcachefs", + "btrfs", + "erofs", + "ext2", + "ext3", + "ext4", + "f2fs", + "fuseblk", + "jfs", + "ntfs3", + "overlay", + "reiserfs", + "rootfs", + "xfs", + "zfs", + } +) +_REMOTE_FILESYSTEM_TYPES = frozenset( + { + "9p", + "afs", + "beegfs", + "ceph", + "cifs", + "fuse.gcsfuse", + "fuse.s3fs", + "fuse.sshfs", + "gcsfuse", + "glusterfs", + "gpfs", + "lustre", + "nfs", + "nfs4", + "orangefs", + "panfs", + "s3fs", + "smb3", + "sshfs", + "virtiofs", + "wekafs", + } +) + + +class _ExitContext(Protocol): + def __exit__(self, exc_type: None, exc: None, _traceback: None) -> object: ... + + +def _make_fd_closer(fd: int) -> Callable[[], None]: + def close() -> None: + os.close(fd) + + return close + + +def _make_context_closer(context: _ExitContext) -> Callable[[], None]: + def close() -> None: + context.__exit__(None, None, None) + + return close + + +def _failure( + stage: ResolutionStage, + code: FailureCode, + message: str, + *, + retryable: bool = False, + details: object | None = None, +) -> HostWeightFailure: + return HostWeightFailure( + stage=stage, + code=code, + retryable=retryable, + message=message, + details=CanonicalJson.from_value({} if details is None else details), + ) + + +def _fsync_directory(path: Path) -> None: + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _write_atomic_json(path: Path, value: object, *, mode: int = 0o444) -> None: + data = canonical_json(value) + temporary = path.parent / f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" + fd = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_CLOEXEC, 0o600) + try: + offset = 0 + while offset < len(data): + count = os.write(fd, data[offset:]) + if count <= 0: + raise OSError(errno.EIO, f"short write for {temporary}") + offset += count + os.fsync(fd) + os.fchmod(fd, mode) + os.fsync(fd) + except BaseException: + os.close(fd) + temporary.unlink(missing_ok=True) + raise + else: + os.close(fd) + os.replace(temporary, path) + _fsync_directory(path.parent) + + +def _read_json_file(path: Path) -> dict[str, object]: + fd = os.open(path, os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW) + try: + return _read_json_fd(fd, path.name) + finally: + os.close(fd) + + +def _read_json_at(directory_fd: int, name: str, *, require_read_only: bool = False) -> dict[str, object]: + fd = os.open(name, os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW, dir_fd=directory_fd) + try: + return _read_json_fd(fd, name, require_read_only=require_read_only) + finally: + os.close(fd) + + +def _read_json_fd(fd: int, name: str, *, require_read_only: bool = False) -> dict[str, object]: + file_stat = os.fstat(fd) + if not stat.S_ISREG(file_stat.st_mode) or file_stat.st_nlink != 1: + raise ManifestValidationError(f"metadata file {name!r} is not one regular file") + if file_stat.st_size > _MAX_METADATA_BYTES: + raise ManifestValidationError(f"metadata file {name!r} exceeds {_MAX_METADATA_BYTES} bytes") + if require_read_only and (file_stat.st_uid != os.geteuid() or file_stat.st_mode & 0o222): + raise ManifestValidationError(f"published metadata file {name!r} is not immutable") + chunks: list[bytes] = [] + offset = 0 + while offset < file_stat.st_size: + chunk = os.pread(fd, min(1024**2, file_stat.st_size - offset), offset) + if not chunk: + raise ManifestValidationError(f"metadata file {name!r} ended unexpectedly") + chunks.append(chunk) + offset += len(chunk) + try: + value = json.loads(b"".join(chunks)) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ManifestValidationError(f"metadata file {name!r} is not valid JSON") from exc + if not isinstance(value, dict): + raise ManifestValidationError(f"metadata file {name!r} must contain an object") + return value + + +def _sha256_fd(fd: int, size: int) -> str: + digest = hashlib.sha256() + offset = 0 + while offset < size: + chunk = os.pread(fd, min(_FILE_HASH_CHUNK_BYTES, size - offset), offset) + if not chunk: + raise OSError(errno.EIO, "artifact file ended while hashing") + digest.update(chunk) + offset += len(chunk) + return digest.hexdigest() + + +def _mount_unescape(value: str) -> str: + return value.replace("\\040", " ").replace("\\011", "\t").replace("\\012", "\n").replace("\\134", "\\") + + +def detect_filesystem_type(path: Path) -> str: + """Return the kernel filesystem type for the most specific mount.""" + resolved = path.resolve() + mountinfo = Path("/proc/self/mountinfo") + if not mountinfo.is_file(): # pragma: no cover - vLLM-Omni production is Linux + raise ValueError("cannot verify filesystem locality without /proc/self/mountinfo") + best_length = -1 + best_type = "" + for line in mountinfo.read_text(encoding="utf-8").splitlines(): + left, separator, right = line.partition(" - ") + if not separator: + continue + left_fields = left.split() + right_fields = right.split() + if len(left_fields) < 5 or not right_fields: + continue + mount_point = Path(_mount_unescape(left_fields[4])) + try: + resolved.relative_to(mount_point) + except ValueError: + continue + if len(str(mount_point)) > best_length: + best_length = len(str(mount_point)) + best_type = right_fields[0] + if not best_type: + raise ValueError(f"cannot identify the filesystem containing {resolved}") + return best_type + + +def _storage_class_for_filesystem_type(filesystem_type: str) -> StorageClass: + if filesystem_type in {"tmpfs", "ramfs"}: + return StorageClass.TMPFS + if filesystem_type in _LOCAL_DISK_FILESYSTEM_TYPES: + return StorageClass.DISK + if filesystem_type in _REMOTE_FILESYSTEM_TYPES: + raise ValueError(f"filesystem type {filesystem_type!r} is not node-local") + raise ValueError(f"filesystem type {filesystem_type!r} is not recognized as a local filesystem") + + +def detect_storage_class(path: Path) -> StorageClass: + """Classify a verified node-local mount without spawning platform tools.""" + return _storage_class_for_filesystem_type(detect_filesystem_type(path)) + + +class FilesystemHostWeightStore: + """A cooperative local store with stable locks and immutable publication.""" + + def __init__( + self, + domain: StorageDomainPolicy, + capacity: CapacityPolicy, + integrity: IntegrityPolicy, + ) -> None: + self.domain_policy = domain + self.capacity_policy = capacity + self.integrity_policy = integrity + self.root = domain.root.expanduser().resolve() + self.locks_dir = self.root / "locks" + self.artifacts_dir = self.root / "artifacts" + self.tmp_dir = self.root / "tmp" + self.quarantine_dir = self.root / "quarantine" + self.deny_dir = self.root / "deny" + if integrity.local_lookup is ValidationLevel.FS_VERITY: + raise HostWeightError( + _failure( + ResolutionStage.CONFIGURATION, + FailureCode.UNSUPPORTED, + "filesystem-verity validation is not implemented", + ) + ) + if domain.numa_prefault or domain.verify_numa_residency: + raise HostWeightError( + _failure( + ResolutionStage.CONFIGURATION, + FailureCode.UNSUPPORTED, + "enforced NUMA placement and residency verification are not implemented", + ) + ) + if integrity.background_scrub: + raise HostWeightError( + _failure( + ResolutionStage.CONFIGURATION, + FailureCode.UNSUPPORTED, + "background integrity scrubbing is not implemented", + ) + ) + try: + self._initialize_domain() + except HostWeightError: + raise + except OSError as exc: + raise HostWeightError( + _failure( + ResolutionStage.DOMAIN, + FailureCode.DOMAIN_UNAVAILABLE, + f"cannot initialize storage domain {self.root}: {exc}", + retryable=True, + ) + ) from exc + except (ValueError, ManifestValidationError) as exc: + raise HostWeightError( + _failure( + ResolutionStage.CONFIGURATION, + FailureCode.DOMAIN_POLICY_MISMATCH, + f"storage domain metadata is invalid under {self.root}: {exc}", + ) + ) from exc + + def _initialize_domain(self) -> None: + try: + self.filesystem_type = detect_filesystem_type(self.root) + detected_storage_class = _storage_class_for_filesystem_type(self.filesystem_type) + except ValueError as exc: + raise HostWeightError( + _failure( + ResolutionStage.CONFIGURATION, + FailureCode.DOMAIN_POLICY_MISMATCH, + f"local filesystem storage is unavailable for {self.root}: {exc}", + ) + ) from exc + if detected_storage_class is not self.domain_policy.storage_class: + raise HostWeightError( + _failure( + ResolutionStage.DOMAIN, + FailureCode.STORAGE_CLASS_MISMATCH, + "configured storage class " + f"{self.domain_policy.storage_class.value} differs from detected {detected_storage_class.value}", + ) + ) + + try: + self.root.mkdir(parents=True, exist_ok=True, mode=0o700) + self.locks_dir.mkdir(exist_ok=True, mode=0o700) + for directory in (self.artifacts_dir, self.tmp_dir, self.quarantine_dir, self.deny_dir): + directory.mkdir(exist_ok=True, mode=0o700) + except OSError as exc: + raise HostWeightError( + _failure( + ResolutionStage.DOMAIN, + FailureCode.DOMAIN_UNAVAILABLE, + f"cannot initialize {self.root}: {exc}", + retryable=True, + ) + ) from exc + + if self.integrity_policy.require_trusted_permissions: + for directory in ( + self.root, + self.locks_dir, + self.artifacts_dir, + self.tmp_dir, + self.quarantine_dir, + self.deny_dir, + ): + directory_stat = directory.lstat() + if ( + not stat.S_ISDIR(directory_stat.st_mode) + or directory_stat.st_uid != os.geteuid() + or directory_stat.st_mode & 0o022 + ): + raise HostWeightError( + _failure( + ResolutionStage.DOMAIN, + FailureCode.DOMAIN_UNAVAILABLE, + f"host weight store directory is not exclusively controlled by this user: {directory}", + ) + ) + + with FileLock(self.locks_dir / "domain-init.lock", exclusive=True, deadline=None): + domain_path = self.root / _DOMAIN_FILE + if domain_path.exists(): + domain_metadata = _read_json_file(domain_path) + expected = { + "schema_version": 1, + "scope": self.domain_policy.scope.value, + "domain_id": self.domain_policy.domain_id, + "backend": "local_filesystem", + "storage_class": self.domain_policy.storage_class.value, + "filesystem_type": self.filesystem_type, + } + if set(domain_metadata) != {*expected, "domain_uuid"}: + raise HostWeightError( + _failure( + ResolutionStage.CONFIGURATION, + FailureCode.DOMAIN_POLICY_MISMATCH, + f"storage domain metadata has an incompatible schema under {self.root}", + ) + ) + actual = {key: domain_metadata.get(key) for key in expected} + domain_uuid = domain_metadata.get("domain_uuid") + if actual != expected or not isinstance(domain_uuid, str) or not domain_uuid: + raise HostWeightError( + _failure( + ResolutionStage.CONFIGURATION, + FailureCode.DOMAIN_POLICY_MISMATCH, + f"storage domain metadata does not match configured policy under {self.root}", + ) + ) + else: + domain_metadata = { + "schema_version": 1, + "domain_uuid": uuid.uuid4().hex, + "scope": self.domain_policy.scope.value, + "domain_id": self.domain_policy.domain_id, + "backend": "local_filesystem", + "storage_class": self.domain_policy.storage_class.value, + "filesystem_type": self.filesystem_type, + } + _write_atomic_json(domain_path, domain_metadata) + self.domain_uuid = str(domain_metadata["domain_uuid"]) + + policy_path = self.root / _DOMAIN_POLICY_FILE + requested_policy = self._capacity_document(policy_version=1) + if policy_path.exists(): + actual_policy = _read_json_file(policy_path) + if set(actual_policy) != set(requested_policy): + raise HostWeightError( + _failure( + ResolutionStage.CONFIGURATION, + FailureCode.DOMAIN_POLICY_MISMATCH, + f"capacity policy has an incompatible schema under {self.root}", + ) + ) + comparable_actual = {key: actual_policy.get(key) for key in requested_policy if key != "policy_version"} + comparable_requested = { + key: value for key, value in requested_policy.items() if key != "policy_version" + } + if comparable_actual != comparable_requested: + raise HostWeightError( + _failure( + ResolutionStage.CONFIGURATION, + FailureCode.DOMAIN_POLICY_MISMATCH, + f"capacity policy differs from the authoritative policy under {self.root}", + ) + ) + policy_version = actual_policy.get("policy_version") + if isinstance(policy_version, bool) or not isinstance(policy_version, int) or policy_version < 1: + raise HostWeightError( + _failure( + ResolutionStage.CONFIGURATION, + FailureCode.DOMAIN_POLICY_MISMATCH, + f"capacity policy has an invalid version under {self.root}", + ) + ) + self.policy_version = policy_version + else: + _write_atomic_json(policy_path, requested_policy) + self.policy_version = 1 + + def _capacity_document(self, *, policy_version: int) -> dict[str, object]: + return { + "schema_version": 1, + "policy_version": policy_version, + "max_artifact_bytes": self.capacity_policy.max_artifact_bytes, + "max_store_bytes": self.capacity_policy.max_store_bytes, + "min_free_bytes": self.capacity_policy.min_free_bytes, + "eviction": self.capacity_policy.eviction, + } + + def _build_lock_path(self, key: str) -> Path: + return self.locks_dir / f"{key}.build.lock" + + def _artifact_lock_path(self, key: str) -> Path: + return self.locks_dir / f"{key}.artifact.lock" + + def _entry_path(self, key: str) -> Path: + return self.artifacts_dir / key + + def _deny_path(self, key: str) -> Path: + return self.deny_dir / f"{key}.json" + + def lookup( + self, + identity: WeightArtifactIdentity, + *, + validation: ValidationLevel, + deadline: float | None = None, + ) -> StoreResult: + return self._lookup(identity, validation=validation, deadline=deadline, record_invalid=True) + + def _lookup( + self, + identity: WeightArtifactIdentity, + *, + validation: ValidationLevel, + deadline: float | None, + record_invalid: bool, + ) -> StoreResult: + if validation is ValidationLevel.FS_VERITY: + return StoreResult( + StoreStatus.FAILED, + failure=_failure( + ResolutionStage.VALIDATION, + FailureCode.UNSUPPORTED, + "filesystem-verity validation is modeled but not implemented", + ), + ) + try: + artifact_lock = FileLock(self._artifact_lock_path(identity.key), exclusive=False, deadline=deadline) + except FileLockTimeoutError as exc: + return StoreResult( + StoreStatus.TIMEOUT, + failure=_failure( + ResolutionStage.LOOKUP, + FailureCode.ACTIVE_BUILD_TIMEOUT, + str(exc), + retryable=True, + ), + ) + except OSError as exc: + return StoreResult( + StoreStatus.FAILED, + failure=_failure(ResolutionStage.DOMAIN, FailureCode.DOMAIN_UNAVAILABLE, str(exc), retryable=True), + ) + + entry_path = self._entry_path(identity.key) + if not entry_path.exists(): + artifact_lock.close() + return StoreResult(StoreStatus.MISS) + if self._deny_path(identity.key).exists(): + artifact_lock.close() + return StoreResult( + StoreStatus.INVALID, + failure=_failure( + ResolutionStage.LOOKUP, + FailureCode.ARTIFACT_DENIED, + f"artifact {identity.key} is denied pending quarantine", + ), + ) + try: + lease = self._open_lease(identity, entry_path, artifact_lock, validation) + if self._deny_path(identity.key).exists(): + lease.close() + return StoreResult( + StoreStatus.INVALID, + failure=_failure( + ResolutionStage.LOOKUP, + FailureCode.ARTIFACT_DENIED, + f"artifact {identity.key} was denied during lookup", + ), + ) + return StoreResult(StoreStatus.HIT, lease=lease) + except HostWeightError as exc: + try: + if record_invalid: + with contextlib.suppress(OSError): + self._write_deny(identity.key, exc.failure) + finally: + artifact_lock.close() + return StoreResult(StoreStatus.INVALID, failure=exc.failure) + except (OSError, SafetensorError, ValueError, ManifestValidationError) as exc: + failure = _failure( + ResolutionStage.VALIDATION, + FailureCode.MANIFEST_CORRUPT, + f"artifact {identity.key} failed validation: {exc}", + ) + try: + if record_invalid: + with contextlib.suppress(OSError): + self._write_deny(identity.key, failure) + finally: + artifact_lock.close() + return StoreResult(StoreStatus.INVALID, failure=failure) + + def _open_lease( + self, + identity: WeightArtifactIdentity, + entry_path: Path, + artifact_lock: FileLock, + validation: ValidationLevel, + ) -> HostWeightLease: + resources: list[Callable[[], None]] = [] + try: + entry_fd = os.open(entry_path, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW) + resources.append(_make_fd_closer(entry_fd)) + entry_stat = os.fstat(entry_fd) + if self.integrity_policy.require_trusted_permissions and ( + entry_stat.st_uid != os.geteuid() or entry_stat.st_mode & 0o222 + ): + raise ManifestValidationError("published artifact directory is not immutable") + names = set(os.listdir(entry_fd)) + marker_document = _read_json_at( + entry_fd, + _READY_FILE, + require_read_only=self.integrity_policy.require_trusted_permissions, + ) + manifest_document = _read_json_at( + entry_fd, + _MANIFEST_FILE, + require_read_only=self.integrity_policy.require_trusted_permissions, + ) + marker = PublicationMarker.from_dict(marker_document) + manifest = WeightManifest.from_dict(manifest_document) + if marker.schema_version != 1 or manifest.schema_version != 1 or marker.integrity_mode != "full_checksum": + raise HostWeightError( + _failure( + ResolutionStage.VALIDATION, + FailureCode.MANIFEST_INCOMPATIBLE, + "artifact publication schema or integrity mode is unsupported", + ) + ) + expected_names = {_READY_FILE, _MANIFEST_FILE, *(entry.file_name for entry in manifest.files)} + if names != expected_names: + raise ManifestValidationError( + f"artifact directory entries differ: missing={sorted(expected_names - names)}, " + f"unexpected={sorted(names - expected_names)}" + ) + manifest_digest = hashlib.sha256(manifest.canonical_bytes).hexdigest() + if marker.manifest_sha256 != manifest_digest: + raise HostWeightError( + _failure(ResolutionStage.VALIDATION, FailureCode.CHECKSUM_MISMATCH, "manifest digest mismatch") + ) + if marker.artifact_content_sha256 != manifest.artifact_content_sha256: + raise HostWeightError( + _failure(ResolutionStage.VALIDATION, FailureCode.CHECKSUM_MISMATCH, "content digest mismatch") + ) + if manifest.artifact_key != identity.key or manifest.identity.canonical_bytes != identity.canonical_bytes: + raise HostWeightError( + _failure( + ResolutionStage.VALIDATION, + FailureCode.MANIFEST_INCOMPATIBLE, + "artifact semantic identity differs from the exact request", + ) + ) + if ( + manifest.producer_schema != identity.producer.manifest_schema + or manifest.restorer_schema != identity.producer.restorer_schema + ): + raise HostWeightError( + _failure( + ResolutionStage.VALIDATION, + FailureCode.MANIFEST_INCOMPATIBLE, + "artifact producer or restorer schema differs from the exact request", + ) + ) + + tensors_by_file: dict[str, list[TensorManifestEntry]] = defaultdict(list) + for tensor_entry in manifest.tensors: + tensors_by_file[tensor_entry.file_name].append(tensor_entry) + tensors: dict[str, torch.Tensor] = {} + regions: list[MappedHostRegion] = [] + backing_digest = hashlib.sha256() + for file_entry in manifest.files: + fd = os.open( + file_entry.file_name, + os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW, + dir_fd=entry_fd, + ) + resources.append(_make_fd_closer(fd)) + file_stat = os.fstat(fd) + if not stat.S_ISREG(file_stat.st_mode) or file_stat.st_nlink != 1: + raise ManifestValidationError(f"payload {file_entry.file_name!r} is not one regular file") + if file_stat.st_size != file_entry.size: + raise HostWeightError( + _failure( + ResolutionStage.VALIDATION, + FailureCode.CHECKSUM_MISMATCH, + f"payload size differs for {file_entry.file_name!r}", + ) + ) + if self.integrity_policy.require_trusted_permissions and ( + file_stat.st_uid != os.geteuid() or file_stat.st_mode & 0o222 + ): + raise ManifestValidationError(f"published payload {file_entry.file_name!r} is not immutable") + if ( + validation is ValidationLevel.FULL_CHECKSUM + and _sha256_fd(fd, file_stat.st_size) != file_entry.sha256 + ): + raise HostWeightError( + _failure( + ResolutionStage.VALIDATION, + FailureCode.CHECKSUM_MISMATCH, + f"payload digest differs for {file_entry.file_name!r}", + ) + ) + backing_digest.update( + canonical_json( + { + "file": file_entry.file_name, + "device": file_stat.st_dev, + "inode": file_stat.st_ino, + "size": file_stat.st_size, + } + ) + ) + handle_context = safe_open(f"/proc/self/fd/{fd}", framework="pt", device="cpu") + handle = handle_context.__enter__() + resources.append(_make_context_closer(handle_context)) + expected_entries = tensors_by_file[file_entry.file_name] + if set(handle.keys()) != {entry.storage_key for entry in expected_entries}: + raise ManifestValidationError(f"payload tensor keys differ for {file_entry.file_name!r}") + addresses: list[tuple[int, int]] = [] + for tensor_entry in expected_entries: + tensor = handle.get_tensor(tensor_entry.storage_key) + actual_nbytes = tensor.numel() * tensor.element_size() + if ( + tuple(tensor.shape) != tensor_entry.shape + or str(tensor.dtype) != tensor_entry.dtype + or tuple(tensor.stride()) != tensor_entry.stride + or actual_nbytes != tensor_entry.nbytes + or tensor.device.type != "cpu" + ): + raise ManifestValidationError(f"mapped tensor metadata differs for {tensor_entry.name!r}") + tensors[tensor_entry.name] = tensor + if actual_nbytes: + addresses.append((tensor.data_ptr(), tensor.data_ptr() + actual_nbytes)) + if addresses: + merged: list[list[int]] = [] + for start, end in sorted(addresses): + if merged and start <= merged[-1][1]: + merged[-1][1] = max(merged[-1][1], end) + else: + merged.append([start, end]) + regions.extend( + MappedHostRegion( + file_name=file_entry.file_name, + address=start, + length=end - start, + ) + for start, end in merged + ) + + backing_digest.update(self.domain_uuid.encode("ascii")) + backing_digest.update(marker.publication_id.encode("ascii")) + provenance = LeaseProvenance( + resolution_id=uuid.uuid4().hex, + identity_digest=identity.key, + artifact_content_sha256=manifest.artifact_content_sha256, + domain_uuid=self.domain_uuid, + publication_id=marker.publication_id, + resolution_source="local", + artifact_origin="producer", + producer_id=identity.producer.producer_id, + provider_id=None, + validation_level=validation.value, + backing_fingerprint=backing_digest.hexdigest(), + ) + return HostWeightLease( + identity=identity, + manifest=manifest, + tensors=tensors, + mapped_regions=tuple(regions), + provenance=provenance, + resource_closers=tuple(resources), + release_artifact_lock=artifact_lock.close, + ) + except BaseException: + for closer in reversed(resources): + with contextlib.suppress(Exception): + closer() + raise + + def get_or_build( + self, + request: BuildRequest, + producer: WeightProducer, + *, + validation: ValidationLevel, + deadline: float, + ) -> StoreResult: + identity = request.identity + if producer.spec.coordination_scope is not CoordinationScope.SINGLE_PROCESS: + return StoreResult( + StoreStatus.FAILED, + failure=_failure( + ResolutionStage.PRODUCTION, + FailureCode.PRODUCER_UNSUPPORTED, + f"coordination scope {producer.spec.coordination_scope.value} is not implemented", + ), + ) + if tuple(output.key for output in producer.spec.outputs) != (identity.key,): + return StoreResult( + StoreStatus.FAILED, + failure=_failure( + ResolutionStage.PRODUCTION, + FailureCode.PRODUCER_UNAVAILABLE, + "producer does not declare exactly the requested artifact identity", + ), + ) + + initial = self.lookup(identity, validation=validation, deadline=deadline) + if initial.status is StoreStatus.HIT: + return initial + if initial.status in {StoreStatus.TIMEOUT, StoreStatus.FAILED}: + return initial + + try: + build_lock = FileLock(self._build_lock_path(identity.key), exclusive=True, deadline=deadline) + except FileLockTimeoutError as exc: + return StoreResult( + StoreStatus.TIMEOUT, + failure=_failure( + ResolutionStage.PRODUCTION, + FailureCode.ACTIVE_BUILD_TIMEOUT, + str(exc), + retryable=True, + ), + ) + except OSError as exc: + return StoreResult( + StoreStatus.FAILED, + failure=_failure(ResolutionStage.DOMAIN, FailureCode.DOMAIN_UNAVAILABLE, str(exc), retryable=True), + ) + + with build_lock: + recheck = self.lookup(identity, validation=validation, deadline=deadline) + if recheck.status is StoreStatus.HIT: + return StoreResult(StoreStatus.JOINED, lease=recheck.lease) + if recheck.status in {StoreStatus.TIMEOUT, StoreStatus.FAILED}: + return recheck + + try: + with FileLock(self._artifact_lock_path(identity.key), exclusive=True, deadline=deadline): + if self._entry_path(identity.key).exists(): + self._quarantine_locked(identity.key, reason="validation_failure") + self._remove_deny_locked(identity.key) + self._remove_stale_temps_locked(identity.key) + except FileLockTimeoutError as exc: + return StoreResult( + StoreStatus.TIMEOUT, + failure=_failure( + ResolutionStage.PRODUCTION, + FailureCode.ACTIVE_BUILD_TIMEOUT, + str(exc), + retryable=True, + ), + ) + except OSError as exc: + return StoreResult( + StoreStatus.FAILED, + failure=_failure( + ResolutionStage.LIFECYCLE, + FailureCode.QUARANTINE_FAILED, + f"failed to prepare artifact rebuild: {exc}", + ), + ) + + staging = self.tmp_dir / f"{identity.key}.{os.getpid()}.{uuid.uuid4().hex}" + writer: FilesystemArtifactWriter | None = None + manifest: WeightManifest | None = None + try: + staging.mkdir(mode=0o700) + writer = FilesystemArtifactWriter(staging, capacity_check=self._check_capacity) + metadata = producer.produce(writer) + manifest, _ = writer.finalize(identity, metadata) + artifact_bytes = self._tree_bytes(staging) + self._check_capacity(artifact_bytes, 0) + os.chmod(staging, 0o555) + _fsync_directory(staging) + except HostWeightError as exc: + if writer is not None: + writer.abort() + self._discard_temp(identity.key, staging) + return StoreResult(StoreStatus.FAILED, failure=exc.failure) + except OSError as exc: + if writer is not None: + writer.abort() + self._discard_temp(identity.key, staging) + if exc.errno in (errno.ENOSPC, errno.EDQUOT): + code = FailureCode.ENOSPC + stage = ResolutionStage.CAPACITY + elif writer is None: + code = FailureCode.DOMAIN_UNAVAILABLE + stage = ResolutionStage.DOMAIN + else: + code = FailureCode.PRODUCER_FAILED + stage = ResolutionStage.PRODUCTION + return StoreResult( + StoreStatus.FAILED, + failure=_failure( + stage, + code, + f"artifact production failed: {exc}", + retryable=stage is ResolutionStage.CAPACITY, + ), + ) + except Exception as exc: + if writer is not None: + writer.abort() + self._discard_temp(identity.key, staging) + return StoreResult( + StoreStatus.FAILED, + failure=_failure( + ResolutionStage.PRODUCTION, + FailureCode.PRODUCER_FAILED, + f"artifact producer failed: {exc}", + ), + ) + + assert manifest is not None + try: + with FileLock(self._artifact_lock_path(identity.key), exclusive=True, deadline=None): + entry_path = self._entry_path(identity.key) + if entry_path.exists(): + existing = WeightManifest.from_dict(_read_json_file(entry_path / _MANIFEST_FILE)) + same_semantic_content = ( + existing.identity.canonical_bytes == manifest.identity.canonical_bytes + and existing.artifact_content_sha256 == manifest.artifact_content_sha256 + and existing.producer_schema == manifest.producer_schema + and existing.restorer_schema == manifest.restorer_schema + and existing.format_metadata == manifest.format_metadata + ) + if not same_semantic_content: + collision = self.quarantine_dir / f"{identity.key}.identity-collision.{uuid.uuid4().hex}" + os.replace(staging, collision) + _fsync_directory(self.quarantine_dir) + _fsync_directory(self.tmp_dir) + return StoreResult( + StoreStatus.FAILED, + failure=_failure( + ResolutionStage.VALIDATION, + FailureCode.IDENTITY_COLLISION, + "the same semantic identity produced different artifact content", + ), + ) + self._remove_tree(staging) + else: + os.replace(staging, entry_path) + _fsync_directory(self.artifacts_dir) + _fsync_directory(self.tmp_dir) + self._remove_deny_locked(identity.key) + except (OSError, ValueError, ManifestValidationError) as exc: + self._discard_temp(identity.key, staging) + return StoreResult( + StoreStatus.FAILED, + failure=_failure( + ResolutionStage.LIFECYCLE, + FailureCode.PUBLICATION_FAILED, + f"atomic artifact publication failed: {exc}", + ), + ) + + opened = self.lookup(identity, validation=validation, deadline=deadline) + if opened.status is not StoreStatus.HIT: + return opened + return StoreResult(StoreStatus.BUILT, lease=opened.lease) + + def _check_capacity(self, projected_artifact_bytes: int, additional_bytes: int) -> None: + policy = self.capacity_policy + if policy.max_artifact_bytes is not None and projected_artifact_bytes > policy.max_artifact_bytes: + raise HostWeightError( + _failure( + ResolutionStage.CAPACITY, + FailureCode.ARTIFACT_TOO_LARGE, + f"projected artifact size {projected_artifact_bytes} exceeds {policy.max_artifact_bytes}", + retryable=True, + ) + ) + store_bytes = self._tree_bytes(self.root) + if policy.max_store_bytes is not None and store_bytes + additional_bytes > policy.max_store_bytes: + raise HostWeightError( + _failure( + ResolutionStage.CAPACITY, + FailureCode.STORE_LIMIT_EXCEEDED, + f"store size plus allocation would exceed {policy.max_store_bytes}", + retryable=True, + ) + ) + free_bytes = shutil.disk_usage(self.root).free + if free_bytes - additional_bytes < policy.min_free_bytes: + raise HostWeightError( + _failure( + ResolutionStage.CAPACITY, + FailureCode.INSUFFICIENT_CAPACITY, + f"allocation would leave less than {policy.min_free_bytes} free bytes", + retryable=True, + ) + ) + + def _write_deny(self, key: str, failure: HostWeightFailure) -> None: + path = self._deny_path(key) + document = { + "schema_version": 1, + "stage": failure.stage.value, + "reason": failure.code.value, + "message": failure.message, + } + try: + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_CLOEXEC, 0o444) + except FileExistsError: + return + try: + data = canonical_json(document) + offset = 0 + while offset < len(data): + written = os.write(fd, data[offset:]) + if written == 0: + raise OSError("short write while publishing deny marker") + offset += written + os.fsync(fd) + finally: + os.close(fd) + _fsync_directory(self.deny_dir) + + def _remove_deny_locked(self, key: str) -> None: + path = self._deny_path(key) + if path.exists(): + path.unlink() + _fsync_directory(self.deny_dir) + + def _quarantine_locked(self, key: str, *, reason: str) -> Path | None: + entry = self._entry_path(key) + if not entry.exists(): + return None + destination = self.quarantine_dir / f"{key}.{reason}.{uuid.uuid4().hex}" + os.replace(entry, destination) + _fsync_directory(self.artifacts_dir) + _fsync_directory(self.quarantine_dir) + return destination + + def _remove_stale_temps_locked(self, key: str) -> None: + for path in self.tmp_dir.glob(f"{key}.*"): + self._remove_tree(path) + _fsync_directory(self.tmp_dir) + + def _discard_temp(self, key: str, path: Path) -> None: + with contextlib.suppress(OSError, FileLockTimeoutError): + with FileLock(self._artifact_lock_path(key), exclusive=True, deadline=None): + self._remove_tree(path) + _fsync_directory(self.tmp_dir) + + @staticmethod + def _remove_tree(path: Path) -> None: + if path.is_symlink() or path.is_file(): + path.unlink(missing_ok=True) + elif path.exists(): + for directory, subdirectories, _ in os.walk(path, topdown=False, followlinks=False): + for name in subdirectories: + child = Path(directory) / name + if not child.is_symlink(): + os.chmod(child, 0o700) + os.chmod(directory, 0o700) + shutil.rmtree(path) + + @staticmethod + def _tree_bytes(root: Path) -> int: + with contextlib.suppress(OSError): + root_stat = root.lstat() + if stat.S_ISREG(root_stat.st_mode): + return root_stat.st_size + total = 0 + for directory, _, files in os.walk(root, followlinks=False): + for name in files: + path = Path(directory) / name + with contextlib.suppress(OSError): + file_stat = path.lstat() + if stat.S_ISREG(file_stat.st_mode): + total += file_stat.st_size + return total + + def cleanup(self, identity: WeightArtifactIdentity) -> HostWeightFailure | None: + """Explicitly remove an inactive artifact without waiting for leases.""" + try: + build_lock = FileLock( + self._build_lock_path(identity.key), + exclusive=True, + deadline=None, + nonblocking=True, + ) + except FileLockTimeoutError: + return _failure( + ResolutionStage.LIFECYCLE, + FailureCode.ACTIVE_BUILD_TIMEOUT, + f"artifact {identity.key} has an active builder", + retryable=True, + ) + with build_lock: + try: + artifact_lock = FileLock( + self._artifact_lock_path(identity.key), + exclusive=True, + deadline=None, + nonblocking=True, + ) + except FileLockTimeoutError: + return _failure( + ResolutionStage.LIFECYCLE, + FailureCode.ACTIVE_LEASE, + f"artifact {identity.key} has an active lease", + retryable=True, + ) + with artifact_lock: + entry = self._entry_path(identity.key) + if entry.exists(): + removal = self.quarantine_dir / f"{identity.key}.cleanup.{uuid.uuid4().hex}" + os.replace(entry, removal) + _fsync_directory(self.artifacts_dir) + self._remove_tree(removal) + _fsync_directory(self.quarantine_dir) + self._remove_deny_locked(identity.key) + return None + + def inspect_domain(self) -> DomainInspection: + usage = shutil.disk_usage(self.root) + inventory: list[ArtifactInventoryEntry] = [] + for state, directory in ( + (ArtifactInventoryState.READY, self.artifacts_dir), + (ArtifactInventoryState.DENIED, self.deny_dir), + (ArtifactInventoryState.QUARANTINED, self.quarantine_dir), + (ArtifactInventoryState.TEMPORARY, self.tmp_dir), + ): + for path in directory.iterdir(): + artifact_key = path.name[:64] + if _ARTIFACT_KEY_RE.fullmatch(artifact_key) is None: + continue + reason = None + if state is ArtifactInventoryState.DENIED: + with contextlib.suppress(OSError, ValueError, ManifestValidationError): + reason_value = _read_json_file(path).get("reason") + if isinstance(reason_value, str): + reason = reason_value + inventory.append( + ArtifactInventoryEntry( + artifact_key=artifact_key, + state=state, + storage_name=path.name, + size_bytes=self._tree_bytes(path), + reason=reason, + ) + ) + inventory.sort(key=lambda item: (item.artifact_key, item.state.value, item.storage_name)) + return DomainInspection( + domain_uuid=self.domain_uuid, + scope=self.domain_policy.scope.value, + domain_id=self.domain_policy.domain_id, + storage_class=self.domain_policy.storage_class.value, + filesystem_type=self.filesystem_type, + policy_version=self.policy_version, + store_bytes=self._tree_bytes(self.root), + filesystem_free_bytes=usage.free, + max_artifact_bytes=self.capacity_policy.max_artifact_bytes, + max_store_bytes=self.capacity_policy.max_store_bytes, + min_free_bytes=self.capacity_policy.min_free_bytes, + inventory=tuple(inventory), + ) + + def inspect_artifact( + self, + identity: WeightArtifactIdentity, + *, + validation: ValidationLevel, + ) -> ArtifactInspection: + result = self._lookup( + identity, + validation=validation, + deadline=time.monotonic(), + record_invalid=False, + ) + manifest = None + publication_id = None + backing_fingerprint = None + if result.lease is not None: + manifest = result.lease.manifest + publication_id = result.lease.provenance.publication_id + backing_fingerprint = result.lease.provenance.backing_fingerprint + result.lease.close() + entry = self._entry_path(identity.key) + return ArtifactInspection( + artifact_key=identity.key, + status=result.status, + size_bytes=self._tree_bytes(entry), + manifest=manifest, + publication_id=publication_id, + backing_fingerprint=backing_fingerprint, + failure=result.failure, + build_lock_active=lock_is_active(self._build_lock_path(identity.key)), + artifact_lock_active=lock_is_active(self._artifact_lock_path(identity.key)), + ) + + +__all__ = ["FilesystemHostWeightStore", "detect_storage_class"] diff --git a/vllm_omni/host_weight_runtime/filesystem/writer.py b/vllm_omni/host_weight_runtime/filesystem/writer.py new file mode 100644 index 000000000..8ecd17799 --- /dev/null +++ b/vllm_omni/host_weight_runtime/filesystem/writer.py @@ -0,0 +1,425 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Store-scoped streaming safetensors publication.""" + +from __future__ import annotations + +import contextlib +import errno +import hashlib +import json +import os +import struct +import uuid +from collections.abc import Callable +from pathlib import Path, PurePosixPath +from types import TracebackType + +import torch +from safetensors import safe_open + +from ..identity import WeightArtifactIdentity, canonical_json +from ..manifest import FileManifestEntry, PublicationMarker, TensorManifestEntry, WeightManifest +from ..protocols import ProductionMetadata, TensorWriteSpec + +_HASH_CHUNK_BYTES = 8 * 1024**2 +_MANIFEST_FILE = "manifest.json" +_READY_FILE = "READY.json" + +_SAFETENSORS_DTYPES: dict[torch.dtype, str] = { + torch.bool: "BOOL", + torch.uint8: "U8", + torch.int8: "I8", + torch.int16: "I16", + torch.int32: "I32", + torch.int64: "I64", + torch.float16: "F16", + torch.bfloat16: "BF16", + torch.float32: "F32", + torch.float64: "F64", +} +for _torch_name, _safe_name in ( + ("float8_e4m3fn", "F8_E4M3"), + ("float8_e5m2", "F8_E5M2"), + ("float8_e4m3fnuz", "F8_E4M3FNUZ"), + ("float8_e5m2fnuz", "F8_E5M2FNUZ"), +): + if (_dtype := getattr(torch, _torch_name, None)) is not None: + _SAFETENSORS_DTYPES[_dtype] = _safe_name + + +class ArtifactWriterError(RuntimeError): + pass + + +def _safe_relative_name(value: str) -> str: + path = PurePosixPath(value) + if not value or path.is_absolute() or len(path.parts) != 1 or path.name != value or value in {".", ".."}: + raise ArtifactWriterError(f"artifact payload name must be one safe relative basename: {value!r}") + if not value.endswith(".safetensors"): + raise ArtifactWriterError(f"artifact payload must use the .safetensors format: {value!r}") + return value + + +def _pwrite_all(fd: int, data: bytes | bytearray | memoryview, offset: int) -> None: + view = memoryview(data).cast("B") + written = 0 + while written < len(view): + count = os.pwrite(fd, view[written:], offset + written) + if count <= 0: + raise OSError(errno.EIO, "short pwrite while publishing host weights") + written += count + + +def _fsync_directory(path: Path) -> None: + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(_HASH_CHUNK_BYTES): + digest.update(chunk) + return digest.hexdigest() + + +def _tensor_sha256(tensor: torch.Tensor) -> str: + digest = hashlib.sha256() + byte_view = tensor.detach().reshape(-1).view(torch.uint8) + for offset in range(0, byte_view.numel(), _HASH_CHUNK_BYTES): + chunk = byte_view[offset : offset + _HASH_CHUNK_BYTES] + digest.update(memoryview(chunk.numpy())) + return digest.hexdigest() + + +class _Coverage: + def __init__(self, size: int) -> None: + self._size = size + self._ranges: list[tuple[int, int]] = [] + + def add(self, start: int, end: int) -> None: + if start < 0 or end < start or end > self._size: + raise ArtifactWriterError(f"write range [{start}, {end}) exceeds tensor storage {self._size}") + if start == end: + return + for existing_start, existing_end in self._ranges: + if start < existing_end and existing_start < end: + raise ArtifactWriterError(f"overlapping tensor write ranges [{start}, {end}) and {self._ranges}") + self._ranges.append((start, end)) + + def complete(self) -> bool: + if self._size == 0: + return True + cursor = 0 + for start, end in sorted(self._ranges): + if start != cursor: + return False + cursor = end + return cursor == self._size + + +class StreamingSafeTensorFile: + """One preallocated safetensors file filled through bounded CPU writes.""" + + def __init__( + self, + owner: FilesystemArtifactWriter, + relative_name: str, + specs: tuple[TensorWriteSpec, ...], + ) -> None: + self._owner = owner + self.relative_name = _safe_relative_name(relative_name) + if not specs: + raise ArtifactWriterError("a tensor payload file must contain at least one tensor") + by_name = {spec.name: spec for spec in specs} + if len(by_name) != len(specs): + raise ArtifactWriterError("tensor payload contains duplicate storage keys") + unsupported = [spec.dtype for spec in specs if spec.dtype not in _SAFETENSORS_DTYPES] + if unsupported: + raise ArtifactWriterError(f"unsupported safetensors dtypes: {unsupported}") + self.specs = tuple(sorted(specs, key=lambda spec: spec.name)) + self._specs = {spec.name: spec for spec in self.specs} + self._coverage = {spec.name: _Coverage(spec.nbytes) for spec in self.specs} + self._closed = False + + offsets: dict[str, int] = {} + cursor = 0 + for spec in self.specs: + offsets[spec.name] = cursor + cursor += spec.nbytes + header: dict[str, object] = {"__metadata__": {"format": "pt", "host_weight_runtime": "1"}} + for spec in self.specs: + start = offsets[spec.name] + header[spec.name] = { + "dtype": _SAFETENSORS_DTYPES[spec.dtype], + "shape": list(spec.shape), + "data_offsets": [start, start + spec.nbytes], + } + encoded_header = json.dumps(header, sort_keys=True, separators=(",", ":")).encode("utf-8") + encoded_header += b" " * (-len(encoded_header) % 8) + self._payload_offset = 8 + len(encoded_header) + self._offsets = offsets + self._path = owner.staging_dir / self.relative_name + total_size = self._payload_offset + cursor + owner._reserve_file(total_size) + self._fd = os.open(self._path, os.O_CREAT | os.O_EXCL | os.O_RDWR | os.O_CLOEXEC, 0o600) + try: + if hasattr(os, "posix_fallocate"): + try: + os.posix_fallocate(self._fd, 0, total_size) + except OSError as exc: + if exc.errno not in (errno.EINVAL, errno.ENOSYS, errno.EOPNOTSUPP): + raise + os.ftruncate(self._fd, total_size) + else: # pragma: no cover - Linux production path has posix_fallocate + os.ftruncate(self._fd, total_size) + _pwrite_all(self._fd, struct.pack(" memoryview: + if tensor.device.type != "cpu" or tensor.is_meta: + raise ArtifactWriterError("artifact producers must provide materialized CPU tensor chunks") + contiguous = tensor.detach().contiguous() + return memoryview(contiguous.view(torch.uint8).numpy()).cast("B") + + def write_tensor(self, name: str, tensor: torch.Tensor) -> None: + spec = self._get_spec(name) + if tuple(tensor.shape) != spec.shape or tensor.dtype != spec.dtype: + raise ArtifactWriterError( + f"tensor {name!r} differs from its declared shape/dtype: " + f"got {tuple(tensor.shape)}/{tensor.dtype}, expected {spec.shape}/{spec.dtype}" + ) + self._write(name, 0, self._bytes(tensor)) + + def write_rows(self, name: str, start_row: int, tensor: torch.Tensor) -> None: + spec = self._get_spec(name) + if not spec.shape: + if start_row != 0 or tuple(tensor.shape) != (): + raise ArtifactWriterError(f"scalar tensor {name!r} only accepts one complete scalar write") + self.write_tensor(name, tensor) + return + if tensor.dtype != spec.dtype or tuple(tensor.shape[1:]) != spec.shape[1:]: + raise ArtifactWriterError(f"row chunk for {name!r} differs from its declared dtype or trailing shape") + row_count = tensor.shape[0] + if start_row < 0 or start_row + row_count > spec.shape[0]: + raise ArtifactWriterError(f"row chunk for {name!r} exceeds its declared first dimension") + row_bytes = spec.nbytes // spec.shape[0] if spec.shape[0] else 0 + self._write(name, start_row * row_bytes, self._bytes(tensor)) + + def _get_spec(self, name: str) -> TensorWriteSpec: + if self._closed: + raise ArtifactWriterError("cannot write through a closed tensor file") + try: + return self._specs[name] + except KeyError as exc: + raise ArtifactWriterError(f"tensor {name!r} was not declared in this payload file") from exc + + def _write(self, name: str, byte_offset: int, data: memoryview) -> None: + coverage = self._coverage[name] + coverage.add(byte_offset, byte_offset + len(data)) + _pwrite_all(self._fd, data, self._payload_offset + self._offsets[name] + byte_offset) + + def close(self) -> None: + if self._closed: + return + missing = [name for name, coverage in self._coverage.items() if not coverage.complete()] + if missing: + self.abort() + raise ArtifactWriterError(f"tensor payload closed with incomplete writes: {missing}") + self._closed = True + try: + os.fsync(self._fd) + finally: + os.close(self._fd) + self._owner._closed(self) + + def abort(self) -> None: + if self._closed: + return + self._closed = True + os.close(self._fd) + size = self._path.stat().st_size if self._path.exists() else 0 + self._path.unlink(missing_ok=True) + self._owner._aborted(self, size) + + def __enter__(self) -> StreamingSafeTensorFile: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + _traceback: TracebackType | None, + ) -> None: + if exc_type is None: + self.close() + else: + self.abort() + + +class FilesystemArtifactWriter: + """Scoped writer that cannot publish outside one invisible staging dir.""" + + def __init__( + self, + staging_dir: Path, + *, + capacity_check: Callable[[int, int], None], + ) -> None: + self.staging_dir = staging_dir + self._capacity_check = capacity_check + self._active: set[StreamingSafeTensorFile] = set() + self._files: dict[str, tuple[TensorWriteSpec, ...]] = {} + self._reserved_bytes = 0 + self._finalized = False + + def open_tensor_file( + self, + relative_name: str, + specs: tuple[TensorWriteSpec, ...], + ) -> StreamingSafeTensorFile: + if self._finalized: + raise ArtifactWriterError("cannot add payloads after artifact finalization") + relative_name = _safe_relative_name(relative_name) + if relative_name in self._files or any(item.relative_name == relative_name for item in self._active): + raise ArtifactWriterError(f"artifact payload {relative_name!r} was declared more than once") + return StreamingSafeTensorFile(self, relative_name, specs) + + def _reserve_file(self, size: int) -> None: + self._capacity_check(self._reserved_bytes + size, size) + self._reserved_bytes += size + + def _forget_reserved_file(self, size: int) -> None: + self._reserved_bytes = max(0, self._reserved_bytes - size) + + def _opened(self, writer: StreamingSafeTensorFile) -> None: + self._active.add(writer) + + def _closed(self, writer: StreamingSafeTensorFile) -> None: + self._active.remove(writer) + self._files[writer.relative_name] = writer.specs + + def _aborted(self, writer: StreamingSafeTensorFile, size: int) -> None: + self._active.discard(writer) + self._forget_reserved_file(size) + + def abort(self) -> None: + for writer in tuple(self._active): + with contextlib.suppress(Exception): + writer.abort() + + def finalize( + self, + identity: WeightArtifactIdentity, + metadata: ProductionMetadata, + ) -> tuple[WeightManifest, PublicationMarker]: + if self._finalized: + raise ArtifactWriterError("artifact writer was finalized more than once") + if self._active: + raise ArtifactWriterError("artifact producer left tensor payload files open") + if not self._files: + raise ArtifactWriterError("artifact producer emitted no tensor payload files") + if metadata.producer_schema != identity.producer.manifest_schema: + raise ArtifactWriterError("production metadata does not match the identity's manifest schema") + if metadata.restorer_schema != identity.producer.restorer_schema: + raise ArtifactWriterError("production metadata does not match the identity's restorer schema") + + tensor_entries: list[TensorManifestEntry] = [] + file_entries: list[FileManifestEntry] = [] + seen_tensors: set[str] = set() + for file_name in sorted(self._files): + path = self.staging_dir / file_name + os.chmod(path, 0o444) + with path.open("rb") as handle: + os.fsync(handle.fileno()) + file_entries.append( + FileManifestEntry( + file_name=file_name, + size=path.stat().st_size, + sha256=_file_sha256(path), + ) + ) + expected = {spec.name: spec for spec in self._files[file_name]} + with safe_open(str(path), framework="pt", device="cpu") as handle: + if set(handle.keys()) != set(expected): + raise ArtifactWriterError(f"published tensor keys differ in {file_name!r}") + for name in sorted(expected): + if name in seen_tensors: + raise ArtifactWriterError(f"artifact tensor {name!r} occurs in multiple payload files") + seen_tensors.add(name) + spec = expected[name] + tensor = handle.get_tensor(name) + if tuple(tensor.shape) != spec.shape or tensor.dtype != spec.dtype: + raise ArtifactWriterError(f"published tensor {name!r} differs from its declared metadata") + entry = TensorManifestEntry( + name=name, + kind=spec.kind, + role=spec.role, + file_name=file_name, + storage_key=name, + shape=tuple(tensor.shape), + dtype=str(tensor.dtype), + stride=tuple(tensor.stride()), + nbytes=tensor.numel() * tensor.element_size(), + sha256=_tensor_sha256(tensor), + ) + tensor_entries.append(entry) + + ordered_tensor_entries = tuple(sorted(tensor_entries, key=lambda entry: entry.name)) + content_digest = hashlib.sha256() + for entry in ordered_tensor_entries: + content_digest.update(canonical_json(entry.semantic_dict())) + + manifest = WeightManifest( + schema_version=1, + artifact_key=identity.key, + identity=identity, + producer_schema=metadata.producer_schema, + restorer_schema=metadata.restorer_schema, + tensors=ordered_tensor_entries, + files=tuple(file_entries), + artifact_content_sha256=content_digest.hexdigest(), + format_metadata=metadata.format_metadata, + ) + manifest_bytes = manifest.canonical_bytes + self._write_metadata_file(_MANIFEST_FILE, manifest_bytes) + marker = PublicationMarker( + schema_version=1, + publication_id=uuid.uuid4().hex, + manifest_sha256=hashlib.sha256(manifest_bytes).hexdigest(), + artifact_content_sha256=manifest.artifact_content_sha256, + integrity_mode="full_checksum", + ) + self._write_metadata_file(_READY_FILE, marker.canonical_bytes) + _fsync_directory(self.staging_dir) + self._finalized = True + return manifest, marker + + def _write_metadata_file(self, name: str, data: bytes) -> None: + path = self.staging_dir / name + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_CLOEXEC, 0o600) + try: + _pwrite_all(fd, data, 0) + os.fsync(fd) + os.fchmod(fd, 0o444) + os.fsync(fd) + finally: + os.close(fd) + + +__all__ = [ + "ArtifactWriterError", + "FilesystemArtifactWriter", + "StreamingSafeTensorFile", +] diff --git a/vllm_omni/host_weight_runtime/identity.py b/vllm_omni/host_weight_runtime/identity.py new file mode 100644 index 000000000..77b769728 --- /dev/null +++ b/vllm_omni/host_weight_runtime/identity.py @@ -0,0 +1,407 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Deterministic semantic identity for runtime-ready host weights.""" + +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import dataclass, field +from typing import TypeAlias, cast + +JsonScalar: TypeAlias = None | bool | int | float | str +JsonValue: TypeAlias = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"] + + +class IdentityValidationError(ValueError): + """Raised when an artifact identity is not deterministic JSON.""" + + +def _validate_json(value: object, path: str = "$") -> JsonValue: + if value is None or isinstance(value, (bool, int, str)): + return value + if isinstance(value, float): + if not math.isfinite(value): + raise IdentityValidationError(f"{path} contains a non-finite float") + return value + if isinstance(value, list | tuple): + return [_validate_json(item, f"{path}[{index}]") for index, item in enumerate(value)] + if isinstance(value, dict): + normalized: dict[str, JsonValue] = {} + for key, item in value.items(): + if not isinstance(key, str): + raise IdentityValidationError(f"{path} contains a non-string mapping key") + normalized[key] = _validate_json(item, f"{path}.{key}") + return {key: normalized[key] for key in sorted(normalized)} + raise IdentityValidationError(f"{path} contains unsupported value type {type(value).__qualname__}") + + +def canonical_json(value: object) -> bytes: + """Return one stable JSON encoding, rejecting process-specific objects.""" + return json.dumps( + _validate_json(value), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + +@dataclass(frozen=True) +class CanonicalJson: + """Immutable snapshot of deterministic JSON metadata.""" + + encoded: bytes + + def __post_init__(self) -> None: + if not isinstance(self.encoded, bytes): + raise IdentityValidationError("canonical metadata must be immutable bytes") + try: + value = json.loads(self.encoded) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise IdentityValidationError("canonical metadata is not valid JSON") from exc + if canonical_json(value) != self.encoded: + raise IdentityValidationError("canonical metadata does not use the required stable encoding") + + @classmethod + def from_value(cls, value: object) -> CanonicalJson: + return cls(canonical_json(value)) + + @classmethod + def empty(cls) -> CanonicalJson: + return cls(b"{}") + + def to_value(self) -> JsonValue: + return json.loads(self.encoded) + + +def _require_text(name: str, value: object) -> str: + if not isinstance(value, str) or not value: + raise IdentityValidationError(f"{name} must not be empty") + return value + + +def _require_int(name: str, value: object) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise IdentityValidationError(f"{name} must be an integer") + return value + + +def _require_metadata(name: str, value: object) -> None: + if not isinstance(value, CanonicalJson): + raise IdentityValidationError(f"{name} must use CanonicalJson") + + +def _require_exact_keys(value: dict[str, object], expected: set[str], kind: str) -> None: + keys = set(value) + if keys != expected: + raise IdentityValidationError( + f"{kind} fields differ: missing={sorted(expected - keys)}, unexpected={sorted(keys - expected)}" + ) + + +@dataclass(frozen=True) +class WeightSourceIdentity: + model_id: str + revision: str + fingerprint: str + metadata: CanonicalJson = field(default_factory=CanonicalJson.empty) + + def __post_init__(self) -> None: + _require_text("source.model_id", self.model_id) + _require_text("source.revision", self.revision) + _require_text("source.fingerprint", self.fingerprint) + _require_metadata("source.metadata", self.metadata) + + def to_dict(self) -> dict[str, JsonValue]: + return { + "model_id": self.model_id, + "revision": self.revision, + "fingerprint": self.fingerprint, + "metadata": self.metadata.to_value(), + } + + @classmethod + def from_dict(cls, value: dict[str, object]) -> WeightSourceIdentity: + _require_exact_keys(value, {"model_id", "revision", "fingerprint", "metadata"}, "source identity") + return cls( + model_id=_require_text("source.model_id", value["model_id"]), + revision=_require_text("source.revision", value["revision"]), + fingerprint=_require_text("source.fingerprint", value["fingerprint"]), + metadata=CanonicalJson.from_value(value["metadata"]), + ) + + +@dataclass(frozen=True) +class ComponentIdentity: + name: str + ownership: str + metadata: CanonicalJson = field(default_factory=CanonicalJson.empty) + + def __post_init__(self) -> None: + _require_text("component.name", self.name) + _require_text("component.ownership", self.ownership) + _require_metadata("component.metadata", self.metadata) + + def to_dict(self) -> dict[str, JsonValue]: + return {"name": self.name, "ownership": self.ownership, "metadata": self.metadata.to_value()} + + @classmethod + def from_dict(cls, value: dict[str, object]) -> ComponentIdentity: + _require_exact_keys(value, {"name", "ownership", "metadata"}, "component identity") + return cls( + name=_require_text("component.name", value["name"]), + ownership=_require_text("component.ownership", value["ownership"]), + metadata=CanonicalJson.from_value(value["metadata"]), + ) + + +@dataclass(frozen=True) +class WeightRepresentation: + name: str + dtype: str + metadata: CanonicalJson = field(default_factory=CanonicalJson.empty) + + def __post_init__(self) -> None: + _require_text("representation.name", self.name) + _require_text("representation.dtype", self.dtype) + _require_metadata("representation.metadata", self.metadata) + + def to_dict(self) -> dict[str, JsonValue]: + return {"name": self.name, "dtype": self.dtype, "metadata": self.metadata.to_value()} + + @classmethod + def from_dict(cls, value: dict[str, object]) -> WeightRepresentation: + _require_exact_keys(value, {"name", "dtype", "metadata"}, "weight representation") + return cls( + name=_require_text("representation.name", value["name"]), + dtype=_require_text("representation.dtype", value["dtype"]), + metadata=CanonicalJson.from_value(value["metadata"]), + ) + + +@dataclass(frozen=True) +class RuntimeWeightLayout: + name: str + tensor_parallel_size: int = 1 + tensor_parallel_rank: int = 0 + sequence_parallel_size: int = 1 + sequence_parallel_backend: str = "none" + metadata: CanonicalJson = field(default_factory=CanonicalJson.empty) + + def __post_init__(self) -> None: + _require_text("layout.name", self.name) + tensor_parallel_size = _require_int("layout.tensor_parallel_size", self.tensor_parallel_size) + tensor_parallel_rank = _require_int("layout.tensor_parallel_rank", self.tensor_parallel_rank) + sequence_parallel_size = _require_int("layout.sequence_parallel_size", self.sequence_parallel_size) + if tensor_parallel_size < 1: + raise IdentityValidationError("tensor_parallel_size must be positive") + if not 0 <= tensor_parallel_rank < tensor_parallel_size: + raise IdentityValidationError("tensor_parallel_rank must be within tensor_parallel_size") + if sequence_parallel_size < 1: + raise IdentityValidationError("sequence_parallel_size must be positive") + _require_text("layout.sequence_parallel_backend", self.sequence_parallel_backend) + _require_metadata("layout.metadata", self.metadata) + + def to_dict(self) -> dict[str, JsonValue]: + return { + "name": self.name, + "tensor_parallel_size": self.tensor_parallel_size, + "tensor_parallel_rank": self.tensor_parallel_rank, + "sequence_parallel_size": self.sequence_parallel_size, + "sequence_parallel_backend": self.sequence_parallel_backend, + "metadata": self.metadata.to_value(), + } + + @classmethod + def from_dict(cls, value: dict[str, object]) -> RuntimeWeightLayout: + expected = { + "name", + "tensor_parallel_size", + "tensor_parallel_rank", + "sequence_parallel_size", + "sequence_parallel_backend", + "metadata", + } + _require_exact_keys(value, expected, "runtime weight layout") + return cls( + name=_require_text("layout.name", value["name"]), + tensor_parallel_size=_require_int("layout.tensor_parallel_size", value["tensor_parallel_size"]), + tensor_parallel_rank=_require_int("layout.tensor_parallel_rank", value["tensor_parallel_rank"]), + sequence_parallel_size=_require_int("layout.sequence_parallel_size", value["sequence_parallel_size"]), + sequence_parallel_backend=_require_text( + "layout.sequence_parallel_backend", value["sequence_parallel_backend"] + ), + metadata=CanonicalJson.from_value(value["metadata"]), + ) + + +@dataclass(frozen=True) +class AdaptationIdentity: + kind: str = "base" + fingerprint: str | None = None + metadata: CanonicalJson = field(default_factory=CanonicalJson.empty) + + def __post_init__(self) -> None: + _require_text("adaptation.kind", self.kind) + if self.kind != "base" and not self.fingerprint: + raise IdentityValidationError("non-base adaptation requires a fingerprint") + if self.fingerprint is not None and (not isinstance(self.fingerprint, str) or not self.fingerprint): + raise IdentityValidationError("adaptation fingerprint must be a non-empty string or null") + _require_metadata("adaptation.metadata", self.metadata) + + def to_dict(self) -> dict[str, JsonValue]: + return {"kind": self.kind, "fingerprint": self.fingerprint, "metadata": self.metadata.to_value()} + + @classmethod + def from_dict(cls, value: dict[str, object]) -> AdaptationIdentity: + _require_exact_keys(value, {"kind", "fingerprint", "metadata"}, "adaptation identity") + fingerprint = value["fingerprint"] + if fingerprint is not None and not isinstance(fingerprint, str): + raise IdentityValidationError("adaptation fingerprint must be a string or null") + return cls( + kind=_require_text("adaptation.kind", value["kind"]), + fingerprint=fingerprint, + metadata=CanonicalJson.from_value(value["metadata"]), + ) + + +@dataclass(frozen=True) +class ProducerIdentity: + producer_id: str + version: str + implementation_fingerprint: str + manifest_schema: str + restorer_schema: str + + def __post_init__(self) -> None: + for name, value in ( + ("producer.producer_id", self.producer_id), + ("producer.version", self.version), + ("producer.implementation_fingerprint", self.implementation_fingerprint), + ("producer.manifest_schema", self.manifest_schema), + ("producer.restorer_schema", self.restorer_schema), + ): + _require_text(name, value) + + def to_dict(self) -> dict[str, JsonValue]: + return { + "producer_id": self.producer_id, + "version": self.version, + "implementation_fingerprint": self.implementation_fingerprint, + "manifest_schema": self.manifest_schema, + "restorer_schema": self.restorer_schema, + } + + @classmethod + def from_dict(cls, value: dict[str, object]) -> ProducerIdentity: + expected = { + "producer_id", + "version", + "implementation_fingerprint", + "manifest_schema", + "restorer_schema", + } + _require_exact_keys(value, expected, "producer identity") + return cls( + producer_id=_require_text("producer.producer_id", value["producer_id"]), + version=_require_text("producer.version", value["version"]), + implementation_fingerprint=_require_text( + "producer.implementation_fingerprint", value["implementation_fingerprint"] + ), + manifest_schema=_require_text("producer.manifest_schema", value["manifest_schema"]), + restorer_schema=_require_text("producer.restorer_schema", value["restorer_schema"]), + ) + + +@dataclass(frozen=True) +class WeightArtifactIdentity: + schema_version: int + source: WeightSourceIdentity + component: ComponentIdentity + representation: WeightRepresentation + layout: RuntimeWeightLayout + adaptation: AdaptationIdentity + producer: ProducerIdentity + + def __post_init__(self) -> None: + if _require_int("identity.schema_version", self.schema_version) < 1: + raise IdentityValidationError("identity schema_version must be positive") + for name, value, expected in ( + ("source", self.source, WeightSourceIdentity), + ("component", self.component, ComponentIdentity), + ("representation", self.representation, WeightRepresentation), + ("layout", self.layout, RuntimeWeightLayout), + ("adaptation", self.adaptation, AdaptationIdentity), + ("producer", self.producer, ProducerIdentity), + ): + if not isinstance(value, expected): + raise IdentityValidationError(f"identity {name} has the wrong contract type") + + def to_dict(self) -> dict[str, JsonValue]: + return { + "schema_version": self.schema_version, + "source": self.source.to_dict(), + "component": self.component.to_dict(), + "representation": self.representation.to_dict(), + "layout": self.layout.to_dict(), + "adaptation": self.adaptation.to_dict(), + "producer": self.producer.to_dict(), + } + + @property + def canonical_bytes(self) -> bytes: + return canonical_json(self.to_dict()) + + @property + def key(self) -> str: + return hashlib.sha256(self.canonical_bytes).hexdigest() + + @classmethod + def from_dict(cls, value: dict[str, object]) -> WeightArtifactIdentity: + expected = { + "schema_version", + "source", + "component", + "representation", + "layout", + "adaptation", + "producer", + } + _require_exact_keys(value, expected, "weight artifact identity") + source = value["source"] + component = value["component"] + representation = value["representation"] + layout = value["layout"] + adaptation = value["adaptation"] + producer = value["producer"] + if any( + not isinstance(section, dict) + for section in (source, component, representation, layout, adaptation, producer) + ): + raise IdentityValidationError("identity sections must be JSON objects") + return cls( + schema_version=_require_int("identity.schema_version", value["schema_version"]), + source=WeightSourceIdentity.from_dict(cast(dict[str, object], source)), + component=ComponentIdentity.from_dict(cast(dict[str, object], component)), + representation=WeightRepresentation.from_dict(cast(dict[str, object], representation)), + layout=RuntimeWeightLayout.from_dict(cast(dict[str, object], layout)), + adaptation=AdaptationIdentity.from_dict(cast(dict[str, object], adaptation)), + producer=ProducerIdentity.from_dict(cast(dict[str, object], producer)), + ) + + +__all__ = [ + "AdaptationIdentity", + "CanonicalJson", + "ComponentIdentity", + "IdentityValidationError", + "JsonValue", + "ProducerIdentity", + "RuntimeWeightLayout", + "WeightArtifactIdentity", + "WeightRepresentation", + "WeightSourceIdentity", + "canonical_json", +] diff --git a/vllm_omni/host_weight_runtime/inspection.py b/vllm_omni/host_weight_runtime/inspection.py new file mode 100644 index 000000000..c61760502 --- /dev/null +++ b/vllm_omni/host_weight_runtime/inspection.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Typed, read-only snapshots for local Host Weight Store inspection.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Protocol + +from .config import ValidationLevel +from .errors import HostWeightFailure +from .identity import WeightArtifactIdentity +from .manifest import WeightManifest +from .outcomes import StoreStatus + + +class ArtifactInventoryState(str, Enum): + READY = "ready" + DENIED = "denied" + QUARANTINED = "quarantined" + TEMPORARY = "temporary" + + +@dataclass(frozen=True) +class ArtifactInventoryEntry: + artifact_key: str + state: ArtifactInventoryState + storage_name: str + size_bytes: int + reason: str | None = None + + +@dataclass(frozen=True) +class DomainInspection: + domain_uuid: str + scope: str + domain_id: str + storage_class: str + filesystem_type: str + policy_version: int + store_bytes: int + filesystem_free_bytes: int + max_artifact_bytes: int | None + max_store_bytes: int | None + min_free_bytes: int + inventory: tuple[ArtifactInventoryEntry, ...] + + +@dataclass(frozen=True) +class ArtifactInspection: + artifact_key: str + status: StoreStatus + size_bytes: int + manifest: WeightManifest | None + publication_id: str | None + backing_fingerprint: str | None + failure: HostWeightFailure | None + build_lock_active: bool + artifact_lock_active: bool + + +class HostWeightInspector(Protocol): + def inspect_domain(self) -> DomainInspection: ... + + def inspect_artifact( + self, + identity: WeightArtifactIdentity, + *, + validation: ValidationLevel, + ) -> ArtifactInspection: ... + + +__all__ = [ + "ArtifactInspection", + "ArtifactInventoryEntry", + "ArtifactInventoryState", + "DomainInspection", + "HostWeightInspector", +] diff --git a/vllm_omni/host_weight_runtime/lease.py b/vllm_omni/host_weight_runtime/lease.py new file mode 100644 index 000000000..7446f0bbe --- /dev/null +++ b/vllm_omni/host_weight_runtime/lease.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Process-local ownership for immutable mmap-backed host tensors.""" + +from __future__ import annotations + +import contextlib +import threading +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import NoReturn, SupportsIndex + +import torch + +from .identity import WeightArtifactIdentity +from .manifest import WeightManifest + + +@dataclass(frozen=True) +class MappedHostRegion: + """One process-local virtual range belonging to an artifact payload file.""" + + file_name: str + address: int + length: int + read_only: bool = True + + def __post_init__(self) -> None: + if not self.file_name or self.address < 0 or self.length < 0: + raise ValueError("mapped host region contains invalid fields") + + +@dataclass(frozen=True) +class LeaseProvenance: + resolution_id: str + identity_digest: str + artifact_content_sha256: str + domain_uuid: str + publication_id: str + resolution_source: str + artifact_origin: str + producer_id: str | None + provider_id: str | None + validation_level: str + backing_fingerprint: str + + +class HostWeightLease: + """Stable tensor views and resources retained until transport teardown.""" + + def __init__( + self, + *, + identity: WeightArtifactIdentity, + manifest: WeightManifest, + tensors: Mapping[str, torch.Tensor], + mapped_regions: tuple[MappedHostRegion, ...], + provenance: LeaseProvenance, + resource_closers: tuple[Callable[[], None], ...], + release_artifact_lock: Callable[[], None], + ) -> None: + if manifest.identity.canonical_bytes != identity.canonical_bytes: + raise ValueError("lease manifest identity differs from the requested identity") + if set(tensors) != {entry.name for entry in manifest.tensors}: + raise ValueError("lease tensor names differ from the validated manifest") + entries = {entry.name: entry for entry in manifest.tensors} + for name, tensor in tensors.items(): + entry = entries[name] + if ( + not isinstance(tensor, torch.Tensor) + or tensor.device.type != "cpu" + or tuple(tensor.shape) != entry.shape + or str(tensor.dtype) != entry.dtype + or tuple(tensor.stride()) != entry.stride + or tensor.numel() * tensor.element_size() != entry.nbytes + ): + raise ValueError(f"lease tensor {name!r} differs from the validated host metadata") + if provenance.identity_digest != identity.key: + raise ValueError("lease provenance identity digest differs from the requested identity") + if provenance.artifact_content_sha256 != manifest.artifact_content_sha256: + raise ValueError("lease provenance content digest differs from the validated manifest") + if not isinstance(mapped_regions, tuple) or any( + not isinstance(region, MappedHostRegion) for region in mapped_regions + ): + raise ValueError("lease mapped regions must be an immutable tuple") + if not isinstance(resource_closers, tuple) or any(not callable(closer) for closer in resource_closers): + raise ValueError("lease resource closers must be an immutable tuple of callables") + if not callable(release_artifact_lock): + raise ValueError("lease artifact lock release must be callable") + self.identity = identity + self.manifest = manifest + self._tensor_storage = dict(tensors) + self.tensors = MappingProxyType(self._tensor_storage) + self.mapped_regions = mapped_regions + self.provenance = provenance + self._resource_closers = resource_closers + self._release_artifact_lock = release_artifact_lock + self._state_lock = threading.RLock() + self._closed = False + + @property + def closed(self) -> bool: + with self._state_lock: + return self._closed + + def close(self) -> None: + with self._state_lock: + if self._closed: + return + self._closed = True + + errors: list[BaseException] = [] + self._tensor_storage.clear() + for closer in reversed(self._resource_closers): + try: + closer() + except BaseException as exc: # pragma: no cover - best-effort teardown aggregation + errors.append(exc) + try: + self._release_artifact_lock() + except BaseException as exc: # pragma: no cover - best-effort teardown aggregation + errors.append(exc) + if errors: + raise RuntimeError(f"failed to close {len(errors)} host weight lease resources") from errors[0] + + def __enter__(self) -> HostWeightLease: + if self.closed: + raise RuntimeError("cannot re-enter a closed HostWeightLease") + return self + + def __exit__(self, exc_type: object, exc: object, _traceback: object) -> None: + self.close() + + def __reduce_ex__(self, _protocol: SupportsIndex) -> NoReturn: + raise TypeError("HostWeightLease is process-local and cannot be serialized") + + def __del__(self) -> None: + with contextlib.suppress(Exception): + self.close() + + +__all__ = ["HostWeightLease", "LeaseProvenance", "MappedHostRegion"] diff --git a/vllm_omni/host_weight_runtime/manifest.py b/vllm_omni/host_weight_runtime/manifest.py new file mode 100644 index 000000000..dfeaf2b95 --- /dev/null +++ b/vllm_omni/host_weight_runtime/manifest.py @@ -0,0 +1,353 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Versioned immutable manifests for runtime-ready host weights.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from pathlib import PurePosixPath +from typing import cast + +import regex as re + +from .identity import CanonicalJson, JsonValue, WeightArtifactIdentity, canonical_json + +_SHA256_RE = re.compile(r"[0-9a-f]{64}") + + +class ManifestValidationError(ValueError): + """Raised when a published manifest violates its schema.""" + + +class TensorKind(str, Enum): + PARAMETER = "parameter" + BUFFER = "buffer" + + +def _require_text(name: str, value: object) -> str: + if not isinstance(value, str) or not value: + raise ManifestValidationError(f"{name} must be a non-empty string") + return value + + +def _require_int(name: str, value: object) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ManifestValidationError(f"{name} must be an integer") + return value + + +def _require_sha256(name: str, value: object) -> str: + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise ManifestValidationError(f"{name} must be a lowercase SHA-256 digest") + return value + + +def _require_safe_file_name(value: object) -> str: + if not isinstance(value, str): + raise ManifestValidationError("artifact file name must be a string") + path = PurePosixPath(value) + if not value or path.is_absolute() or len(path.parts) != 1 or path.name != value or value in {".", ".."}: + raise ManifestValidationError(f"artifact file name must be one safe relative basename: {value!r}") + return value + + +def _require_int_tuple(name: str, value: object) -> tuple[int, ...]: + if not isinstance(value, list): + raise ManifestValidationError(f"{name} must be an array") + return tuple(_require_int(f"{name}[{index}]", item) for index, item in enumerate(value)) + + +def _exact_keys(value: dict[str, object], expected: set[str], kind: str) -> None: + keys = set(value) + if keys != expected: + raise ManifestValidationError( + f"{kind} fields differ: missing={sorted(expected - keys)}, unexpected={sorted(keys - expected)}" + ) + + +@dataclass(frozen=True) +class TensorManifestEntry: + name: str + kind: TensorKind + role: str + file_name: str + storage_key: str + shape: tuple[int, ...] + dtype: str + stride: tuple[int, ...] + nbytes: int + sha256: str + + def __post_init__(self) -> None: + _require_text("tensor.name", self.name) + _require_text("tensor.storage_key", self.storage_key) + _require_text("tensor.role", self.role) + _require_text("tensor.dtype", self.dtype) + if not isinstance(self.kind, TensorKind): + raise ManifestValidationError("tensor kind must use TensorKind") + _require_safe_file_name(self.file_name) + if not isinstance(self.shape, tuple) or any( + isinstance(dimension, bool) or not isinstance(dimension, int) or dimension < 0 for dimension in self.shape + ): + raise ManifestValidationError(f"tensor {self.name!r} has a negative shape dimension") + if ( + not isinstance(self.stride, tuple) + or len(self.stride) != len(self.shape) + or any(isinstance(value, bool) or not isinstance(value, int) or value < 0 for value in self.stride) + ): + raise ManifestValidationError(f"tensor {self.name!r} has an invalid stride") + if _require_int(f"tensors[{self.name!r}].nbytes", self.nbytes) < 0: + raise ManifestValidationError(f"tensor {self.name!r} has negative nbytes") + _require_sha256(f"tensors[{self.name!r}].sha256", self.sha256) + + def semantic_dict(self) -> dict[str, JsonValue]: + """Return content identity independent of physical file sharding.""" + return { + "name": self.name, + "kind": self.kind.value, + "role": self.role, + "shape": list(self.shape), + "dtype": self.dtype, + "stride": list(self.stride), + "nbytes": self.nbytes, + "sha256": self.sha256, + } + + def to_dict(self) -> dict[str, JsonValue]: + return { + "name": self.name, + "kind": self.kind.value, + "role": self.role, + "file_name": self.file_name, + "storage_key": self.storage_key, + "shape": list(self.shape), + "dtype": self.dtype, + "stride": list(self.stride), + "nbytes": self.nbytes, + "sha256": self.sha256, + } + + @classmethod + def from_dict(cls, value: dict[str, object]) -> TensorManifestEntry: + expected = { + "name", + "kind", + "role", + "file_name", + "storage_key", + "shape", + "dtype", + "stride", + "nbytes", + "sha256", + } + _exact_keys(value, expected, "tensor manifest entry") + kind_value = _require_text("tensor.kind", value["kind"]) + try: + kind = TensorKind(kind_value) + except ValueError as exc: + raise ManifestValidationError(f"unknown tensor kind {value['kind']!r}") from exc + return cls( + name=_require_text("tensor.name", value["name"]), + kind=kind, + role=_require_text("tensor.role", value["role"]), + file_name=_require_safe_file_name(value["file_name"]), + storage_key=_require_text("tensor.storage_key", value["storage_key"]), + shape=_require_int_tuple("tensor.shape", value["shape"]), + dtype=_require_text("tensor.dtype", value["dtype"]), + stride=_require_int_tuple("tensor.stride", value["stride"]), + nbytes=_require_int("tensor.nbytes", value["nbytes"]), + sha256=_require_sha256("tensor.sha256", value["sha256"]), + ) + + +@dataclass(frozen=True) +class FileManifestEntry: + file_name: str + size: int + sha256: str + + def __post_init__(self) -> None: + _require_safe_file_name(self.file_name) + if _require_int(f"files[{self.file_name!r}].size", self.size) < 0: + raise ManifestValidationError(f"artifact file {self.file_name!r} has negative size") + _require_sha256(f"files[{self.file_name!r}].sha256", self.sha256) + + def to_dict(self) -> dict[str, JsonValue]: + return {"file_name": self.file_name, "size": self.size, "sha256": self.sha256} + + @classmethod + def from_dict(cls, value: dict[str, object]) -> FileManifestEntry: + _exact_keys(value, {"file_name", "size", "sha256"}, "file manifest entry") + return cls( + file_name=_require_safe_file_name(value["file_name"]), + size=_require_int("file.size", value["size"]), + sha256=_require_sha256("file.sha256", value["sha256"]), + ) + + +@dataclass(frozen=True) +class WeightManifest: + schema_version: int + artifact_key: str + identity: WeightArtifactIdentity + producer_schema: str + restorer_schema: str + tensors: tuple[TensorManifestEntry, ...] + files: tuple[FileManifestEntry, ...] + artifact_content_sha256: str + format_metadata: CanonicalJson = field(default_factory=CanonicalJson.empty) + + def __post_init__(self) -> None: + if _require_int("manifest.schema_version", self.schema_version) < 1: + raise ManifestValidationError("manifest schema_version must be positive") + _require_sha256("artifact_key", self.artifact_key) + _require_sha256("artifact_content_sha256", self.artifact_content_sha256) + if not isinstance(self.identity, WeightArtifactIdentity): + raise ManifestValidationError("manifest identity must use WeightArtifactIdentity") + if self.artifact_key != self.identity.key: + raise ManifestValidationError("manifest artifact_key does not match its semantic identity") + _require_text("manifest.producer_schema", self.producer_schema) + _require_text("manifest.restorer_schema", self.restorer_schema) + if ( + not isinstance(self.tensors, tuple) + or not isinstance(self.files, tuple) + or not self.tensors + or not self.files + or any(not isinstance(entry, TensorManifestEntry) for entry in self.tensors) + or any(not isinstance(entry, FileManifestEntry) for entry in self.files) + ): + raise ManifestValidationError("an artifact must contain at least one tensor and one file") + if not isinstance(self.format_metadata, CanonicalJson): + raise ManifestValidationError("manifest format_metadata must use CanonicalJson") + tensor_names = [entry.name for entry in self.tensors] + if tensor_names != sorted(tensor_names) or len(set(tensor_names)) != len(tensor_names): + raise ManifestValidationError("manifest tensors must have unique names in canonical order") + file_names = [entry.file_name for entry in self.files] + if file_names != sorted(file_names) or len(set(file_names)) != len(file_names): + raise ManifestValidationError("manifest files must have unique names in canonical order") + available_files = set(file_names) + if any(entry.file_name not in available_files for entry in self.tensors): + raise ManifestValidationError("tensor manifest references an undeclared artifact file") + storage_bindings = [(entry.file_name, entry.storage_key) for entry in self.tensors] + if len(set(storage_bindings)) != len(storage_bindings): + raise ManifestValidationError("manifest tensor storage bindings must be unique") + + @property + def canonical_bytes(self) -> bytes: + return canonical_json(self.to_dict()) + + @property + def total_tensor_bytes(self) -> int: + return sum(entry.nbytes for entry in self.tensors) + + def to_dict(self) -> dict[str, JsonValue]: + return { + "schema_version": self.schema_version, + "artifact_key": self.artifact_key, + "identity": self.identity.to_dict(), + "producer_schema": self.producer_schema, + "restorer_schema": self.restorer_schema, + "tensors": [entry.to_dict() for entry in self.tensors], + "files": [entry.to_dict() for entry in self.files], + "artifact_content_sha256": self.artifact_content_sha256, + "format_metadata": self.format_metadata.to_value(), + } + + @classmethod + def from_dict(cls, value: dict[str, object]) -> WeightManifest: + expected = { + "schema_version", + "artifact_key", + "identity", + "producer_schema", + "restorer_schema", + "tensors", + "files", + "artifact_content_sha256", + "format_metadata", + } + _exact_keys(value, expected, "weight manifest") + identity = value["identity"] + if not isinstance(identity, dict): + raise ManifestValidationError("manifest identity must be an object") + tensors = value["tensors"] + files = value["files"] + if not isinstance(tensors, list) or not all(isinstance(item, dict) for item in tensors): + raise ManifestValidationError("manifest tensors must be an array of objects") + if not isinstance(files, list) or not all(isinstance(item, dict) for item in files): + raise ManifestValidationError("manifest files must be an array of objects") + return cls( + schema_version=_require_int("manifest.schema_version", value["schema_version"]), + artifact_key=_require_sha256("manifest.artifact_key", value["artifact_key"]), + identity=WeightArtifactIdentity.from_dict(cast(dict[str, object], identity)), + producer_schema=_require_text("manifest.producer_schema", value["producer_schema"]), + restorer_schema=_require_text("manifest.restorer_schema", value["restorer_schema"]), + tensors=tuple(TensorManifestEntry.from_dict(cast(dict[str, object], item)) for item in tensors), + files=tuple(FileManifestEntry.from_dict(cast(dict[str, object], item)) for item in files), + artifact_content_sha256=_require_sha256( + "manifest.artifact_content_sha256", value["artifact_content_sha256"] + ), + format_metadata=CanonicalJson.from_value(value["format_metadata"]), + ) + + +@dataclass(frozen=True) +class PublicationMarker: + schema_version: int + publication_id: str + manifest_sha256: str + artifact_content_sha256: str + integrity_mode: str + + def __post_init__(self) -> None: + if _require_int("publication.schema_version", self.schema_version) < 1: + raise ManifestValidationError("publication schema_version must be positive") + _require_text("publication.publication_id", self.publication_id) + _require_text("publication.integrity_mode", self.integrity_mode) + _require_sha256("publication.manifest_sha256", self.manifest_sha256) + _require_sha256("publication.artifact_content_sha256", self.artifact_content_sha256) + + @property + def canonical_bytes(self) -> bytes: + return canonical_json(self.to_dict()) + + def to_dict(self) -> dict[str, JsonValue]: + return { + "schema_version": self.schema_version, + "publication_id": self.publication_id, + "manifest_sha256": self.manifest_sha256, + "artifact_content_sha256": self.artifact_content_sha256, + "integrity_mode": self.integrity_mode, + } + + @classmethod + def from_dict(cls, value: dict[str, object]) -> PublicationMarker: + expected = { + "schema_version", + "publication_id", + "manifest_sha256", + "artifact_content_sha256", + "integrity_mode", + } + _exact_keys(value, expected, "publication marker") + return cls( + schema_version=_require_int("publication.schema_version", value["schema_version"]), + publication_id=_require_text("publication.publication_id", value["publication_id"]), + manifest_sha256=_require_sha256("publication.manifest_sha256", value["manifest_sha256"]), + artifact_content_sha256=_require_sha256( + "publication.artifact_content_sha256", value["artifact_content_sha256"] + ), + integrity_mode=_require_text("publication.integrity_mode", value["integrity_mode"]), + ) + + +__all__ = [ + "FileManifestEntry", + "ManifestValidationError", + "PublicationMarker", + "TensorKind", + "TensorManifestEntry", + "WeightManifest", +] diff --git a/vllm_omni/host_weight_runtime/outcomes.py b/vllm_omni/host_weight_runtime/outcomes.py new file mode 100644 index 000000000..7cae2aaad --- /dev/null +++ b/vllm_omni/host_weight_runtime/outcomes.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Typed store and end-to-end resolution outcomes.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING + +from .errors import HostWeightFailure, ResolutionAction, ResolutionStage + +if TYPE_CHECKING: + from .lease import HostWeightLease + + +class StoreStatus(str, Enum): + HIT = "hit" + MISS = "miss" + BUILT = "built" + JOINED = "joined" + INVALID = "invalid" + TIMEOUT = "timeout" + FAILED = "failed" + + +@dataclass(frozen=True) +class StoreResult: + status: StoreStatus + lease: HostWeightLease | None = None + failure: HostWeightFailure | None = None + + def __post_init__(self) -> None: + has_lease = self.status in {StoreStatus.HIT, StoreStatus.BUILT, StoreStatus.JOINED} + if has_lease != (self.lease is not None): + raise ValueError(f"store result {self.status.value} has inconsistent lease ownership") + if self.status in {StoreStatus.HIT, StoreStatus.MISS, StoreStatus.BUILT, StoreStatus.JOINED}: + if self.failure is not None: + raise ValueError(f"successful store result {self.status.value} must not carry a failure") + if self.status in {StoreStatus.INVALID, StoreStatus.TIMEOUT, StoreStatus.FAILED} and self.failure is None: + raise ValueError(f"store result {self.status.value} requires a typed failure") + + +class AttemptResult(str, Enum): + HIT = "hit" + MISS = "miss" + SUCCESS = "success" + TIMEOUT = "timeout" + ERROR = "error" + SKIPPED = "skipped" + + +class ResolutionOutcome(str, Enum): + LOCAL_HIT = "local_hit" + REMOTE_IMPORT = "remote_import" + LOCAL_PRODUCTION = "local_production" + CANONICAL_DIRECT = "canonical_direct" + CANONICAL_FALLBACK = "canonical_fallback" + FAILED = "failed" + + +@dataclass(frozen=True) +class ResolutionAttempt: + stage: ResolutionStage + result: AttemptResult + action: ResolutionAction + elapsed_seconds: float + failure: HostWeightFailure | None = None + + def __post_init__(self) -> None: + if not math.isfinite(self.elapsed_seconds) or self.elapsed_seconds < 0: + raise ValueError("resolution attempt elapsed_seconds must be finite and non-negative") + if self.result in {AttemptResult.HIT, AttemptResult.MISS, AttemptResult.SUCCESS} and self.failure is not None: + raise ValueError(f"resolution attempt {self.result.value} must not carry a failure") + if self.result in {AttemptResult.TIMEOUT, AttemptResult.ERROR} and self.failure is None: + raise ValueError(f"resolution attempt {self.result.value} requires a typed failure") + + +@dataclass(frozen=True) +class ResolutionReport: + resolution_id: str + outcome: ResolutionOutcome + attempts: tuple[ResolutionAttempt, ...] + elapsed_seconds: float + + def __post_init__(self) -> None: + if not self.resolution_id: + raise ValueError("resolution report requires a resolution_id") + if not isinstance(self.attempts, tuple): + raise ValueError("resolution report attempts must be an immutable tuple") + if not math.isfinite(self.elapsed_seconds) or self.elapsed_seconds < 0: + raise ValueError("resolution report elapsed_seconds must be finite and non-negative") + + +@dataclass(frozen=True) +class HostWeightResolution: + report: ResolutionReport + lease: HostWeightLease | None = None + + def __post_init__(self) -> None: + lease_outcomes = { + ResolutionOutcome.LOCAL_HIT, + ResolutionOutcome.REMOTE_IMPORT, + ResolutionOutcome.LOCAL_PRODUCTION, + } + if (self.report.outcome in lease_outcomes) != (self.lease is not None): + raise ValueError("resolution outcome has inconsistent lease ownership") + + +__all__ = [ + "AttemptResult", + "HostWeightResolution", + "ResolutionAttempt", + "ResolutionOutcome", + "ResolutionReport", + "StoreResult", + "StoreStatus", +] diff --git a/vllm_omni/host_weight_runtime/protocols.py b/vllm_omni/host_weight_runtime/protocols.py new file mode 100644 index 000000000..c5572c9fb --- /dev/null +++ b/vllm_omni/host_weight_runtime/protocols.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Provider and consumer protocols for Host Weight Runtime extensions.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from enum import Enum +from types import TracebackType +from typing import TYPE_CHECKING, Protocol + +import torch + +from .identity import CanonicalJson, WeightArtifactIdentity +from .manifest import TensorKind + +if TYPE_CHECKING: + from .config import ValidationLevel + from .lease import HostWeightLease + from .outcomes import StoreResult + + +class ProductionSourceMode(str, Enum): + FINALIZED_MODEL = "finalized_model" + CHECKPOINT_STREAM = "checkpoint_stream" + + +class CoordinationScope(str, Enum): + SINGLE_PROCESS = "single_process" + GROUP_COHORT = "group_cohort" + ARTIFACT_BUNDLE = "artifact_bundle" + + +class LookupPhase(str, Enum): + PRE_LOAD_SAFE = "pre_load_safe" + POST_LOAD_ONLY = "post_load_only" + + +@dataclass(frozen=True) +class TensorWriteSpec: + name: str + shape: tuple[int, ...] + dtype: torch.dtype + kind: TensorKind = TensorKind.PARAMETER + role: str = "weight" + + def __post_init__(self) -> None: + if not isinstance(self.name, str) or not self.name or not isinstance(self.role, str) or not self.role: + raise ValueError("tensor write name and role must not be empty") + if not isinstance(self.shape, tuple) or any( + isinstance(dimension, bool) or not isinstance(dimension, int) or dimension < 0 for dimension in self.shape + ): + raise ValueError(f"tensor {self.name!r} has a negative shape dimension") + if not isinstance(self.dtype, torch.dtype): + raise ValueError(f"tensor {self.name!r} must declare one torch dtype") + if not isinstance(self.kind, TensorKind): + raise ValueError(f"tensor {self.name!r} must use TensorKind") + + @property + def numel(self) -> int: + return math.prod(self.shape) if self.shape else 1 + + @property + def nbytes(self) -> int: + return self.numel * torch.empty((), dtype=self.dtype).element_size() + + +class TensorFileWriter(Protocol): + def write_tensor(self, name: str, tensor: torch.Tensor) -> None: ... + + def write_rows(self, name: str, start_row: int, tensor: torch.Tensor) -> None: ... + + def close(self) -> None: ... + + def __enter__(self) -> TensorFileWriter: ... + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + _traceback: TracebackType | None, + ) -> bool | None: ... + + +class ArtifactWriter(Protocol): + def open_tensor_file( + self, + relative_name: str, + specs: tuple[TensorWriteSpec, ...], + ) -> TensorFileWriter: ... + + +@dataclass(frozen=True) +class WeightProductionSpec: + producer_id: str + outputs: tuple[WeightArtifactIdentity, ...] + source_mode: ProductionSourceMode + coordination_scope: CoordinationScope = CoordinationScope.SINGLE_PROCESS + lookup_phase: LookupPhase = LookupPhase.PRE_LOAD_SAFE + + def __post_init__(self) -> None: + if not isinstance(self.producer_id, str) or not self.producer_id or not self.outputs: + raise ValueError("producer_id and outputs must not be empty") + if not isinstance(self.outputs, tuple) or any( + not isinstance(output, WeightArtifactIdentity) for output in self.outputs + ): + raise ValueError("producer outputs must be an immutable tuple of artifact identities") + if not isinstance(self.source_mode, ProductionSourceMode): + raise ValueError("producer source mode must use ProductionSourceMode") + if not isinstance(self.coordination_scope, CoordinationScope): + raise ValueError("producer coordination scope must use CoordinationScope") + if not isinstance(self.lookup_phase, LookupPhase): + raise ValueError("producer lookup phase must use LookupPhase") + if len({output.key for output in self.outputs}) != len(self.outputs): + raise ValueError("producer outputs must have unique semantic identities") + if any(output.producer.producer_id != self.producer_id for output in self.outputs): + raise ValueError("producer outputs must name the declaring producer_id") + + +@dataclass(frozen=True) +class ProductionMetadata: + producer_schema: str + restorer_schema: str + format_metadata: CanonicalJson = field(default_factory=CanonicalJson.empty) + + def __post_init__(self) -> None: + if ( + not isinstance(self.producer_schema, str) + or not self.producer_schema + or not isinstance(self.restorer_schema, str) + or not self.restorer_schema + ): + raise ValueError("producer and restorer schema identifiers must not be empty") + if not isinstance(self.format_metadata, CanonicalJson): + raise ValueError("production format metadata must use CanonicalJson") + + +class WeightProducer(Protocol): + """Synchronous V1 producer; callers do not preempt an active build.""" + + @property + def spec(self) -> WeightProductionSpec: ... + + def produce(self, writer: ArtifactWriter) -> ProductionMetadata: + """Produce one exact artifact synchronously or raise.""" + ... + + +@dataclass(frozen=True) +class BuildRequest: + identity: WeightArtifactIdentity + + +class WeightRestorePlan(Protocol): + """A validated, one-shot model mutation plan.""" + + def commit(self) -> None: + """Mutate the target model exactly once or raise.""" + ... + + +class WeightRestorer(Protocol): + """Validate a lease-to-model restore without mutating either input.""" + + @property + def schema(self) -> str: ... + + def plan_restore(self, model: object, lease: HostWeightLease) -> WeightRestorePlan: + """Return a validated plan; all model mutation is deferred to commit().""" + ... + + +class HostWeightStore(Protocol): + """Exact artifact access whose deadlines bound lock acquisition only.""" + + def lookup( + self, + identity: WeightArtifactIdentity, + *, + validation: ValidationLevel, + deadline: float | None = None, + ) -> StoreResult: ... + + def get_or_build( + self, + request: BuildRequest, + producer: WeightProducer, + *, + validation: ValidationLevel, + deadline: float, + ) -> StoreResult: ... + + +@dataclass(frozen=True) +class RemoteArtifactDescriptor: + identity: WeightArtifactIdentity + provider_id: str + manifest_sha256: str + content_sha256: str + + +class RemoteArtifactProvider(Protocol): + @property + def provider_id(self) -> str: ... + + def probe(self, identity: WeightArtifactIdentity) -> RemoteArtifactDescriptor | None: ... + + def fetch(self, descriptor: RemoteArtifactDescriptor, writer: ArtifactWriter) -> ProductionMetadata: ... + + +__all__ = [ + "ArtifactWriter", + "BuildRequest", + "CoordinationScope", + "HostWeightStore", + "LookupPhase", + "ProductionMetadata", + "ProductionSourceMode", + "RemoteArtifactDescriptor", + "RemoteArtifactProvider", + "TensorFileWriter", + "TensorWriteSpec", + "WeightProducer", + "WeightProductionSpec", + "WeightRestorePlan", + "WeightRestorer", +] diff --git a/vllm_omni/host_weight_runtime/runtime.py b/vllm_omni/host_weight_runtime/runtime.py new file mode 100644 index 000000000..4e911e04a --- /dev/null +++ b/vllm_omni/host_weight_runtime/runtime.py @@ -0,0 +1,332 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Policy-driven exact resolution over HostWeightStore contracts.""" + +from __future__ import annotations + +import logging +import time +import uuid +from collections.abc import Callable + +from .config import HostWeightRuntimeConfig, RemoteOnMiss, RuntimeMode +from .errors import FailureCode, HostWeightError, HostWeightFailure, ResolutionAction, ResolutionStage +from .identity import CanonicalJson, WeightArtifactIdentity +from .lease import HostWeightLease +from .outcomes import ( + AttemptResult, + HostWeightResolution, + ResolutionAttempt, + ResolutionOutcome, + ResolutionReport, + StoreResult, + StoreStatus, +) +from .protocols import BuildRequest, HostWeightStore, LookupPhase, WeightProducer + +logger = logging.getLogger(__name__) + + +class HostWeightRuntime: + """Resolve one exact representation without owning model restoration.""" + + def __init__( + self, + config: HostWeightRuntimeConfig, + store: HostWeightStore | None = None, + *, + observer: Callable[[ResolutionReport], None] | None = None, + _initialization_failure: HostWeightFailure | None = None, + ) -> None: + if observer is not None and not callable(observer): + raise ValueError("Host Weight Runtime observer must be callable") + if config.mode is RuntimeMode.DISABLED and (store is not None or _initialization_failure is not None): + raise ValueError("disabled Host Weight Runtime must not initialize a store") + if config.mode is not RuntimeMode.DISABLED and (store is None) == (_initialization_failure is None): + raise ValueError("enabled Host Weight Runtime requires exactly one store or initialization failure") + self.config = config + self.store = store + self._initialization_failure = _initialization_failure + self._observer = observer + + @classmethod + def from_config( + cls, + config: HostWeightRuntimeConfig, + *, + observer: Callable[[ResolutionReport], None] | None = None, + ) -> HostWeightRuntime: + """Construct lazily so disabled mode performs no filesystem probe.""" + if config.mode is RuntimeMode.DISABLED: + return cls(config, observer=observer) + assert config.domain is not None + from .filesystem import FilesystemHostWeightStore # noqa: PLC0415 + + try: + store = FilesystemHostWeightStore(config.domain, config.capacity, config.integrity) + except HostWeightError as exc: + return cls(config, observer=observer, _initialization_failure=exc.failure) + return cls(config, store, observer=observer) + + @property + def enabled(self) -> bool: + return self.config.mode is not RuntimeMode.DISABLED + + def resolve( + self, + identity: WeightArtifactIdentity | None = None, + *, + producer: WeightProducer | None = None, + ) -> HostWeightResolution: + started = time.monotonic() + if not self.enabled: + return self._finish( + ResolutionReport( + resolution_id=uuid.uuid4().hex, + outcome=ResolutionOutcome.CANONICAL_DIRECT, + attempts=(), + elapsed_seconds=time.monotonic() - started, + ) + ) + if identity is None: + raise ValueError("enabled Host Weight Runtime requires an exact artifact identity") + if self._initialization_failure is not None: + failure = self._initialization_failure + action = self._action_for_failure(failure) + return self._finish_terminal( + ( + ResolutionAttempt( + stage=failure.stage, + result=AttemptResult.ERROR, + action=action, + elapsed_seconds=time.monotonic() - started, + failure=failure, + ), + ), + started, + ) + assert self.store is not None + deadline = started + self.config.wait.coordination_timeout_seconds + attempts: list[ResolutionAttempt] = [] + + lookup_started = time.monotonic() + local = self.store.lookup( + identity, + validation=self.config.integrity.local_lookup, + deadline=deadline, + ) + if local.status is StoreStatus.HIT: + assert local.lease is not None + attempts.append( + ResolutionAttempt( + stage=ResolutionStage.LOOKUP, + result=AttemptResult.HIT, + action=ResolutionAction.RETURN_LEASE, + elapsed_seconds=time.monotonic() - lookup_started, + ) + ) + return self._finish( + ResolutionReport( + resolution_id=local.lease.provenance.resolution_id, + outcome=ResolutionOutcome.LOCAL_HIT, + attempts=tuple(attempts), + elapsed_seconds=time.monotonic() - started, + ), + lease=local.lease, + ) + + local_action = self._next_after_local(local, producer) + attempts.append( + self._store_attempt( + ResolutionStage.LOOKUP, + local, + time.monotonic() - lookup_started, + action=local_action, + ) + ) + if local.status in {StoreStatus.TIMEOUT, StoreStatus.FAILED}: + return self._finish_terminal(tuple(attempts), started) + + producer_allowed = self._producer_allowed(producer) + if self.config.remote.on_local_miss is not RemoteOnMiss.DISABLED: + remote_failure = HostWeightFailure( + stage=ResolutionStage.REMOTE, + code=FailureCode.UNSUPPORTED, + retryable=False, + message="remote artifact providers are modeled but not implemented", + details=CanonicalJson.empty(), + ) + attempts.append( + ResolutionAttempt( + stage=ResolutionStage.REMOTE, + result=AttemptResult.SKIPPED, + action=self._action_for_failure(remote_failure), + elapsed_seconds=0.0, + failure=remote_failure, + ) + ) + return self._finish_terminal(tuple(attempts), started) + + if producer_allowed: + assert producer is not None + production_started = time.monotonic() + produced = self.store.get_or_build( + BuildRequest(identity), + producer, + validation=self.config.integrity.local_lookup, + deadline=deadline, + ) + if produced.status in {StoreStatus.HIT, StoreStatus.BUILT, StoreStatus.JOINED}: + assert produced.lease is not None + attempts.append( + ResolutionAttempt( + stage=ResolutionStage.PRODUCTION, + result=AttemptResult.SUCCESS, + action=ResolutionAction.RETURN_LEASE, + elapsed_seconds=time.monotonic() - production_started, + ) + ) + return self._finish( + ResolutionReport( + resolution_id=produced.lease.provenance.resolution_id, + outcome=ResolutionOutcome.LOCAL_PRODUCTION, + attempts=tuple(attempts), + elapsed_seconds=time.monotonic() - started, + ), + lease=produced.lease, + ) + attempts.append( + self._store_attempt( + ResolutionStage.PRODUCTION, + produced, + time.monotonic() - production_started, + action=self._terminal_action_for_store_result(produced), + ) + ) + else: + unavailable = HostWeightFailure( + stage=ResolutionStage.PRODUCTION, + code=FailureCode.PRODUCER_UNAVAILABLE, + retryable=True, + message="no allowed producer can build the exact requested representation", + details=CanonicalJson.empty(), + ) + attempts.append( + ResolutionAttempt( + stage=ResolutionStage.PRODUCTION, + result=AttemptResult.SKIPPED, + action=self._action_for_failure(unavailable), + elapsed_seconds=0.0, + failure=unavailable, + ) + ) + + return self._finish_terminal(tuple(attempts), started) + + def _mode_terminal_action(self) -> ResolutionAction: + if self.config.mode is RuntimeMode.PREFERRED: + return ResolutionAction.CANONICAL_FALLBACK + return ResolutionAction.FAIL_STARTUP + + def _action_for_failure(self, failure: HostWeightFailure | None) -> ResolutionAction: + if failure is None: + raise ValueError("failed store results require a typed failure") + if self.config.mode is RuntimeMode.PREFERRED and failure.retryable: + return ResolutionAction.CANONICAL_FALLBACK + return ResolutionAction.FAIL_STARTUP + + def _terminal_action_for_store_result(self, result: StoreResult) -> ResolutionAction: + if result.status in {StoreStatus.MISS, StoreStatus.INVALID}: + return self._mode_terminal_action() + return self._action_for_failure(result.failure) + + def _next_after_local( + self, + result: StoreResult, + producer: WeightProducer | None, + ) -> ResolutionAction: + if result.status in {StoreStatus.TIMEOUT, StoreStatus.FAILED}: + return self._terminal_action_for_store_result(result) + if self.config.remote.on_local_miss is not RemoteOnMiss.DISABLED: + return ResolutionAction.TRY_REMOTE + if self._producer_allowed(producer): + if result.status is StoreStatus.INVALID: + return ResolutionAction.QUARANTINE_THEN_PRODUCE + return ResolutionAction.TRY_PRODUCER + return self._mode_terminal_action() + + def _producer_allowed(self, producer: WeightProducer | None) -> bool: + return ( + producer is not None + and self.config.production.allow_local_build + and producer.spec.lookup_phase is LookupPhase.PRE_LOAD_SAFE + ) + + @staticmethod + def _store_attempt( + stage: ResolutionStage, + result: StoreResult, + elapsed: float, + *, + action: ResolutionAction, + ) -> ResolutionAttempt: + if result.status is StoreStatus.MISS: + return ResolutionAttempt( + stage=stage, + result=AttemptResult.MISS, + action=action, + elapsed_seconds=elapsed, + ) + if result.status is StoreStatus.TIMEOUT: + attempt_result = AttemptResult.TIMEOUT + elif result.status is StoreStatus.INVALID: + attempt_result = AttemptResult.ERROR + else: + attempt_result = AttemptResult.ERROR + return ResolutionAttempt( + stage=stage, + result=attempt_result, + action=action, + elapsed_seconds=elapsed, + failure=result.failure, + ) + + def _finish_terminal( + self, + attempts: tuple[ResolutionAttempt, ...], + started: float, + ) -> HostWeightResolution: + action = attempts[-1].action + if action is ResolutionAction.CANONICAL_FALLBACK: + outcome = ResolutionOutcome.CANONICAL_FALLBACK + elif action is ResolutionAction.FAIL_STARTUP: + outcome = ResolutionOutcome.FAILED + else: + raise ValueError(f"resolution cannot terminate with action {action.value}") + return self._finish( + ResolutionReport( + resolution_id=uuid.uuid4().hex, + outcome=outcome, + attempts=attempts, + elapsed_seconds=time.monotonic() - started, + ) + ) + + def _finish( + self, + report: ResolutionReport, + *, + lease: HostWeightLease | None = None, + ) -> HostWeightResolution: + if self._observer is not None: + try: + self._observer(report) + except Exception: + logger.exception( + "Host Weight Runtime observer failed", + extra={"resolution_id": report.resolution_id}, + ) + return HostWeightResolution(report=report, lease=lease) + + +__all__ = ["HostWeightRuntime"] From 70bdd49402004169b7d93f7f8620f7325d3b0b0e Mon Sep 17 00:00:00 2001 From: Shin Date: Fri, 21 Aug 2026 14:15:37 +0800 Subject: [PATCH 100/267] [Model] Add HiDream-O1-Image support (#5194) Signed-off-by: yixiaoer --- .buildkite/cuda/test-nightly.yml | 6 + docs/models/supported_models.md | 1 + docs/user_guide/diffusion_features.md | 1 + .../text_to_image/text_to_image.py | 2 +- recipes/HiDream-ai/HiDream-O1-Image.md | 113 +++ recipes/README.md | 1 + .../assets/hidream_o1/hidream_o1_baseline.png | Bin 0 -> 1468077 bytes .../test_hidream_o1_image_cache.py | 67 ++ tests/e2e/accuracy/test_hidream_o1_image.py | 65 ++ tests/model_tests/diffusion/test_alignment.py | 1 + vllm_omni/diffusion/data.py | 16 +- .../models/hidream_o1_image/__init__.py | 15 + .../hidream_o1_image_transformer.py | 572 ++++++++++++++ .../pipeline_hidream_o1_image.py | 732 ++++++++++++++++++ vllm_omni/diffusion/registry.py | 6 + vllm_omni/diffusion/utils/hf_utils.py | 29 +- 16 files changed, 1621 insertions(+), 6 deletions(-) create mode 100644 recipes/HiDream-ai/HiDream-O1-Image.md create mode 100644 tests/assets/hidream_o1/hidream_o1_baseline.png create mode 100644 tests/diffusion/models/hidream_o1_image/test_hidream_o1_image_cache.py create mode 100644 tests/e2e/accuracy/test_hidream_o1_image.py create mode 100644 vllm_omni/diffusion/models/hidream_o1_image/__init__.py create mode 100644 vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py create mode 100644 vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 0c5d94d81..986047bf6 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -255,6 +255,12 @@ steps: - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model mirror_hardwares: h100_4 + - label: ":full_moon: HiDream-O1-Image · Accuracy Test" + timeout_in_minutes: 180 + commands: + - pytest -s -v tests/e2e/accuracy/test_hidream_o1_image.py -m full_model --run-level full_model + mirror_hardwares: h100_2 + - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image" key: nightly-diffusion-x2iat-performance-qwen-image timeout_in_minutes: 180 diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 58bca4809..beb70949d 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -109,6 +109,7 @@ th { | `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | | `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | | — | |`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — | +| `HiDreamO1ImagePipeline` | HiDream-O1-Image | `HiDream-ai/HiDream-O1-Image` | ✅︎ | | | | — | The `Recipe` column links the corresponding deployment recipe. `Published` is preferred when a corresponding `recipes.vllm.ai` page exists; otherwise the diff --git a/docs/user_guide/diffusion_features.md b/docs/user_guide/diffusion_features.md index 34b642b9e..4a46c6b25 100644 --- a/docs/user_guide/diffusion_features.md +++ b/docs/user_guide/diffusion_features.md @@ -124,6 +124,7 @@ The following tables show which models support each feature: | **FLUX.2-dev** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | **GLM-Image** | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | | **Hidream-I1-Full** | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| **HiDream-O1-Image** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | **HunyuanImage3** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅* | | **Krea 2** | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ | | **LongCat-Image** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | diff --git a/examples/offline_inference/text_to_image/text_to_image.py b/examples/offline_inference/text_to_image/text_to_image.py index 2d1f9823a..2ee09e178 100644 --- a/examples/offline_inference/text_to_image/text_to_image.py +++ b/examples/offline_inference/text_to_image/text_to_image.py @@ -89,7 +89,7 @@ def parse_args() -> argparse.Namespace: "Qwen/Qwen-Image, Tongyi-MAI/Z-Image-Turbo, Qwen/Qwen-Image-2512, stepfun-ai/NextStep-1.1, " "black-forest-labs/FLUX.1-dev, black-forest-labs/FLUX.2-klein-9B, " "black-forest-labs/FLUX.2-dev, tencent/HunyuanImage-3.0-Instruct, " - "meituan-longcat/LongCat-Image, OvisAI/Ovis-Image, " + "HiDream-ai/HiDream-O1-Image, meituan-longcat/LongCat-Image, OvisAI/Ovis-Image, " "stabilityai/stable-diffusion-3.5-medium, Tongyi-MAI/Z-Image-Turbo and etc.", ) parser.add_argument( diff --git a/recipes/HiDream-ai/HiDream-O1-Image.md b/recipes/HiDream-ai/HiDream-O1-Image.md new file mode 100644 index 000000000..1864324b2 --- /dev/null +++ b/recipes/HiDream-ai/HiDream-O1-Image.md @@ -0,0 +1,113 @@ +# HiDream-O1-Image + +> HiDream-O1 text-to-image generation through the shared offline example + +## Summary + +- Vendor: HiDream.ai +- Model: `HiDream-ai/HiDream-O1-Image` +- Task: Text-to-image +- Mode: Offline inference +- Maintainer: Community + +## When to use this recipe + +Use this recipe to generate native-resolution images with HiDream-O1 on one +or two CUDA GPUs. The two-GPU command shards the Qwen3-VL attention and MLP +weights, as well as the timestep and pixel projections. + +## References + +- Upstream model: + [`HiDream-ai/HiDream-O1-Image`](https://huggingface.co/HiDream-ai/HiDream-O1-Image) +- Related offline example: + [`examples/offline_inference/text_to_image/text_to_image.py`](../../examples/offline_inference/text_to_image/text_to_image.py) +- Related issue: + [`vllm-project/vllm-omni#3479`](https://github.com/vllm-project/vllm-omni/issues/3479) + +## Hardware Support + +## GPU + +### 1x H100 80GB + +#### Environment + +- OS: Linux +- Python: Match the repository requirements for your checkout +- Driver / runtime: NVIDIA CUDA +- vLLM version: Match the repository requirements for your checkout +- vLLM-Omni version or commit: Use the commit you are deploying from + +#### Command + +```bash +python examples/offline_inference/text_to_image/text_to_image.py \ + --model HiDream-ai/HiDream-O1-Image \ + --prompt "A cat is sitting next to a sign that says 'HiDream-O1 vLLM-Omni'" \ + --height 2048 \ + --width 2048 \ + --num-inference-steps 50 \ + --guidance-scale 5.0 \ + --seed 42 \ + --output hidream_o1_output.png +``` + +#### Verification + +```bash +python -c "from PIL import Image; im = Image.open('hidream_o1_output.png'); print(im.mode, im.size)" +``` + +Expected output: + +```text +RGB (2048, 2048) +``` + +#### Notes + +- The pipeline supports BF16 text-to-image generation. +- Requested sizes are matched to the closest resolution supported by the checkpoint. + +### 2x H100 80GB + +#### Environment + +- OS: Linux +- Python: Match the repository requirements for your checkout +- Driver / runtime: NVIDIA CUDA +- vLLM version: Match the repository requirements for your checkout +- vLLM-Omni version or commit: Use the commit you are deploying from + +#### Command + +```bash +python examples/offline_inference/text_to_image/text_to_image.py \ + --model HiDream-ai/HiDream-O1-Image \ + --tensor-parallel-size 2 \ + --prompt "A cat is sitting next to a sign that says 'HiDream-O1 vLLM-Omni'" \ + --height 2048 \ + --width 2048 \ + --num-inference-steps 50 \ + --guidance-scale 5.0 \ + --seed 42 \ + --output hidream_o1_tp2_output.png +``` + +#### Verification + +```bash +python -c "from PIL import Image; im = Image.open('hidream_o1_tp2_output.png'); print(im.mode, im.size)" +``` + +Expected output: + +```text +RGB (2048, 2048) +``` + +#### Notes + +- Tensor parallelism shards the attention, MLP, timestep, and pixel projection weights across both GPUs. +- Image editing and image-conditioned generation are not supported. diff --git a/recipes/README.md b/recipes/README.md index 8b93124d0..1a384d08e 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -47,6 +47,7 @@ recipes/ | [`ByteDance/Lance.md`](./ByteDance/Lance.md) | Unified AR+diffusion: text/img/video gen + understanding (Lance 3B) | 1x B300 / A100 80GB | | [`fishaudio/Fish-Speech-S2-Pro.md`](./fishaudio/Fish-Speech-S2-Pro.md) | Online serving for TTS | 1x A800 80GB | | [`Helios/Helios.md`](./Helios/Helios.md) | Text-to-video, image-to-video, and video-to-video generation | 1x NVIDIA H20 | +| [`HiDream-ai/HiDream-O1-Image.md`](./HiDream-ai/HiDream-O1-Image.md) | Text-to-image with shared offline example | 1x or 2x H100 80GB | | [`inclusionAI/Ming-flash-omni-2.0.md`](./inclusionAI/Ming-flash-omni-2.0.md) | Online serving for multimodal chat + standalone TTS | 4x H100 / 1x H100 80GB | | [`inclusionAI/Ming-omni-tts.md`](./inclusionAI/Ming-omni-tts.md) | Offline + online dense Ming TTS/audio generation | 1x H100 80GB / 1x AMD MI300X (ROCm 7.2) | | [`k2-fsa/OmniVoice.md`](./k2-fsa/OmniVoice.md) | Offline and online multilingual text-to-speech | 1x AMD MI300X (ROCm 7.2) | diff --git a/tests/assets/hidream_o1/hidream_o1_baseline.png b/tests/assets/hidream_o1/hidream_o1_baseline.png new file mode 100644 index 0000000000000000000000000000000000000000..38920f301b20c42b3fd591c08d834190bd5d26aa GIT binary patch literal 1468077 zcmV)DK*7I>P)Px#L}ge>W=%~1DgXcg2mk?xX#fNO00031000^Q000001E2u_0{{R30RRC20H6W@ z1ONa40RR950H6Z^1ONa40RR95000000Mp&-9RL7807*naRCobY-CbhjIFqI62uYFE z)mxdHxtsm3YO24K6!SjE=#e>;I|&%<=a|6&f^@p;=l}ix{9nI+{QmLd_s{PiZIyrj zto-}?=W0KH{`~R#`^Vc`Efs(K`1tYu{xgnPNBRBT@wV?DsrdQx?Pm>dN=kSbw~Ar19I~?RT28*Jl3~$IoAhzEvg}?Z@|5n%GEJiUT*zX-AFkt35d1X@_cT zq5m0dbUW(b+Gnr>v@pZ}`}>Q_)W)C0^wEtgnqU--q51y)`br?X-@m`UlKjd}t>gXu z`-`cXzbgD7n9jMH9@yR@7M-`B@9*!q`S$Vl@$>D!K7RfBc>k~WpMSl5{EG7N^Vj>^ z`+JRV9o`E@j03*_rQ}Cu!vO8)XWkdgYV9+=kp8TLKGIMxyuJPP>*M3&?Y;8PA0KnR z=;}b-15eeky%+Sv7O^qwf_k!!4{>{3x$urZRinJ8CZ~j4-{_AERRQ=cMBm?^-#^<& z3}}5G?1J;3FV+8i{{F1+`SqXg-+zC7|L6P1-^I5me*XTFEQzm6(o5+ATS$NN_)Cv2 z?jaAGkQSwn_W$+%<1hR7|Ml_l-+%r5uaC+vofB$;>?~k|t{_FjJf4u## zU+*6uzyA8`{jZPrzkdDo>(|>~)vMK;wS0u7&J^8HvMi84e!ssL>`Hdk3#9kTm!j2U z6;!eP9#QT}t9sx^sj&8??{ro#ILkWi3dKp2;v|23L_vPqFN{Tcg1S<(J$YO8;=eyd zy;<%TlR9zzSzZ6?t*_4?f7gAVpPv=Ke}De{cOk7?|Ni{@|9sW-_4oJJ-=Fn$g_5`Q zC?#tBOzJ;hpa1ju<9|N?E))Jw9q}FI`%^|8irkRn!&GO5K&E!z7Joj3g-db3-XDxRTbM3CXLszRK(DeSa;@He?DW7_o?buq618RJ_Y?Nz~RXoXwxVenl#~ z-+xK#vaJmKugxECS!@zuG0PMi+ot*Ix>OTKTs2jcSE84O83(+PI1yi|Mg+rksC%qL}3Co zmP1?R@**obF3|`u1?7mji_YkNQ|-TG%Ab3OOwvvMT98^BR4R&wt96xti}+Ss^INbB z(ZQ~0hI`^zjm5Yxcw5_M(24HJ5t+QOVcfkA60uD$s1`-3HQsM5)x(u`)o+L`_I4P!AA#6`Ly9U=Yk{nmTo zZL#lC?yco;6UtlkbL=_hh_K9sMz_!|37^#0RP%n6C{jnX;~!YqVk7`){-Wj)p)C7fTNFdVj_a@ zv%u-9K)n^If=F-qSf+;et0D0H`~B;yNvs5pT31;TC)DcxW9yzS;eUU~p-@#0Mpt-M zn|nSpFArVr{Vs+LgzyyqH2;uhI(+%fm!bpVg0}v>0Y+EO!QBx)0jr{L5fAc!PZp!; zfU3@H^>{JY!{VEq&X^Vo1)9ci^mntgs(T5hQ@ytAFt~qP{i|NjCOe||%23p&?=L@7 zS-|8V{edbxpNqw^aRatS7Z_S&-t4Fce*gHp(@2-o=`6Btlf1nZ#5T^&mugJ*H(zs1 zM?$dBMAs0c74TK-jqUc;g4RZ(2n0VyUrjg1wKN=fh9It&+IV8OM9+pT%KSD>^a z^gb4FFDT4z2Dv8a26k@hX7ws(e0eu*HHm|icqGS>-&V;Hi_|6i3k0khydF>LsZZ*s z=mDoKtL$c>+4J$Px(~%;BMIn`GxETxksfMSn9ENm zMA%r>7(e`4z?`-bXO;X`r~aC0+(jr;|Um2=AZLvd>Cp46?#k!xh^ z-#X7Umv04?rE$Et31=0i2N%L1jIvY8nH(4WoSB`AZ}oA_?%7ZKSWcDEfc{8n%(W-% z3+(vy(0rN&5yhy&S_M=T0!}6n z4k~32AqzkZpJLut&a2`iq5drW##Z$ZErTYP^8VOQ|DT|-P#5B7v&Lr}VcD`^dJ>t& z7%I#}yE!#fi!CHnM^4FNy2gr>wzTc$<5FL9BA;`hU^U%0)pi!|&Ru3(seQTd{)*Ju zx}YqUUzXHwT}_tZ3ASX4xr{QX;*%hmD(bDWWM<-vRSfIpDUL7iaG;bT(|Gl-y&wAh(s0I7Kqn&Ji`+`Ha&vKRC?Hbt(7a0&YGo`WyD$|mbSM@ zZDd+krbuw&k%XjLx>$16*gl@!i>m@EV%X~M_XhpXZuh$5+b0ZtXw&|$w=M>{)!)Y~ z?SK8{et#GHKGKQa%D>frt26_?#eYYpcYlSqI+lB;1%1`zP#9W~!e!hjwNF?N=oj%% zM;$=yHW1UWHmFIBRq?ffvMXqU<*n)T#* zXvWK*(1f1{N*9X9LpKe^yoQ3oK@OSp8yk_4TqVpAL9q1AB-7mD(=JpL>ppVctv z*)*T*LL%N0t4#UKB?G5WQR_xZtlgp7fJBt^89wN@!UaIkN$kOxJKP1!i#N|;uTeU7(&j=Cltw{JsRRET%=>u?#XM4y-Xhy&` zhIs<%^#qYes21aC9W>;Bl>IxR)3X-4$K)Yko6#B^7cpbvqna|S zwOw~=YZ`diX^wwrbjL(5i4&Mg>+78i@lh1d^d*#82oZ;oZl2fMmb3Lv71)larqF1Q zqBp-y`OdsCM(bvjSv;1Ew(_sB7OCUaG$=B)m4;>b8r%nS=PzmZ_&aPebV3lPC?IZ& zvf|4D8U~$L3#Wp}GI#M^5#IUo3NttH?EUQ1^#pw5W**Nkx<1f&GriMJ>=l<8OMbnQ ze_`mp9qs})B|Kkpt>ZOOg6EK=3;5TWs;;#Q|HfcWPE_yBRF0E{?VB}H7%V!v(Tsiov_m%oXl;JI%ix97_Qj` z=UL0bzj2v1B2#bBE^Mn|0eo#Ja(1p?A3fEhXjmq7zRcVd_(ThnB;r^-&{pIKlB2nDy(FA%=)l|3(^-O1$lM~u6=?ND&b z#eE>!pm=LtpTVy;3;&Y1hZX)dQ~rGK$)9FVvtA3lxVk2ai_Y$@B-AGo|B=&mSk9Jd zvaDV$*P*N2YB&syjdj^20g!PMdv$smH*@>!twITsXG$qpW+-qrmFIB*X|-vTi%>Ob zvF$0;Fa}(Y-72}(bnqhjAdNH*mjo4bv!hAAgLk_D7_X?#oW#s%-WVi7Sx*4C zQN#Z{_}NF!()q`GvrB$^a_Yy=U&FntZ&%m60}5gmdA8;^G{` zyzTH@`o*xn;2lPLl_1v@iEfPF$fmy0*1pZB5?K2t#_X= zRVj-$@qn6&Ha(`WPGKld@?12TS#_2x4g!$1OmaolbDy9V=;`Hje~%tq^tQcrIT`;P zpB&jWR}hLdBTcEpaYU&nx8eymuhUW!iJ4Zf4?$s%;dVy7qxZAT#zZF#&>W2ApS z(CZNdNO7x}HKiM%A{<#W7)%V!OKv%UA=Rk_jX^wXiU11v+bN8Vs+B37%(+@^+^Kj3 zchrF=(7tlaq)*kAk7a=y{ZSK3qE#-E#M|cAG+yJiP9^GNJSmco=OTX-!iW`{6tRwN z_$b&I>5$!}AqtiB*gZJb&0~8tkX*R2F~QHL_$U0R*Z8!I?}4F%FEV;7L)F*)i9Gmr zxS>Jv1lC84oy?YRH+Em9`0kanAgz&iwlF_HSKS#FNN?t9)P6f;8{_sGGQ0h?P5GdE zRAYMLuML1~ZoY&hFHVjgh)f}a2Pns@pColkX#gZ{*}Alt=1tZStg>sw_(;|xX>TZ4 zolV?+B?=R?9^RDMDP*5uJ0&O6XK&5DzfHs4a&jhIz*w%Iy7}MSMK|u7xq|EI-!1<$ z3yksYDNy^JLUw9Yy|D8z>+SqewfBH>vzE+{yjUIYKl=r27$!^?Ap*isA?Kv(6z8`m z0-`fB#T|vL7a&B}nG%JYfTCW@#4!IcV3J|fd`@i*kq+{s%wZy0JI$p})KO29`EvRd z#(GX*8_LK)SrW|h;}WQ%wd;f!CLG0Nwq+Gv$B8$}kHl zZw0Y2Z4Rwn{UbqL1ageq9{g z5}`<#Qr2IoMgmQbdhSn~UKbzt=qvO)i`!rT^!4<-f@2-1|w5}$6denW+x4*LE7Rxe4Z*VhX16F(hgKWDN zi}_Hww@_9|TV54)DCW69)@1q?U5=FJ_8m<3>W_xXw@oB5k)8hfZYEaC?qiW=qo(;l z`=0fxYNj{EcRVu!eK^F@XiJjfs+dX3(EgT&GP5W*!NgsgAiVWZ6;dB8-~Ik6EYR6) z>{@y6%NSD|)kh;&_5sUuO9N89b3EG%<;C#jIu=Pkkae_(*^}6(6)>R2IEawOU7fb* zxihEIAbetI7|hR_P|T6|s_BIr7`rWlG7jhzh3**RhN8-*Nlm>s3M6yvtpDqP{|F*T z8=sZJ^VCjg!VmZ;PjzhqmI(a|3x!iAHHG1+J4)L}IQD6QvEm(M8m; zjnzp~!g}6^{e@>8Cr+m|-YGr{XK~k4k*kTuFvTsRoSIuNq~k(-K;cPO1iB`y4$IW7 zO`}$-Dop7(v=a44OaMWk2ZOd9l_wC?r(+?)wAxgu%+KjKm=EbhE(Q}}gogzr%LiNWsYa_MegPTeYR1J# z8U)iHMGr*w@s58O%z(*{@Pr@lWKL|+lQUKokNEU$r}g5yk#%`A#qt zepeo?;t>v$%I3pV1WmRsIg@_;)_#=}{5SmLwH~7N2@Ll0wD5YWE8eFuv)za*B&$*TwRegMAI=0NS4(p7-!_uZiv`(Y&>5stG z^@%sL`5eH<0d9#crovY{mNthP_{k9z+uRIv2G^nLYgp3*b6Cf{6CsDgPeVl!hiL7! z1TEOqm@PMRbpb%6X&ek9<^Mn}44v?~+h&(g=ixgKqP6A6nun|%y^*2bO=R>D1bYD` zg$EkNc(ZNoS?Ui$q*}0q>dZ%{aydmRSi(f7o5AzPVu!w|rhb+9n(A|#rR(COQ`2@g z_-dgtO+L4t(VM90frqj3vIjxBrPdH_Sr|jh_g%z5;FX#hT_yyqby7*QEXksF=g-?1 z>m78@oLoP_U=c7inJT%uKdQrx3w+~)W8LNLTEgz^B_eo-7QELycW^oF={ z^-9Vbat3U5rR~>@Qa7i-*epd_K2D_XJ?p>+5tQ~*_dA7jlGSGJy3NmN8Y{}mW3oz(< zUHOGWC<4;h?HiY^z5ePlpr`y@1MHgseHma+`*;M%^Zp+B@8KUyULOH8sCV`6GeZuU z1-8ix6pN7S55A+UR$&MNxuA?}hDZui0j##zTr%jSn(Dtp9MxDNVC^-zI2yT4L6%O~ z^x!GE)QM~Tx(-Scp0O=fRjFway4An2>f+TkZXaTmI^Ci6h3}qfYlMK8>(=_7Y{T~% z&){k&{}qI-m|K%6oT2;M)$pr5Ebb8 zAzAvs%~zAscGLIEt(5S(oHB??n4+*^oRH#`&m(&EKSv(t zGqjj#i9CAH<&P8PBJWI4jO!M05y*>=K3!D6Ra(P!P^O~Z24uN(uD0*j#fxjj<-Yrvjz-GrlyQCK^uR zdGgd#xH*IvRg_?arYwVIbtJ=PhK^FPm#)`HcTs+-D5-qF&b{G!kZK~MTb5gFr^3dU zl^-CGJ<2t2(ivNi^0HYiCUE z_yl_WDL}8xox;`f=o}^?nB>^lHo?`i`-hAQ=|fn?=y(LuGEwDN+CRis1~7b$90=o? zd}ue{<4sc2+S{ukv83UkR%*uHO_VC<5G6#s9?28Lw_JW+oo$ejsdk&)%K=p)zqHUA0bW0r8Q&>^^+9QIGi{c zyQkl4p=oExDXhlkSQ1bYms!gyewrMr!h;w)%5J&QmwrYo#>3McX{%ZUZ%TLCPmQxd z%nS)ch1pBG<=VP3s{2OQOiCv?8N}S1fd>Lq2}xSqnH^lcOM8kJAX0jI;&EK8EL=zc z*quc^??Hxr`>_Jm`|!GAYD4CN5Qua01vT$xJFQMvZa=5Flj_^g&(>3&g!`&#P;5KJ z?jZ^H+xyv=&eC?GAgCeSOQ1zke0G~VW3@UnK#YnPu^Lv3jF>=O3`lxUAxhYmAh}@z zrf~YH!P`QBcHOavKS&s3($w-iSYJ8sP+EgDVv3xFLxo%rfH-$P(>ntY)mI^Rf{trE z8f%VWNU($dY%ug2rZiRGd&Dn*a zE3~ZiNm)w!$RM}%OSk`xn5`as8uQ*PTTJ`Wg;1{q) zAvg-w5a{MZvd zX&88#qG07zWsplk5$waNDG=h27aBlnFDoHS0&@J_8qj^Qx9>JVA|!2cb8)E%WaxXMX9SyKPinkii9z`zb=yvAw6QKR8t6-8j&QD@mVN@HB5KA z{BY6}v&|wGk!qRTz8xS%s03L>roKWJS6>tny@!2e?ax+>^;or>09$!Zh)df7e3V)- zXRTfr3M?X#*>OICk2qa1+(aoXQ=g}NyVp3co&wpIz_hZ$UMgZz8bk*1M~Vi35mbpJ z!k8;n7?lO(+&Ce@(Kq2DL7j=(Mqoqz=90;aN#05OBugmjcuus#31N-U@WP-tZEIeg zrNICY!#2iiU}01)(h8Y8))FJ0mqdyU#a`ol)F4qmOq5M8K6dX|+Z2sY!@uc=0XO@( z3GT#k(mRI^##&FerWaes2h|P~?e*&Lbe}vnZBOmRI2X|VrI77Y@~3oKLl;J>_1vkL zK7DX1F1iP>rnx}%BroFEomBtv;^B`mO{rL8yCvCBU#y<`DZAhc=ml!742aROS1Pi( zC?38nZU85Q`Uo!dR*k=5%ZX(LE^22T!}53`Al?ftxD!%~-?id=ScHUY`j{j^SHxux zOa0NrOpOXigHiDi*7A6{kg>aQ`>zPn`~YZLGFQxeGyG1CP0Lwq(dNTdld0c_F}o(@ zJWjTIEm_$pxCMY4|JwwuE$_A%czk#Gv(0$BXfv*0WfId|R7F$uT2L#A^T}vZEI3nE zu?^wHOKww{)umz`?GxGBr+S4I5pyc@awr|^;zhG_@}e%nr=Qv(Nn^bJYV(UZ12Yc< zN~?I$M->AzSpkF)>6J)C5KaN@wwYN#;;^^`2^_l6|?xet4AYs zC$}^cK}|BiG6Rb^4Ewq}hCkWbQfkf|gL9*CI}3YMg_SmD%9aj=Qw=8CYkO8S$Hr2m z#i!WsH&|NncjMplJ+1z&_$~0eIXca$lDCjsh<%jv2;_^xAO>f6!m_1RQ$AAsrTd%}Hsnz>I8!hL8Su4;W1xN=QU z75`s)xXZ&HA?gNa>&TvN|J#h)t@J%-RxJ0)tAyMT(Sw|r2VxS+wq0PbQ;QK`_$+nL zrFl#4wwr6lZF=K$L(rq1X-B!nkCZi!a__|kF%Mwq8_#CWm)k)6A%$O z$KhvNZc7LB80{_3KgwB8Ks8(&@ky1XbI)At%fYZj`095?T|GhO%+u#D%|tyJv|xlF zxNSxDm*=wTJ_ZJJGSEbIY1N*?Vj2^l435mZ*>AmXd~7H;lKtF;3;DKPKVu+eI90fh zGf{Fj)rLWaE0pyu^Y4O6!d6 zrQHf0$1!a${u6)MW%9*#>8kd)6?Db?rzh5NLchXUJgNBVD7jJi&1g7=Y6Yc~P0T)_ z*dlN`-m^5&$^C&^8P9b9*0#f|G(;LjP+L}kJb3903dINJQg0Aul9{oC1JQK+gOL6u zPV^~Z;Lhq*O97_N6)h(n;asAVE}9QPbeV-V=0_Y?9Bqb?U`~?+9C^DTUH1IrD~XWJ z%!2UJL)VmFG*r*VBTJ*?mY8)V>l+MX#5s6^=I^XjTQ^MBm7pXzy|g|7dZw6+Yt*kA zo-Af(5>HmqZlWU{zg6y74$;BVb+Ku(!Sj?GgRxXe>l9ykIA_rqP51LAEhg>9{b0s2 zZ^dF7LxREBUT`QcS0iI&osLJ3Lz?{Qy@i3w+2B;nVCp$9rYPBwhfGb0f(D9a#zcex zs*Nd+p?BBDyp~nQz!1No>%7s(yK_SO6{m4#FIMnOIY7;0$$~suO-;i382wxs{*}4t z6f{IkTd|Q9m*-X*F-_?iqv{--`-$&YfM#<|cCBF(n8{pn(l@TM1?E6DXf#WKCt!*j z)d(-?O7n~gdQOqy3#0D5G%S>~4cSdq1TNBJ<5-UB(N6y&0NB*CKy)XhqRm^HfE9<+ zI;e>|VfRO0EF0PjVrMuVn%+!okwebem$ED^&^reuwc8s#-Ow`-oUod^I0->F0(W*y zjbJ&ADY$pl!IdJVSwrXleqbqUHTbUiAAf$fF6vYJPOm}GBRO9P4avGXg8;V)Bakv- zE)}cg>@LaAU4oWKduh!bH3+ju(a&uKd*ZFRsL5?R@{XK7G# zAX<^>Wij#eRjUg0RGzTxA-dB}Osmxu*^j0E0mRnDVS!9Gb7U$In(u7529qw!7Sp?z zp_#>o1)HfTlnQNWX(6K>jlJ>I6GgOKZ7JnyLjPIu zSL%E6y2o&@S4@dK*iMIJUU%^@cLKixo-sv?vc9VnX{mep`($a-sMA=s zr7LSQJ3$orl_g8Yl>ewr~dSv)ILP$ zV}MM#3aD8(x7DweK@*zx)|jbW*tjX^J(_O<$Kvya!dGF_Ne)Vk6ertzu1t{-OzlS1 zMixV<>t)@)<>{^gS9IgM$-H?cCmR6s*v|>fBd9s(l?zQQ%du_=_|PFmL0M2! zpDc5YhTcq9u>ZF)Y^oy`(LY$GXeusvJlq2W~`#IQ>N*CL!@{uq%170XD<%^uoeR`bR}h?8(`^z zemuu5D7{8(61fOHiZPD4xZaj{+sq_${j}jRyK9?mOlmgP?v*kcYuoHo+k)FMkmJFB z3Z@+Oz~@rVREZSDZY?KZ0(;35$@3|y$o*l_Q?(8J(+?98$iOR`1)<`Gst&ViUIrK?RhlUR8D6jyI28(>909*0K?3~6ve zj(-|?f}U`>t+cJ`)I3sSk=ZfM#LL-4))#~_GFKEu2Bof|^$B{+7q%H~x8q>W8ZFrE z|EN0Bv{e-Qbi82Wp1@hCVVw>>+vXCic4y(A(vWF1UGRURa{}T+r3uJ&bUk~l&_$1^ z3ojyM#(C^(wmZHlnXaY(SF07eGBhdF2pnD;cjkT;Nm(B)2p_%6{1LV zmk!?q*JrmIbaJs&Cene`Tx6YY)}Of5MD-$4K@yzWoarQ<^gOme$X6O{Y|*sp(ahja z7dH`3edi4dP9z0sDi)`J5=)P&)TXKy+=BQ7GU|;mw!*d?;>ijl{1cN+zAx|3h7M+j zTf}xIh+v)B`p6RGeryA!+t z(c|PTdlq~Sm3|b2eR*A>j2gl{JYl6!tYgfeT%&9J&ranOjLfMh@L&l}73+kmx9;cYEwzW^b%IXXI#Lf$`a&QnMG>*>L2?;laiN!W#gnz*F(o*bnH95M$c>47 z{O%TSgnr<^g?uY{v*crMxM15a)dN_1YclUo8RqCDB&3{ZLQ6 z>$_M!E)#=Zu+jn_=h?|Ty{eqO5wJC-*6^l&lWl$s@?BUjAepvMf5tF*soc_f1? z{Wg7GgzcHxD}?oav8la4fRVNcV<$yr9kNjR9tMXO96{lPmA+3r&_kKAc)41}ZdB<$ zxIN(?%V$sewBYv%K;kXwGtk!pn5tv^B56I))}u-LX90?wilMVsBLi@8$lA`&a2Bh^_{9J#ZHVIQD{y2Hg{InbO|pKl8tVlbvGH2b>!uE(ax;@v&ew zg~l--1OUxvFHRwqb{@@3UpUbLeK@p7FR4vX3=4g_^&dTLIiYCcgs74b3IqDmn(9Q{ zW~;sYlHfSTUA*gznB&gN1?wWSa4lYEr+q5*80EFD#{ST3Bs^NNF#q6~HRekED-SrX zZD+*3s&~4VrHpI{M`dm00yP%)anI*y0{E*n12UdsK}Z z*Z8QN?5wt%0TI}?H9;Kgw>zeqd0h3E)NNVu}nOTOCSY2NbVDcD2&)a9^*5-UxYmlEx@)h+Lzumr>SV z5J%K;HkMvIM;V!b7NU#7Nm6z112b?(dFc30gF>290Yq^~1wiSeJg%eHh`#;iaBebl zr_i|Jplk(Wk1US<5Sw|StWs5@ionlilp>;EC{}F#X>Op+OkxV^zEy-KEQ(_{o0-E5 z$PUg1Lz_xL+&HDrlz)zh=QM&ccti%}eCb?(VX*ZqE=f6+4mu2V4QWnENV+A2d5&S) zW)kVE3|lMUtJXHb@m4%bQ?NK@U6Qpkd#$*tu)*o$iUo7ESsB+9*{NX`js!Yi2s?eG zXKtdp3M)L5yST(*BBubvZD~^45Q*#l#*X|NOAjD|{%Ytj207&d?7XRuqGn*{XP~TO zVhw3h?tI1&+RhuU_?*=H=AN%YmJT)i{_H>Sa;!xd%}V;Z?(gzI4$uwQ!K)!F-7bZ& z#$C;HSyrp;!R(3N!qi!wMKRd&po1k#K&?^6-icusnRJ#^G^1~~D)3qYM?_B3un9+O zAa-6{6o?rCBTvi51N#ZxpGftWmy&ITAl7cNo@^t-qr*kSUG~LgagS7MA}*OgAWuQM zLuyFzOpI(bJGVOfqf^D$_{C_RMqIqOm;hD_%eI}mXJPL14{!RfdCHjY^(c;q)~+A! z;h%2&yWhWS0Ly*P`rM+wMO&kPD;Q&Zg&X#>vSmzh7HV(Kolk=uDXUE^?B2W&#<5|q zywoK2h`ux|lBbd5iR<1(Dv+vVSBj4QOjebn3TQkl9p5Ms&iEoe+tVb*-orMg;byXx zt2-FL`wDB$Wy7{TnBI4|Dbl6%JKd=odKs_G(+#3|3g(2X9Jh(J!|fNS4v> zKYsNb5v0jd<+h9Q%eV8hY7;ObpOlTbtkfUX9ZEzWNwC|8- zCh}+Y7T8{0X*A}lbV*P_*|?Jxh))tEv&#cP8ie*6p9Xmz7!J(}m(4vI^n0ZFvWEYvx&!)h^DXYw|sFz+7JwQU6I zk^07%F=P#1t9vW{TlV)Kv-_n{^zHukQD6LH6G!5`c$_UgXbs=y%Oa+iE?ERkCiEFm zg9uRs7p8WJ=}b>p`t}Icitk0Dd~)I~5Hd?W19q}C$qx0JXp3c)bt<0|IZ#7rJ~n;S z0OB4}8%<{~2O=ggCt`KASaop`pDNTnMH-?#{cyc;0S0npbwB zHBV7@GOd1!1Y}Z)4Z_5pzKAVD#}Al5u!?nRjj2Dmpy3bwrD;-AzIZ$gGkSha&oIPg z9og&-Ff@*9w}<07XnWeDgiJetg}D@&i_L9VqK&nJ*P|x?B*iG3wJ}4yKH5xetIeMT7))?ZH+ABxz6m)@ghf&|E(Lc?q*-x}u?cbVb9x@o7h!+fiJ8XGsk}5`h+kM%VP-|CLhYGVFfI1y zG=2@b53kX(#b2ehkQ~C7N=IjsF@Of+q|bKri&``aJHa~-*kF$UlooV042dWMzti#D zTYn`JCq)D@P#+CZwPjrG>GKca48eJ^$9M1Ml!v1W4fUDx145-Mv<;jQv!u0VN4;Bq zZ=6`4uonCEVvC++Q`jru4|ofYdMnnJOAdaURB-RIg>@Zb9b02&9Qmr)-#QH8Pi*Sz zm@kQ;jEgor6?hwse4BY1PC zxBorx}I7TFIGLrU8~Q7>ZL|L!r$U4wzzCxL*Q~+(D@2`k$R8g3ht>vJ9iK zMX-g1k7a4{Zgl&?)bqX8v(_*A_qC&ze1+O|Eswf^c88Py6(vLEVq-I{<%o zB~Z@$>WoW^?vD4xlFUImd}&xUf-OF=c6luH7u|+MVOf+n#>kcyBt~rks{5zY@QI$Y z{prFbK+=^>RtA}mfLhXaRkMBTcAf2UAeSn7d)yuOQb!P!5^~3`0>a~3Aj7VT3S-Yk zwXAhDm7!iO{j=AFx-0(OY=U~(V=Ms-o&?u_Ig`%beT$+=qYndewmHP70A@Bv{CyR} zJ(@)JxKtAL&(l^kUn3#4UX2*rp88kz4Fo^-`u){^dg!n3W>;>9^+!(9-#{l8#=hoNBguBGm*sXH$#U!hNgYA%IxApBx%m*cAm zILAdQZP5ytgj_I+Jrd%A0+p3zoZ%4K8=ja$qLWNF9onWuq)6{=cg{|sru04YOnpe` zKtQ*57DvqBDPiF>rt{U-%*pKf7!~B2&$XCpS%};5v7Ki43(QmH)Tnp%7+o9lg|{M7 zwe2o!+QDdxW2d&MhoKhCSSI<@_?RWr@jDiQRNO z2J_Yd4DNPT@4%+auomrAEW-W$H^s|4iPgB}=yxqBuKQf3NA#;4e z(x#rHGq_sfFRbf;30*j~fOLl9GKk|}To2E+#lj>`gk&&As=%f7#3L9{jE6drDIC&H zLwyTtnZ*ExiCj#CfKCP&qMMgzWgwO3I4BhdXXhJ89(@Ec4ze=a53!PwHma5f1ZG5j zMO%WO@)#ZuG0(Q|=7A zAQsd_e#U0|_y7RjY4Dey!6MCuioP0m6gqH{9c9JrK`IsD?37;VYk z`Ly-(*5{dsuFdSS@flK}Ce&G??iIuI?VvdDTn1vMSoU`+mPMW2ZpNm$j`3MRC%+aq z04_?M&a^q*MPQYhcI$$wMe5NQ+x&BTH2D$){?3xSd)UcutBrjD^Sw_WLY7}nvyc43 zDMW6V474C=t#bVgb$|2X9G+19#?!EDx!L|8QWDaex)+RUqapA7#gEOQyo=lR><#yZ zWziV(Wwm$(sYDqov<$aP`btTSruK>nZdIM=y#K%$=-Rv{#Q9$1#$0dLGSPIJ3<6~v zRxL}NI1Q_mxAb!3zkmJfX`i0;@5=yw?(;qk=tF@0(7)rYH(&d~>Q<$d1S}~UAcxAT z)|Zfo?yH05N)YDTojRb#HgTk>O3Atfr?Taoa@*tkYHLPgVj)#BI`HvvO*&zU4QO5; z#S?E~E1Ru*{ftuoKkU0F zta|3>@9$RF``M-DrdD~szx)5e{eo!4mVAvT7s+6^t0am7YVnvyZYNtN3J3Q9TCmzl zxe*(RjMt{;r->RKaB>r<##9Pc8i-Y?a97}4{a5X>hk;7~X7?l*36&in1fdOF;fr8_ zn1KQiIWSgv#=AwYo{AVxdD!Drtw5U;hsN6bFvqg1tz0Up^8rQ~h*H)4|L**M{oRE?9|82sOg)9< z>YytCyxA7QWOff@(~hqBsPAX8&j9jG6m3PSg}t9qRuqPfmvzTN26>||hw50N_Lxq- z^kcl~4pNl%l{fenv?-Pm^ohe9QdhhQ6+~p)IQ6>EaBE0jQq>z|dwb$;E(uFAk8oxc z`HIFAEoyev>gObBSp%JGV_-gNq~O68lH;Xg3@g-%XQalS&`y(%N#J?aj;(!>oA5?y z90i>HQy9aQ;ufG9u;aO=wL0_Di`4OvAHy)tDv&QRW@ZF}dzZNBBs~?5z1Mh!vAw1e zVsYf$@T`3J25g{@C$TUpw;KuM2j&a%6E2a%czi3VJ%R>QTe~sbE^MQwapUyD#1CCO zyDUGxJnB=hcsiNjTAl85U8~Y5=8$+7o>8dseuQ3q71uwwS$s*4@f4G)x|Vj2mPP$S z&t|Mrapq6?u{IRS_ihE4i8j#F@Sq$r4AUu$Ln$OC@V5Gi52s#rrZ{x-$RZCJRTsV? z7?_-?X;jT#$M-5}<QQzU}SQ=o|0OIt;z7@7t#r7 z(L7mF&FJCpe2^&|l`^pTX_hvr%})s6hqWgA1bROaAFXpy^7O_$u)YKw55_`QlXDS~ zGzhSkm)eDg(8lHY&UMR?5E-3zLUx(x+eUHftBMI!!b`H%a zMVd@rEr^wRb4ZXlPR_gHmH9nWPa>hefKF;^CU$_FYy=xn${$@!^F6`^WF6u|dtK*r znb?An>CPYW&1s($0bg!|v%K>&L0bgGw}1l}-Nin(~M3siyuIc2o;Ue300{vlse~s*ZXO7x;JLJ9ZaVmD14);oLhT7i`oOf;;I#^ASr~l(gQfkxvWv|91pb0f7Gmn;3)0ZezixN(3sxM{Ip}`J!4|2Q3RFL77gBig>gxNVUJn zN(No+Cah!qb6iLwv(M(X_1^_Zod7;S!M`Yr4zo5SN2Yyvk8!MaDXa(YiQ|e#!E=g) zEwnd4!iE1<*334_3!INI(=J;dWX`4M*!-bl>^N$u($+md)wK=Vu%u!p+X-AekJbPH zKmbWZK~zE}ny%VXw+4t8AGa;$CY3}MJpzsWOW=FT@f9o+-YIAF_!B(Q=Ri^X+}zB! z_zZ81U?XoPV0=mQGCWuK5qamLfrB0NCor1wgJ+9_;StNN>MK-nvT>?tX*kC!9Z3FO zR0HC)8rb+E33U=p+O11)o%lP^(~*HtSgcS$*%qiZiMTUw68`=4{`K@GrB+NuQjB8Y zNh^~17wav-hAt8>ibp5~OxHO9Q+V6Um3gEoq*`^Hm{eI*6}MGD=*=JG!MnB^bXD}$ zW=Kk7yFo*5rrB3dsjbjb@c*==g%T55D*R1l;NEH{koQr7u<)N-c$58!X15Z0h9B0+L~|Z^gWz@9L|# zi%JKJ&KNh;_JeACD~g_k-#AZE`O~1>A}d~Kc_!FO`?nk^50wXg(%-_fFNITY4Eg{F zy^_^=WRcv_l%g{t!C+R^Jr>cZy>k_^8Odsm}e}@~aU_-^mLa-?II3cb=d!s9RU+!^>zG4pVzC{k za05B67&Zx;!;|;!m2k(?t9*uL@{DVv6Bgs7D7>~*myl}qE2M%!glnmM4JmnX;>8!C z?#~So*vTi>h!of!iKmAB&z8`tC$RFhvz~8H^yoFRE?xM^?qP**(LJT8qK9XyPd|y% za?o<0KLWH8;5c9JN(4}*rgLIyMg|&L7ubV! z#5ikVYixF#5H^Vo-RO_GQ$yDzPL#R_3S|Lbr=3m3+dyr*wu*`UkyQNS@xS=TkkO8( zru~U7Xj88B8u`E0`6__(I!LkjyKSY2$k?BtKXZ3t4GNN7o!2GinX<1b>W^t^AbONC zq>i*#(rX@;*{n`svx^96PGNDj$kppKJaAGYh#Qutd5w?p1u>59HgHnR7IK~pVDlHk z)M+R8&u%srjR19C*jZxCTf|ff(z9|V#ltz){?yL)y8nRA$C@;6x8s@T@I#-^({$}k zFqieGmU08+Sfp-cguJh)YTBQ{DT>I&Qq@U{ugb>1an&Ld@z$K9Mbk$0gBqJQ1TjtGp z&X}{rla3H|6hdQn8!C=@&D)AYyy0RAPKdj$=Gi5xXSNMbnwK#U)|O z9qRftt(h|p&oOmt)1AI#yELsW^XX4P>qB?8n%X?zPfq#Rm8D8m8wzol6;$WsBp>Wi z7*FD{_b($z2du>t_Vkz@F{AA$T8LF z&4T&DllVHKcTlZ8(>YGf4Y(Gsd60%ns7gz(t^xX<0ZjrxGubv1j}?Dov`%(8VYzQr z+nW=TL^(uTwZ@$U;jT&qhsyTr$(1fK7=D*f#j6-n>Wq8Sd>ZQ* zGfyl2npv^#Z;g)-ggxTpKPxl`cT>XRzlV52idh)4(8_s}){I|&IwM!=FP@5a^`pxG zL;D`|iR!r@zX{ss09OC)aqKI``0_oaAvZV(^WLkFrqC$5Kb2SkXoB5Z0?+U}5R zs8wpfH%D;J+@=Lz1bb>qai)$IiuFMh$~uSY0Ih4ZryJvVAN4q*FU)vW$FXgJtp>BE z%?>7&)uxzyHvgqVaGk<$3rvz%J79Fp0$aU{{3yC{^x=#Y{Rp!s8>m`5)&zLl#1A9> zCthQwPV|UQSof~@>_lwEe4;Gm%YaSCswY^97Mv->$t&%uqssHRt;cZVD{rqY7MxQt zH4hyRax#y^)e$ML_ShBt(p7%aq6Z$KOyYesF^L1EZAjZaw4+qd=kc{0H(fAh!IYst zou!P7ogUUTh99*x@N_CnRg$DSGdMPTJz-&)34e_1>e)%gO!trVrNIJmn4XASay*GV z-A|@%xH1^zDH=2y)7fD?1MS6Ow03aMNr0Rhnmiu~jvO0`qA6y6ncz~yvuG7pg@gky zs631U*&61aSXNKpoMU#-5T+(oqO(J6l$=(0ru-jelO8nPLuT9NF>P~iBq|fGr5kO& zPQ{8fqyN1lVT3NlJlhpWIk71@Mz7GOz|eS3>{G||~!I-kLHhizBv5LOa zVaxzU+f123epaJRCukG*98l#zuiimPxRZ2cPcB>87fDa&`*jmHtZMBv8M_mWoOC|x zZAD@_Z$i#qJWvSlJ_O6xa);_yJ?X`HLpg18vD0)y&Z*$ssLoc7@qfL}&mvPVhcr0dY%D_iyuRgxyht}I;w4%F zSZG*FF34%3qL(mnc}Q{G3gUzcV>-fuFEc4AhE$5b_Y=#3jTN5aZ-KFeT6@d3`N5-J zvS#VID+Bo}Kn0U-H>G(KM`D(w{=4tbUp)qt-mUt7?O*=>`nZb${|jsv0kdgc!-R!m zrCh#ynduiveUcM;YbG-+sKTTqHXVBKh-cMMgNJ(E8O(falma}dnt4rn`9om>6V0`y zM&5{aOf!;b*}2RK;<>-R(h2~D6%vq>>t%sXZ+iMR&DMD^ne$q~@uo*jT6A6(F8Sj3 zT^FNb5c4L5VUAx1G)9^j%KWeQzL;7$#a(gz?qcBYz7FhuwOOQPWzk=4;+oypcf?fN1?&23S5wY1YG{omoryM=sCreOh;iVKoeQhN6>XyR#0 zO_CD5YF#VET#ZzyyIgcE<7&k?NBNnj`p(3)@_L$fcphSG@nkwV1EJ-~iI_#gU)yZ$1A583UXMJzgKJh&Lx=?Gcx}58f6IVJw%RH(3>UKbP>9 zlTAe;K$T!g(4XIV-e@fk-YL!hSxI}M#k9cI9uTpf4ENE+yu}&8OUKB((#C(L@t4ldh^FHX_2)iYocSPFu$W*Tefn#&@*~rRxhL zVv_S9PWl%o(*Y>YE58RnRSl!HP+j!HC)e0epCRX8NMJcr*fzh@D6MUeg@d1=q{;B` zh8jE6!e22J9WKgKz-HcMe9*@^fw{~Xg0^H6eb{12aAwd}6CZ4=!Yu4$GlcfqklHjp z_}4bhDLaHo$09t*mQdKk8I3X;Gocuv7KCe>c2u*W{g{IDaN^}Ca3leo;e)buGaP>%eWr&F$*ty2`z}A z&Hje$uvNEAbOkWwvoQn!vrq#o2frDsK3^n*g26TvPia!kAKlQJ*)@+3bXL5_{K$du zbnio6msG}>nf%LQ868mh#)ZRR5m&MHEJ%V%S7%E)O+}@aJh86BXJnI~iDz!Y*W2MH zkcqZ4Z}=b?8#(3&D+*BU_>|=0BsN>9o?-fyn#`PmP96DLB{$+dV&$|}eRNf&lSNrK zQ>mtoT@bXzZj`Fg9zPBkh$dJy7LC@;+MSE0mW9a{$Rv2kSnQp%FjY*4TUF4cW9cezqHc8m&P9O_AR_Nj&|Q(Z(BKF{Buzboiv$snnUG69+tz zrMEN3@ioG#YEY0A^P{@g-g#Dmc05+XZRJ!or5cHc{MaWpo;kWw5yQdMm{{CtbEos0 zTmu_IN};x6f-kz|058KvM$gyr+?n;MM?X*xW^U0}JaPnrt5%89wAmIa-PchnakQ6i zQG$w6va1!%d80A@4qFa=cmLUP^PW2BGQcU+6De-_d-X(6FZcep@VnJt>qE;z>yb2_ zX={@xDo42RndBFwT0(-K|Hm>`kn7kZ^1mvtb461WG-FzC4v5BaK1ts)*bFMfp}tQ?*bsP zx+L-TJ12%KgGq{1Gh?OxZ40Wvw)r~g&b*6Hf!AfJC}1f|TX&ut@lo6_$&mE;(Z@k+ z$&nS#F+vMVdeg5z@*d_*k-YoWLg!brhnI7-?j+OIK(ZPO9S^T z;5Bc;-@K57JMn=my^aDUWJkzHEet**l&Zud5n5sxI<`!8ObARso z_qq6fuGIn`lff645_bCA`#SJVThqYf)cZue32_&QywdmIZ=aro6#t^xzkSBh7en&a zpzJCukoVo?+L;5cD4Rf8{rLU%*}vnkhH2*Poq_eT0J@}kc~w#8FH#fTWTTUKLD#me zU!sD?gpp;_E#@4a)Ioebr>m~qE|D?ln^PS7Q=o-`?db4c$7@n93t+m`6Dz~FOitPI z!6d$@XhU>%8KU%N0+FSSGU0W1p?+c_iJ^NiHy#&`tA^}a2c8Ut=Ntq3JJf;8F<#2D z(pv_JYXtFC`Xgcn1k|Ms&#sdO6gfV-Cyvc9hnI$zCVGpLH5TnTG2fY3Z5j7CG5;x@ z@Swn2>+H-dIUULUO1WKhb0l=AF`fr^fZq94O*kgGezOjMeR`hwP{|o_(Y-kz6XCeH zQQOu25sZ8`8IU8(Gyi5sp9$WPm^%i33BIYgR@iHp8-|TGz~=V_Pv3Uh0uGc|vn3&` zZeztK?F-zx^%#Mvttep_t7tL61DQsj*38Blj#+-?<@kmOR~t=J_VRi;vsEJG10K`3 zW;4ENpYi7O+7^eMU;z>EiE-<`c|3_|g>ZqGo3$Ee2A-R51KJWH2V5l^R={%=jxq0c z-WG)gd0zoZW7!Z(oC4rOd{OIy3=B%(sT^u-W$j$ze;C#%VM&xj|gPS5B zG+0s0UgF&N-sO^kAg=y1XFdk3^Cg7sV$V2Z`A@!rh z2DAtghHdU(Y+>dw#$-hjH9`|=s+|92ZN>bF2 z&0A{soZ^2bRfH|zr^qrr6)hHPD%$l)S2d$H&Jl9;YPsd%jt^koK`JXq0}7voq0 zqLP0LAviQP$&eN>`|&MHC74?@g1eZwhQGWoru!Rt$FrVvNUICl>r%}m>h`mBXS=hg665v+^6LR$ zeU|#oJ@by58w@uIl5%oyIsn)(x9*N!!rFEB!Sv#Xf)6Iy4iRgUC5HV+Hd(C6ms{rNwc&E8=ts=9DlV zZbdBeX=d{=6!ocFKh=j~e%U{^ShM@6p3rHJoUa^kbBW9htS=(30Xa(@xMjAbV430! zL$l!BgoqK?L3E7q=?-vTThv0$4TFusVdKq&rePYTNGigYS^^mEpijmPK*4Nb0qE4t zP=Rx1Xj+=U(-^iCCgP9rnGY&cG^L)2HJ1KvXQ+yOf0ryNRPKaCfC}{GP;6_YWhCGS%3pr(Ly-^mVg?A(%V9eIp#TyY(AG1TYsPN(dsJFdg}2#|aLx0@MP(EoJSVtrKT2UjyE~{uuVyf7QLZ z`or#IBg{Yrm?N7W^Ru6Nk|gy@#9FhRr~D0Nn6WIuiqPb zZNgu$dAr7Bra!ex3s?Y(3*(W*2DUaf+tMnXnu{Yb+Ml`)RRWh@IX!9MD|Fy!qEn!A zB6S}IWr;N=hOUo>W=`XjWf>I$22dV7i49u>PR-95JVWE#XQ)AgB)I9xN<(b3zw_`` z@Q|+Q6aq5M2f7NVrd! zH#HBogTW?R?aPt#fhl;L%g*e$BBDMfJM!Lk+OO#ZYbcx1HTTAiRo%N9vntoLh7|>T z#A_Je1OL%DDs861{#f` z=gF4AGFb?BqNKeic^aZ^<(Ay!FN(B~$@)12%j|A9w@o#3f{Kd})DKsstUUf)GrMNc zG_{*3tQnU+uT*mXfp8aMab9B%JZxfAeUu~?$riJIi2(K z0Q~jwqptwC9`M^Qex0V(pM9%UMSIJXmW1u~$pt?^K?>m$cu*FO2h-BGfUO-5S1#yt z^++{yFs6K88vNtcMDR|>opcOR5ued%)ZE__iW*7}Juhu77HjG?tsP!gw5e%IXUgl} z@TIweP6$}*Kfd_bRUdek{<{*;?QuRm&aVUZincB@eMA>v(QQ&K^kQCo5!-|s ztL=$jUKG}a6Zj>0;+65k7k`rkSz9adE{0m5Bh_wVu{=jgXZPLfzRCgYMt*VdQDfhk z&Rno|O+o!sZMomt-%Wr2^UJo?|8Da4qd(QV*ywfhl%ce%vm4zZh$6Tf07Ko5-_AD* zwVCGDi4Ibt-A=dDR_EjpqG6&5vVIrJ&6leAp9@7`KdI0^k~T_us;EFEB__GNZcHaM z(P(RpX@YtBWAt`{AInTIie1oQC}69tXS5ww39l^6Pfa;wBxI&Aloiux)`?=0PAMz1 zM6_yUw2|ay{T&6NfwCg0W5B$5v8r1{6aZE;2pt#JhrJ>4YJBuFqOr(gU2GoGbC_sS zDV|Kbq6*#ZX@oH2=LI(YF4P_N2iGaQX=VyuHfv!!UvQn;RcDm~ZS4~{LnKx_>M=Z8 zcGz?R=|2)lRs(z-A_qX*QtI+S6;TT?3+a*OfqL$qT>H%%X`-wKN>BO<+E?O6?LCewuOM9Jh%5J_;X zrJ3j$W(PY(3vJ8~UjiE{kDKxPJFSN{PBJkXGqf&ev!Lr#=^xAmrqYx}HjUQVZFVLl zYSd4W`Qu5pM29l z!+e^BLcdQihb85y;B{@z)l>L5Y^nsvX?vDH8KOnztW7}tD_CFlcZp-nc*J-b+&68> zBhn^#x2{{(#Mx)udyL(EofOo_MCY?DSOA{pEi@9|1ZO4(SnNB`%m?OW=XeN_t5&`dZX8_DD~>P^4Ilvxfu6-txt-dAag>L8 zc~oU>HO2HOc*Tcw1}NJIWGh-eE`w-hY(}0L21S}p)tawW($kgGx?`Zn4G>Gf0QAQ0a9ZKd$aX!F41@vS?Lh*8KnXPUkZB7@YC4 zhYFFt00skmp_8H%0^y!(VqFFoMmuZkTC8<;U%lv7dyD?RfB);R-~Z@hV7K;bix#FH z0`F0H-C<9&MB!@)i8d>8U(0PGuU*E2%}nY+*6toUKfnCH`~R}L3@BgOk;ueOxyMNa1d%5Qua^jGajU2`$8&#VX8v3; z9L;~J_%OF-ooBDItB6Q}?vcIzU9_82n~2Q0uE)XhBYO+#vfwMqlM-pSCq+io!{UC& z)Q_#+(I27qq}9B={NeZAm|*nGlzf}*G-4V?9gA0E1;Z5qbp8j#eLj?h?YspOR3j=(ndizl6g`a_n? zhz|K{L-M0?ns1-z`9E1&TNJ~Z{0>8>{G1fE>1Szd#bxZS;u7vL+7B)i zdNxxQ7~liD;*kb{8@OMa(sGZD_-b#TdI7zQ>S`gbk3P*@49O}b?AMfQLtWbqm#pMi z)A-0B&})-sV7?97Tv89Ra;e9S@YA`mj8qyOu6Q0U`$HDw){7H0;OE)2YJfVR&*>7M zb3B@*&EMcM9k!5BTz@8GD*1`hf;kUiHwZcyV=)dE{O%VNi3Jkmq)FM3r_ccvnNp0F zUi6eqw|vS>i}j3=B5JF&7nDPWHdljNE1VdMm3MBTQH^n4dWy(Z{N?9nwX7z38|jk+ zt8_|;mHoSq;?=@H0CQHlE`xe18a0vO{WMFpCtP{tnt1C0Mw(M;&@;2cuSiwFk6u}* zH!aqL*}RJHSat;OBbaPOH=5vc3|MKM>a+G5_xuXEV`laCKN8~D}qbKUz_m_29U-W|7M zI^vy9_{f#FLh(s=+h#Mjxe|4jDB_+^0I$iMsT#7GF7teleh$-r&aNMZZ#zj2aEMHy zFzSC0sBvyiCRJ9Y;gUeb^u?N6Niql;wuhbu+DwSv)7QF<>OlkSkD0U^Tn;CW+ zS)ozr)x!dA4qhA1$l9~Htzi0qp0f}pJ^Zr(p}<$srBpX5|IZekeF)IsJ@eCw-(7!k zuD~V0mi?8qtsPnbNLxMXOnDPt+eGR|*TWR7Jki=9;qnu%6!VEqaaD_FDQZ|VwnSX% zRYb162Gj8R?~j$$?s|DfDKVedV_`SpQ%G8cE-(yXOJqCY-Gmu~c&-|v;mZM}RbG33WJgvoaM%FM67wo~XWA{Vty@t54h zVHjhdK;#-$8yiDme|`iBRR}~}10E0(AyzOM0Z{4HpwAq3q9Idn= zQc$|B!g9`~Qz;z56Vqnw2BIR#{z^!j8QPW`6NS;5%cSBM<1kjVH~@79Ak&Uha;LKb zEwq*q66Luw;la`n>K|0j@;E<#f~^nT@M6buV~6V)oySx7K%Z7Yep|K&RW`5nlAxk7 z4adc_w)DbyiE2!PkG3ZOk{aTye_4fDc18ET$R$j zS0p2s0)9GqeFAH^uNiQ8?$E4G#GN}x|&`X^vMe(@`jg?$eKxy$f(zpmTT)=X_J zW91skvM|3@dyDqMwJ7CE1C8B1*lL}rpv#F0gUA_Aj4dps^1a&Jfb7<;yUI+=ii?1f zTE8KgHSXAHYtfN+SYW~zRsm$I8Ka@ytK=l=>lIR&n64k`eZygpc&*#Ssp+G-pz-Kp z_mqNp?a73h^RF(3x;-DkPR~s^T?6EB|E=quQuown4tJef-M{|sDXZ~!121jNjII8& zZ@=;174<#`aO3~`w_m#;kX75w{|x2=_ntGnU&|cV%z1IAZJnXxMv3yGO10z$h8Us1 z%VJc#CLw@bD7wdAS7-w5D-F*5El=uYOedF#Jl0O;(svgR9Q@v*i|?5$E9h-ZjDNI7 zQ-%BgA)RuG2=-ROmHo{pXJnTpeFwX*g@iRXIAi=>WaeUuO@--jy{fR;eq9;pHZaz)+_O!$#!Nlf6cG#72e;&3a9sEG(9G z&Q}+pM#>d`nAOf0l9ds@!#baHU*(_Ylyamnk-cGgoUX?H!B1?E;p-sV3FDIy*m5(g z7|${*ywaJP6u+oJI;Ei`oyJu1>WT)1;YUQpGIm~{N2OLiqlbSG+aFv4wsVGg#R?ab z^q7DcGR#aJHqn&1zFaEl!*fA7LlF#T z1yg=D3$tj#>3B4fBVxta;ijF6b{t_zxH zrn@FNkwK+4(}N~bsZMUXtASc{_Gy8K(f6G9J%X}pfPeq?dw2eeyeC6y4#^I}Q)9do z6I(~HH&T##HIp^0rUT)(M=G?xN;taL8L>Uu9DA?MhejR~BA1!KY!pIGuro@r`E z^t`WPM^K}8pd>~?H5n`(7Gq??Hzdv@QtQDtr4|cAB+uHDEk3dDPf#oyqP@ys`YffT zlmGkc`JZn8SJEy2Klf2Uzrs}F_kZ=h&#oz2>k;T~c0Fd{2)Do99m8!nv`S3oO#U

C@p_psBUoSm8oG^W%@?hGVDy%IeoA8Vu> zEeJ^kkKHahXHrj6nh-+efN~DIO3Qj*eo0yVOh-t+tJITy#x6&f}{>7}foX?&RV$uo-&vi2q|l`w^NgK^s~-Bxb}JOPUFs6Iyw=`}BL>C!fG- zpUiT0xdVuuL?kW+l22`mR&|NZp$>{?lJKEF{L~V{=MA-iPJaAScupP7`3y?M;0JkR zsL8(LOK0ybLU1s@KrJE1Dj*Y>E_gB2d@FS%JW~m&bD?Rg(Nk=HgLw_qTgf4^C179e&iE-p+d4<(K9X39(eg?FR(E1}!X=V|b{tAbBgMIRDE}3?S-{hHe z&1{4wcKs$aEZCLA>k^BQUSYDRsH&S$OCe5bqvAD>9yk3VF5>HQ^a>80ImF1xhF4+G~K*NQjY}{?@nL zwsa>rj6Rk3$#sJ1)P*~X@WcvA!K|gjPN|BftJ;}N#Y?H#;Nq|Dlhi}0FJMwO*)7~qI2tFzk?WeNi1BF_J7Ul3%w_gmST4^JZv1 z?SB2l+0n5@-Cc%a89!88f~sp}UKsl~Q;&3)1e=~S_n$7N{@90+dDogTi2OxP5MYGj z;r@TMz%SJo0Ic@^=n0^f`x5S=@4EqIC&Oxm0W1b!+Oin>k_+H`5Fb5pZbVts&k)aKfP=TV@V$z>c4c`0VDdB9k=Zw{S(3cXYI>bjsOG0V4g*ol|5-cYo4 z(4&aY%Ji37yl-6S1}9J)wNM&eXE=+wX2WWYlsOIPBB1t0{=T)gE|cA?%{)d&Z;6#w zLf~=4Oeet+MR7nlxBZ1-3ZssXOT@-Xi76Z;W6H_ ze5LKGhqRGD5oNcXN@+nokkQKaPYDJxtUP-FhosvmE!gLJ-cGK}4CNS74%}fZ9FCC+sU>*uf^JpSJ zq_<#rNH`f!ZK0eO^Kt33uVULI=Fzw$J%N>sTjr2)@OReLEIJo5I&IlpA^B(P1>!&_ zkrWG%B7K<`fn}bB;J>UvK%uFHzh_LDQY|+s+P4DgkK=33JFHn!wr(8B;inPqb7qJj z)`k)Zj>Pr!au$1<$o?(6K3}KSI?IKGRs6Ka?u%7NoaZ~O!PKv>eP`$dCWRKMiWaOX z2U;no=W~G;Yo_Tq4k4@(UlqGkG}TX>Ac<`yngCu5VZ1ao(u*|wIaN_5Ln-yUc&1^_ z+^oe8z%R%nH8ZR&@IwICdqdKq)+|yj)d?~%BbA>)gZJu`C|m=SdCO~jov}vvEQ#^# z5HB^dYdA4C^X?q7%6UtBF=CsWPK?|^wr!cuRRk+}ow7^>Pf9mivK@z_Do~}&gVz>( z#i2dsBy%LSORumpU2l!X-K9HoCBr}+zST?%K?HY}4zV>TBk*=Q(P?WR`!M@>-+Hs} zCR!xd?gheR8JX=@IBk`qQsspJCetEmzOjui_2sMvUg;GOKQ2j9q#LoyG*e}1LaIm$ z&E57(CtOh?_BzQlGPX6#iBF^?jNzR0;h=4WLv)Lj{ZM~aX8a4jW)5bb`yV^qvXXIX z-vT8CJY8-@B+j@p(y<&r_J3Z*(AO$jk6KXfnICukD|wd#9d~)475cm2r?i>nLaw#I zEIKcG%kf-Az6xv9bay2;kE&XSG85*}dRW9Lly(@=#YjV?@JjQB$Vse*$9ibRN8~Ce zF##%vyA+22K1{|>D9HxOJB`0QoZ1Hbiu!}?)R7r;JpT&o5+|K~`t>{@!GU~9Ft5UUGJ zUBG;?aEnWX{MjnT2SY#@kL<;QgIA7!0FRKc=~!{%IADB3{mI zvDv1g?&m8lT__+kvN#yNTip8`phtijKfCdtv0kmA74-a%D}t^PjH6wH?iqbSGF8nx z4v6$ZG!mCtNY|rXgH)!|)IPlo%5kh27RE7GgXK29n`6YTXY^ys^*{PjRkK?00H_V0 ztLW8U#QUFPu2#hCTH}LEXY%dvt!sJ!EKiDSrhT%#Z%}l^6`OV-O|r4WLDb5dX7-D9 zA`_8pu^&KS8T>KMKatUx!nHr`bQ+F)sP=u;yA(%=Na0M@%qCc(K5G&aDVMlus%;*p z9AwQUtf#-qPlEexoMFIg#E;eNt+&dLH+rWLq7W}IgJ6RDZDTB<2Uj`^ykqaH9HXZ| z7BoAAZ4usknF|aTz@1q2Y(7Yd#cClui47o7A!;(ftC<%I&5BN%`w5T@3b@?Wm7-|? z&!ywKj_?qYR~RA9Q{iwPrxPE@KUm1qew>zoKd+R|Hmq%qID9qgokVT%65k8Qj}(p6 zkBzS2PpehFH}&VXQ=6JzqP%on-tnHXHJc!RHPcU9`wrwKFFSBi6m{ELz!Bupl^MZ} zE8HfDP!%z=@JDzC?6W+3cO-2h!fMmva6>U0ZC<$%TQP|yQHdat0hdL?dBCRf>MPdi zc9@`H@0GUU#{VfkE;4&7e(#;;%PN^G{JRmD^g7UK4=;mp3OJ(}s6yc=g0F5A7|=8c znr9@)C$ht|D+bYyf1KqV8G^n$r|g}#nn~4N)wfKV^dP5swa4uFw?!%g}&knJnaIXhepXwl2u#>baUwv$1?cD6WXpt7N#uUQcxx2@siH^_>lx z>KU2ds6mrkw>e7)TwpdE>LdmSIj4rKY};Y1%HCOqjK2Gk?KqBo9=dU4*&3bB9KNO6 zxdR!!L~qvyvd)XUzuA~fI>v^c#cZd%PN8veLb?}H(H$=?--(k`+s68ZSV0vzD^yaP znrO$@WM%2ty~=lZp}G1K0V0^!7N?S*v-3$5J1Qn9aXEP{vROdG?C->5bd5dHD)Qfa zHKS|A(N86R%NqwUUsWKBB|a=rDb9*3C%6$rdly) z){BKhy3kg}s|B3yy2cJOgEekvKPI+CVV;c6m9dJ;s%fN`{|lzE!lOLq)vG(bc6}VSk$9yGfXl#d6l^4A;rKaLFDU7w(fyKhy zMrYgD4sE-tRPMli8}X=Tp+;nuAHdJ@@#semsymF+nj}{9MvpmNGbo%9uF~k6*zibC za_MQPXMt8_U9wxL08v1$zmsGigYUrm zk9LKZS$s&Lruv5EDOMUNDNh5m@d{kcCGTR@Nn?qIkV;>F<_WK@sY;S~oh^=>Z`hgjGd~*Qlj& zE?Yy=*)pj0;y&y5w26Dq`!c{5ku5@ZN#IFBjg-+={>Z*`td;7tX)-dDI-VpsuA~MW~#)O>~~dD^aQI3R-}L!cs}DEB_S< zi~2RkVpK{in)14Hk6g8nGxUJS`Zq_2ztJCo3k}8IttCe^EEjWV^*|Um-d+ z)~lA`i%OED+qvBzTZ65qeOlf7C}6kz%Q||BMnOp*2ZT>Aa=kZKUlaS(Lg?e&YrUu- z#b3;n={s}As4C611i-|E;Bb9Fx)BgwDruzI?mX?d$uu7eTx^UBGg9w!X$O!z3iy9| zYH7(}LKgSP;6JKaL94%hG|6C3qNz4A70#ip-`wUW)@_1n`qZL}xGtRNhS~+`jPoY_ zLw~)Gu^YS#oIuRyj!B~N4k~(S;scq)?KVr6w7I%~(xkwd99Xzy?vyge7E@g*il->U zqeZL~w z3F-o?l4DBxX&g8;8^e1aSl|;=bOI-Y+$1ylR8~Fwp?qu>J@&iJq9c_IbhsDC z^p4W?NzBzw==Iz*K3-FW=1lWOB57W%lcslg4TZumZ*UKw`eJ)LiB&>^!qf?oY*RhY z2a%+psMjP7WcDVjYULNF6EoQX*N2>2ZFN*Rjedzi1@CKJ*k` zT%Q{K?yt3(^2?zzpR@xIbvBVYOI1`nKBQN=(K;!$^I}y~w+P@GBWgSQOnarUE1GFt zu$p=*P{XTZqksK_G|c^(xdAE@!_^R5!S+KWU82LOE_HHIIeC>+wByKfHW@OWx@zD={#LBXP+FJqmc9%h> zv)VWGT8~<)D(P>lRVOaloof}FiFUdbPxI8rH^6xUe6h2m*w$SxKg_~ zMsB+T@Wgf>UAk`Ab*FIXQ{|oiiN(qFYZ&B{#sx#=RlIlo4H5Vg+k%n z>Z^j`e?)i^?#@-G70%*ajJC4uWbc-9e?R}As;B?0N8O9Qj{&T`_RWi(O)u)#Hui1# zIr{DP%B)7th3ixqRc=>uC%xmoSXGlN{$2nFsC3QHQP+FjvsT`048BgHr!X_;;Lx(A z@G34$S2iOhS)Z(awzj5-!FC%9%N*W7wSWyfVV>V<{{Yn_-?eov8a=>aO{KV%yZftJ zPY14R_p2`b$WPY*`?%%5ZuS@d%lc;afBQLIzo^@&=+Pne8VDTU@WpIlXqZ&C&~m8q zh7%eBMEX?7%WMK%w=A2a%gWnWAmxFOByrziPAe%4gG`28-r3dO##bTI0e7}(u(W92 z#}6tMM<=C=b!wsMj7HV0Lhp1|q}+lws@s)q_3tKfc#4Mu%~~l{~Jak*FN<{#j3w{s36L@ zBK`-yNF}G!u-NXnS?K2m@78c?B}He#YS(ET)}Y)dU2S#mbQ0C{x*EEemzu40QX*v$ z{p+v)>9+2zl>a4HAbung-F6}+IY6e}VkISCgJxvGd%o_xCH-2T_>-;G zl^+i}+0-L0sZX>$mXyBtKj^G1e*Ag33?QuyI{p;PLavWqPrSNmY231HYbd!BPv!BT zdSVYWMe(m()e8~{!QDVRmjfD!8Irm>C@cStWzK^Ys2t{?NkV094MP&+2fh>yCAaM; z4AWQzsCx9H>I`1z(H~Xr7x8w8jf?bym?T%f{J0NZSz_}=PuaM5 z<7qWxEVT)xMCfDH$eOZw>OhiFseT~`HzPC4iYzD<=~&9J{+NFBW~8QPVOn$OB4c`- z+D1CKZzEY6GP7sl#j=d**;iThY&Q9J*I?x<4q${5>@~ROE}in50w)9+%n7DNdS7tZO8~4gHfHJ#|zTnw3=fSs{Mir2QFqs@w$5D$-1FLw72N|*q3b+ zs3|Qw3Phl)gFwLW&I0FkU7G7qtF0p7ipf(X7ebQdbS=lB+S8rU;Ve+QIn%hTDufH) zjF3xRI9=SH`mfVS@&&xr0~s6>@R~g8XU^GqR6XiYzb@UjCxAXhV9*pXWKCo@-G z$#f$-!SwKHOXUV2WHPg`U5;pum8`w0mLHExyxNngrf@ieuJ`~U#OfifSIjD7-wL1I zdcN;S&%mc5Qb=fL%LGk>VpjAN)EzP)o*www^W`P{(4)Wo5Kv1}=>@xlORud~+F26V zHjP3FUkhbKT1|;|y??@VwO6PYd9Uw}=A9lRnev5?To$&WQfGFoLeA<6E^!h~ zZ(R($=B5 zht1MiylV0*RCgr7ze1r+@t@rad9sBP^&#$f#WV-_YM-Z+=3fT`w*5unY|0Kbt(m*r zlyhb)j4WyAv13&X5=(yI|J(!R!&THG1aJEi9eI|_ktd#;Ib6}&MBRfx0mF;K)WQ#R zkt!6=64j}-AI<_Q?A)7Jy$heK-~Q|{UCNg9JFS&uW4^q#vAQvakMtzy7P^r__H_u= z7o_S=cC6d&M7T*O5!bKx54Nst=4X)o)l6e6hO!lQ;9XmgmOA zBP|$ciNg!^2Cf3}F-)%|AkW|&TrxB!8ONx<&9`fv6IQdX&*!LMg~8WLliy<+Zdn5iHH;( zvJZ?HHHK^IPkDxbRhsWQ!HS;v5IzgyC%p#mJE&sAe@c#0jLhnUhZJLlDktRQsr>N9_JtQ!yn>YTJ6lwK0mx}c0a!aixHumB+JbVtYom7iYHO+qC;$j zA@MNwo#L1pK9!5v*nrsk*JSf?UXNwJswN{&Vvxe2UPc2W(_5UtXL1E4F=-R<6g~Sd z7;D4(O>%3W`WxOj!PIX3~MXXEqUSAl~k*kc!Z;8)~oVck-SJBoUEGSV@P5~w7m^O@gsH!^#4$-s0Yc};fC<({?U2|@ zM6%Z)L8=yGsTWtnbON%H-{j0uclZCc;!3e#`(4K#zu5Ia-vscGVvAot0rcm7GqbzSbCs zfD-CyGY)b~3oJBij*7}fub`i5?IfLLD^!M^?u3-WUGMW{C=~_QFV9$n3aGZI$X^li zqp6|9VfrYxZC)>ngseGs|%8Q{U&WK zQFYUl@lZ&*rmj# z8$e}+KoJ8{3B=DP$HKPN;zJg{9f$DbPuxVZATN~afnZ1>DC*mm7GMI7l`=IdkHM3k z7FJ01B1`2+u(8%1gTefYTydT|3$40wfXlH0p%0&#YL7e$!#Z&~6ItwL`ZJubf*O!w zZ+{(6U!6(_rK-Ag$pnz(#5RH-4h?zr6i$uXuqUSUDJQvT?estVsX(o{Asn`Xhj+Bp zOn$v6*JoFkL~o|l81K4XGe!M*Uk~DRQQehQ-LQ+#qZ_yR6x6!aI`*?abe^mx_%Y+e zb2KUewVzFUSeV$Ii4G$+dnfv-(Cn|>^l2j%%Zi<*^zbfXqlsEXzyKAL%J2|qmIxr>vsrL`t2tzp+H^{TZ!n(Lht)L#Jh)w>E z=SP(naxkS4K87Xb2~#^m@PtEUe~!*UHyCN>#p^ufN{^yq@^*;yqcT%|*EPWj5iePv zxfF}jBFM}O+yffYl@7DXHFf<8*Jj~XrDtm76@uUddS`npkY|pQsdQLD|EajM77ES0 znruOM{Qyiwiz|he4q=w%1WSv|XO6#BacXgeAKm52?p(%AOY`I~aJA5~Bp~w>ydMNI zcQFC5ZSYGqBR@D~oIW>-=K4OWo~Zj)`O2{tAW?QQo|A&_wo>*`;bL|hvFR1s_MMq3 z?5^Ri0f5LX($%(Yb;>mt1T!M2bVCfp{X^M-N``y@dA&Ht*6@f|iZMyWwyuDna0mun z25Rx@TsFJIxIhsQ<+j4felGV+cdU)As&q?x|GreG2{fK(Ersq-0Im~NW&zjHCN}^6 zt(%nTq8_zu>!yr7ld~E-+|GPRwM^#FnHAUcV!JQ<{_FpAce$m1%d19v&AtU`@e*gN z0T%&b@Y?FEUlX=$qhocU}LRU?MOrw{757w z80agkXJURs2wcScPH?SPy}S0cUGkc`j_+aVR``GY?so4VzxO}B{{FrEAMSX2aJ377 zI@Ep-R2%u_d9%u52fW+S9rWDib8Otit?+HX`Jcj=iHptC^_uBFf4Iz`24Xjnx!ld} zjhMA@mwSd7OD{;#lxD7uRiaMPJS;3Lt3frX4r=Z*8mg`hs;-1;)h)n9v^AzuW#Ttf zdT6dQ6em-3Q9%vUMT~B&Jg*;f6Ng?AW^Ma==8l+4gr70I7rC&qnP_zdbUD<5EgN;O zKl;}T60+M0Hm4SBkH?g}Mon)}b7I4SKYS2K>4k3rEM!W!jFh05`zkSkC7y~3u2K{h z%uxO)rf?;b6093d6KL*>_& zUmo<|S~Qc71a-OkipenQnb1-h8jDSs^HUj;^Hs|vH`Y8#R;)fZ`D=?nW#$Wd(ru|4 z*lmT!)YW`$jLrpJa3Nbly!FX&$PCa9>sPSZWoZKbuD!BwL6u;2O{$N?c8)N}Ddg#G zWoHSw9Yhfla|>X8J(R0rA-If+`Lu`B&bnzPkY zUomlk%P2E2uDl9n48<8-aZb{nm*+}JC6kUXsB#aY_L%$=Sx0=trj%MSVP`6bI0w>W z_hQEWp(^uAqhyV%XVG3XPlMJ|I1m~Y{R3V9NJ;;0@*YX45~eDhwJtGVB2afGQR(2= zMK{!p=5b^FC^=Lv07$Kc9p@rtRc8JrQHsK(aOpi|#Y5<@$wV7YQl);6sbA8Gl~ zPPyZWjO9-MPletEmW?7{yrE~AmgTW^%!lK!k+mf@x*8Rh&JqEV80uD!1RAPgPltOSgkD`L)2rBP9+>_p=D8ET0 zg_JkBUF?G+&5Bk6Yb)U;gsR5%P}k7oK*?G3R{SsFp?^;Vy=#EG$VW!#xCf(408pNQ2Cxab_iEJE zr&+50D=wjWEpJXRc+r0KWsE;z%7814-~Q}u)@0_lFLP$%g{!f^Vc0o2NFiV}G=I)5 z9))tgKkk3~;HA6T`K5S1xf~^D!QZ;qlRj?!TQqN#93D6g>l_nM9r4AYgHSVW5#Vka z*xfr3C?}?I*~L%3$W9tjAzU0MA}HXemOlM2{nhiqrINiO80Qkz6tc~Zq8h@>oLK(!8^Ij6IUFT7*(hjQikIjkHA&v1y!$^A1ftWaMCQ@ zN%mAjA>xEV1ITQ^sVqQhi_0+RZ9^j6i`X(Mj?AE7wf=Ju9ijIfEOs{gSL_%HrA{Me zM3PR?Z%dYu`GcvgeW_Hn`ry-YetmVLFq!KFAh_6`YK$^@pCt|b6dBQtebUS(mDy9R zDdN_q346`=j_aA7e7T@PZ)<&#pwk(gK-yi30@(J>ksjMpQqNfzto&A^$*;uRx21%F zi)1M(!>(?D3?FLcad`1z=YMUtqEnl>0*uI8;EvmlwC^%SQUGNM0aNF(nF4(MrHfJ5>3Ue1bPrxg)CcKJ`e6tcc`Yow&?V= z3M(zsp7mDolwKi2CPKe7uSof|fXGgGWT5t%pSd)Sk&8|8Sjv=iIq)`=K2q9)RMr%! zlH7<r{l^!7|MS})eJ<3s+3pRuSk3kSHX-zr zp#3(Ziv%NIs}3dbppe4;&M-wy$6%I;QCoKy_w@{nrX~>6b&%%Dy8Feb&QP^=<8`)~ zDH6-wv_ehi-T1n7Q#j>Jlbs9~vVo9HG&QqUfoW9xamnO%S4*VKYQQa~t{=GqG1J%E z_9?OMuUSu~LZx@S<`9$XPQinDx&}rx63^Kg3qaebQ5abD7ryd0=JNr~OCDIk@Ee~(4!iS2(={>=f zEYZXXCe>4Le5OD;)56TRwlbF8XOYjqjXo@Lj~QP%BfQe#<^QI+;W;v#xA zRrh1=yxr@;mR^jUs?8~#J47pvhg%MIc65@b+p+ZMC1o@Fa`Nbm*NLBG`y)XX5fg1F zG$2UMk>(Q`%Sg=bMCN5Z;IT=i)@r3eo4UmVHMCiwdYh8bY}&~?0AvJ5TK8O5v;Wf9 z3#w1ECv1_rZU$?uIczq-JwH~p?xkayDpnuIfJ~^eWT(>Ve@uQD-JDS+fAo?sIJ1c9 z0z#fuaM!6Oys1$WPVO0h@kwE{geX6@&t&@*lDFt>o5XA{vKK|NTGAk-!$9fQLu+Wn z>ey{vI9QSDU5IY0GDE$y4%jv?kV`^F(TYjT&og-h$9<9nw>`CLgmcQ3moCx^)==do z7SY!Vq}W=d_E3JlD>-ty>!?>>kw$z?un%wO(bqXPNiaVFP83#6pZkei;@}p8$Ab96 z$IKcV2Ec{U9cL?ki_32R?^~z-1yw83E($7NfQ4I9ahhS4*CwO!^V#!Q3L-*pPMwUN zwcu5ww)bTLt&gg%+!V3|E9;6dAL0%PeG~0w+`PDQ{vF2lLOh6AHbs;Z27waD^6Me1 zm?7m9ZPTUwlclG$#;#wjNv_M2nb^X6HY3vCED^q0G^m((o_xH@E`+-mRadv^$pyWi z`?T-tzy3FH_xu0;-S1KQ?t34@6`@wc7L?9&un-r;<1g&GppbV=#9(&!SBJpD@=c9A zmim$2%Ew<$AKI#?cBV>lZJSOcWNR?a4e%=^S*-1abP+(AgmjQDk>OM#QQ^#>NARmf^>j6&-5vOZbm1*Hr*Qk4Wb`1r5 z{0HhfYAJnR1gq*>>IH_M3o2g$cuy@h=TF6a;?d6z=|%nugH^S5DUe$2S}=3;W?ad> zh;baA!T=Z&RP#U}W4Zyq%M+cNHA1}|VUkrMz5R-Yvp%%c+OiaW?T+I~4>f?%nJyq^ zK;oh|(Akw}>IQoZ9?VgK3Zd6e#~(mZ=(ydps^qW-D;ZWr087i?Nv#@nx#{pQQ^RY+{`AuV6rl|mAl%I_xyz78GL9naKSp=P831n+XmFg z{_G<_11s&p;T%kCES~RtowDKf=>K%W>873Pz>|2!j#dgVGjsTIO;FTW=6os%OfMAv zEc>hy%k^_0Jw6}g1Q*23N}no#31|h??O+9BW~{b&5LiE3Wu@maxvJ)F6CQ8J#{*J} zv{oog18rL5h^qF@x6@qZ!8?Nt-yR$@itM~_nZrR0O4V&uUNcp{hOX+`eFl>@Aq%1a zV%p!FK$syXvbBjxTM>V<->ZuACQ$=W`Q~b%-Ml=*SFl#f%|&$TOeHaAJ=*Gj8vYl2+jG!dA{45e-|OPu)66iJOA;8}ZNs zP6PmLXtO0G$EHM%uBzR{tFkE!dVS4#P5dE{r=IjjcapMlzxBD>|2`17r+_@2yrrpFCs$AlxR4L1N*-SWT<7Kys&nkfOhiEy zt6h99Eg;aYV%%4;aroq)JC);o+!@(S08m-Ji2+|{eAbM%70xVxiW?5R*dTpOhcP6K zM~`jNz-(*AAy<=#O`}k~CRou4nl^0L5)IAnj<$O9!xMV^w1$!L?`aBpsEMaD334)cLVT1_G# zM;S}W(RH~eg8hdy@-3wMxVj*?YUw=a+5NAT=G{#msh_V0Ygp{Nr);X(mjTVT338gV z4P@(N<5?d$GIg^#lHxFS;De1!e+FJ2Db>35Fl!EF)zEG%>zo;>Ve(d6_5Swxw@#EorOxsgyGNp}sQ2M3nbEHgL1Q|KDw>;0A8R-y=Wdgb4O#=AL20B^BqTVi?_ zj4vojt(8RdAXpTUaZPQ?ss?A;3(cRB1^H?2d~QrzA%~}X9Gi6n@x^0>{E}a`eOA`Q zNgUcm>a`ZBPkvf7raxaP4?E?m`|IP9_6jGyN|_T}mEAe9Ec@rugmpIp0kV}b4z0t_ z_JE#9^#j6ar}A)4VSNoM^ch5y_};IW%TLx6y38c9DhWC-%#vJ=}+k|9kvf*r&FFo+t(Y+^yQoUiFq0YS- z`G{om;nAe)yfj4q5gq#Nv7LMCso9@LAyilgOnb^_mi0{rVPs>K^B zC(hEgOI|+osjM&V#BUOv(B_;cT>30dWHCxZX)FOdB6>E@Hab;s2(cOd$eo*6-=>w) zhGBgme4|&WF5Xpao@($NIlP%Q95*jXr*r-4WTt!E0B-}!%#FP(vS<8l%n$wyjLCd3 znG9&T1v(Y~QR>`NkNRXL%By;P^6UNMUb>m|n6~HN1dhRM?!G%VtF<;#jAUEM)b%KF zoxjR1Y8_TlW!^W3jgAj}o z+g-_|ewE~8(^HzKFj+DeO9 z`?tvNi7ej>|IJr&YoIpQ?_CVUC$X*p{_tZsS-YoZw82=7oQr<{4el~m9_1X)l`B0L z@}z9<=9?hM^gSP9qSq?82SWI}ZT+@Ok`(NQ!^dVA(3yLa7RX7JD1;vKgUMRwLU1XBn5IDVx(+$wPy+9ZZZQPUx zDwm=8j$4WCC$-@mKt7Gv0yU5xA7++%HXjs!&`p}5%u2d2DFSXR9=4n7D8Z~M0=*Zz zd5KOVjqYPyIkNJS-G0XVo!P&t8qQ3}mUMi^ef(#tIL z>u9a`^<5vccy6g{S3lD14!o%D*1!9Wv%Fx8fL544!hE5@E^7I;yETYmyzsv;E+<$B z7sCY7SOD#&`ensdmmr)};VZl+Pcwvtr;;mN40IJkF{PiSqiXL@n*l~sU=qdrHrm8E z(ZJ6n10t-I)=1RSJ58-SIdoQ5B^P=b-=DCg3{W@#uJU9m<4PI0tC@i@p|?4D7d z=}t{)8Y?-P4bl%)p{Lm{OBjz9hW;t>RJ%gj9?-=o#1xO4}I+1~SgIYoCjON~dvvz^Vuc zLqO>rt#vDz=j>cYcPtwy$26UksB);@qxySaY6VH8dBTOlIp!Bc&5+ULD!e*~>NndR zgI6!J=%%B$j0Qy>%%@qObg05!rJg(C@4T(>0&qe6U@`gOw#H&FgN1D6oM6GW&`0uk zBxb0R`d(R?*GE_tf#fZpZe*raaqY&dHZ79SHRdc$KnmyfJ8%aa=|*iBBRQ`52(T<$h|WOm@!)+yhMaP zqu}A){-lI=KDH-n!v6pQUTD3^<2W53f!;nxt%MRZVOKl~kIaV@q+&NCKO4 zl|-nO-~6v4V{KF6`~Xf75;2gI$(Kq3DzC;IRb8H7X8ANeD$y2|s}X^BVgh74{RvPf ze_f@bh)Q?T??)wDSvGKuV5`jj_91}V%vJkQpsEb|xO9LdK?^%2>vWQ<$hNR(tj#$h zh|qcYlIVIP8(N8Njry}Rjevt1O#4#MhrMhZHQ#})f0n!c z*YPRCHrj~{=)bUaG2n+mJ(|^T3e|g?HC5Fi_Vgj|Lkaofg{A(TT{iGzWU~pYF{(a}O#@pH zuKLY2Yh1!9E(DZGQ-kE#q`8@CVVH$UQ9|LI=48|uWOcnb%sCE5G>nvL*MJGvrOtjB zA%;~pqKzdtrR!kEZo%pE5LW>0My8`FK#YFSiI~8Dx z4mH`T{Cx?f%h$l63+TD3@QhLDdc2f9_-C<60=0 zT4647T9d7577KB>I*b!iN;c&sUy(wuHB<-NbSJG~^@^11ZmXH#nqcBH#$X!u3H5jN zy4^pJ2N{NlXp^d_am-9Jq61K5oPNr~n{#z;C-QXJIx1>8w;bC^{(spSI<&#gBYGrVmDUi?Q*1=4(y> zGd{VcRvRE0qO);%*4X;2VS(7s8%}36 zZU{6H{=Qhl*z8IX?o0yjU93!wa2JaksNEP-Kle+W+5JGl7c#ZfAHQ?<+QKxtsdFKQ zv%b=C!WVK8>Mg_dIgk>5=!Z)!(-Q$KZPuLvFUG3^QXIx|g0-=qJ<%ODQ?hgH*vjOg4|nU;yMB+deZ zwT@pCbRFhEmuvsw|b=IYH z%EgQWDJd4Gj#IC{ufin84+9{+$9B>8fdrdVtK6MpHk8=yd`79+69Es4G*WuGY$4)Z zVYitZKdwG8S)}iAAi6|~Lm0(o-+)NWJ>8c4@$I94{Y&KX;ATleTc*4I0wRMt6pM1% zsia*C+u^Gq2_U}Ks2H@*#7Op-93Sm@Jj+RymyvA6g>oWdC!i#S27q(3cvU?r1 zCm)*6i)_5AFBAvgm|=mz$*DZ`Z?9N$Uz@8;B29i_IPrfRC3UnKrE1{wBl$s3!A{;z z^~r=ullYkkdLdN3Qt}h8Lcs?qZ8**g#?8W~b7v9f^ta{w>U8_L&1|)dZA(!M+Ge8N z;sdWmOngz|htH-A|Jpd*#5bxYS1n>A1Q{3P}xn#R)Oh88&Zov`ZPR;nAr zO4lmxVV|wnAY+ZaWitXJy~KQ-Ehcr=#vkx)c{bk#P>9-5t<0Pwoh7H@F+A8TnBckB z%lW|3Hr6Qr95|m;f#H1 z1m|PrC?u4Y1pYH9J)N?L%#ZbEMr~oSN|zoK_{wLccFf{jbjmz=LcQunCPLdn=P(4~ zNp6^#nu(Htt754M>0yuS%w}SuiG_ScX*>$4oG^shVlni?jX#rBZkx~ejNHO}tu#9M=D!%ZYOtAlpClt zsya-lV>_(jn;y(i{xpI~M|>H&(j6WXB&vSQIuQDJ8R7FVyf_}Mllh0eilg1xOu%!Y zCxQB_bz1k|ZvpwdWlw7Lm4LpLY0{iPooZb4;!!MHQ6bnIv8X}M3;eyebK>)xE;`t@ z+R_bKesrx_r&uqVKF?2CC;iSz8v{-c0i;VrBaF>aqSJOr0};k}RM9zbJ*Z!pVY2AB zd@atK(h?Jhg|V6h);%k%2NEgK*Ht{5UKLl&)tjV^zcOrJ|J>|Z=(uX2yKc}Tnu9+! zH|cNUVcT`U?)z7>rT@?BU#)fjta5r620BSt z>$)8gUk?@B_=l3yJmZ06eFKH7c-8z2Cq-aKZQidU$B?Y5L$ET!`#h;RPK$>D4#7?X z00Wx3*lGn;S6+rUu9Jl)yccpJ1QR2!qWIQ5&gPt5K(9xUqH7gY+F*EF1Yk3_-3X1w zb;;Rd;%pHgw+CfVSvEl#5=U4&l~Y<31jDVWB=sU~f+Jb>6{4$rWoJvwC!rR3Dk+dE zCvaa;#V<`pT~hN1w<&eJbS?2L4{y7KulDK0VkzAkSmc-gxDKElS|80ro#}@k%5c%< z*S2$*-%4KTt~Q>)LQusbwX$2UlO_Hrm40)$CotE8F1ASwwGhm}8~`T}wHZgk*!v)> zP<73dKU1Hvkn%tbrZc}jwK=`EL4O9w@MURCTKE;F`0O+IZtUQ5XsaR7Hn24 zIh#PMl?C1QQ(!z92hUW%;*P+F1dvzF*ZnF%3#n|f-=pE zNdkTaNDWD{)t}}mR}w7x3)m3rP{F*w| z3J&Yg7@KNM7OCncFtx=K#=Wz#mU>oe#Zh7Oy)m^y6D~YllM={v(YoJLy>@Ih>E<*_H)-~0 zN!_XGsB4`?$V^(Ft>+DC``DA>jbe^#7d^2+Q9R%GE1!ZXwkVlJyHtV%NKs>CnK^Vj zK`~DJYk^84ce}U?1%FPnuq>2=5uwoa3o0qqo~X^cQ+1W&D?54CS333!p!3g-^1QAxnm=~^Qbkql{DR-|?gY%v^ufxWtW_$TRo130-VbGpA$w8Eb1 zXrnHsHDWe;X+pmOuC#Kv$#tPK{Kxk!kK*&SDY(VdM;GI6krs2%on>HVN- zUBm?QEsL*OSHFgt{hbJQgGlt#5m+R$Yccn%9ni|=a%O=5lTVM!i{jNRw{B2{6#y|{ zy{@gxiKh^_d*KLWprueSF;}zD6Cd=V1`N5e00LK8nhlTzOmzkpw_0rLg9*`0wz~FX zwBvSYEQ;-asfU7DXVn^C^QFflDjW74Ui6xLv6(M88L92HP8%{QX|$E{bD75owU9eS zAK;49AUxC%rcP}@D)n4Fw)Ef^x#PK$a%`?|rF+hdM3JCY28gWQP*Ne0RmN~<^NW`FIAvX1-2Tt^V*Gl=bw(nkl)ng*Kw=8Y9sumY3tP7sUew#)_%Z3 z_iJ#}Oc-b#7TX*Ih5VK6bgWEK+H0+VS55HZ{}~}PUlB0u9F`7_2Met>rExZX8829( zHjIx^1S+^^VrN4X5p?aPp~URWBOv#)r8p^skSih>$z| zTNFm%CpOoXbg?=&)l#~BWqter7ACv849+B$BAD?Gf^z6+uLuAl%>xeN2fI~irbt%>p-zt z&_E|}tusSwsntQ{3!3jPV|mQDx)YMJ}K<2nUw`*;RM7KU_}zpliHjAIAhkUs!5HA{#%ei%gi|G={&5a5Ft1( z*ZGaDwMckWLl>h8cbMNBej~PaIF_7BjKiltLzZe&zX&`ma^0~ymmjs6OID2d;t=uq zB1vbjd4Ca5y7v>OKGykvI$!Ud7=BH^AWsdR!nXLFQi*fzne;nlpLT>MzL>?MQ%fEZ zWbW=u$q2U~v7;^XiAPHCD?T$TU1<|IPuG{$+txXR5Brb+06+jqL_t)j2CD!L4boei zfYjgGW`>bf6V*xUdvpvI8 z`6>Gp3193~g|kR^Je1Ks3&Sq-Q4-Ic*4saoV}MZ*w1swVJ5*r8*_dP|B{|WrgzAhgVC!E@3JXH? zGwkwbT`Z@6`}ZF$%exBljp;u+g%h0m^sbA6QL()!%k(AJ=7pUt_@(2asj_XZW{u9S zFU^XT_r_FkCiiR2z`r0wVOr8Bo+#n*pUdu}Cs#<;XbmcWQ4l8D`~x(!_FqSy^{PIw zQCMX%F|_d4zAxIAGyQ!3r=?5vouUk|hFJK$*qAPoBfBl zB5dm)c@OP0Z^{AmVl7ImC)*$6OF|25UG9w6^wc43I#fDfO?SaY5gWXdE7^{)tpKl= z1)Ew`e4Vh)j0aPE!I5B*oM1MF3V z!r2{p4GB011d7^-nTJIu#9&vu^ObXaV z>OuVzqoN|w8V*{if+|=Ba#c)?Tx1;^hi#Z|@?#_EMYiNsEwRWTHa)u;d2Kf>I3u)% z9N`b6BVMjOa4Fjx(Mb*VX;!?J49dru_f@N;XdwU0AGOPYT>k%&bua4C8#mLg6Ti;+ zpYMw8tmiH?Wq;RCX~=FgP$-}`+{|cH;b%tdKAyo+ISr6)7k?7-fJvni0p0eq7=N)y zh>L+)RL|#dVvE z&gSufJ=u-fp=MQ9jsTivH!p^R)dD1`YrVxopp>v*BAj)J`tl_sL{6TtG}UlwUxWcb z$?(p))i%W~SHOlik4G<2+l(9y$^qyAd?l>YYNwK8guFpgeUhP`2Zm1huH0!=ndlNH z9hA|2meV0}u4})#10`McGnft{!>2P8P9R}I-fp>>4rg#4%W9jFk5CQVY@6bSCgnFI zge`F%?2S>XwH^K8W~Q^o(fKecF<9t}2>mpigzuzmIlJbIi*s&(Aje7wBe5Pt@**Iu z1NC8W4ti#mcn)cVqP*W-V(vlEKJ(*K=S>W60k|UAZ9v?rUr{K*4x&GlB&0CYQa+q?k`1n7B^wH#U+U9b+bMh^ zsx*&9W#Lp^%JBB;_ux6gPl%HR%c8RGXn$8!NnCEFm6ZGSC7Wfbf-E7Ju{#BS_*mK1 zzf`xXF5?Pa;NIT@^nU^Rv*51vOSKEq{Y`MU0ge8;dCz)K!-mV_8lmuogI+C{h+MCJ zMR@x8v!bo~KQ!C%C?)#*8KI)?aP{TTrLKJ#LDF5zR5qp}*w^@?opbdn`U&Z8mA zujkMefWS&7cH0)25UaZ~==gppq@IgHsv^+c6O3?pZS!B_(v?nq!CYusrFUwpC(DX| zP+@tDCI}fF+!x$Yt&QuFd4Z1F7=Dag*Ux(^e0NtD!9v(YPJZ(5R~9bqdsB2X9kDe> z5~1MVdlWEH-2sC|Q?1Tj1eQiu2H*959h!v@TH;zFvK$5zCBtiI9hQOBrqYNYJE_5= zDMkoi!k|54Fjlxm2Y*Z%zu%KhnBw1!R)AGG5vyFtM#Wls{Zsv)C}~(=XERr4p!g5} z4NZ*sn4>&9W;Tke9xDSRsSu_PdZn7DmZxQpP_QF*v5e(|obpU(rP|B>A3L??i!pYiD=3N)K{{d8pgVsEeFN@E7{^@Wp*bmnsScEQJX+)IYb5Qyp}E=NRNk z*GUz1!7^HLT@GIpyd^+Ot4cq%hxXh8H+Qh5r@-p8E|Wl zm=YiBTIX^M%)2ibZYdfL3e~wmolRrG;4v##UAC0;!>-rRPVkyLvHTnhd;V%o5V>3YLLcrzP=Wd+jvqdPbFX07H2&)?!7jMOD$(BHHqF zu#R6@pgD&-i!z$kGaA2=i5d!Z@eNJbfo$D*WJdOHq=QTg{@S+wPmBd)t+jQtz#sfv zuZL{r9ZGe8ZaZq8k2RleRxq`}^CDD|(Cr~&oS$H~dF_GEFkLcTGbxcoG`3CH&LW9hk z;sIR~=OfvGSv&8bbV0vzm;ox6O!f=*sRILIz~DC1yJnh0Q{hb^=fcxq1o%lRZUzsC z8-SOD`K{KaMc4N(A}vuXbPW%sauV_`0_xhFrp;TzY6~tIZE`PfK3Cx{yBo1tsOD(X^T0Y`SzFjHbB?u;Yb0Y&r2eBsq5+ z>-QX;9GFpYNLtQ1rRK5CYAGyA<@{y4 z+qiW4>5~8JgTAn}OZ`3(R0nmpmb^D>+zDg_Yr%{>0&<9o+l%Uo{m#3)P#x!IMwr#Q zVLMSnD3yvEhto8M4`~e*7t2H7R`a$kKN{K+u7@1wrN{i?wRIcN$JV2_zfmP2r{y>> zIjNvWwq#WI_p)Wh<*|3IY_u+MDQ{%wD?Q{u4j<2yTi8MrqZ# z+p@F)_0w<|DGu)NBw-YgBZ6Z~z6`89$)hvRDe0$^Pm@LU6Xjurb{E5N;Sq#zv0Oem z-bOW)S|0@s2dI?sk5AH5I-RHWqki?NFQDlk?3rQlH{Y%B$PJc8ud0`fldtg;t=XdZ z#d3Z2jGijMe&ymtw5(2;_~qz%;`AG{FE71Ic&7rZ27@p4Do1Zd>xdlH+wDV1Z}Y4k zNirH5s-S7m8{rlY?NdGX>!6S+4GPyY1^fsmI(1EY#kPQmwQ~BlSejAaR3(ou^|T zfx|m}beZCY3y5>R{doewBRwFW`%Ll#Xc!gJ%BQ*?zrkIN(QQFIM5!9+EXO(rwpo7$ zFasW5vQtR%;?;4Ghr2?U=7{pm2bJSFV9G!0l}>rkkv;m_4|aZ~Zy(mw zk*}j)_sxEfErTfT!7Z1<6>5?_SDG^k*_M;LZNP|7f(j0q6HuECjj5bupE_40TUrr5 zf$;)H=95|+rpGPttLhdjZq<2w`_QyYHc1I~X!FG1|7Gqztv>2nSMt|o{YSR$0sKio z^8Soze-}_&9`bGyOY|t*P`)HAFIu}8OnwUZ6wcNwW#0FTClCWCi9$AWfTB(a88Ad_tJbvOR-YNuKOg=JNMPkbwu{Nxh)wEOA zyX)L>)b4jppM-E$KM6U=h*n7J9PtvG&U;?xX71|c8b{^rQ_<^`-;Ro^d*MNCAQ&Gy zQO})X=Qj#A#AsvSpc5;s8hk71mtupFh;_H+1wAYA1OFq}e$*oCn4Pm@iqrcyZv6$70iX8?}y z*^#Z(t{+IFa6IZ-3BSE{yC*75(J4;j#z~^R2i`zv(=ao+0{A4QD}_zixw%*mW*$&s z{<3h{m6QoU!gZy4+dp;B=ls%|&9y&cCZx7#__(t|5p5=?^(Cvd$0deA7^8hk=JgbA zM780u?jdS%%3|~lMuz1|KR`$!SKJ#0?fujyG2oUU&lyvl5(UK>*DjAW_^HcNPjkvv z5Hx7RxzRfwYxtrmyns%985!3l#ifDTkYFK4 z@`ix27#An4RjV=k86UajUP2Pzgu#U22qg|T5;Q#J?Kp%}sZ7Ju-FQxi=uAYLt`KVM zW6yv144x$b;w+b?V14EPT%EUCU4nO*{qn0%eW^^j_I8b}zuC|!eeUJIeR$EO|Ns80 zyGrTj=zVZImU-q_$<-Dj(wv9r?e_3i(1Q1&(n~_^>&@|?O`fUADYDrE^;*IvUzMwB zOXe#a?4LQmRIRZhXAW--@5YZ+5#!aN+9iWestyUGvGP*#N80|9#WKBIz4Xarqu=1JR_nqG|! z(~fiE>*GyCw{q08(_?x-4&ym^UF4)!f`a5TmC0-^<`Ufx<6P zu&S>x%)`HMKA>O-F68DJd(~bKLa{lRROuW{XA7zxtruE?!F!r;|Ejc@Df#vUp9oI|A`n)m>D3uQI175>fE*PiY>(6Jh?_c&^6tZZ1 z+)#|d221|5=x&RGl4Tr^=WwgV2rNZol$H>dO_88`xC3DtiBVCUFQ=jToEZxMS{K%L zrW6aw^6P`Y#o?E(NhxKbRjr9avL42G>sk7NVb-ot9oDRr3y;#{=hGYHs(;9dU z;a=t?#2|0AT>qr9&n+O7@;xPT1q@v<@yT9@Bb=~pnqT+Vu=T|$?tPk082InrXy>9B zLQ1o1p>n}^ka<6JZj`V#$V$}R==p?UMVG?)H{Drvhul{+)uMU4 zXWMQ%F*JH;Z&v`ooc6>PKkSyktmtTv1!m#m*XXsIks32TI5 zSAE@+0klC}L}bX6tnf_5 z1!Y=Ze%hObvobcawL74SnzPwoHB_lZyTub=T_SBFkSJ@)uKG=^%dK00qKD6b5W8iWE1@IvX)!WyKSjU z5wVqm^?R=TWk(@NMEGoj*8_A3evVn}o-8&Hq+SUz{uA-NE&F=IpGaFQI2Nn}foT>U zf4G)Gruqbc&chAdp~7208~%-z?u^?!x{m35@E6R7XyI>;A&&rZjX&*|j zu&gbB&p{}8f$@37eHrUEGb2Hwd75;Wc2%uAI!QEr#9Et`N5^;8Qjoa-y(nmq<*OO zv!TsIT{_o7u$ZdNC{10RJd;2-hi8S9p$2Vp0Mazi7dJ1$&K=BYo+-^OF4GzHsI`Pt zv=%NjIoi;A0~)tz8HmSmejxT5?wAYzs+2DSBO*ZSsbNI@CbvMVv=yWzX}UZ`z0QjfxEdtE#o(2D!TlD5S?SM>FMmGn$%#^$*sgA$yMUb zs7!C+{OFAB0#D8M$aBl?9D$RRgdTi~pvEwkWbyHEwt6B39^t`WqbQtxmz7LZT+|8g zQySx{-0?Vot5gM;Y6Q0S)0vM`OMB)Nxt0DuOH;|#@M8}{63;5-)CSeajNw8{MxI)c z+ZFp&uvM}JPD>JkCoJmhSyn&3l?tE>y;WJr6oze=vI+=g5#oL1z;orh*l&^R`jU0W z?`oymPRJ4<+A#shF$cr{cE?9oQNGZ`dIRjJB!!}1VIZ`Q{?b=v-4eV@R@Vh1KYa$w z*kLf;@Qd4DP`nBHTak{Vfs%Xg7aO}}PDUpiA{W!YTY0#!` zgOQM$*rpCe%ePN+b6^4%YizH7Xzl>|gGX*rtntUyDKI&kD=liv!SqmI`{ns9oXf}B z(n9)p93{b^+cbHFwL(hF%LZA6j@z~}AW@B{T?b?k~7 z4?-yEwqSU<>}jMos?A_a;kp?lh;ifjddw1Z8<|xg6A6Lc>)h)3OXrx$Sw9*NcFClJ zmFQ-eLcca0%Kd97g6h0%vfv^si5>uNP=>b3<`kp3)H-oMC1Xx@fgyZm_On*{NZ{iJ z@{B3rmUBu~b@&W-No68fSth(GJ+0wqM*ANWn~| zh1tvv4gsu9DA6=PS#3Kfion90r>{ex!zagPxvMz6=^wh3Gz$4H#?#ZoD^WN0hds1` z0_pdznvc&&rM|#v63q9V9EJGKwau1!gWHT=PfNnWe&JQae~EFogkkDL5wb7|3XL^z zsL1Uc+m$Sp&9MP-uvRgPEAR8ej`f-mIl~pOJfp%9IP$Yv`BGbb8^t?Y2YHeXTLIKs zu`ipD7hFtG$OVQHcCJO2N{{AxieEw1E<^cT*jdcgJV7SuQqD809!LQXXWXwY?p@(u)J3B)s@KX$*c9nkg{sn#% zbg|~K0ejTun)tM|Pm#Abn7&j7x~$XONULyGFMujDX9`~oSe)M{6NLCv)}L$;UJ|5u zo!l;g>!76p+Oh#EnVs&cqgsjm4*>rOSbwEo!ByaH*m_=Dwf`>Fjhr_J3TfARqv^qE zI9to}hS;r7E4e5vO3A%O$rQtwD)p?uxiLu%f`TSPeNktPBCpI9wyZG+azl54tbJhC zja%%2!_#HP0&4DY4D#Q)vHAD^{$u4_Yj+0JV&fJ5nRRT$-rFTz1p7Z)MzXQvgDsuW zn}f9HNq!gS6()mPa!@(`<-z*c2eVGZ5o%#*>I!>-g?d6a=DWcI8qBQ~@5=vg!oXu? zuOI8g@Jzchy9PbP-c2bIP%qeC<*G!8)Z z$vZ5;H+JZ+VF1ZGs-I3cQ$YI@v9Em@pXLw-c?1t(K#O6ci#gzo71s%us@1P`iRSxW zqrvrKoKz8hrJQMcg5zmY6JVn=A3~M-#_;`4mxjirXqwB}U=KaupI9mlh9V+7aPgzA z_12BXkaVq>d@_nlq=s26i9`cW6&A>O^$IyKAK@ur_YDh^{A+XOLh|cbfm| zqsAVtw|!Bu1ZG-ou{mojrXYC(px5}Jph;&z$_X}Yqal`?YDt5h-Q7p*CZgNcMOHHbHwQpWk7|9Ei z^!QzUHM{kzyn%JbTgc1dux|M2>6_i>f+56N&*S_Kn+)qitT128*CsQMb$wM|q2yF2oq}4_R8hjQcOx1b zA<95-kZ4AGZaN?PAx`?}h=)gqU|quT9Yu%1z6oHhZd z<~X$_i%)GMVb{jW^+zR_B7Sx9D#-Z7(uGUXbNbpSv$2Xk6STMvM#!zJ93Yq){UQ05vd0E1nehl ztTjafc|5I}Do=qrr>Lz{D~PApj+$5|-EQP&ktdOT`kMbihuuJv!)iNN1XpfXoP;F3 zp5g}L>HL`1%G(ds7@dLrI3PLoByAMJC3^(Fl&txm?lh!3Do89UwIA=vlD?p|%tW># zHD6=*Qnau}f!S<9=~G;nBnOytQGYDwRHwkMP}exnjCuPJ9Ow<##hYNEW+RX^bFh_Y zNnk2>{#|(6VH)lh?xvTkOTUQJry@NU&*Nsyf?zJS@_O6i;`ee9dP|$u1*pMF_ke!H zTlcdGnO-W(i^7l^!aJ$d=!pz>n|}i7S;gPEUo?MM^v+|40;txw@2%&bZ6}xljtWoT z==ezI)~tD>=GKvPvI2$O#nfS;!0qFO*Qj!*l)+ZJq&V~=5O0nSNaM1*^l%UPj3OHP2h1jMuN#HSkTYG{6RyvtdCQmdp4Tmo zSqmsp*gv$mJMIQ|DyS_#9bE|QBsO{XU&oL+2KzLQ#k#W3y#ZY(gTS7@ktE=iU867NZ6r2slvdN{FoifMAQ2;RQmQcJF9D!XXv?*3of$+; zSq(PZE37#bWD?!YHwuv7##V%nUc+bJFN_D9aTh1;_wi!p0W>6LipraV7{rTP6SO|@ z3FeSD-nN^H7#f-qx<`N5dY$JYA3KbS{BNeq%JhoA^W)sl@ zPn&d=(lu5om=nRz6%TE$y>2%A7G872T)vAEeMbcmd4`$5Qd|XpqK+}mXrE{R=pWBcU39@Y*$ED8Cg40W#W|Iu06_Ye+?COpyYZ^SZULjQAE?S2Li^N}-likV;$~&Kz+>D{?RdSU@*sry zV02H>4Lm3$FcoaaIe;_~3dfTQEE;o#1>t4oa14a>)4}I?WH`%B=Cj!zTPSco>`ExS z^&zJ4PAlA{<2#(3wwLXP3JZf($QoxieWgdP61`_kTV@7CI6=qE2p`p%0GxQ}x0va{ zR#{iE6+Ag;YhFr-Fwk2rA6EK12mKJJ5$2Ht!nL2XQ@Ax!Nb$*5jwHZS~DYSkNx%vr5VGhnq=Rz4BEbRkYnpkIwRU84H zbLHCJP)YvxT2w=skRl~7&eti?zD#}@NiGNnk+s!Azm9p4#zzca)+#M5KpDJVJC;)d zb;?WVP^P7Uaj(x7$6y-a%}JLuSPSAChLapeEMrPar?6Sng)QmJG@gzE(^y8f2g?Jhe zwk<5)<}z^&7Z6GC>nV;5G6`IsWC~9p*@Fyd_*P%lY*cy{Lc7UX{8x@k!(`Jb@n~v1kB)owY zPDznt{gC;{)fqkpw*K;QAd}1>)`Pu6pm3iN>M43_t~Z1O%7f`bZ;wVUjuETPnthq; zcCt^<)LMWn^O};-6Lg{Jb272&iuUAi}d`V#h znBkKU3*L8f8Ff@_Y8GbP!zSr;-)Ags31y(7c*X##_P&d6X(3B6v1t<5mIt(yoCRbJ?8Pr=3rJ*v#B zH~(OOZbfTXYDS4J6S7pRWICf8K)G}i{V^ZN#hz@ehCjb=C#t!JTM9z`%y>AX6SlVz z&F0lnn;kyGxOPTCT)!(!afvqUw=ICcvZoIA(5L=}+vVpBM)e#sn7eGV1TY4Lu$Y3Y zqnr<#b<5cDlIFC})ACUxg zp_haemJi#6{K$YaHa9jh0#1 zR{_*aDNW$na1)yuR==rKAz)uHt>JlQ`D#`dDzB|IzzvEUfPu$gO~PMlGF-!$S{uIk;&c2n!X{NqhEVxCCJk3B?NCH>$2L3i^86l)bdM3Jphpi1Gv!*%yaq`7rK+JmpF9c*!M$e2J zMEY39>-6W$jdu&NeAOZp3j(uLw>GF{5(?u>T8+obRp5jzHX$tn%HkJzo4#y0bbY^H z4;^`mooA0-OZgayD*>Y03d~rw(mE|6r$XEosWq~&)s?s`U zZrS?TATEIsV#LQVXCWX_Xo~h+ZrD@RWAfFnt~Jh&pj@Xtu8dSu8_{5voAD>fbGJ#3 z`tva)85ey)3Z8&}8p6s&bC4d@Cx zqoU|DucfYr8D<(>*^!osjn+VF$EtEa>o&RJ&$8RsPPBli9eEAwwW-SY*^4W+>q{Yk zus>t5sM>+hY;4t@?j*96_{vYYmI3T0Cl|4$UZ&G~7trY}eX@u}IjJ--cy(m^8$#gb znx~9p>E_X-6a$qfO{o^sPC!SkdQxIR#pUKgMWPx?Ocs~Yk=Z$LC-n2XaE>i-Aa~sG zk`Bz;?bxA<(4Jh5{o!gq67z~h>2TH&OzaK&;^lsgLeY%b1vT`jc8dN!y};%&O?{r& zqX@12X{?+GQx__veseuyLaR=Hom2&k!Sj|2ERyAXdc)B!b+yw2^5#@RBL zC~nL2Q_{mIMVxycL-%MkZ_|ERsm)c|IG6){NoqXXoq0{Uak<~MzFyqEGboJrygzX# z(-c_Q^=qEU#>{!=Ud!T%1vO63RSmWTOZGTQUvW>!U@Aax8`SJ8U7GHyy7DP>$kZ^2 z-gC=Go20_f-*0yEoauy^Qdo_mH8xPW8$Avv-QcNkR5J(ju$94);%-^E?u^g+?;!XZ zmIBxj*H-Y->Y=ps2lc)r=q&6x>($ns*go>PXm|eIJ=(wjtk=Nk9{uFrprii}FrkrM zXKe|A*oDz8nq;;uT*6o>p``?wlxaF+$$-z~*9FB`bmUb0Y`Ypc*$qyFB(kiA7mXg~ z7w*IxKs8*(bCIEkM>~cbjl$^Y<2P?BhPAI+@p4$eWwNb7JAV7W6eYl?>@sc;_a;fE zlyc8$HqQt{G3=Uc)4SGvoF??Wk!apF&OI{|S~to6yYFkQLWFUpM-IiRfo7kdnhSMj zppjfIigan%hde)h_%m{V3v`TD(JQy$)Xt_z`VvpMSb2+v#8>Up12R`BO)BpCi{L6O z)*;m|co3V2nP6T66~#reKwoh=EV2Vnw;1P?a*V&5LN=!}A9Eo&#JJVFmm(hcY&(;u z1FT|)9}hE53$YHVbAM*An?^pa)e2wPN-c(GPjMb*S>)Z5@R=&DkN1cq&~ZP^rU+-Y z$9Go$)QSoR7Zuvm=fOB=AdW2x^SCHRZfT&6&O0b`mx7^kOxJEVr~?RmwYxG%@d(f< z27T7gkhM*G_Q?i6M|Ouh0w7=bfi!q#a(0LI+#+@~rK_CF8;ub;GYojXC}wf4ctrE7 zJ$3by&4{V$+cpXrod-6jR-wwAy|`7;xsB#eY5l8t=1#sOOLdaX)<#4V*R&2!zguTy zIh!^U;FT*S8^T?nQG6?I4SXEQI~g(F(Be_j0D3VR#f_?}o4|~zsoZ=xw9Rj;@`&ZJ zc1$2$pof10qLz{3t(BmM7#oDR@%hr}n@`cuyEE90{tr*a<4nnh){Rf%F(7mUV0tvk zOp3l0&{mY+gi?e=N5DP(Gjmpzr zcCsso8zM)U#P-Hd`n%o1=z&xaASZhT)IboGoowY3$)#a@HL=)nDlApW60C_)FYQ{d zTFXJJ9(n`xT%T40G~LqMo#|%hM!`8iLtOoAfdfq&R1NbasR`cP7LOWSrSdUd=V_;k32u5p0zv5p zM-Ia~~HyY9l2_iUG*9c4YPx+9FQ{ngtJ&C4IM(f7I#OrWO1GT zqg*_9=EY!-rY##8m2E{<*oUU>1 zFj$5-j222-S6u4r%QUwJ;G>&sJqHwRc^pSx7yl+P;7kda%nCkRd3C{O>LE|-XZgfr z%jvS7OP%8JY(K8dS378Z*EP_DdP#=z7wOFs7xa*pZ$qvd0_zLEm3HQ;t{Fd3^!qq5 zP10t74?+}yxvv()jAn{*E#%vdMqypjE6`81IQx_mXU=nbPbg$HXQNR6x#BNp3;0R{LXRn&pk$Ny+55 zAc59w-+3UZ2J>P|Vz4KAi8yoggYJ~13rZ(Ic$6m*q2b-@IMi0lk~=}lbSL~tGz}=* zF0?5rZiHML0O%k_;QDJl@=yzGAwWE*=kNmuH^KrrpEdP+0dSX@?FQsGBFHEHdxwl# z#}x%Vqya>PKq`!~dH88qLD$XMvW6Ea6Bvg%E~ihr zSm`2#*4dppk4u)dcNJ<)4mWLSdaqTlZgh+U{dsBt@Y*ZXE!CBxJTDThz}I}Q>C{g$ zu*W7x7eIy6MiK$z%qaKoJCtwA?*jUN|4nC0GR>gN)iG0ZC1nc0F>ER!*Y98bZ7`a` zAVO*Y?T_(tw{@kpF{w!e(>7>0_^qQq+l|b;{?QFa2xfdriz`oCQ0L&sr=-D6lj8nC zz=xm3yMBB_`c1fo_`f?LbU;J|M`?-oe;fpMT_?0!`_c)WQCVA7>l$a;q}JftU{x#F zw0;la0_+&ZwPqg$SB=h*S`L0Ooc??)vxBs#*n-)BK4 zGuZ5B%;FvSNx}4TpkNy@g{mi~lb3#+D92~r`(t`67u&2dd$c+jL?T!s4s)%$_ccW# zyeXmM>H)EwITGvXr473+qPZ3Jb;rw+T;$(nVsb+iU(g+H41TU1H9&7}biNng@#T;< zUx=tLB2ibHi=m#uREL$yOy>xVjmot`s!j9D=_(f)cLiAowN0L&A9EQh2hcD( zYf8hXPuk0*p*T8gSLJt^SQGVLGqS5?YvYPyZ+zdZsFklVT5nZP-}T~d1G+}>n7OTdPEGK67UAdsj>@&kh8h$_yIls8+OW6=5xAyDx9l@F&gb&K*0i4d+^i~ zWVRM`o(39MTwJF_jmEWAGPd!U_@O4wa*!i8)?!UK}`~l=f4z7OtLCvxF zXhhp?%};rFG?|9u2}DEog+jVMW+>E-vw)JsXtV94G?|rcgJP6x;&Bso06EA_nI56R zgM1lDx-I=z@Ml^&{$~5;I18ISw|*lv;R zvLTJW|E&dSzFu|(gw@mpcIJTv?;VCr7v9M6%W<1RtH+fPuc<6}w-(i-tJNY?jeToR z(~{kEJ@0~JXTWH4gCZPPjQ&gNB`;lfMT>13HtV zYAKe7>DF3$1V1{Q%;Ckn)fV_NoJx(OgGeO|J1W_I?T_cORgcgEBDD{RmjqGSEskox z@q$zbu7cp%44CmvvyS6*9XslFFZf(ce~eDkei{hZij(UEvrrSjMN;82K2GvqQ}q2V zYXs$KZpS2n@Y;PojP9b7*|9l|_UCKaoBoMToZz0;!3j99HbbPtDBwtVUa4WR-R%KC zO8TH;zfX@Qts0D1=TRQ62f-)Lfmw&QOhKl4kNutLJT5h{I4P6v0q#|~Narybp65If z-EE0OGwLHQA+C{^Uc!46fm*r`3Ed-7!uQB;;(LeaA&!I~YOmxE~ho<5G=q^!K2d?$yOqsb3P&2T9n;@sdG8+d ztw7iF;WES!X)r#czYZBIW#kvHZe29rvcLhae%z|$-iL3?19$EEaK95vcQjlKSYb)m z%^3rPyR9KuPInHJBV#57>vz5CQ$3AR##<^7-H->l1e*bR7> zE0#enZD9v`X(X)zpx^Z@Jg9?uM?Bk}FP|AJgT;|SD}{uO{80}_CI{&px~7Bs6m!6$ zU0&vT-b{j;yq5CXPDA2o5VTGCYTOXW@aTV_*hM4xzxyW;fBjwpKV#vt5*!MK0sK43 zc736i9atf!sWh#TO(sh>zVI20fU;q6t^hx9APZPQqNj<2*Mx)oOjr2d-ALc=yWPC9aww;g+x=wdR0N#G30{AfXR0_g$$l<8WBCGJ&MPo0e z0O5k%eC-iHqbPyJ5Ls1|IY^@mVn*!tFzR{a?82Jcjl`Nfp+{yPw5~q$LhMo-op_z+ zX!Qp1&P#HA!`PJshG{8MpEgo64Zr_H4z8WF(_Q-IajTkRR=uv-A*>17At4ke!MWML zaers80t~}4B>0Q3zkE}!h7Q<@w-`w+lIQ z{Mv=mHXJ@l=6#aO(s@s-jtg_=n@YD`ZpLoj(7X*!c%itHBksqhU0rk|0D(I}CpOzsBCpx| z0>3D)ZecLZ7f%gi|I)?FbmhS|`xj>pPdH1M7xaa#>3T3NptE1(YI_|5&9)TeiH9<55qq zvt1X6R5Abz*Q6-7*v{mlI( z=fH#jb>_!*LuLqTYD~N_o8LU{!Y3W1vV=TeJ94C-JETlo+g{wv%HctY>tL!&%dxwckh3`}A6mw_UcjeOLb*ScREEmUZ2J4RF509O!613Ni1@ zlEmUoAAfc`d+OFM|I-gBCr10m~r+TeaeB zh!{`M`UeyW>9ujOULPC6q0D^Tk^TZlvmn)u@7#<3e4Qlx}8a*)BR8GB9B42T7DO=z{y=W;+(e$JJ3z7#$nqrv#8YYdlKtvT1nx`9hxQPG%+ zpY6`E-q1qJLyI-mtnc_;v(0O$oM?aaOr1J^5I1<+eQK(n!Ox;kFD zBbf1dpi8Lo)R~u;ei#(V3=E_rny|FaHIKozW$pd(+>YtM#>IFHkM+3!O@3ovFrjVO z=C49{OV1|3C04AA=UK5rT!u`Bc@Fl6crF{E={+Y6fwEi>l?U`4hBNC2Q|olV0Ttkg zh@ir)(4SA{w8D4YpvaMCev%L1Gz>*T4ul1ZB|ra1@u@G$pl+B=Ny}30xPTzOrkR}w z>)D+1xQid0h-5-h2K694Nd1_}BVF4hbU6WWB$=j@aORbA)Bw5JAE#QQ`Lp9K=);$r=t2=Hc9}2x|&;i)@M7F=^1_ zYaAfK*zsICEd4jIYS=`b<&fLWEWDC44(I&9x=4IXH@_1>kd`Y;h z@&>XMG+OSBH-EgjoRGp1fiCZ1- zLjdxms}8@ev)T+tumX_AZZD)lIEw|wt~uD|p%&*U0aKw#nhxYdyH6Air!&5$bTrdS zXAQ^w&LkYdM+KP>JN9!d_!a3>*b+Gq3Ik2`Sg6Jr!~uY;vJpy~4+lO5`Xm^alm@Zq z5(X@fC@56gR3kkaKdq-2zWW_s+~Didi=Lj6&AtZB{cRG;19Q~wO%3@Z_!8I_JD=V3U!?deoo${bkR&bs5E zKc?57k|1=+=Y>Sw(lDi=u!Y#jR!6GFm!%F%1#g7uq^mFzwKi)86M{wiU9(VWv`oEy z3&{G@Vs*~lW%_*O-g@jXk0IxL9_axCGpt9*NcEC^OEimH8Wu&m6k7L^T7tc+)iNFJ z_b2~N^F0l5hHCzMtF!?0zSSTL5&GiiSuM17yytT9hV(~I%!8Mw6p0rr>~-qphZX<3 z#7z~z(zV?zSp(5UOS{_M0noDSCb4aRtn}&&v+~i3Ij$U1%JcQci=sN5`a#KZQB5>a zARHH!F;e-g=$DyY6eg+p2&5{X7$us~Mrp-yvZ(7Pql6QqJ>!I8`oe)nVz^IzCP3$g z6I}tLp^@Eo=Mw(R9S6dUG&PWpLkgpUOHlKv(oN>3%BNx)G}dBa5Zt($+e1K3i3p`8 z*mdc1HTUP_vNIyG)0Q0JJD%hSvjS<*$IJZAIZq22(sMsJ0Bt_|wFqSBerq|m)KMCh z%0mM)SJYRsM$pq8{Y%q(_TLP@-#z|!7of)sq{KZF$iT0(Dm}#W_X%@6EhFRg4Tda6 z6}?>-jT+&?b*$mKPJiEmiLSU%Z3~6^woJ3pR>R)CC+X$)En|!(*!~};Im(9!ZX3f< z`ABZ8O>%*z|5EzA^78n?0RFvJj4XkSYj2BxCM3ntU}JwuuiXke#!vmk07mo&htWaD z&E}SEC#(*7XcxTUcSY6BM1RiFsM52^bLngn3-1#(J}ck;QSFqIWEY#2a^vhUbs->< z)CX!3Yt}VnEzo^Lw6eS|_?eA*PIX|+M4gD?+mZqTT&tvnrqT&bG8z)dwt89h`W7 zP}d}1u1&?{D>eU%jlBbwjhUw7<6aeju^(^N9M##}j@i+(~@TrLh{j+(*rsljfM$bPVBQqFn3maPniv^2( ze0Wz`xA6QK=Zv?l<>*hHBpvzbNJ{YbE3Q(*wQ+*`BR>3BTgr}&B`%drrYE`V7CYkK zR_SzjPIvj9r`5?`z6J!f2CJjOir73U!4LEG<&3wed`{80ST==5G_9oK2lSFu>G&pZ z26@uJv!u6>Y`-NZs|9<#veT)-Hz-Fx5&<5&h1}y3OE4xzT3zCjr0ul6LUe8sDSOS< z8#@D-i&|T4dSA3dg+F^pQf^MQI{gd^oqXV}mr6 z1yL%x!Re0q8ahlNSEt%1Kn=S{uOpUN4*tLYy1}3x5N_9m0;_D?7zO#a_Cpc#dBRzwoGVH}L2keWl#!C!D@Vz>r z38R*b{Iuz}vwzcbghRsmfggq46#e9KG1R&9Dcv{(9+4GfDP#5N)P(Uq{lbGHI2_;qM=RIEejPe9a9skf^Oc7eF(rtyfm>qYok61dGa_YG{yP|n z++U_j6;9P}TNc~R)6mJW!#PHGHD0As?GX$itnx6FQOkZSWQOeEFyN`Og>%uI!SK(A zJug{VxU~ym*RX4=Qj$VD$U~D-=}GK5eX$xRi%U{9biXRSWa~cz$jv}scW}L#LSKJV zpXw^_$z9Ke>zVoi$AsfJB+ilFML#|#G7=Zv2sM&>xZVY3%Y!sYn~_yI z+X)ukl0cGD>~5MYB*}lV3hbz%<6L7`uAa{^8_|S~QXRn$^Ij6vKS&mpoGe$}ANiu_ zjdnQcxF%*n$R;{CJhXg!q-2}RBsf+3( zv|+@f^Le1%4K@%6{*1ZQo1T5W>C!!{V|&@TC`6(g@w;+77Amo<-opYno;1X^E>XVf za`2gJpQfFbUPu;Gm4e7v1Ji*~cmxiP4TU?{Z(xp8SsIUoO}i_8B({{$;?BN6R-C&7 ztRso^>lQA3s5jcb?Wefz7_-aH{C`TKxbhoY%*JkVxYv5K=yzUEnpc#a~-F0AKw z7zwtDt@D$7{Jvv0hRj(yY{6tYR$LA+m5@($0=xCDX)^5@9hrZiPcMc>7BI^4p|Er> zo}m+P!J`R1e{VO_A2u|4=@^tQ0Vjg#`i{y+eYmp{dD-jd+S*AByWdln_M^u0BV!SZTUg_U^?&=G{@7W_+Ok{oP6YJQ zTu>`?m(lvc`oRy$dvA&IR24O8lAWDg{`+U76K+(~leHW=y(Y&87?qaOK)QXYJ0H&2 z16=PhnCg^?gDPvu=G6qSB)8l-DN7fucBjFGC51m_VH{D4la0rDJ*v9_xUrMite^4j z7ARNKBhOfK=wm+@+!&?p{g`upsFAg$8^;i4_E;b{6dum|AvpOJBqZi962@!4ypLiE z^I=qSbGt?r;MFD@c?k}U^Nxp0Vr0yPtE-(GvYG1TbzTW72x}Yy8xo(+i8#$>4$co3 z^E12!V`{{%_OLgxDq2XUJKNRK(#}qFEXz8)`*J@0B zUN0@rNZXh_7>Ad=G2v|h%SJSV61QB(Re8b1J-tb5Stw1s70445w`+%VP}|pbWS?&7 zC`mh=H3-0ma2VIw8ymnixn9$0Ls3WYS#<1YX|7<#bZM|l{N4w5sr2t(JKh8<%=-TV zC|%;%3kJ;UbF{9AkRZnF$X&68<5FCc=x56L{XCQ9=9-f)$bv?j&=m?p<~O>h%G)wT zNk9A_o!N2p>}ovi!aUG4Hqs>M01YlqtXn(UfmlwJYzsW%7s6cYo?|vjGj={`7FG`P z)L(hLLxrTf8V9L`M+*lo)miqM<|fLd{1dj-gO)Y|1G-LXuc2bS=_@e(iBEg|SC9LX zmykV^t`Iw!Z^`zF(hEJ!|MgWUt?U`>wrKQ6=jPY-#-*lph1x~gVAL@A8LnG zrNqa2_7`Hy%9~qb5yzbNm2K{0{`E)Qz7#Bn>zXjPg%jAWHl^Xb&!=gUQO~y5wZHXs zK>ZC{uiozZ7mexs`rCl(uJzO+ z#`!K^zsRwZ2jr=kXl-`Aes!JRal5_$c7KFZ6#;rgT`ZH!2sp}%m;5A%z@l1-&wtAfDb-B!b_j7dI6mp2^+knmssJtYGmNAqHr zailrc@c(QH*$K!fY*)%S3|DW%;(=jG3X_w~XtGMQ+(G_mtUMeRL&y`%&qK(E=m3Ai zM|tkFU5u-Waa`6diI}IL5fu`zs@pOyXoBTq1nGX@SDJMSgR){QcMJ!4rt1n436Fin z4UVPYLmZOe5^S69KPz}Hf@LF=8Kwgs_uyPeMv__2&{HMSx{ zrW=Rm((?2P|J6GHVY#emx!8yJNAwtHk|ILsWY_Ze-=Veq30-4*3*oK&YQk76%zyp) z2U1%r+wtM^HX*_jZM(x!OX^RSz{w#_^D`}+HeVSAx<@Wy4R^L_0}(0eXdrWSlxa&a{F8AIow zoF3Jvf0@)(2e-r8?&X#eYvE-w4(oWK6{`>(zPsNU+D?>YY?ZKF|jiOQAhL1z^A z0#-|I=E~Kh&|=jqh(<(Sg`U0q*3uBYF4d+>8#RrOtI#Z>KwX2fN+I5@I=}w>t#@z* z(Px${OZ{DG52$^w=pVAQDG?vPo^|Cu-jZx647X!&rEN*uJh)ndhGm@-mEV5-`R`x9 z|Lb>O59bt{HjpNS(2y2hybS8hf{AYK_n(F3s?_e+JMi9=_^U~heDBBituy6KzjmfA z%eoPHly=!HIbGDZE*IXfU%&s}v7I~a->d)j8;Ok)!S79xw%rCa1xt@CS@I5O>u0D zC30F=PvsT|>jr`%(PeGz>dKvZ$V_jDe{Rx$KA@GneiVe$_S>)D5q~OjJGBJ~W2&@ObYVel zV(Qbms^<3ien1AXx^V?(QBlWI`%svtq31Z|VJla@)?R(%+61rIfo;cmCEJ#RMWO-B zP?tk>sUZBK1waut_*dN+O{oEtdwuU*lI=1(ZbN>?BQgep9zI0z0r`?)**@0W53Y8G zy_k(c#U-{p-f_)r$5OLNeY7QEmhuS-%SCaiJx^%;ZJpYgP@h#`f~B+VZwJk|zW6cJ zb&QUt=6A`l@fb`$X5t=3KIzHg@~Z`SvTpnj~|P%?;7#i1?}m53X6?YqjY z_9nk53QozNv$Q~73?;a=>?5aLFk3O(rSSQ&!flIH!d=GIn``x1F{R*)kV?g zP48q_>qcYEm(0&e8mX0qjnr|4^XNEvj7=_vpIU&e<^9R?B#0>u)tLa>_QyY15m^hn z0pP--P-x-%@3D0T&V!!x>XQKx)ELUmipO3#Ux#E^y72Rdy4WeS6n_a%0UOR3w@sw##>6-}=<&WDA(^AE5yowY0gn?`2_Jd|KM6dw{Jf^|@L%8mDv!7y7UUw>baSM@ZV%$;7AGp}AC9#y2w?T^nH=F_coFOku9E2n^Bd zkz(RIr0TB^Yg=QOb&gogRd5@H=$^C|9Fh(JmjUyz5>H-JV!utREyMj_Xj_CF26Lcu zJg0|b(h9h3AlnK4B=Ql}=$K}L=!yAZVY3B2xDK z!^9(L5C*eEb}%aCsIC54Q9?49yN$%=LtwmjG5PC%zk8p#Rko9<|1mIvLGaEV-VX41 znt-jVQEs*-%JTdz-sl%kBcrIhSnpV4Iry&+z?(CQ@<MG$yvtIz0p7-TMP0Z0a#ZgWtk2i)L(IW<@t?EbfA6Zli;i9ASLlNO zKV4ZTA$QpA03G6gNAQKY5Zi$-Po2c_*|O2z)qHpTMcA!W|NG!K7y5rD?2^B?0%*HU zh+1Uv%_P;a?&bkzP00+(0Od#dMq2(c@hU^3BW2~Pq{`ZQN`B?fs#zhT)2UIcdYV@m0=uy&YF{P}wi~Byx4-DmR3TsdOkt3cJ;YXxOaq zBMW*g&btwnK8MD?3MVcQ&E794?L;PD6*Y75Ry%txb2#x3xpw zr$`SIi_Huk4&gM6pS4xr@PaW@PMj=X=RAF_=g)v;{NeJt%bz39d^Tx}qbnl@=!jq> zRl`|bb1y8)UwZ@_FKfKreJ5IQv1sO$1E-=R$y@sH8yT%BJ8g4%o3Jc!>Z4!;^@c4R zD^^moGsGrNH~9;ub?c^R`%Pru>Dfhs@Ac_TfUQFU;Kr+^6!jK{3QfjuT5~(6)(;Fk z6F{ULLEZ$+EU8yp4MIyS-YG*#JK6ODH@e8Inl^Eg8FIjbtJw8U-0st?g53bxKy6*x zA4cXFsmrKCeF14*FjiI4vGC^R?rCW8bDQ&~)&8NQvVN+yKRbv_o)T>8&JP?Xf^Zi| ziWU0RQw!h$r_nZvco$b@tda|8W2I2v@QHa`EVQ07Ej<}t?}VH>ADlQvXS`LXhOn04 zut6{W22OaQqGf!QNgWeL%4pAg) z;+>;mlX&{F^Y8|+=oX`^IoM})j3E=s$5VcrRJm zxOxIA)^{tAPR*&!kJdMB--@FdbP$VUL5~sm+s;RadMSJi{ zyxKX`D223oDf84~xnkAqYyx1nBt(NxSFmO0G94D86vOFEQiL3!P?!}v@-MZw3^Rpj%v!`MOCHMbeg(UfD_c&Nz$$v9!_ zduN#v;Ci-Ip>FD0P{G;91tzo^gxZMf`wLp}? z#k|hxrw%EsuBW}noa{IMi{Q?@T~+U6w>s5rz{Xl*zf-nej70XOzSt$m%}W1cvdjLi z`(66G+V72kTvtdU345*FGwNEZ+iEE#Z!*C)a+<(>_}?XaL%yp4mmmhZ8-f0+d4zxM zO^faadTRLRufJC4wn4WSh95&{JdHl7{}9A3j*VbFX`D!m6y1JnIB^Z-`Tu9@Zq!`K zZ6#gX?)KdO^`71CJ>MH()>=N#WJ#GM2m}J)Pf{wWD&@wME`um`=9QKKCw2MVxOzyb zT@}t07!OjFL6qk5UVF+!nsb7F1l{@-=qbg0hN^-0EG}8{ZSOG^)q${~V;sk2AT40V zQ^$sRi5c$AKXwBhI;wZ@xv()qk&Ps*x5jzR{G^%Feo{Ep`4)HG0B&MNuxz9101 zSZ9Udi04kCI*e+phK&>TCiBAD`;u&rp z(&2zeB0ILNXPIsB)?M3N*YtsIQ8xeiPzWjC7^CYSbhgvTtpdw z>`CAa(IB^dS)xF(OK}MLdA1hl3I={D)wqsq>Vs@)s%Kb?a@SjZ(jVtebC2=hr--SB zJdd87her8gg{Xa$m0xc@Pt0Lx=}?*jcK~(sL@W525LQ&JKETmGDNPnyD`7HPMQq~& zaGkSaT8kyU1HbrSA2Mb}T|aXvAvd)yLbbVJP+OTg+JjgCNtXy1STUeF-z4Err73kz zs@gsn@2C0BjC(GRSJaS^)FwahP>8v5lUe!IHKd;FD4mSfQZW9H8>XYJQ@uca>15IX zVKaq1t+EcoYv>X%galsGqGBXo-YQRVObkUga$5{<6}wI3(j#Gk^|Pc@JOkB^iQ|A; z+9Ou(x5=DX_{&SqmXN5((Ntp~HC_z0XU=Ex>m=PcMFqrnTmsy-tFHOUxhezKv(l&w z=VMx(42YA2c;mWBCVlLF(9&FW_?gJo$KZ#v3<@StzK+@o87CPgKF5ju%XW)|nqAeQ z(N&^rC7)(xO1Qj~m1k=p7?D9M3+h9DdxpeTj-K0-8GvPB-KU`rZda{m8x3XKd0tu* ztfCT>ULt=py-&sGtZj3fdY>ZEZ1nJ{ISP#mFBf~~zd^FS>vpXMQm%p;jk&&&l2Md~ zAi&wym4#ciee1vDB}+G9i48G%UyUn=z1osKaYjQDI%KU+VcGHj@yEYy?EmP6 z&8}Md4FJ(?_=TzdBwqh`0T8LetUrteIV`e%ClM1HdR@BU>)w82T|>KzM&y6qtC#({ zz$VzMmTeeHRIg#m!=^=HidZ?$=J{V{ARn6Zmg-OP(9B&hXLK;;18MV&K-T=?lmIF{p^5W%ouf$FDCc{NNYM1I}%ZrTPS!_&J&e?^282~uU z+IQ{4z}C!`l&0BV3C+BV*JN$B?q$4r-)n%seDL1C_^QE6_gf?m^mBy#DpVFvLwJ?A_OMgqZ{#uBrykxf6ATV7fVqAcJ=b_p*^OGHuCDtT#G zf{q9h8(kIxvp>Roc_q^iU5Vv@tKvPO?jV|Y0aQ=8@o(tB+HwMZAR0jc={=BcMmm*j z=xbqlYFmM-tYfpW26oS^WL$Obl8*7fB*E=?y1rZ7#O84}=h=+oTx!=Qi31r*3K7KcPPFIf%ja~@3qn1qK7j6kk-s%^>}qx= z;vaIdk12u5gV(y5HO&RM_EH@>rt`w;_}++;+6!2EKdmeue-sytrsIhc%};+0w9bi; zxEno`CJzFOMOve7rG@a&)ic>pWpGT=tXWwDYiQ@D(wsoPa+-?xw$QjsT5qyxt~nAf z5qo=KpP5!|rB9w22h@*AS*o-{eAjepp=82>GD1}eDerv~c(e~8g}8Ro+PQL>OUI~5s1u{q?p;zNSoWLoZI3xutx=pmnPhQj^2GiR{-NSq{wrBSe+yfQweuPiY!09&twy|jk z(qnH|E3|^;ysIARu%lb5yeaI9#Vx8)s@46>-n?bmyYmuLe`H^fHunGQPS!ivcOTj- z3mfDA^RGT9vyaW#5TonNC6gj4HF)10S%&-#(Y5#^K9!>p^1s_3>x`RHU71U8XmV|& zH++}q6cxtU7_WJO@TUjqBXDK7@LzMC%}8 z&kIJhPw-PH2T!4V{`Zp1|DIt?=WO}qeW0BcKEcYo(685~JBG;b7zy%WTYyfp%Zi;B z&vi~(OX>wSmFYj_I4{=unb%8Z(LZn21cO?f7OpnxQq#Ob=x$aBM5y*HPIG_su4|i} z^FOUh3l9DFR!>(w=8c=Ct|`|DG09PvUpCgCy_ok)fX(ez?~a7wV~d~E-DL}%UxLNc z*nNFNp`QR)JTo4amOL)cZaS~=OILG12MdnKEU}sRKv0~k+9|obfj+_J_3~wX49M%9 zRHznMY${oI$>y6cMgDqL8fDke8KgR-^yYPf0%ahwBc~a~(Uq6qbiHQe-IV zUAM?64-5bN$Ku&>4k+E8d|6gF0yPTGHFgn-1pcgmjDOE|trrad%F_og9=Kf@aVp*v z+(~)nQ4S70kxrm%bz5AWfDCuk2G7_}*bk?RV^{+p53SQn6cE=BT*C;m&{wM-n)6C? ztNje0m;kESpbl83Pki(NP_c(OEv-ZDWYi_C|6U01K9C>3(mNoHO1+EM|6qlt-XmH) zRv17U>$j7dyF7-qA)TgiF)Z0cSOldwuL_Wdms)A>ep*X6a=VP(QEll~=#*{?P2Rup z2XwUgwX%?xU!8FrJ zy4E??)C!&8!GBt(CN~DE7lS#Z*z*wXvSBiEQ>jU(M0*{kzuKU?U;N!pvjw4V{Lj{QHl8 z_D*@xrQ00WZc)W-d2c%GP3<}HK81$fKI@#rICnq2tU~Wu$`n4FqpTBZ9A<^K z{Jolpk`;9Ydu6YvN~#-f+wFZ9&`S+y4O4gOd1ySkd&hC_>r2!X`e=Wf`!@%R(N2Fa zu8`Izo14Oy`c0!UWa{*&xAGmGQsA{l?jou~$5TQYKdg3J+9Wl3(knkK{c>QxIH*^8 z?SgKmrQJ$Xs^Uy!0GbyTKQF4%yL3mXv{g5^EJX%i7Q`SmWode>XVjF?WV1jF3pOrp zBJ~ZgFf0G&?@yMc`q-As0pE7*xz%isFM=%N|i)*4}u9?T5f6gXT181Wn+Aap=- zJqBG3M|ZA8r%Ohtc8a?DiY0Xn^K5*4M~L+0v(juQvcju-zv(>EFyf?NTN-0OB$TE; z-3{QQ)-7*Zm#kD486@ScB^XrcGLzg*AlT%c8iBlUN#if%;N|M5SS+VK9f zH|h7gf=hGRSR`I-{PAbI|6T(`e!G8JZQSO!0nAPBW^<;$y_sKj3dKk8dspi3{xi+K zChI+Nf$wqZY)9|thZ5$x(@ga0pt~Mzqs*fAg^I`$3FX0XnQ^26^{NZu5JBeP&JO`_4-gy3w7L0 zWh}hFL%v_-SX9cwG7!(E1BOX^M=@bP#a057Y}1g3+Tk=*3S+;Y@mr0{L~X{J-sI6R z%@3z1sHDiL+mwZ=Tab)p$+&c&UM_Y`W6Tc>@WLb1aXrtMbCLIfXua5yK~-+&mfASv z;dqWujZ1`caqY$#S;vt`#-MpRw~1eLWP>iSF$8>OkUO}s*vMxsRXarp6X;_EjdF(q zMb;;zyjh?}SUFslL2HMYR@Xv1!HNrAeqS}Awh*txYFtzgx|Ug))bmDlJ`%v4yz^0U z&G|ZG<;=l{Jk^X7Tp-E`MM2;3#sePfqq_*;FghQtvwYR@$9HyS=jgCvIM5|4xZ_;= zw9kML)Y+{CFG{N{qu-cFaXI<%J<|XTnIbP9T|J`5sB^W2$kB5x9jz_M>dz%9`=e{b zn+PV2>;zyhLF;+?QtV(rj;afmU?%)h}*im05Fo zej_j$u5&wSMbk**0yJp_HD#fKNw8oOo^=hWEX6GkS3tG<(X4AU63&ApKA#Voh1_yN z=cz-=gVl{Aoj?NvY4^)wjLf*%)3(T_g$m*%lr^2vYHyNWUSo!IeV7WHjEA2GKm??4 zy*BDStX8l=$b%s<4>PX%Je}pE@j)z;nd&-bA$D+gihuy}xjs%z^0CU&yLI5|``2-9 zBl}5)imJse6q*|v%ZourW1%i4G`N}Yo*c!!tX6W9Ct_mi^>x*yy zWEyKhEX}XN(9fD+&UlQ?hFY-H?nNv69SJ0cRKBu0E$4HC!0h-s{?E-hTw~$j86Ut8 z5x{vSQRzVVp1Bg~Io9$X7_8Vy?8VDQ#vEM5xP0zryVS`A-9~D2q+h3Y)QN5v_Kx*( z1AUw7)`|Ww=xyv0_JXi=wan77HR=on~eQGZ|bgw0%F2 zg-k{C8!f%FJEr|AVNCwxyT-;~QKit58z$F9v6SS! z2$Oj3F+cNCuIS*WZ4Jx}hY6mL?GE~{!SxxEo#t4jtue`uJjGQ99u<__Ea0L%rVk>4 z(pfy>oF9LMb(YsIoN))6+~N)Y2oi=*BNET;h4d2|&yA}i@t9;Vo6$JWDAAd> zPBU|UWtKZGo6AX&ZFiOY8>VAa{Xmzemrei!-*^NiF*htzG2c3bqpL5d3R2%=n~%LE zvf%AV`(U9DE!HOZ+u>APy@{OIYFize95snE!@-hmDQbQpI+gi44x_0)U#$Vb9}!3! zV;dCX79#Wk^Kns}-_WDI%|%!e2Ik3>_pq+|O@?Ufa1tlp*?@cHT-GoLkO*d%s}E zlJSxHP{uL_gV8308xQM=K)owBIy{XawN=I}8O*J;9B8pS>C)2!HE6~`_uOa+OJJhw zM`Y_*>Rg=dE-x}`f!~5PE=cAiwBdqX+>2sqxy}Q3U~^?4P4Hi5U78?UY&tj6TzEt# z)=mHQ&UD5kgA^$bU#U?y;Tx_hwHb>bGw9sUxH7u^l+1f;jk;s$n2CWYK@;v$3D?Ck z*pvqdHtdba*_DdqGtHC3eGF&mAV$kUm&(nVb$2 zmP0Cso>uo&>S-~N=kyGMn?-^X&8=Pok@bW#l6X-uzI{q6^iQ*L?{bVoqU$R9+#joK zUnW=_uQ3_w1WE(kyC9Dt#_>!6+`Offr+^!|0^PW~4%d-exiWgxGMw_xgXr{KYs>Id zbNEU;&XZiN@$sdM5T2i?>)3SOn?W>;-;K4LH>SLg{6D&~Vjtgjw=vS%EBEex?^F9o zhk@Ot{USA!GMQio;Dh@1xc&H2n=p-qbgUNFN!5G1Iu|~p0^_^@hy^9o0bZQeD}5rE z3*FqpJ0iWyWgMoz=?ALMr-$v27p2a&@+bP2whiHN4%VKMj;K ziXkj!gJ#Kt5!?O5y{>AruWM}d1>}W+EXn95n4&<#>e=Yl`>UXeq8Zgu_yn1+f!5hq@&hliHqwNgjW)NXWL=y?jP*so(=W zr|k{B2)sqA0wL`AAJ@ep&&DI5&pt=|Bm(TZI52X!EP^8!i7gLPo5C!3%%$@a4!$-M z>&i6@GMx)0!RgCsItxkuYM=K?1zu~7T{Qr?$X2`Zxi#F+IW#R+71zNJpd!S1Mho2L ztODaIa&S#2n!UF1#zKiwS`T_i2ltwHaZ3M`VBYftQaS!QWW93GW{BP*U zpMP^t^RG~O*zqa7>P8^H>aV-b&zBb&_}w@)=@|Bnl3H`L&C2zetkD{xHDdN%Oy% ztgon{pVI)bjw(|#CdgJ$soc-c`i)pr?*36|^?JS^s&(j1fJ91X4b4zWGVFnDaknsE zwz}lC8=V(`xjPO<%A;=&p0N14Io4LCGm|4FVEtAU4LhEj+FcOl0@NTnWX?kK)b&WlvalrFk04t zoy()XzBsSxELvXA_w3gT;6T9~9Khks1z+=_O#_Us-?*J*xqbb|!$!q!lxfcX<=|Lk zzbJ5dHwLf24%}8HPn?zXcwVR@39W)juuGj)#ksz;o-BkLV6z+7Ru+-rFF~y|rG6Wc zMD`(?AXsA{=b4)$Ia(9UXP>%AujcuT=Za(H#5&Qk^fEQ9+LqtzI7lLAyIjqT)etba z=P6nwP&{L%>w=vNTj(7-rLy!Da9YFWr0&&78Cd+WMg?DzSf%*R!R z z>nk_E9<(is{)}8SYJET3e^1xFgfkHvh^~^-#rO#}5sdD|B}VGxyTo~Gy;YZtY2~E> z71hAfq1@Svr}XaiOx4Lb*0|(s4n6`GLTp=#FPE`OZHcV-$%qi{8hV_q3ETP|CFFHV zFSz<7g1-1ie}Tz=){>A~E{d$YSrb;@8YzkN_WPpfj2jo?lM2o#ye=O|K)wIeyZqJK z&6KTs$G?)z&0dmtRp8ioix}isE_E(48@zm}qQsd#nBO$ss{pRAm33+~Jq_Qbv`Q@u zAuS2X=WoW=NHLVQbOgC$Y$l)6A89(H!RE$8_8E1m^M>?;BBaB{S}Y$6=sbN-_KzQU zKxZ&9{IB)1nNyXE@2n0cTH>hT3bl>lNDgXAd@dIUH_za^Tq=I!pJF-$U{m8mgzBGk zC`}1jn*0%&@xuLO+T!xjA3r)HoL9BGyteAcrwn3sV~QS4Z4Z-( z6EIU;6v8@p0k8Rpyea=J2NuU4*r(k0-Ch3AQ?I!b?wf9p05ZTME_><{1RM zAR!GE$nXm(=hXSwc^Vhn%-mMeP0KdvX?2Op=Xkg|Pk*~7&_`4zE;+R^ss)J(T}yd` zCQLPLlhPR?i;+sFpUJI4ZUmECXvo98w0#KvZK||rpI`PI%{^mx29tiShpmy?)?h)S zHT_z)q`CRRSpUA3P(d&w85jKKJemf8R)w)kO+U*f7BgGD#dpayuEGgpE*7@zAfq8m zZ;EAlKg1Pjqim#9qU|m+-<8JFZGsZbS*6!QKC%*uAN{-l1V(XannT-DK1>H#8~{J9 z!Ompijb|=HX5P}G|GY98LBcC+UQyYB&LNNIVyLsGiom?V{4!d>PUvx&xz5%6w2-ax z#wGM^bs%@FvnsJ$udN3=wR#kJN$oqHTj}gn-_|)nF9sTmZKlKZ$?!gHS>HP*E*HOXR0D4cwn=~B6FMS9FdH^(W|^eW18uz40^ zw9l96Ts=3EQ4B^qEd&5KpW0%IR3zmOuU>@s8?lA9_ZAx#&C5P|K*u_hiTo*a!>Hfp zA>+L>$=Nt`3bVBA4O`BA0?h*N-eudr*8wA^alR!C(=ZGXgEkjg1T~)$q8qT7Uv75v zD&h)=d8W4PC5E-eIFc5qKL7N~)#1*R_i};Q*vHoBqaw5lZZr@drVGaXjJ{4_J@J>S z)z9Y)=_g}%(jitW;^AuUd8m`jJ^eX)T5A~!fABB_on8al$fk-t>QG4((xKXc`A*tU zhLdd6m{MX!si(>h?Q2Lza>|3C4Q}nup^SY*k6S6UolG!sZ*VRp|8Z-5=T9O>fJ3X8X>-WqM{ZQd^lw zxG`^r*7UlO(g@)S^KjEJxql8OP=#;VUJ#3B^Z4`@-1O+W#blse=*1w~nNU~7P z!;g^n$a9fOP$tXPY@t!dA(WftDu|w+w>;e5VeEN&)Oe)37|eFyB*^T}GKK!b<=k>a zdjY4g#o4mP@F*8Sh18tlZ#^Fd8`f#w0-h1WT8)p2eAk>-KdY|aXII2`!>=LyTNkJG zR;dBhu*mqjK{&9SE@6CQAm8O9q`YvrIM&29jcB=LsqlCw;956q3u<*b2J&H7xlVgQ z4irEKO*!3{UjPqIlgjlYMyL9^{=rs5l=p{IxkvB+Q6H13)(m!vwLJ5=5Z8<^13z|i zzmQ>4Sz_w0tgrKy#)fz99);qdY{=P!=we9|V*ve0}=N4+K-rbzmDOTmL*3Kfq zz1q5XrF0X(Sj#TysVDsMCz$lQcy_9R%T$URyy}>AYNU%7%nF=db+0fr2rmd&McGx( z*pC3^H8D$bqw_GV0GaScgTY;%MD7q85~Vp;ovK&bKNAEL^HZ7|Q8K;*)cQ^A7t2UT zsbup1%TFjbC}x5-mrIuh1Bc+pCDH0oVCnWsiYL!$k2=u15xp6Ght8{`b#ObF_RU7j z-mmC8p+U7l(~_ulu<|0xQ65kqX)X9$Q}n=xYZ-qthJs_P#XoWPg2>^hf;R zm$@$xo`o`2Exh+Ax(>|v#Dgv z#G;LY0e}I9fA`TOe-gGNcib-r0=ZBZyM|ftc+9Ehqrk-|^u32ECz$%c6`rCwMzy>f z(m^DZTKAtaB-!+S4y`0yFY5}^s6clwwfddK{mEo)g=X(=@Kn3bt<>-neJC-q+AS$* zEy+Z&pim1ZQdVj1B|cyp1L(}I=L#AX93)r&EtaYAS0nL2=PBLKMJ-S`-Dww^vK?ei zJjPhEs3d(BU=F&H5sGTve1=@=G0KzV>C%&6$6SwgFhiP{4@Qm=ScyywXwtnJ)pWH!=EEi6IPA_%8kF@@s+iam**AL}f zs>?=(C{WUHvs&?qb0Xbw0mvmsE!~xmEop}PcfDQr$b2@}ru_n3Qbqs(@;c&Nt!@CU zMZfRPC2-cA@5KO17EwaWd81^#r<;!47ZKkUHrYf9Kd%kUsAG;~9L@`-iam_%RXb&$ zJ|q09ZL_M6BxE-OFza(YmhIx&;IrC>u|It=L`{hZ1L`ohw{doB;W$jUsD@H!+}s!u z^Q^GcTvjj=lw*t+e{~@&j67x4F^5yWNgL`JYUV0@LF~*xaZSqO^-yLoJ~doAmYkAg zwc>`^*L=S;@S8T%FKXCFv(O2q_frlwK{m^&Q_Dzo;o)Z=rXQ zNL_?RLTBU)O{`a=wd2497A1+SSoz0dfmXjUIP~m_XAK{jZnU2iSjHZ>hRu5)xsdoezgx=PFL zt7O)itwPQk*44FeU)ddKkDtXigFWEIcQIrJf5^NRMv~wRT2Nck_`@uXR5!52&J!0K zoHr~_)>%3_7K!oEtmOpuYe?ftBKkdKT$a+@j`Diu&6IsAsIkGDWs1Vg477z*lg`0KL>0@@wiNx^!ao`C zw|~S>ElEwS7}1fEjwStl{Z!+%&O?^`9PIUr;6ZdG*;VP#m!|pt!G?4_nHgvEXZ|J<%&r*4=<^%4t;pW+H!6{tyr6`jp+2J5gF)$^}>Dq_sG9apK^I z z^C=Hchs2QV!+9gKC1fU<3VrFEel>`_!+KwgiH^4%25NK-6`uEdsv83>V;1~|Rb*f9 z9dRLU&f~~Rm?BCntxses3v}pQIW%?-$Q|n}25@?f;x)2cyw{fd4TG`j6_A4Jr-ru0{sFW_Pp2MTSADJno^?p*A zxa#Jbx9K!Z+1;YnhT>sXRX-r|#NB+ovDn*`^-6)24{vses3$?{dd%hXrf`+Hzu5E? z#6~)vk5^K3=5$*?OX_)n7u)P- zP7B6qjr6O`CBX5nV3*V5WSez8lUxIr@gMT1#2YqG3+|X=RJCvxJAV+pyY-b zOn(#7oDkze;>ijxJ@Zvjvt-=Rn77X@iiif%$Gx)>x#`^1zZDxMl^_aaHxiCEdQ!uMr0bUqI)6TAfaTvx4qS z&Sq!?47d9iW33;E0fT37gNedCGB^k++^v>olvah{3HGU0*^*&#PTFz%*?pGA%3jZ9 z&0)6s;S$G7G7AVplYgj_^jn|v056Ks_8c>@4F&(aT>fjU!J?Ah^~Tt|U^uUzOM+IX z^_&3XWWIXw6H8oiNsd_TdP%SV=KV{PEZ~Ed4Bk^uvg3xlMo*`y>&w`hl5f#F<%SsBy^dH)9!6573s6NPXve|haowb0L-nKk zqju^=eUc~v1u%?GD?F;S7mdaqGMU9}7tfi}I_dOI8Pw@u_2Yu@d;=6jgJGabC+i~8 z03Jy2x4g}AZK;%_C83ipW4~;sl-=cLm_x!qX7ieG{qC}^o5Hg=4R~1KV9YxZUzxh* z+KH%%q_)d~=8hq0-JqLq{-~O3S3L8i>gNcu+HhSvh9!-wxaEJ%~bO|VxQGXLHJhaD2(~Cg3NY?Pyc7Nq;m1dyv_Y#uAaINZJ1Y+(Lf9KK^k;MFzHW+CIUxAc?#E7 zna4rnr{7Yb(};6x(&bj7rLl6r;+JI&2KQ55ydJN`bo8EwI&`Znwe~6%aKsD*>3PV;xAIobAZcm<%p$8u`w9&IWPfd zPS3ql$CCx^j*LwvT}BWX1#z6LcjI}}is{@TR&|!!HNpe^o*%3LsfuJxs$)!-qdjN1 zgc{d;Aka_A(l$R`+<~(hnfcc`pd$qGFsE1;Ge3mE>^EOE&dId5^Q$!79DUx>@J0n|2{qR=x_8VqNtq4X!YsG!<(5O!6JBTNB{(cFtbaSMj-MSL-4qz__ zl@cwooEd1Zz3cgnQOyFa#adP&I+up-pEbif{aQ90FGowQlZoAj*k?%bHkGHD>G%NL zz^sd?Pgr9z`E>AoV#ynYs|MV(l*u6I5+AX+&m11s9iqKWiSo&&tW{{!)yedp@8kzH z$!Tb!d_UiX3kNAKpNAC%^ZyfF`*ihH8oIR?b=^5iU8R4$n*uqBjdfy_d=0eiGkI*@ z+q1f>VvQSIduOlSf3Ee$xJcRv?$V10!>E9Kpggc;!vMVTEWs>@my7XE*<8x#8rPcj zL@%6VrrpjN>))up-224MVS@4&b7A>Pu}lo*45SUU<$Hzd*w15Eg$?1$`QRr`dG6kx z@?@j2B}0c7w~3cx#5kQ`{xtso`m_Oo9or!$Hbm$yQDHs@_O`GXNz zZI|f5v7DPzsQM&?kZSEYIR~lzQjo+cq?)woV5c&Q^cbizb#Jw|g!LQF4IoaKIVu22 zsUBF5=3oD>X)3RHo@!+C)80)Tv-uy}A51W_pNq3N!cAzo(LAtGo_5u^VE47H^2NDp zEek+pt-xlmmaBah7wIKcGu$mh_!usHJ@%tWp(lPT3G7FuzA*y~`2pueiYc-{qXk$w zwGsKTP0Tv|v84;Pc0<3FgVP};8XyBtM@#t>YHF?TLw3pf2O>IcN5J}^>G8eQqnvgV zBnO))l^_$aNiI$6vz5tQt1$<53%lj_9EYW8VeOXT-y{3TWlJGm8Ku#>70tR*h7dx1PS^iJoGQ=C zW2%TYSpydOF^9B-HU$k2quJg0b#eVHl00(A6e9Lx3QlDiK}D`pgfSnlxuy<2Q*U{% zFer>hF@LApr9!qg8f;EsNf`t>Jef18ZZ0Gk3+M{MIFAG6d9c!v{An!?lMsdHU23Zs zOvgl=bC+#J#Xe=BXAQ}b*SDi(V6+2BAQw)CyRq69*{dIGCSh{3s zC$Z!_DL&E5Za7nfj2Sfdy7{|tgJc~cgS8Jb#eD0b3j1|@rP7FFZwFU4eDk|9Rf~uh z2-*cx)%;xY2|GGxB)BVmWv_HrywVOm4qbs4(buE-FP>Ie=^?zb7Boi|SxV!G{_$-3HLy)v#d@55*CtbcqAz8-_ znmcdM<)wzNUtRIwT9k;Yb%LeohOdF6eYRRmOU`CK&hqu@*?jx>D0V>Q3JjyncK=Av z7QU6!3!8mg=iT%4@a03!o6dK zyoPqL2UR4hl3iq)AhAP2i~{~p`MF5X3q7TQj)2PA=m9FsQz~>`w31VrMXev`+Ri+` zoU=b=#Z@~o)E7I}H5+GZ==9L%aAjkDV_^KBVFM=zjVwdp7;U*zhBOa{k>oQvpQ`D7 znkO9NWTSo>VyHXW$FMc)fS}LAP_mA}vb0Q0YnGRRwWl(lkB9Q=k59e5<7c8g^l9Kij_E#g3Yc5uIWi;2Fs^F*rV z-u}7eiq44NfUD)IOmc&&L!p-K4b%-tW<57R=Wn`AsA(?RWhpg!Lmq=~1f66pr%`c` zJak+Nm82{_g8yFM8^5F2JtL2uFnDb-R8wjBf-q4aKmIvIJ6f)Y&!g(X)XaxV9(V!f(Lg%xA>#hnyzIe=2dTG zoh~wtRk{vt0HPYr{gpgB8Uxi@_3#l0L4E+p!{xqu@~-8)wN;#S$*%5R?KK-KJGhPJeSr?UhRs;k967s!4ajW{jG4A{{ioM`(=KJI7uCpK z$_hCA{_IHqPPO+5_D@y!KS&ey`H&EMeOieAEV+NWB%3WMQ{S6(DgC*qtbT zWwa!1sVbEGSJ{9Rr}!mTwA&*s?1tbpb`hmoiqkBJv+&#f95I})4FmSF`}$xW12DmE zk2O!(Q8VMF-WmLk=gAAQWc@jGqzAcT7HhZ=vM`S~I&Jbc1LymOfe$ZM%HJY2B+nCK z=T8quO+RZiS@qY3!N$?#l#Odzc{38ea&*S;;70%ATZod@Q9tJ8&nE2NdF|DCzXjO) z;&oT*5U8x@`fK_(drDIcjUwN+>Y?+9XbWq>C5ZpSz;VMRm9?L8>iZc-KyPZ=zE)oB z5^1SB-{M>Y2BF#k^RBBmR;pzjDg*po9ObXE%Hw%m`n8K7A5+n(=QTI8?6joMMWBK; zO!#MfO0+hdrd#_!Vh!7bs}Qg-h^TMYHB1Ap?hVblKyWqI8_9@dbwolCld)vCjP6q1 z0^zML0Mb#EZsO@dPUQlVh!R64fPSAJRGPK`i${y8&>)+j(^zW8Jljn~d$w3jNzNAt zS!H0?RJ*?pm>_HG*+LLYMUN=fm@hq2^%AqcR8ZSRY;^9}662pBR8V#Yla=#I1*j~q zVCvVn#KowF#_AHmvm`zZtbi0B{e@Ymm9iPjCY)^7+=-Ie5af(H->@UWK=Y2n)D2Zt zcA&VGz%Ay)EKZq@UK>bIn>k1;X~;bqIHat8(D5@6G)mJ;vg zN9QTAbkxs7p!HSuAHiqdIMDCvME9AB*!=&E0i=8ACYnkp9S>6{`O z*Ja~@!fybeCV0s6F(;JZCX9;7yfG5&wc#(%PJ?OGYdJD zs(L3+{!6;`zE=X?Ip_nD{cm!6gZ>3fx_o$Hz5~DJvlU~cTF_Pi%ZK}d$!AoAKF&?n zZWso^2+55!KYCNUWKNV)|F;o1)OIuN6?lzhWn2S>i&z^~B$W{$l$;pL<{p`f_hKD51e+#1`4mc_m!XR&aFb+beh^B zQ#$~1F;OS2uxXhUHgj>ihl({9+zv|Pv@v%+v;h^|B2Mb&<3#5d;QmqBj z@NXigUu=T3AsaP&iM^^gb^A{*0KBF4{+@cHZJEms<~iiQql~&YmlaBGHc8ek(7D0> zrx&uXe4?PL9^s3wf%fXxw7B-NQiH|jw1;p30vsqTD;nQPZlR@?rRwlz&gVig0;`btj3kF$$;uf{=wm%72~1=boq!(x}qGM)P&pC8&E2Oe^99p^j1eHeTH{I&sN5$l|5{pEIBCNJZ4sjhi0 z!^Vg|VJ!-v$S#CU8x>NnU&GX8%2z)7^Kq3PLRo|)JLwd*ayS z8Efkgmeq^Toz)PlFi{L#?_kP9YkoJffp;pTRBqf%)h(|e=`G}?D6_uCS6E%^Jzwd| z_7693FSf78bj1RhzaR_|Oz_L%n4}T$+2?jw?R#EktF79}BOy~ps|Qf%j;ODs!E1&^ z9cF4ybL2ElOx-5qTg-pT<>?{RLt0x;bYE2`!h0yDUt0&DXa%djHA|mgtRI**ng2Ab(GT?ExJ0W9cHjI5n_> z%p_reGtK1Ej|5g-j21LfIP}kU4TO4*?yVXN|0EjMtD`fBN=Pq16D&Khsh)Pl&2mXM z2{7gKg@jIrswp)mujy;)IxQP5{-%G~*}zRw=StrIX7);RdgU{Ha*R;`y(>z!txnwI z04JQ!Y$jdf<vK*W4#vcnHgzb=(_ubGMVg75i9i-!5-Q?s|QSPX0TL6g$f zS~6Px3rKIZ>@8>=#VKxUt^&4+PWiFpxZdKF1UL6hUv8tEzpF-gdp}9?42nqIQKIf z;eoLphuUcwKI`W*He9Lb{5cPg#d`O)BC{s!NEbBNS9Kvjij_uQe$wE2-!-f@@_bC1 zK3SvrpqdZ2&68jFjGgA^yG*b&U?Xk7#@xN6u`;jN5I%1Rp7Qip-N{h+)Y8ZnVdG-m z;)39O5Yfh`q6aT>xt@3(j?Jk)@Ha@8zet#;9Bt3A@vSy$4V|^jg@McuIO?6OwJy)i zNG`04?G%60(Ni4FMr(RITy0r~JKmj^5AMQk|J0v5oZ8|dyq3-~>r^lQ^FRI8(eF>{ z`297Bdf9!V(I)A3m5t`A`hY2+DYCTH5UhkuRHfNUjl{DP_b)4Yj9@e0ZxbAQhv_M7 zpllY^kC9)Z+9+3hZci4<>0t=e$=&WjjFAV#7lUixQo{>!_wEo zSYw)dI1%)yxf;3tI#;VLrug?7Qyb5E-fx>~m$-J3x9w&RW=QLgbeld~axyG1AsTGm z(Pu4oR>YlGq(fvpqOhBF{D@P&l*sj*(S^ZJrIZjrjC|2)U+pREjW4fT#%5~g?I{6D z+SA+*tzv*khH9W3ZQ`rUpkggknpVy0kEFClKP+*D_5vPK&BGZCR8_}wEv5iwxT$J{ zz&vQUa#V7q!VIZeWK_4F7jZrIh!?51};~)Ro zZ^$=R|I-X=sqhicmRQ}s(d=!7igj;qC$uKDQc6=2FBDo+IV;A!-7$B0+pes&YBAB> z@z|^Ph{?xG>y@1YZz#E(oSAwl=9bF;!$rA>-c zc$`em{T4ITIi9yGB5+HMzSu%NN~y$*=Ny=ii)Iz~g=*5{9?%*)KUFUh-fFpB%%p1o z$o@#H%mtf{7fprwI)6XHm8uoMWSyRS^q3A|jB3bIU=f?zmZiC6lqL%iZB8xH!D@U? z&y8v&d|f=}?Uk40d7qWH+1^l{7pm;#ls{H03?sRoB1QK+QKK_M*;s~U7UwQKB%Y*K zfg`obS|9Ewo(+?d5i_=DFf=H!t(WL^HXb=QBd7W7*ZB$3o8!(u8lozTHr~1{5I3JK z7Q>*LV2*Z`#}UrZB>3&IclalXugz5oJdw|ArnNyHSdGh`ah4KsUP0mko(BXQ31omwK*-sDlV~RWzpa zsVw1_Hwy2@&Dg7tPC>3SjtP4CX!Z6vbFLUa z@8ss75%;IR(rx(A`Z%VyxF6-_AOT)rUf*j2ZF073Usv8*w z0FW?lG*wwdi>x-L8(*Bp!&j~odMW06?TynP^G!W)?mWjha^1t52VOEHzEyZbscMKB z7r{7XvHb!@i8UsGgph1ZZai}$Eo@VCSzDKW>bROB4!7$mPD|QKMFXT3%C)vsq+L-r zjFo!sH|o+EP3jI|BxvMO(6mxRF{VyhG||;O6SMg$nHsgk4E%>wjKQaVqo6hsrgPvC z_Q6I-ih#n)oShRdGms0kzVW6M#)WlgFMA0yy$0Exw~ca82Y6TIgP@HOMvIrf2qH$=P(%jt38MTbv~0d13Ri=<}0FRoK*pU9Yc@XyJ^`vii`O50JvF zs>{N}8V5MIABupjFOf$MQ?IF$bti{GKopGtg5!s2Z z{VCVB)+Ovyn>n*(u)^-oCf{tus=R)l7hx@Vh|SY)f)(fel=z#{G5x8!W3EF+36ju9 z1NC~wOfHl2HZLKR7gOQl*oRcw?@M~^ge)+Q2 zId>-8rP&%l_HOAfM1pOBMUKq^8TZppOm&GmW7SjTXI>QgIlo2&jz%(uNBIWZ>=InJ zgYHfS#pgt0raiNDS(iZ4+3+8`IrtSrD?~sx7MEjri3F})*UJlt4%AlOKe?0GQ%Ob* zU6Nfaca+;HxK{DrR=oqXi*!-xXG>RR=Z;dLd?fWn0G-Yqe0L3)pPKa!0Y+BoT5500M9 z7)=|9l~s3uf>q^QEG)S=hu`=-qNz-`s{r(1beE~Q0JCIcbWYJINtf~XgP1F{dq+xw z8I8gQ(ZtXx4jorc? zjtIgtUn)?Zpb{^exC-$z0+y83=h;l2`Sm9HAeKU)Ylb=#Fe}ZAwm{5>4vUZl=q_Ft zwdVJRd>hc-J&$S);oR{0@VE$)N0(I(ZPn=NdX!l3>P0BOs+}agPRdO++s{i^nt|Ct z*4|<5%iv z%!B^tU+W6RcC|&f&jEExTOyZC{RMD$r>-dZ8Q?2^c@jdmc7G9srsUs2`zXY`ATY0S z1H!5rZOnS*b4}u#jdvW9WqUoLS&R*Tce#5rirZ~KJKp8}KO8^~*;=MjEaJur9cwtPo=nHH-4q4bK-S7lIZJ{*c8 zJNCzigEO5k>8F{h&QU#1Oea>?Jo-~+-jSqI&o!Nh6yj=AdZ&d`;~34f25jXxbfh)U zZ~=_U(4Ox#?tU@QlQBNXaqnyUcBXBgD~~cXbN=wan9R0c>baebJrWRP_s`HP&kNc^J{3=%!5? z5Vh?`#xyLxzeLygLBGhfh+($^jg3mfoQn{oZF#$>;B&wWs4jDh@vjhOIoqWGyEOAd znY;@Wz<@9uTg7v_47hc#bIl=rOqu;6s&s6Qq}F9HH(4EC1{)hJ@-~l)V6QaAIOp6} zZV)Fhd8#+STvDi{*n`8V%vq{}@C1E{e3W(@8rku_=y4}&)6$hJxE;#}9BQFf+q zu}sqhD6}U8)ks{D8frE_Yr{Dk&8c<-YQzH@ z4;MdqT?hU61d=uH0d65Vqca=U5J%?159pzBIK@u46Vvv!<`3=^wgtl}U-z3I*^H*` z{!54$!r^35iNdnsHq`FwzRMKzdu6kDU&IV9+y%MlMl4sg1!NG>>ELl0o;O+RFm#V; zkeQV8-le}b3X*d@!UG3;qko0*UDuwc+KxuBZbrGZ!Y@PD>$jEk?S`5yqCq!)sx3h% zkPpTP%03Mt2zV>5kOwPW9I9973g?pBFWK%l0JSVkntKhXUv@C@v&}*o$j#tFlhN?5 zXSwcW>FT`#l^d#j2Z-0!B%ZCAYg_u9wtC)-%wUhRHUhXmpWs#&J;G*k-|@>?>EpZ! zUrPtnS+V{gCo1IKU!vs|z=H6CS~fHB`#De|zLe?q<$B1yz+58tqA;(uS!Zr(SuY4Y z7X8@#*W^sL*8<9xrA4rSVv21Kg)=3%6jJU7H@QIZctw2r%qm1>Huck3slGnQ0B6n5 zQN?)(5qw86b8~(OB*tYt6&#BX1*(TI7UM;?2c0ur-iI3Tq%VLq4uQc3bEr%=Q;zdo zrhn|lXPG`O;@9?wfTB2PZM2T3AXK0yN$YEWIXz{T8&^Nglp9ZxZ`dra9Cpaoo@c`z zGKY-$${PtvYe~kqau3(^f~+*!KjLYD=l}&~7CicaBbJ#??f-Qei#4PJpN8<6iYHssUHO1wBH|Fi_BpM6;HurQ~ zKI}N~x&|&<=BTBehe3F@N=e9 zFo==?*X3+1x$CfG!#415O4BH7>26^5-D#e_{;ljzPj%u#wmM5qb}~?9m?<#|{W>58 z7ICaF+ttZk%QHA@c6!@jFy{6xCJrqIZjzjO!(v= zCZQ}H6i6|i4)`&;YiE4!Jw*yQCCjXy>K?No64g}kz@%e7j*K+QXLcoVz9vd#a@s&2 zg>;Z!*Q^PjFX`HM0i3p6<8GaWajvALk5ESW6 zaLlULvTVo&NP?34ZnuWI-0ZPS^%CiZOdYI;P}P*LK2Kxkvq;O07^kLp5OICQ1jVE^ z9kb#TasizWQ}P)tsl=bhYqHfIr>Y!blm^3*Ox=QaKDR&F-J3tksnhRPMP*-+`l3R- zh^JX2P%$4Af}^OHe&SioLnh3>k5Ve^&@fE30n|}Q`xy6RI zcx|=6zgqOjeYSyYdd3=!;MfqQ8a7D~6R*1~Sb zx^BH7M2V@P@2}3|Srt zPz`$Z8FV@Q>$6(88r7}~%%`6Oczgent|HRJ70r>VbDyBqSFhNw^ln~aFr#x(&vB>k zFeY>PhGvy*Viu!p>}T*tzB(1C%qn-Al86bD*Xe#iILESoN8F8%_{7y8z&V308TFz5 zyx;c2eKt#f2I{z{B{x5qC!RJp6}nhHD2Cw`` z{(z+c=ut~H+es;|O$}15;z%~*hxKD}`c>0^iqlKNco_nZ^&?Q;1Tjdi=8%rzJ`&TP zL=f{O`g%;fYnYrVH&4f9Xbau`<`Wzne3H!qR7a=d^0sN*jM33*;yk4xX!USZSh4mE z+0&zr=tfH#hfk&t&8!Bvb-<2pt=i%g+ig2y_tz-wovWGWX1T1Q?$K$_1u==L)_cjZ zm6y{6vw}Fa@9cDeH+WXbLt&t?H$0{_9a46(kH~}hN?E6yS)DhdVxdAAZM}6Yz~rOG z0;dX=4g&U=t1&yOuV)Pw5c4yW|Mic+P6>L1Z&b`1EcY6Z-I30ZD0oH`4+MabS+#EH zy|!B_=fp-p7>*D!dq8755*DJ(HtuQh9S{ z)>)lLfu*)R*H*PgQX$mI_%fQ_cwv#ow1_XLL$XGeTOG^ zFr!^Bq--6~Aj>;@uOLC?9bgj~Kl2u9qkzJ^<$z!Ay7VlrrEAKG^|ff9eRgp6pqHta zImR#*ZqvUh2`*Ttife4o#A?D!(Oe5U(@>v!)fH}TTw6M@I?=Bn155vk!5&I#GZ=4K zV3_}&d+$`)TPq;SFCqs>1D|j4bZ1fbiIzBf*qGzZNzP?YFiqpP17^;bu7T<=;SAARDCnh)Q4|TMpTcd+-?t z#-uzyKF4m2n%36}*pxr|=}JH+zA`_2gClJ25^OA0`!X05lTI_R04dkJ7WcdA47@VZ zb}mlnT-y;2X8D*bX@iz7B#Wsr490l=-~afSIiEwPg1obVi)la282QU+e^#T0H4YVS ziVc&~Uyb`!d4YcXxRD*VZGLV~>w-^uOId%jH}4{6MeK^ht9RM3Oy2jueXF*k&Zjth zy|d(V!xI#gcdUkXUIgEc-&VqN9-JJ@zycM*=0+dV>nn%`oczSIhAlE_=e)2+q01d9 zdZc2g7IuH3Ov`a?F*XN3e-~%~4Wcyi5eaz65WnK8YmIbjS{p4ZU1uz~9O*or`5{4& z^|TDHQnoCE*wEPp9k9=P$y;I%X2mgrdRm!gQ+PT1?y6yga!8VObrdU7d;!%h^EHJH zuyf0_68&Am_=)Cb#`H!%MWSOoyW%d+89yLMsVpI#uX^~SBPG*VNR_{=u?pP?s3;Zn z*e?V=R-Y^-UeC|LgqZq}uLK)-8B&*zTvFgKG)$M5s9c^)o( zuv0yq&M51G-;Sq39h=MLX{c7}`Ak*L^m^ih(QGU8)l~)d--1Y8ZV!!e3A!n|&L&_w zpB!zFo1eL>rAN!lu7+E-&8baf?S^4YP3ANO({*$!HBc5s5VeHiXcipHh#9_@yQ5;c zgf^Vd;7y3ZV{?AG3Jd_jlgrd35^QE+(^{t6$k8!<&`H%U&{7Ivq(vfQM4UPzB6A&u}U_V$D8QIVZ!PrI2=$*MURGoboa^<|;!y_$33`zL%6h+_Jhc#i99 z=~DA)XnT&AXe_l8b~qdLeQv{>)fz-jT zMvnTzs)j-VqStw_qiXk`ak}mfcSGo{A+mzOAm3mGIX$`QbEbolLHC)ZF{ywMo15f& zZ->W=qr&_j4?aoMDrYpt)E&dzt|hIQh+u{BdYyZ{vF|j@nWVoaG)o?|K!E@NKmbWZ zK~&GVe(}e3@N`60C7+CNu43E2%@#5vWEgn?K!eJuUIr2R8vgem|Fhm;)g5%}bm0es zy4`$5FOf!BRzq8sd2OpM_oKI-s=}Y3&Ftse8)mh*qw9h+ctC4GECmEh)rA@x|LVJS z#C=;dJsu!C4582KtDgXDt!P*1y3s+h8kZ~9R3N=PXl2Mvt^GlgfBx?u(e|;Qfobq1 z+}4@KOmZ$qr4mwZx?R@}!s~~`wl0oJ+OIgobOc&i{91z8sKg4x=v_`rZx2{yt-q#Z z;>()LOtmh`cYgr@RzRu0t)VMR@SrGm>@q_`BBi5#DVur4F`<0E8Pze*m%d4!GtZ-Ouq?=e zWU=OHPL{p~ke%*wLn@TthN6W+?<4UAboiq+hrbNqr02Nc35F(>6)%}DlA}klL1SFF zUiZFW;eh-N7ak`)KAO*dZuv<2{0ltXai^5H=XRo#p5o4D%uW~3{3s0Lnqx_MjPYF) z$rxUa9P?WI=HV9ZM9}(LF00X2ePN^{iVbGhxjH5IzNmyP9WpM z)aeU-Le2wsC!iH(f|*s-Imu@=>z4U+CvUXnmDRLwNvg6q`aZINYR#GBAY9<_QrTko zoUDCa4Ii4dTk@a` z=?2F<;{EZOpA2NnQz_$YZXFVMg>OYWm;oH;$umqxeel9_ExmZL%DBLQXZI$B(AE5P67V{ONQ>X3)BJsx$c9s_xB ztMQa-jnD1@M07lJ@OrqI_U2r%3+2thQwukQ#jH_gLhoVDsPs(7eYI5FMr4r)^|~&* zM{v=glt83-X3zBKFGcVwjOQX*2uaAx(e-0?+xZM#)2PsT<11P82vB6b9+)H`I7{*w z)~LRBR38*?_$>-FLr&}Ej~6gQ`M`ME9#)4KK&%<`nB=bKe6a%n)cI6q`G2{(7d5%D z8bR+ZZYljX6r%nodj5-2^_t4?~Q|&?4tg zI}k5y=_#mW)mEsh7@*!USJ&vo)lm;%3PY&iXWy?=Y1i^b?%B*MiFZw#-~4~l-R#Wmy)SIV~d(DU_S0xYiPYSG7u!zEci)Sn(b(! z;n_TDR$5y12m)017$Wu9&yXgsdf$;|EIN2RT*{?ZOf!5w>W-7?Q6Dr0l>*M31EH{l z>CIAvhuj;hM+3yHXo&{<#mynY`0N(l$|v$43!i0 zM!`lNTG{#bYd7OWCV7z<>(SaIqL&)A7a&Kk5x>4SbE}@sVV&bsVtku2NewjdP-Do_ zOT1=JqW20Fn|G*v9T6Q(~>Nj3Q0MQ;ME#C;mZiJPUW_UAR)jxWU>%cVGp zD@PcHzE?>KBowJxonWYo+v#Z53V?UmBF(Phk-|t*ECo+LwF6$Mv-OnKU3hV%4X?NT zSd)Lf4Gc5GDz!iuBQfP!n-kME7|~Cl{{wK#esRn={0kGF+n(dXTujjm9PVD!5Kg*O z6I{76^NY1Z8LtQFX@;uUNH|EKz1R071O)72S{>E&h_}5asAWk;fzVK*f=jO{NL4hw zG#759CNl+y@DH6Q2|t!zVnoiZA-AKCJ3)a{ltGi&z$(1b0k=!wLA&w|NoGk_HGgG? z`y3b2ObkmW8Af@%JDhW!HV-E@QbWvKFHu+>0`#qtHcHR(K6bbmw(lkqu}74hd*XGY z$bz7APd?gC1|Dm2ivn^_ZDoqadQL7+7bRLTS)Z+zfVMiI1DWVA3s0ok%6a0~)Qf~* zJn{mA)p*{R;FfUAayt$S&3-{I%g$Z>gQvCyOwhFKbl zms0YEdV8R|nRqo^0&{ddJpcS%tctcn*79lbK97ON-T z>8XDLtqw(7;@6I9;jD9*T$fH!{l?D(7Pd}M8^~m9w5MgB28XNg)i-8lI4ER!EKcf@ znQ)D+jT^OjNxQ;xBB$YP7giNP|M>ZRcNFIrH$X2AGdK76^0>LV`jr=)XEf-=K~#g! zD=L`0ejn&+HFRhse59ZFu0M9t06WJp0KqDb@n0a;<0m03+Y}|SlTry^lMcrD;lNOO zWPde458f!{(p(XU8EDt}tM8Ma6Q!To$+T`J3{@=K0w~8Yb7%x;fB%XT5P=oF#Mp3t z=ta%$=HZWgH2Az=!()iti2f?L{(4=ehyvWNK}~iO_@z?Zd|m)C#Qm{QK{?=@jM&ml z*-i>~3RlF-hu1`SK>eyqo&mF2W!0!;#F66i85`XbSn@2FJlL}AHo^4Xc3sj~c1j;O zD%*H8*Jfi3kyiIYT$tt&r&X*f8Mz%$AJ6$K1Cx%E^t#k3K0&K8QaWAM59k#rnBI37 z5n+H%aL2U)D?C#<6~Q2XuE|}RAi4I+h+MFz#n1^%>N10w(bSGjMNg!LFUF7TV;1yb z=hDj7Qi>+X(_W;GOEqC=(1(yziYVSIJ(Y8y(rd@@pwau`usJyuXQuXWdNV%+qK~hE zUu&PK2ocv5&`Z?c**vZl%?{TR8;=7UK~!(Ms>@vuOYEuSZXq+$UtK{`j!>3NUSLa_ z1&Ip;1Q#Q`Tf4JWIQfEqH7-m5! z^i-X^8Q`UvwOv9@B`%de#--t0$>A*XXfJvdzl`cvYk`D@&uYpbk4~lt97*CQHj8%i zd^?^0SG%OmjL=sKZm9#3M0v+{j=gORfjcwkNzOnLYf-A!$)Q#|o zRq1mQrnLFX?hteU?AOHe7xJAzM{7ErUj*z`FhD)#>^_HThbbhCFU+hdbDSB^JD7cnY^BW zMsIz?ry?G~FZ?fv=m=aOh?9Nw0ZHXNcvmg4sQeN2EHd4zab>)kjWR1{>!duMHuxKAW;zC0wAkAf~wvPl9SIUQy z;7>W*T}V|XRHGGm_0_u5)e$H8ZA-WsX|xhFAC+P!t!md=q@L40v65VTq;OrjBOma2 zSYPhqn2?a};qr)-{SX)BDqR-@Bn28!yE_yX*th3@#RN$@Nby?Csf1J#@M^$NsZPO5 zM~T)c&vzIl2stpKNIk*V0ZQoyNvZsU^VC%afzCP+WL2n5dP-fnt@n*sC09vcy405> zjM*;_-~hidAi)5wF<+HZG=_2}cyTYX3;JZ(c{WB4nuAg=>I>;CJie=+<^lcHsBU_7 z9Tq9v$t^R~6;wh`>t>9%x1I77bis`mVv{Cv5G80#OWoS6s$o`q5jpWk>_LdnwA#%! zVvpa%VLT;|&A>}-zGiO{q)Zks12sqGg?xwCEM%J84^pI;G_#^{7Rt$_hU;>??)=#( z^Xp`@lzV#9YQqC|fb-*7{m&*d zK%mUjSyZFvLD_T49y<_NPfnBit2^YBL^{-1#J#Qk#scIqQiA2(8K@&|`PIF_Js_3l zeFoo`^mLyG1(M~AZJjJRG>Wk&3q%Shi>eH*zihfNHkreY6i_s;2U{4R1?so{F~?Oo zY@yJBTJ_5(B&)#u5=|o;qjFUU?{!%&RttOb>Tp%xcI~*zF%3@c<7|dwG|(v3#S!J%=~y7gn;Z-9_F&^Y>=;Nj=-8B zaj~G<<0>)%tR^k^@O%znqY@iUM{G2!or{%R1nW>~o#aVm*eUeKpIt9b?=c;sD2zs) zg$T0XE<)v0EyF7IoX~(I8|Oz*6BN0dTlBC6b_9@JcsJ>dur7O|RlwX|To6We{TX;FeqvpNpb}g)Wz~EQqOZ04r|0 zVZEA3U%M)2Ei`-{JBc$;S4ORx+k-id6>s9J{ZgA&EwJ?qQ(8Ow>k^bp06*@)UfVQh zHi3|i!U>dC2pt<;)_Q;p4XR7ZuJCf=GRx_@51@&X-+mfgdSC%pcGxdq(ctEZ9k%o= zQU-UzONrpPbN8x?^^U^NZ7ZF9^BFo1k&Z1I{vr)JRNz*Z0b^1xP>psvF z@XPwDJ8)*v&%*r-D0q@n4jUy-MB)OR#!F^$v&Zj@hWLSgNySxQpoY-!NB&fVFX*cz zajZ|8vI)5y&bBf;L0s%B69LaN3XJ!}UdJ*l%Q74Ym{)D3a{9peJqtwr`+5{=fniqE zY_L<)F5OB_m&}rM=7;@AsuHxq*Dc2m$d6pI4=+XR*bio24Eojg)0x)G3TdV{L?p{R zEiz{Y_ZL&~K_c<%57ey&@{AL&ZKsLV0T7M;it0)0k@^7=-bkD;)zdOGMPP0?NWCPB zoFK0l50$fB3ia1(9lZ*#K;cTu4!0($1@ea4!-7BokaNdxV@F81#|oA-fO#Jr$j~~i ziwZDyiprOEyTI$%NBC7HCV18|3=iz&^T`5+6e|RSA=0~?2=z*Nt1agJOIIBsopp@$ zp?dOd{THEl`>VWvDx}peUAg8!LEY5Iz`QMgVqTEDEAqq78n)8?ku@B3aAYD?D%&rW zD}_?@UN+=#Sg|fpK+5Pe1!@mpR#isa1|uz8?gg<; zKnHX(D#m_+o-3z$d#s{Hb%J4rhp^pDhiw~y>nbz0hfbU=R8t75K7Q-t{4wC4Rq$bx zKy}g;y}I-O)NB!afpPI7(!GSHz@eXJ!UDHJo5*a+6F(0dn*z7tra@0NehW)71@(bm zT)5VI10&&V)Wrl|>jz6y@nU+9#)ym#u~^S{*N*&P)nLZwAU=)**({)Qw7D@6DQN^! zG7^C_M~dcERqyFR-vLG=Xi`B;2@bktRF-{OO|({U`7^m%)ZP*qhY zM2xijGAm2%=z$RTIh}o+i@JW#y7v0)fxt%K%?Z{VrxdPV0_uG?|NX{Ku4B~uEGj}; zIXz zj&yao{D`dRT20MlH1!#nxYoOEwJ!uASbcT+{Y)+{Ds;DP+~N22$rt-G_4JFuF!*@z zsA?|Sy=*uAm@0{Cz#|sfDJU37JE{z7_hZqX=2Jm7tq1=@g8sdqP#*9hlX@993LfNH zJu0bgJCk_-3iFj6MRFN`gzJ45yBR-U2t0m^$|R%oi%2D*zYG$8f~Tmr1{nkfsP6Sd z3Lq%_MEKX1_07Rm9=$mLYpBzj^yq(RN|1jD8haI0vcjuOL-ppoN^sgNLb!C5>i4GH zNsiRGM@EdAbH>6Mn~G_H3PE!r^FZa#K5hNr$&w4Yzs?Sgt;fefJw7ICu${pZC2b1x zZvba+`o{c_5*q&5DT&wSLKX5)^~^L^D7y95oHOS}rMM)1&=c51Ux^U#l)62qU;2l? znsinHJ~`trE#UO}8PHNsWSSg1om3i(s0vc=jTIqQ*K0B;ROC)1drKg& zg#Adhzmg0I;VNosm9t%$*-a%!F&K$knN^W1DODy)XOBRV%St~iSZ1!j=lrN-17dX9 zU--xAc8osQ+Lb^=iqk z6g*A9kQz-9-a~ESt2pe3K}H_+SEOmq_WWPiE4ks6u=_->`TH;uew-`M8sdwOY@4Xx ziQTDvm6)kn?a~QNSTRPyXx>G9Y*k#<0z#=X-Fihv4Tx%S=|H(->CVHok<)GOsQ057 zFlm!fwlwSO(~*$eif*elv|b(SDz`?MWs%sH&=Wrmw%;L73e50d<|=u@b2F- z>F6Dj+xRFxa)^h(=%+WAA6SUp{t6$$S$)RP7N9v;lRkZ8lWJc<=e913083?UWl1o7 z&xa^E9#QdH=OCV?crJdo##uGLG>d(e$6w(zLJLeV2c=l2Q_!W4_;SwSX?Xc%$z*)I zO12I3F>|${C{H05^tvpaY^5oys*ni1r)*Lx%F`Vk;IEGq9L4Q-k2a^e>?lhJ&Efc zi~MskSx>1=2)LV`CzxIL5ybAQB){V9>{%y~Livm#yKOEEn|Z-{yhR!>eU^^+K3RfD zJrbK>*WMMZw8Pb&u?La-O(m?anN0#Sifu&Gs{gLBMkR~GyfM3R6SU#uZ@VIJvrvNL zs+5uqNpj^g>AhXK&JOiA35ALik4#y|{HsN<)dbe+Hl(u2?K}o(~yH zbJFZyEtID(B&8E79yYzRveI@Fwol^HWpw>l-+39DsF-1u$T;9KZ%%dgOV*=z4oqgQ zo-j2fblVi@kU`PC{D6n1(%yWD$u~p#;7ADMw~cl`cnh*Vwngc zwVfc`5}E#Md{SO}$F+xoI|+s2cF1BjU}QnJ!>43Q$ED(~Kg4XfS3n+Dk|+!S6zR4O6;H(eR~yg4b?rzjYUeYRP-bs| zXhsRCG7Q)lzxKn|{bwz{O5(J7WzD=kiEqme{cL`IGwY!-FDouPba0!L&V74cz%7kb z0LmWAx2&iYC&WtI#PmV;PYSN+LmD`oN;Z4rDzBeC)YRTqv+Y(ogOw7yNu$&=^=O-f zb%2*Z1&S_z;H)ccmn1$co&&-in5yaUy;ON`{e4wOPN~QSGVOC}V)T6q&MtjqtERsK z9lR_z2ae9m&Pg7p;TO28rfd=sP!ynF5irB}z(}!;mL%D|;gqL%tR%*wvJwWw^^GS) zxi=q@gZFafAlfr~U^_&EqiOZG<5^p|(aE$bs&z9924Ogev|iexF|J{eNysU*@jiPR zVnCFaX=P&(-PhopVx37-s>EG7vJe9JaD5Dq=$+eVSpyklmad_p8@W7R95E?6_DP5o zaO8Tgbhkyzh~Y{rpVKk0_JW53qUm}v0O=kj@XO$m%ifJeW^zQMo-z5u%|4_diqPHa z_u~lfKYHHcM?Ze{<^9j^n6UuP?OIi7SE`{nOu<&u^KJgUgQXx44NH|FT_B zT<;ePlUvhgbF`~~oV)qDy0Qcu+7-m5DWc`wgXg`{BcoTPe`;~@m<0fT%z#-sxyhE* zkiDHm*)|EB_&iBn#6%;fpH!g_uZ=d~}0LhB3ks#HIy*Q>>{d>kn>9zQXPdTE@ELI5;!*ytDE+pz)H}mVh3?9 zGuN<5i93wdpWd_L=a+(JwBODz4Yd6@DNnj~3ug>iuO8uKdTF_sRB8YurgVd;{yXI% zF`l)CPUh!A11CGs$GD_e_?VtTsU6^(O*G4_ZjH=Q-8*3)KVM#Y?kCg~Bw{ zbABCaA+1^9o;!(@Sx71=8J`_7dOQZctqLu-)?-eZeTQeG^Z@Gd48pDGk)NqS+{iyG z*G`}k>vgaW*9XD$9--~X%{;sM<83Q`)K8`IYR6Hi2SGLcWeqdhgs=n_fqC`d)*Hs@ z<@-yPNr3l75hCnZQq2zs0`eR5r8XbX+s1Gan&sKPV*MM0-7vAmiG@eAx+^jorPPE+ zY5Q~QW}RwA?FF$6xDQ#T9{7i2s4TeMGLu}wf0l?IB&gR`i71Zsgc(5V7eJ}`^q2B# zffTmQaVSj#y#l}X-$*d2j3Wz`$f|Frf#}VBA-(@lcrAOyqe~W>H7K)V3`R$-KqalK zc)V5h2$x7$5PCO6S0PViJ;&IpfFVsg5pmOmrg3qbfsRhDiMOXR5WQ!agsP-e-b}Iz zl1jl%*?DP3$Sng6Auqb*%`+IU2Bow6+oyg7=*dR;`rY)w&xwoG*;4SNO0oypR{VBa zhPBv@EjJ_1a5Hc;wJ}2a6a$yB7*tA77l?;f$}(4eo+KcY@mz->Vas|or3D-rihO(k zI`oQzvrrNm4s^b3rlfv7Xw`d21RLh8B&*9kh(YpUfQzY+vU~Qc*C+Tp`-0Rj>h46N z*vy+`3fG<`iOjj8ffYMS6mv=37~zXpgTBT~Kq4(JC6UaX#QLe~>^L8LpuJcUjnS!2 z^c0XsM|8XyL?|b3OAP1n$w*lR39$`zCC)eoB#cB$?gN0L9aN*XzhntV!eRuZfy`1aP5Gp%1g*0AowW#>6V_q=Y1 zk;J)Y-+IhmL75gR&iV^#h;1nbE6u+h_?mRT<;&LBFR5o~3-=Li_rSsIB|oLLol2fB zwpm^u$Y{>Di?^)>ICR$q>q26*ZaPFfuIH>R%Cv!Oab1;h)b0Q7`~UdV2LSK;2|=sy zAAW@KS$oxbVGjMvmoIN`Z(rX&zrAIDmjGh70DgW$AwQGkMn`u;ds)K0c2$+@vxg)W zH&ty&0)QkMeh@~s%~w>rAmD=x{Dg|rE?7vY;I|%H2f!Z>p)DE1>ivY% zzN=vnAY5A($bK5K0WGR&)LuX9L=D20ct4u&O#bmZ;@3LhQr4a46DtNfNGt?73n__* zPM<(@gtH#=+*e7ZigC}@;zr_hA&$N$v_AdVk3CgoSPEMV%2o6DoOmnkxjWaaMa#}K zLBQcfhvV2TAteAE5^h(RK~_mU;u&5iQaPu;eKS*39f2YfeZmTCiP`mn(7X1eZ@Lts zw%Fsn_>s`-1$0QM`JIV3dx4BmkJi@-n79bjET)_C<9ERY{E+6fsBZ+Prf*iY3C;9T zo-su87XK!0iqu zYzt~5D0#8y_?P#FJ5R)E(xzYYdU0RTV|^w!^(W9%qiqs_J>8@K%hL?eP!Z@nHlSwkbq4i&690$(EMsw%!S$z()7P)LGX72#65Of8n(37!e}V?hVo9xyGxg_chn=2a zlvXvJ@?2}@A8~YS0O)uYWyu*cn@ULua>T08!S#&8046yfhtEKBhV;%fhJWT#I!z%R zLe-_k_>2|O%8mSmFqk((b2G;0{kM~<&J-2YC%@>&VX6bw)du0X)?RFvy-X<$pI2gHSEb|f> zydGM4#LlO|Xg*FatS_u;PUKEn1nY7uY=__M9TmllXyp&PtW@4ySgd}&mo5$jPb@FQ zM#)Fw>W3B&e!SpjfZ*JthG-TyYX!P;8ZdECcI8T{?$CP0BOMi2P|5M0;1sJ^I5RZ$ zU3i?d{VCE^yX?$qgY%)nVC`3%Tuy)a*@~4y*#*AJdVj6qH7^ZvqsPfGp#o(LN-8i1 zRMf5}DxQ|m^hz&L+mhxG55vUH`S$!kRWz|AiQoV>y8vFOzPM#?-TdYA*SF60Uw6|y zW1D`PaTo((Kx>}@gBzI5ySJlxvIlSmm`YmeZT|NFh2YFgX3a}AJqjJ;~mQiZI;6-#stqw+_e zXBZk?4Gcar%r6w$);q|vL8@~+qM~7qcD*%58Bqsn9;<$8>;*hw&>x zq8DSM+YVc}kw~bKJTJkR7pZYw-SRFY?_YwU;)}dS`LEXH3qKxHysTtI=1zzQIbIxF z-!x{FbRa?G#9sax*H6U~b68aF#65a$gEbGCY4YlThG$l(hxRXjFFx%B8@8||<7R-( z(7D3(LPaMc|``h`P23;q@d#gRR*&z9yuSOIL1Ri#ja2X*bZHCY^t)Z1>VR(J{+6wahFL z=8ms}&|ojWFiT_g1hfmtp*GEbsh(pu4lU9d);69fzGXK1ld zavjEg23A`!XwHCP%x3?*PhzD5T+K8Zes@&^JbS0i{&X6Q9ak#8jmFM|vod)%pqeTl z;VOaENkS5!WE|X?;N#Pl88PnkAFN50H*=+h-n?u2Z4O#&Rsg5}12BQo)IxuOI&IF^ z>PI(oo6sT=+}**-*geuo@1y0UFoS(BEQwJRFh;w4t$ zq}Be?TQ`Lq-~$J8P!Gx@JR10%_M$0lSCX-{4uhLbf*W1_CAA1mcYe-4y*pFuq|Pil z>1k<=wrC9WY|d~kUmSN%=!j-Jww^>aTjbGQnKkU>HbNmvi&3FTmDvi)|&uP*nT~zk|PK1FIyl`f?cPZu?vhoSH(+fmL^zv(A%@WuGm@k+Vuge zwk1J@0ESXZ@b^acl2xrXl)0DAJXvkBw2~N7M9Yp8(wMik7|lTqo5d+~T4`CI)-A*9 z|I{oM48XST@pb3?+t+VypYNuh#fMWrq(#a|U38CufGOxz{a8p%4d_;;4%XiemNRK2 zatpzPKn&DZ66ba$AoqPsE%m3TfBtkG@x4Ct{`Q4emb=-iH4GxF_MW2q+3Nq>+i$({Pkx}NFIW#0wx^Fyvi>OCEBKqb`1Da#i*IHG4O4H^dzXHLTKMyYwsg5Fl9 zFDJCW3jlQ?=qF0;t0?i)hZuJW@Z)DUL`(W21BBiYAhcH#hYt#j&h-`!KcP`y8d(R$ zCdRG<7S~#JM-pp_SU;-q7Hk{Mx@w}Twd}43Z+HU$UZ6BjvW5U_S;pfK4d10+`^}!O z22ISw>8UF6_H;R=t5m#6q~X=FGH{Ji?FL&(V)&2V97w55#rlzwlqL;-vEt8d$r9Gr zO|vc;li>=~v@nTYg+6zfxidAjRCl@o8_~s$nrsLQEc1hhnFtOhe)qZ~V)XI4ZmQ|^ z_gTx-UGT3}I zpva6*R2A>ap)n$KqHRC?iKix%+h$2RK^yQV!)Do}V+O$D*ZnpmH6=CWcF@sTn45%7P5lY2=xOIrW>nvL zLVrxU*xnAljaaFNg7oDMC&~KPNmk=@kV+_>1UvQ+{xi;xbrqLSVW zVyK@>+9`gb^P<=ha)S8T?`;QlwaQ+QOB|Sa!h)@Ka~I}h=`mpd1ky@jfT}0i{)TCz z+%Kjfw020sx`Lyl-2@xz+-<)06hxx4w6!PJ581?DE9>eD2)yy+)i2AKCHtmy?$j@5 zF#UCub%ck}X5z%^al+GqKdI&Qx&K5e{8Jw4;VuFk$MBI0CjM5yqrKR!esrJ+xHf2O z$6fV1KV6)}rAonGf`Q3bT@~SzaJdKz6ofmOm&gVa!XQ=t?5&S<*Jp&P+>CXVBRZ4t zj`on4^19tVI%Wy#%slWFK$EK+#6;|sS0rx~nk(l|f;%Fzw6hsylFs*uRwV9~RBi^@G%$Slk`nOkB zDh`zavn1$t*4CI2JDgQDoK9~F)Nac#bf+H9irY@n=TsepOKdoC22rAcwT#`UqsC*9wdC5?i<@+dBT|pI;HSkl((;^Ge$cxL0x({iWNL{`1H8_qRR@ z2*YJT*ATlDNIX}<}E5(bNHrH^6$(DJ1wlL^f^~oQF_6739=Eq?Hrnct@vq8K9P%$11FOm5lyUDp5t5< zGyOBG-Q9nxNIR=J4LQT?-b?K(UpK}fN;V(iVN2SOlyNATiUGg5XP{ytJ;Ns zOy}>zR3HS8BR7h5Wc=42bFA7cGNHT(RK%1kV8S;;9E%}V#w&~({S7m?7fGR|b+1e> zdwVo@Mv*1ASkSPEp5h|eLH)K9RNmVl7GohH*}w~AA_XR%1HH?t*`D8Bz*W8;u>K+> z^b|WLsYfLuj+?jg?J9yz;IOm}%jX;W#4CkT_Gv=tk8KsxczCK2en_Ubqan_cLmZmq zN?&lXSj&$u>4|+rDlp26;zJ&7+D@dizF|wM(h1){si2L$^?)DH`mMsH@VjE*MahnZiX$M>z5-?>DCNWP4m_FB6Ll9%KU1=THwd^ zF|Bi#wB6A>mVh3-cwTfTc5w{?!&S#;Eqsypd_!FilSm;NW}`Gj;3=}A`n1=mPAgnV zcz!49KAdM|&c97mDW*qzp<^M_nc6RF&n7O`TJR?80-z}Edyg?L8ihjLBElXf=%lmb zXeykwQ7UzWtwxn+-$&BP>=eet6-P;6MXlEYT^%+DODtJ-Dec{k^|GUOv0Cv17HJND%}?L zKr+jS@M3HSkITcHsY6&94@Ym+>l=$oFo(qeGAB3CY4CK3XKdCWA`VX|o;ngV&-#tO z@|jRPAv@7{&ehJ^IUxK{S^@{!P;2k{q5)X&u&4C-x0dxi)8k>7ukD=J z1>-ks&;Gu*7A>s}w1m$AJT3hPU3xK!o z{eSsexBtJsef!qO82&!@m(O3`_7g;YhzKLy9z{uZ7r_#|xhYpXOJ;7NHf^j%sC#!) zrV@+x|6}QYWkiSbl2NRn#eSa^wEOvc&o%jC1E^9}S6UmsSK8xF_fy7OvoM{5UH05L zGrx)x8-E1)X~ zOW!|pS$Nm+0PscUM;aY!Ea6r%p5i`fl(L?QK7C)FMjuM{kof%|~;@n+@QxeBCx7JSFR znt?iK4K$Oa6oxA1D`sV_FguTR%|p)TbqTuuuSS6p+O|N=;xZ!FCUh0b6beP0?zn=I zyqT!7r3g(-I9Iyi+(T>y^y)M{K0G9G4rzB1sdeyIvpwH=UVKuXLh_e-4v7wG;G={} z<8)R+8Eirs)m>(^BXzoX4(dp8I3W$W-gvO1c?v^UN)Ow4FfUIPzheYz8B_Whk>7U5 zIRm%%;#d7sHc8;W?ckyz1jS{m06K?!bXc zbpR>bj+8c;?L@ZXi}?omBRZwh6P~V-RQ3Tgdo4fx(lDhJ3l9R1`YO1pbSR~d6-nNH z_(wp|Xsd+t0MR{#2RLJ=IQeP5%s$4&cje3nH)7{TYu#66<>Zz7t1p>PGJO3C<_!l@ z#X&z@?YeJeY1L!0uu~OFFxtohocs!o2lrPsR;y58luz8jCaAsn(3F{m6{F$B68<}1 zCy+=sIz+E=aAPxo=QKbgX+mbPMQz5yFwohxH-zBM!xH@&nW&tlNwquPh?(bT@wlm? z({<;WnP8jdpKs4_Dw)F7{)Mx#%bT4ZoEZc-vPR+bXLRhxe1%m6p0{nL@r;7!V)f_I z)UDcLazKufAbMjn4Eg6oCVXYIs_+ht)u+C;7KyWV(2JiINe{bG37pV5qnu%3ycp-5(~vSJpBaM z%M|60-$qGfHm^g8UJxR9#npMNB*kbLPG=!v%P!A+oMu#dL!vMQ-rdxdXSMTv&%C$j z?+KXi-CuMsJVz8p>V`zujV1gq9|pVq?`ilRiD^Cjx&_`{=a!+mXQSqinE43dx3B$d zZU^7@>k`1W)pH%Z`c|B^QHXbUgVVeYmEkp4ZuqO4w>6de@^wNXNu& z%p7;InZ_`#sO|w0r<%n)NmlGSYLAKAG1$So zI0Qqk^yfb6MD&|2$m$AMLne55LRt(|5B4wJgl)F0~CLBPRWK;y*j z%dEOGtLCA^n9R%N@SN1KoWNwNt!P+`U&T_BJaNSb<3M<7Dxbs->62_bHMN^1R+7l% zYl`!)0?jhC(sSz3Je*37(^RQfz2JoZOo<=M$4vxD3>&R!?*zH@sw$irfX1qsW*Q2v zSCoZ*GePO~M1ll--KaLdd2PL8^tlNv-L{AP0bXwQCQ>R7z}ES2d15Jb0%SLIINNCK zfP=q8>=B%)lu8Wc0e=n|An|LyiFtasTX2iJiIC6)sUgBMFL@alb<+=WN{OaD5=>^ij#uqY}xM^HHfi%W$W zxHwNn{bE_o%pF@mXM5LFL%iyR=FVm_H}2vi60j>>NOUtS)U+L(5Emm)b9a7Fx?G)hQEdMchjGd-uEUQfw=XPyP+~mpyns zVl`{O46~J3iz#1%Z9(q}px^P>;vp|P_F+Lw&riR7ck5Z_`uz6eU%JF_>h~eQ+t+WO zzkIcH?$o+PGxGDCn?BNXnNj!8mGun4OJ*WflyDc(_;M$}>ULrD`z@j(AJ=bhfBXIGH=h39yMk8#J_PvuEe#L< z6bmCZ(}l;@A?(1tpVkx>e;t*y0?TM?hYGr1-xosF6tS9Rl6~ZIiRS{KdEu@DT64LI zNOTVsRsPCp{Gc0tk^$o|Ea!*rulU>v`LO${Dsw!R#){-gM)L5%8y@m@8 zT1p!!Ks-&7=ndoQH-c3VY?}-sbxHZwodIV{^ZN^{Gy7L&`A z6Jvsl0%JoP$mZKL0+|f-8epw{(a*o(?Zlh+iS0Vw*v?3weS+i6b|pc)8oe1y5`;~} zM6h5IJFbb5{N)t$oAMkmYe6gm@&kIS%6<77qM2WsMl0Gy=OyBlsg{TyE%Bz%q z;2keFgZedjRHpK($B5c$f6;(BM-GG9tnk{PeaP@=JSI@#+CEWNy*4g5E^^g2FKMg9 zsxNagXMN&ShNz0u*gX`-UNL{glH}D@I)#q2n1tHOvR}$}8ZwXBRVOjI4v$;{rN-Z6 zF};CI%{;JH0Yl@7{VS>mVn2BLrHAt>o4QCU*8yhj)xZ}pvJg6txR*6cj#CrO%Ably9JARk+}H|*k3F>GsKY!g`d$n{s)_Q*`pfeIk47< z9w#W>nF|$OMZ&53u#*1(RKiIdMr1-T6DHP7-W8Sld`z*{?;6w9pyi0%{?Xi@9xlGnW=zo6y@w2~{ z7+>B$|Iw;6qi)>abwd{b{k3o}wEH9?rU|Sw#x<(O)jX^wAMLkv(Hh}Mw@+)26A0`+ z74UIgh1hg>;?2=EzIF-oS>;2kLALr&Y=ti3rS?gHeJ#+kKlv^HyR_*d;m41f>PKI9etZA^_50f& zpa1#gyV`nX{Ro^-g0gvis?chiiW{XRwmecW54Csx z{OWDFuhPD{$~yvBYs%Tgs7kHe0DW~!bs0}5TD=|$wXmPPf;sil>WXpW;OTiS(p1N! z%9u`C5gkNVk|9KOnnRnhtF)i^ah67Ymd0kY9|oBr%j?aJLI8gQYEA_q&pz7=f+p2iu*=Am2?a8hE8DR| zp%16EgKOd6`o;0jEdYHQ=sviWvAh3!K<0ao^7kEpggP_6TkW@4)|Fb8JwNWhh4`AZ znx>1IsiprPJq6TL=zn(8x&N}|%fX%b-LiKLU=7_r)a#oBZuVMFg(za{qb_W{IB8p) zSCEw&I`LuQLAUjH=t^B*A;5PrF z&p^rMgKD}n|L1Q#A=PI9vDuY?zGO4QcYV?#eoI8_dM&nc-?hG*{#*2WTu6BuE!re( zZ~GWv*8st__V0VoKi+(^3N=8TLrZne+mywca(V=PSnc6)M|Is1!T0^;<*@#?`hVXs+i(U zOS<)d$V_ToO1F}GDABr6kyvHvR!iXWrNAIs&l&|aV>6s~dh82BA3nDz>a}CD9eP34 zCLs{=rdqkE0BxsaUkFf9jYGS}lLcudzR28&q_S6OpUAoL$eEf-hblQEL1+|E6_D5e zFF57ufyR;ZM(NP?+exFd#2v>P-}Q%tD=y~I2_CLMXlY*TetF42ljta9HksG{HF{T( zz`UEws6Kx}k}>^1KQhC!X&UTm3PsqC^_W zOe(%!MR~0_7PzGImyVzxZFM|Abraa(Ogz1O`qRbH3EQ-|hOlqtgPBNMo z&6*S+p$DOT$4ut+W=VYoC#B+ANYTm^ka#i$cjZdgz*Bhvl;P>fRd%tbdD_pBUS+Q> z#N3^P=(at!Ca?m**X@q&hfF>`!YiWrste*EF*h*OtoDkL+xB7xC|-4=$H5Y(L2;r& z^WrWDiT?_qu2b$@CRndu=w60E7~|}Z)bxk>3K@~<&_9NM8wQ9Mv%zmBNQ#F=>USh(NJ>Mo!mE#(A0%g{c4vQmOgfcyo(3X}+@liem zMRO6EUqzMnT!sm~l=X{MffJTsBoC4Qr@qoxg&@z;Xb`C;gi$&h=kSGb6-f}>oN(-i z5uvHqI75US&D}|H*16_`(3VK&79KwP`(DXxLuX!QswAY9Uz_S^vu>BxBnKqP%_?}& zm|xvWlPv3;tFRS$^%kB#|NLhiv{iBAz5B+$b%)%w)*nBAe*N>)@2x;9_UXia^5*BC z-zuRSBY$EXMIGL4iJq|OlZ5{M`TqXLkN1A+{P(YsX~B69&3D)Q>u+Da{r2{Ci{^%s z$7Wi%2bPO4>s^ChM2hOQV-{QPSCf!c>(Z}1(bLIk=7?3|S7QpVN0)REf32tcETjLl zTGuHKE_#0SwS<=XKl{H+t^L2|#MeLj?Vr^838U8i_Zh&g{@wex`v3fU>;B*P($asQ z14L=-{`j>>(f~zUZ-_@RzUikpHHNcykRU$hMjfadP&#XN0SDaiFukhK$0T*NSj7Rr?dw@Hg+g3ge`2BxdsI zPw#gnz?^G76%`MIE~xDlRelvqz?!A2fuX$IE-jKqbq8W9_d38Q3)H6jF;_9agE2@I z1~Gq04UP!8RZgG69Dianc_sS9sxlv1@3vfBbI8wa2I5&{wW}tA-_LMpf_z*(gHug( z!Gjl1C**p`zO9msGZCA+3ZTr5_NX&nk>0opu8$)>X_PT&_YE8uKzUz{<$o%71zG6= zCO(Uj>&<$RTJP->De)4`=&YBYJ+>Pt8Ne&HNUd%mGqMo1yJFhYG>}SQL*PYT=K(<> z`%|hQ$7cfSDvMJKsM6JRvxEBWs$I5Rl?zyQN*J||9k6)(Mq%=1{J=Ch6wXuA3 z9hCptUglmAQiDiBiPl?I(2m-(wtw!5pE9xb;bQzoWxEX9T;+pjmGDgW6;qEU`mImZjG`ZGruC=o`e` zi$Cr|Voa++np^nqKWuzsRM_Lt_Wv?cFzRS*cHrIkZ_i)5G~N;S99i$n0>| zH6wiR_8-9+yLA(uEHFA&B058~bZpC%X^6&YI8iCo4r4-KG7HR7PEDX3EGiQ53P6Lt zSgc|-bGM4HXXKUFA1a{(yxes0RS1l46R#%kPbNV_pg(xmOb;-7IwmsOVNLO56`sN& zYqZ_94PzsrnKsqU0XGT{W|2x4p3{7{*%I%5^!uNmTKa$g*-v+V>H?r&%KWyqq=~P0^T&_>?(en=f{71fnn=@ z`&|k29e^%o+WXvKYao$Ac@017oq24Ex+I!!rjz_H&|x{m{{@w$K(pIj3-lI`bEzyg$(LAT8yF_!@Ix3FJRX557?UBC-j^UYaT zdZCGofpK287piTJFZCz{F`f3S$s2>L$)m0WBFFpKPn1X-OQUsO>2BK0r-pINH?20` zaN`m`=W0O$fZ|&J0n|*~L>Tu+I=z)lMTFa6iS|q3smpbBiEBW%CO(9T#4rOa59Z27 z^U#K@3UY_no5w?)0U4Nj0|VDhLPm$y%-`jK+Vh1z6h7iOa78cVN^4RbfR*i*_(%aC z#)tZ(c0&J9IlT505<3=$!Pc!7D|&fYhrm2%CObZ_FddP@;79sd)anwwnM0pD=nb@>xjNvW)MqEPyp~>tdcjF1h0~t3Q{0ozB4tg z`<#L8+3O5tPOFMc6W!*V2y&_F2Vx!LnJTFTEU;h(AuTF;pW?QC?Z4RSzB4k)Fkp3ERV@@ec0VRof&Cuxp!hQ@L-JE<&n z=;GacS~rm={^&QWhaoiQKAViLONcv?wzQu$nC|n;5V@&dbd)W^+aKU;pTMG@IPZ>Q zcy~Tw3(#?2V=tMaUb>p_&Ll>0XYWBL3u?IFP5jxgu_Pu-OTfc-CBa;%$95lH>qG>` z^8o?VCdziJ-_RjgwwKPmrcwLxH?4_NC%6e=`!dl{iBa#^T^z!(_B<7~>uTkaZqylj zqS98br@#BJxGniS)BcBr=%;Ug{`Pej0N?+7Yo*zjMZ1ms$ImVjTBUo`r&Z{CJ;gOg z_x^jzLe~da*!+>5Nd?O4#JK zSoMt0*2?dHe*Wg03O{?=r+*6E?=gOD!1o`(0*`IiEj{_;>g9{oKj!{=1n67e1OBrg zAxgf7>HAu{yxnx88x1W)e`mo%t-Vr~tPUn>HNXhC$zn^FVC zkW<+Y;_T)RM;59|C56}~o76r8Ft-M>#t=`{Jn(0iMa7qa@>B_)fntA!%W`#vSI=$y zdl~-_cWei81@o+wz^Jv;o-<6OR6Z8lZgj+eD5(y9`KQj8t=xo|*tmvul8=$qoJk@2 za6<@ho1zM=k{D8l*@3S3VSTa{JZArbA>Q~uMe#cx=bRcG!OhI0_h>u1+-XxZ5AC72zTnetN~z!^>S!+h(!ddCcp zDx5Xm$QWVAz<%@a1WxME&{TUBb(bp^#BDQs_FkeJ5nS2?4Y+#k!`w5ge1KEJXklVz zIg*L=K2-wS9-z`s$>Y{5$Bz{q(sUH8Fd`cZBbb1OW(2p7ad6TUO)uN@nRKh!@WK5R zY=hBCkmO)WT>S;=$*vofCU(LDHb+%hOvdk2h)ia_p8w2$=q&>Y__c$HnAxcwlQR`$ zQ(}#gmm;f_xn}EZ_qW+_NCzPpqTYKG8_3}*mqPVAj+woQbZtZl1@0_1zAX57RY^6b znkudqBusE2q8Jh9=c}kVeok>&L_K01WMst8yCv*>X1*4H|ILCC11xU7v zqm`%_i4=;nu9c2ZSqXNJ`osBUYnBJ>j3eR?1p+jBJCQ@c9Fkh4i+^Naw|eMjbsk9Xt`M06y|C1HG831IMm?GAoph(#|@;zRo!Xg|rrD%+$3! zk$I$gZaOSoLSNf!K`PYg9pXEK*<&mzw0FbNL?9up6ZrzvfnV>!j=xed9fBUHz zeX4kVd;9UX_n-gl37;MiYMttmAl(*~fBx~_z5nf#*#>xVXn+0Q=K&gLAzUl+*rOuFQ^N>cPyWc=(k7R?G5*_cU%La^WzHA31N-k= z{fCFFZ53{%=YM|f`K7PBZo$07CrkexWcv2mWyIU3x38j8I9p1Ke8ntjitHCzR#9us zwhp4R^36h~caf|`TRnMNxldE}!y(zMIf!xnY~+S~f_sZCZKLdV|i3#o??2 zFti~dFw)*PxX*}M0_w9g{go&`>LkGsI&%{Qt!3G^_UKns$>>;eJWsyd^i!F~ z2RvFz6oQ3FTuORSABujFC!qS&Lb%RR5bFUR)#KgyLaN!*#eA*pDlMoka0Ppyno$a^ z%}^ZOIg$%z(HX~MGS8$iHB(H@o;;g*%~QtnBdwaW=56fzznJA+ch09>tYkwYw_Chx zwk)(S@2`H>Yd-=|RtVaro%kijXb!!osgJ^K5)m>ea<>rFLF@CCDjMy|+oE3n#Mc-I z@qvuL9dy!C3l1p`LGw!hZ9z=Q($A-i3-89X)Ci}l{7?d5HDVKWoEYTsC%X+1vmK3@ zeFQ1iSKDju_D}7lUMRXj?A9QaJt$AD$t6D^#k#}&m8CQ3kSk=~ftf2fYj7(Wy%b*P z1QREF8}H_B*~9s$UEChYCyX=rOIvfNJQ*f*rgAI6es(LKz~WePPM>iN;QcbJ6b|xF zMMue#KRI=6PuZ-TtpTZIccGo25VeiX<5L_SfV$2BE#A0SF$<^kggTdAystb~ATnfp zt?XDITqTdyk3~WTCV1uLCdnMeVmG!>DZ5kDaW&XWD7co$y1_xQlHR6x5D!=dI?mJ) zEjWV23X`L{MOZ=BQ+k`jz(V9Ep;I~}u__N~2q91&HaD^@r= zSm7?;E0>xZgG)cBeQ3HNeW=?fhVm3nE&YQWEZb^7gwBEjsJSuUsdgT~JT!@^xy7}h zUQDXOH|6>grtLU`Au;)F;BK23Qy6SkA&JyXiOc|}yf95Sq|s;2^g6pJ&3sjO?bKk3 z1F{41!gu*~9=d_;35fc|eFZXrVVUFat{#t2xivnEGIhL8F$0(p4o)m#sP;(_DYFs< z(5*!@g$mU+sVDJ@)nbRshtP#Fg!9lY;Ku+5J@hY@Bs6s}oiwX-rCZND?@tz9o#xXYttUVK(GUIqv87-j-3kBO z*Z2SY{{HokmXhDY^;Oxm?jJwj|Lc3J|M&m=OyY{Rv?yQ!t|8#4=rz~~+R?jJQ4e^#I+of4D5noGz8C8YS#gd0LjB%0Px>XD--4)zbdTy%M()(kl+wXfLVBmEkXx zH|$e3*RBgxS^1zQS$RjW17aoBM*>Hpd0!N+3&nEcC#jNG*rY|X4} z-Bw`~T?1TO?D)jEC_JLGblljKEw?u(`O%KkPv#@g2@ARXqy|zhWH@oGq9{j-_d_kn zoi}xTrzsi^>jGes6UlU_t}g@PH`i9&frQ-baQvWHkVNVUbTAcwoCZ>}c>c|pQ5F7p zPHL(T0_s|K78p{%8aCU`v$31@Qb&x?nf++}QF}33>7zg32NV^_Wtkkx!?5(3n81;l zz!MFmPUCzAa*RKMA{4{7w>IoAv)`n81ilXBjk;LFM+1Mxf9M2;4SyUQC2bQF- z6B%0RSZl<>^!gA}P5N2nPDSP5n?tD-mDW7utU(G<2PdM}R@D)sG`VC4a5kY$Cf%p_ z<(2G9ba99s3s-tXsG|NTt6p-Y^qHR@9RoH)IwOR~K9hhh!qOF(e=7N?L73>T5|B0I z#4}l9KE(qXt}As^rwEDVR$^I~G$|$}Dv!l;8o1h*%taH9Qww}ce+!T-3?l5YK$D2M zxU@*J`uYp~PBUvPVq?+18f&;w;nT#iWca3!zO4{^}OV`iT=cK z9MlSDKmkU%Ky~l+GL-2SZSgEB`?{&MAaD|z!J2m`0AKyLo z?-uc`l;igE{jIM7`%k(7wM>bmFD~y>3kkpbxyyil2Gj*i&o6D?5)oIGd7m8q@#Fg+Jpt6kMc?SC?7KCXVIuAtA$WFs=*!QE?lPeB z2fXM-UssQkR^ZCkDFWPU1$^WtB+;&%@3>c?TlGqj{}&4)RTww4G;n(=E74Tz%nI8$ zukHk=W(W_F8*b5-EgsmFhW6B#aqs7VR~EO3C6y?~`_QQRW`?w*D;)}?6_y`r*?8!D z9F|Mnl&|+G?M1})%E8w@W51e!;j_aUOJ+r4B@3h;@@Jz&PfbKDK)6CJSx@fTzLA|# z{igc#Nc6GRVC;^;5Zl$`E82-G@n`lZMnRZn`Z?WgVp= zSEW|cb%b@}6jR|sySbZE{j2ffnLn9+N9-;J%J_SUJxA(7LEr1Vm@9U%iKcylC=!=F z_ZfH2sqAAzCFC?j$!pzg*ot1;Y9|xEavBY)R6{7E|In&v%ID`0E<}nCV8dedK-LmP zOl5@?JT_OU^IgFK5|Mwb5tSTpJLy7}mRAMD8A&9cAH7IEhH3m2UIY&RsYtM33)*e) zdnab}NF|?{@RZIO3U>nJejnRwpE7FQ54aKa2IYCW_U;Tx2ptW1x?Y!(7oj z$u;c^Evf7W-tjb)zq|3bgj2$$n}wP%I?qUJ5?3W z1d|V3AV=ZcSuo14_hG~tcbCVK6*fg@?<%LfJ}{6UiMXH;@0TAk)-vgikzjznmjx8#E zuAjZG4FY@n^u135zV+qN|8Di4$rk)xSRxG+F5pwC^zZ9&=v<=nIj}3i?p2^yxSq$vS9>_L^$$#4ld!xDwsX-6Ai4 zw4bUD_FM7i(3;BTo~m>i0q0eClv*vaur06z7y<%6SlZ?Xt>e7iJ}| zW_7G_R_6l?set}JrtWP~ZX4Oww97i$wN88O?u&l^2i$$ptIE=p@AOu0BSNgxo1 z5rF^*GMPDbI`efYW{6IyU{=N?l8eE>(<&2iuFI*tuYh!kBjzHp_IMVV_KO9g}Xrv;8s zSdZ^W^X|xh_k5S%|LIF3uK&LG+*z0VemAymuPget&UM%ZG@PZ>eCKu338m6d7ZAG7 z30X#t;F7NxZ_a6#>{?to7l!7Wci+BsJ-=&BnUqEj(X~^xOR?nx1HyjWP5M3xsBib- zlY&}2>z()t#LZ2DD4s@va@rS4wC8$t^UIN@`8B=nJtaqlCN-<>|Zbkc3jgJLBo)6PLzvuy{*AR|tHcq%!3a7!b z%Hf=Wg?K$Q)$E=XhE{8*htf#wf&k65zBtD6v$E#YI5k8Z7-m6>6s_P=`h2ICbq^z)#!5_3$4HD>J^hr|87isAYrsz{$Pz z%(L_MDaXu04YXa{D2jQ3pG%aTjfh&-e6(Xb%lMAEkm?TH6X#3YB6KBa}|z z=bnk0VT$kwlk^$*tW^RVsxh`73MVI&#?ti>Mh?^sus?A#)Gn>3pUb*yrsB?tr{i`? zXQn7^gEF0N$Ldke*G~?v0elD7Anw*|3-lml(m8PGJYic??^r2=ziirL9GtAd$McI{ z9(3^S=jZ-AZXW`4H~G(B z{`sfXe?9&8-#-6e|MlM<{`vg7=ljk1tv?9{x=6MG^!jk1DpYFQ_68o-Qw}8qtH)pQV(?exlj* z0IzB({0$gn-L)`myr)J}Q)ZU|?!C6w*KZ|8Yq0oQS9=gh_U(HDfBV+sK;K*^1oB(I zeAH#Yn>;i;nk}rq+BOPW5N3YXjlXZU@4tNO>*4LP18449V9rhPY7FI@%)iMhp0?Dq zE3O>QpVbv2wJjRrOdsbEaf_Qcu0wsc($B@tCfj7=0D;zC?INyDM)75_GQvczft$fG zwkA@OQtA#Ac+E%9?6hF6S&OWgB4$Yq^R(-SLa!IwE~Kgf~Htyc-gmzJu{#p14~>j??y#db6@K(UtElne~QDBC*`uQ?O7#InwQw6Y)h z)$wO|45w8~IOow7kKOcQ zsgV~0Z!`=myzAejeeLhMgx=gR7o(qRFx5LR?j-4=|NOu?2wNzyV z<02XBcrPZSn3lRc)j|1?&Eu!PZBdLb|#7pnG|viR#2Q^bQIiB-L>hiert!Jet&FgFbO zu?YbZr`m>$L@|j{xLVKq(N8De=`!410K7GWcw7L6mVjS>`SH(p-#@*5fB(nVlyzZJ z(tP>w=f`fQ`&YQXeEHLV|LsTS|NZZO|BuhVf9~6}>xHb>Ay&ttA1P>+3Lgmiryg8~ zy+E;OHfe?Vr6B}0mq6V$=-rm#c$I=X9~WPJ zwF%~~*HU%elxR6nQ!ZBYQ%p5%?;&YBuj5xb@ih zcfa>3NO15K)IPru5V4+JI*qwHzkliQy&rS>_Vw-Cm#^P^c3=%yTo1cxrx#Uq0{8A6 z;DAPsD0QE0!Bww?2KgD!>f|4%r)9O`oqFu;Cd7N?Sd)CntLwPvTXs%jn1~t$m}hKS z$f2a3es3o1C$(biDLm3zSRIUWYHk%uAB^a{%qTd!nzPgcX||h~d2gxNWrFb3o%5}5 zJee9_CN`yjmQ-6-mM_a`qQWMj$Nh4A5q5Ws;&`S?KW+1Fx-T4TIX?a zNt9&i@_u1H3xKsr!K`L2PAA(zYRrkNt5?sG-Fl&GWC`D;$yCj1ma@al^?V(*ktqgx zARnSQ^v)4M-bN&r7v7Gk0A$#o-IQh%_$_T}UGd!EP#U~bYeZakObdP38|bGKkQrpw z9JNgaR+V$a5#teaq(THcbM`y~Gb`((D597D)1P;j^wlcR{#SrTnyy=+OzRl}z*djg z_a{^gXBIxHC>-&p3=Uoo5U7(Ku=%9n7_@O_Dw&uzjRG_-RurDI2mtCP5maYj{x?tVLt4jd8l>4E^K0OAYR90EURG>AgU$!kfovhK8G) zWF%Uu;kkx^sKvbbxas(qpW;~7NpZDRCQtEstOgBoG(WHXwBlvx0(t#iU@aP5F<9KS zP197g6Z*s+Q>bmvs=1KQbUHB&%KVTPP*C+K5V?a$)q;URPshvztZUMRD_Qf@q6e$1 zI8oykYfRHq^`+;rDv@w40D!zoIoWwkV7DdC?U=_jTc&1CO!m>gN{p^11h9B*4h~tK zFBOuq2yu03H8N_(m`S-H++`()oT@F=`X$CI1piJffW|(zD+xs0>z)A)l-DsG%w*SR zN7uGs9nOTVk0Nih%c@dTEB(-kzka07W-XtX@SQB8+MJkotDJA~iOxoY(@7iMi+*RY z8B6og)=aHCZ32&%MD1+k?}|;w04-O#k)jV;2D3qc4g~8(nYx1km?izxInX zo)w_09{_#-{m*~+`Y-dT>r;r&-+z4U_cQwrU{3^DjsE%RiqeTv5Cy06Z=9&BK@ zOx0NczOa^f>a9fmcM0paCZWs)b|IUg%5!YQd4HohJ+)&j#!KsrZDpT<_@H`XneR=2 zi-RITPwgl8XFvR7A}eEhprS(7H}#C6n_`U=w=9jDKE7xlW0-^4z6*m?75omG)&rX> zXaO#G{HnSfpH#6(R#;=T zEEX;LGuxM(`$*({UjqK+Z(GAl@SZyIu}Ltv9hrqp{P}19_u+K`&<~Bief!>5I=+7U z;o+{n4&oC4y*y@!z|uG2wQzq>h86jqNESc{tD>G4{7Pz;}9%_|ln{jMqiv1fydw6V)<4 z!V=$PFrb)HVQOmOGrywa@Z>-l%3GadS*2PY$2Yk!!i=Oi`lQ)qO$Vi~5S~diU4RTX38^5%dM^E8huK zfP3e`V9R5p?xkj|r+*n`fpcBz~41LawXY@!@H}HN(vzBaNH-GY=*1*lyNIn5hed;j6J%Goz<@2I(;{5H+Xc zuvV%#338PM!^uMaZ3!3UCwS@TkJgwCCn0F)XU-t5pyf1TT7WK9W^yrLWjcp@%*1XJ z@EaopOe;Fc0=jmwqCY0{&o+P>DKopqO4L7Wgw~_a$Q+DkZm5Yb1GDnfr;1KD38+zy zi>rKQQ>B6CI{}kzfc(?Op9~z|#6(!9VjwXr)Sf%Z;-}>4^7PZ) zUc;Js5b;=VE?WqYOoErm2LbO?XH`dukOZ#}N=9eXor)|t)&jloJ|9GX@Av7t57n}= z)xU3(dXQxcnx28#Nu&8YZ?kWL%rWx7+6u%~oN_8Jm{9J`2_VmtN|t9?9)wDq&BloF zq-rsyq>)iI#z3DfT&yMY>Xj7{h6E6QEMhyZH3_dE(f86oE2e3Wp=#lCQtRYW{<<}1 zvAdo)3n`B~z|P&lveN#wr`cTXfBv;^_rCk|&W-MU>f{2qPXKzJ=l%D73+P9`2ig{f zo(1m*K;Lzz(P_F4`d-t=KfizIlY(6q+_OY2wcR}a=GFIxTSoS8r}}}N9zAdPv`&>j zaECM3_P|U5c5|K2?fqpC)((hS*ZMep<7o9FKHjS=;(1Dnlg%WGy|>(zzUx)@{`(lo z?aSEKmYxmj)H!`DVEUM=YF=e%i)S?xKx>?J{U(D%^xl@LC(gGB>EIE&GjF(hh7c~G zJ3gm+nG`^Qrmx&1$gj`|$1U)A#QEcMb5*uV4Ro>+8e&xHSky6z#8ATML5a zDrfEnJOZuURLsO8QB8!6rHZ}h4FpvP3wP9d%Vq_0iPpmJ||b%TmS7S|%;skcSOI{RW4yhywL<53-ts zA&sEL@}%C&pSZyUb}Yi;lGKg(0CqX%oB~f=t+RVhtN&C7%6`46H1p3oX4518O048X zEv%VJH`7)den%Gza`#9FjhGhLlSoBZAD**u(s38ClrvJw+`fen+OYW-UbGrD2`zS9 z*8R!#w%<`%R&Epaim)cN0E7m7sFix>)*zwUfXa@Lk3sti7r4m@Q#>;>O(~D5o-`nx z)a1lsR7pQQ16&`S>ydI``&jkL`UgHecquW{id1P=;&tYE3OSFob6Nj zv*EPkZ2QanN^)bIL&Ig{M^@onBsd_onl;c#4F02aq5?j}anv5iqw^W@HZ zfBh)4{mfi(35_IO8M5RdzG)>^V(iz7g|q5sNcgB2S>I15Rp94SB$; z;I`ahF(^p=l}^!Wo30n08kQX%v{R@(c#?f@cRyf&7aX77Zp_%jtxY0J;>|k0uD!DnPf*&)w?lToIBUnveUE^ypwOz6w*n8 zxG9D9JZyF2D_e}`=U8GQ6Jx?7W*HnE&|c_zPZO;g=2+4IRez{Qp4Bb=*c7RFI_R2l zuO>TBv1%&jqCUg((z`xj!D0=Q$+LN15=)NS^PkYCrO<LKoa!)GSB9)(pBy#un5neruAXA{d!ENjPOljg0xrbK#H=(MDP*D{R9FR zb%jWGa%YiCTPmkz&Ag}RSPHmUd4(z9y6@=;T{5nCqi$%C!&DHlW{yE;K2cde^RTu4 zFTRlb{+E{ipZk{X`@X2#^VTrC0BB|3Ka>5|(?R_YDIW-QYqQSiW^fPuzyI;>kMI3> zPA9APts{T@^5!mjzm=o^`d?sQ|M|xse|-4#v8RA~Bnk_&Ty&T>ZJ>T*1>|c z?zi~Ql}3SE+KQrX3qF53s_eBwx*~dAWQ2_B_PX%G(1JuHViJTY(_I-TOTqA>`fgR@`Mw?tx@aa4$AcVSw<9hEx=iL&730P<=K3zd07hxC--i{gbFxxjWX8ua9P2wd{+ZcQ zt2}h|{jpDf-ahuY*Za3`?|=XN@yoZj-~V|3j}Pzv`0}ma5^|a0N1^ybNqeCUl>)ek zNQE|XCC>c|TSOfrtZ~`d!#v5NA)Mk)XUC0Kjqyf|_f>dlJrXfZ_0YehPimzUg;|mZ zkCbX<1Ui9DGyN+d&4Z{1HQ^DLp<7ZL%N#6puTq8kH21KZn#u8s#RbMS9c;S9777u* zvQ~t)8_#fTTUN~p-oQ2o+poCkj%(GV*ErAFIg}?xwRnlGRP|huh3uRoPur0Xd<^*hGF+p$!;bxwLONBH_d1YgAfD^qsXbF;I5Q_XJ!x3?0!POr zc}e$QYS%om1;BiCi=5#BZXAu7n|Fd3$9cH?IqerVyy`%cG$zX?s$O$4qdjI`ygK?9#mM)}Y-1JH9%(2!K z7o;A)&03(SxM_n`b2%K;7K5MH{Va(Xh#kT->0>+!!BNQyhbry88KgIKR+6!$EXuLc z=5u`208h`PN0?yf9g~Sy3`tTw`V5a&kWj{u87u?`g+py;L7{;G+JpK+6p-Q*@P6Cp zZ*O{I;4Rm!{yVu?bnbx!xtk8a&(W>$8(GQs2(#P&)|!6SyspT3y@T0`h$n?2Zs!aW zGq~rLcs!&Vr{+j$aE6ZQwV$FS*Ihy*x-lQHIZW?>&UjO|6flEdsA;-1kvcIh`-fDz zEKuUS_{F>=%)sTtD+C}s!zP$ncU3*JDZJ7;P=EA>vXJ<`LO!O-LVvG!fBW=n7Xa_N z0Qlq=enhpwua)Ucb>I@_@^w zieB|uomRXxoJws*6yBwl2v2A~6E)ZI7pSGO)-=U|->%5n002M$NklknbniATAGGJT(k8~B`nK_fNiOutF}FdnIJb^**qOs=4tJVtTlUW^QDGR z)RSqiw|04ah_ajDO~j7RnzFA6iw0Z11`xpDB(|>zr#3zNjh1xk6_%I9z>Bqx`p~>k zclSH3eck(g9}0c{*#CR@_UXg7{)^jhpFe*7r&aR3oOt(B#SjTX*44GE~^id zto|u6S3h56N%`|^8CG$ z`HrfJPwyT@liqH>cniLCpy}CGx)EDP3;=*Yf4?oJ0({6f;!ofpCXlkrPg-wqAsX9Y z4w%X_@J6ahn(DXkECNvFuP(7o@tmZr2rbBKTnSg09@gl~pQGYhTh10LmMEFhDb{_H zJtEK>81bUmeyY35XpcF*fA8yX02zY8`Xy%DMW|urQj)k!3 zv}bJ9Ygd%&jTi9$}j!W7Q~n9PcrHVzq>A}njROfTcB*{;=8kQ1WWE5U>M=+e~b6Er&P z=J7e3swj(j;e95OW6@IEOI=b)9{&t$CCFf2*Qj;r7Q*20{$EU=)Tpl4fS9)eM%soTIE60)H4Ur9socP7gwh70->&X5M8}EMpg)TM(3k?vS{N zLz)lCL%40GDji)u)}~n$mY(X`9}tgVtrw3Ro}v*VrhvNVtz2zm&2^4BelZf3G;16d z$im$0KWk=0^poEt`^*iq`;L z+5_6Dk?NfgV-=nhz=htX#H--PI7hPV82AUkQh_14VgrITO$V%7JF;g2!8D6q1oAz1Vl z#V&DD7wXL@i%vT+9GVsMkYAH&t>_^lzpnEBTOaRy>uVkFD)k$#Z~e^E$B*ypV;|`B zp~6>DKb)Bz-jHtq56@!+)3tN#C%?YBE40n;fzcL&_g+(WB#nVx|34f?K(S8yE_$=5 zA$CONZG60N?J$?y#l3NA59BlEf<6tn3rZuLvxX-Ww1$R4QY-1aYCutR$M~i^ZsUTj2LaPF0#n5r z^tleAP-)Z&z?C8pe&*e4H8SeV(6o)}y^)sZIRh zC#6`^2E#R^t@}58xX8YAbM=8TJV%G@LA2sSogJEgf{Q8ZT)hZX8jSWYK&xK}#zl!{ ze-XB+&c;EhIM$~zqAK8A9brz^umu+HXU(g_VL2&q==F_QQ?6M6Wj515>dVyfMv=C${`7iQzaRwuNg?Ns6c zhb~p_OhS05rg4%f<<^Wq{xt89%FYSHFjp1?{uGjAHt`cl5q>#NUG9OvO)fdVu2RT6 zB4M>Dc*&eq{)qu6$?#TeQ#&2bpiC|z4+)!5k68%YCd+eq5&)+`(?|=LKzkdJi_YWWX%5Mg0`s4ezKfZte(*Eu1|ND>sFUvg=U_SAaGA+IH zr0tjYAA#=k2DBJ@V~@O@aDxWmy7i3PYgxCC-n~x=oh9mBqnPJ zJ7rn#t+sIL?VyCnpxw(FM%AYSjMc^cdZyIdh3c*Z8sZVH!IIr$&lOm21-QvTCl+it z{KYkFb;~;Ms`E+T*bIu4L>qM3GU{b;hE|A}u6VTW*`7pXpL@TUq%f?`&lZREFeiqA z=HhBe(-yfEYst`B%{K%*Qr>jd#h(B9_WqX^|6KyS^O;(ukNr$z`w#E;ZEr)(V;(XK zXyFHi4??wp)U&b ztR5z`3w&k^68qaK;dZ`3vh>ezWu0fVEInY4Tzg8m7{fEm@1eMqU+S3VN2xbrxC?+L zuh6k+2q(W;xmYz_xuVQYW+kT??m$#vTa0vRPT97v$HcL*SPXSA75g|QUkheaj%mlz zK_1#h;xbeh_a9FI?BC z;T)1y1AW{q=1Vsv;7Fi)H#9DqCVQ=EsTnTK9Gj7tAml$L};E%d_Ohe9C7%Py6S_NmV+F5KHrXO z6mATSiDz*v5S6a_8ar?{x+X&GrHR*jPZlz@+M%T4a<3e4dbzfmscU-k^mVfmBsJH zB>~9FisfMOg6HBNj|U-A5+?h*i{=cajB(43b(u(`bm7(1U(t4eG+m1r6fz5QH@!`I9{kZFM#i z!?>^;fVoNo9n@c_7U1Ef0KHZoIe$+dp_8kxsMkG*_b^~Q%gN_vT!pBGdN*-D^l<;X z?YjW@@ayj%-u?EWr-0t|)JN9_K1s0z?k7cl?e>2^{L*thR9Sg+kNnTy`k~L?KK}22 z|NH;``|tn$>Eq`XhFhnU{2#uzbo}<`*I)km?c4u+`BI!)3H5_O^-Ox5+{LMc(O(`$ zuiTQd-!ZPmgFzj48s<)s-ZCgg5$AC*WVVouuo$f20#r9IF80gE9dv&Q<2s=$nDU{% zcQN9(!@FJULkmwly~-`&+&gde)JSMi^UenwfotJWiguOKW7|DzQA|SE6+&Pt6gVkW zcht-s{ioN!U7m%BE0Scm^CDCb3-0oO*>NGT9(#d`YXn}m)%h`QelR#`W2S`Td<8jC zkc(7*KAM?Tms(pv{p)#~<7NQik%K%qtvMqxMM&oKsize$5gYP7Y zh3MmJ@Z*V`{I$ZSn$px{ef`3S<#smqK~6%&AT4J2Ku{y6=xa)OGN~`ydIr{t-=hD= z_mZi-%U%CR*)~JOctWdmNXZ_5+nLx02qkBcr5~OE|IMgjc)$dpvYK@85oigiNB7qq zc&gnV3A10}EVgxh*BM=8`;=5x3UN!@t{L;OQ?}F9#*=XQn}-6G6&TNEhwH8NyxuPw zU?~7a629&0n6>gbG% zUWbR@aWz?xnP4#st4PECs77mRXd$)JRml9fX7jiC{eZT8VfS@ATsGcS^@H$7lb9WDpyqX_^Cvg4D~-n#UHk*GbHs z$4fe)o$R`MN+WKaKqLa4I8R5FP!fvGz6`3k`FTDJK%^<>C!S<~Dw?Kc%;~-U$Pt{P za2xH{GjvV*lxn`}b>hSxp|xC`*Qx(P{u$AW@xfZVz*msdr+D=-JWdZiU)a5lKDaNi zF&BZC&heb*s%zLTefGwET`enjr@WJRKcz>o< z^J2Ws!UC|sR~50kp*FK@*y*+xA`*BU3?Y8@gUR%Jn2Ys$5_)C$5g?c6>;U40SLgc1)%%eG{m9;z!mT7h4sMS3=HTk(dwF4;?)75jUvQ zR-;I78ctO@3%oeP zPoee2hEDC|30#cAtr7+BYpUIP#0FUVM_Y=8k-?lB8vOx9??HvItE7JHzbF2?>+eGV zp8)voZ$I4G&vy0{(8r$S-vTdH^+F%#i*Pne@LoCn)`wl+|Lwzv|Ngh%{`>de{`1qP z-#_-0h$n|yEY({{=jYcyeH75+wg}&4Co;suNHM(h|t<|6kYK5|Fb^S`b)oJ)MM5hZE4rlM4w%xKCKp6!X$8)U%}G{ z1GNi>W=GhU72xWY%Xq!VR;}~oacA|&+LFX`Le;P=Q7(7I>HcEhS$U4{!Ky}>Ovv1l zMtkE`GgOx0l9B2b_5=QS#Bd&=N%u_d3z_ijfVbqzS_@#U@ z8jh(#V%TUZq}8T}=*ti$xK<|#@imJnO$ZNk@H?SQ=4l!p8po9+b=p!zYKIpY*U&zA z)Q9i_^;iMm(Uzd%!d6A>xlG3>Mq@KExLe8WV{Fq+S=gR3}zFmx#Zd8Mk0Wx`@}5C7}7s=Fj-3 zp6>2xZcbxBY~p-UlMTeQ;ogxwm1-z(q+gptkZ287v(VF7F^T$h>ck7egR)?30aI(j z1!=1gI zz80{&uyk?eyY=6E=Clq2le1O0Jg0@D7w<|mBWG?n)4{qZEd@AAGvpktQk6E#hvOXS zltt$P#tF3-k949*IpXqcPkS(~F>Kw5Hm(zXaAMXqa>Ho@V?dkt>v<IPlST{tp6~0KmtXq+f{*tC`RzylPOIDh<(_MNVQuIc%7BNUXNLNqKq@S67%qIx zpYXTqj~4>*_;CQW3y!|!H_-D-|5e&+>4`yOGg|gArqEij0yY7WJ{!+T6kQ72WtNf2 z$DBzy>#5!x_U^Cb*}S&Cngn$b)34b_G<4g*Yz}!!H$v#k+00E74d*uAB6jvEJRrAB zAH;K$;Jly$|7>CyVt zl#qNM`mRyt%$pb2V)XvII>ui}jk%r&8~{q0buKJ)_?T z0hBx;)8Eg1gyj)6VkiXkY&oJr9>6(?n7zM#b!GgEGrsprC6>%*Lm9BAcjXa%o8fu#Q zMLHrdw?&5l@N@4_#J_lNn2nQ{SsRtWLp&^1>Taz+6+awUDet{Ki198UKUr2gje|J} zv5wIk^V%_U3S-N1nYdnLVB7R&fwvnlg@JIEMu8r42Zj_cFc{7ibc6jT8@JoQsC3FG zfTWgU9IMCxApkT?phIlTnV>J-VqK?PAQ!KpI9-p?G;R$?@1s6N%_6B~dIsi7^X?pI zx8qFGHQzS5MPdw89+GcAv=l0Pwd?)9N1kK38)Q*j**M1}&H^R$nA^5NG|!RQ@~*$W$=WhTcw7T`m>k&g4Q8mA`%tbom@HzHE;d8h0(z z;s<7&Ixd8oC+b9+7aT4gtuWhDtT{!3!J!m7IVOF*CS}vXflTd8IZk5Euo}HrI43eZ zI0e`FadW_2qlO7$4v%cNlbPNY3&l53k%z9Pi#E-;H5}8`(liyRZvz9t86YW|ziR*w|NQ!{=lDPF86ggcNO9<1s*c$ z2b=VwaKhJ9G!6ZC-IedYq?7S;9-lLdqqFHS)p&7TsHMsitHnC&L8zCVxvQ)>hD@~h zu}UkoT4MF6k6$hS-p`k|RQvkpx353GfBEHG>;A7@27G&~;6Gz^0o3B4Gjq={{q4us zZrA#%}}Dx9XZH zbi9ljG2|aq)F$EjvC_FRMUOk9jD^*V=-4GZ7z1*0P^$JGm6A#adz3saQ2Y_E#g9gh zYUynY?xQ=i)DqI_16Ap~=GL6mgUZ|bI=DfThYUvAI=!Bn20`B4{gR=CXx}A4EB<|Q z$6i%?HFj_XzbnUmH(nPGR#^GGj#wO^ZNACL2bcLN zW|~YyPf`A0eNI8q%lYomDC`cw_r7jp!L5($D#Ai5OwScKQj}Ehmlc)_u!vAEBmS%^ zRV-R*ci5zCdKNr9oY~)mV|_LfM7>VyzZ52V#`Nwi&%u=X$H%(erh{{0feP&r`9G)> za3pPW5ROo2>n(G&U6frP)fz=}U9v%2<5EUIXA3)R*v=XagMe{7JFg7_t6GD;I82!$ z`|;*TsO8mS{nB`?OtL%6aGiEOupGijBEAiY+%$pLd%Mk0)$;CmaLhd2M6e;E9uI#i zT|-S{9A((Sp1JsrlUYp2DD8F_4be7=ecQ`wRmEA7kONSy?btgENMO;f{8zi-6oZv4 z#S4o0cAEqrCSJ`*JHE9Y#0PS9xqi++fn#e%@l!wh;HbnsyK7>4>PsoxS$K-r+VFEh z1+0pD9|0WOWH3C1T2V_!q|IO&E3YAP(~4R%$n+!UBCN%P8Z-2w2HZ=Qxu%&~>HTXo z4Is*lZPIey+4lHdIU~b+A-$MRPi-b$;bKesb5>11e4&3>D!(ic`2n&F3NS@*v z=cnDaQ?AI9+1Yr$MzNjX3k|UoeQ8)saIN2|*PgG7!Qp<&CwblF$Kc({bf2JyEqkAS&qzWJ_QgOk$1gAKtBv2 z-Ou2mQndzi7({TNDO17IwjQLT4H8_>IW(!&rhIKPo*?_F>d<7O9H-MiOdE2cO;3rZ zHL6bbxoN2uAT{KUJL#xaj&Ij!$YKy7K(+0Ck|8gUyv@Q+IGsuRE8fwBFNu`4=uypT zzg?5nLQuj9`<^v`k6Su`6}_z^P#{B73VvAh99_2sdt4M?&>AfwrB#N`;oh5?chV1A z%|V&lft|#i<&&T2!&ru#*7a0ngLDzf2o0cCHyq9m@%;1Gxmy@0;4%Y14{tf+Y{l7Kx>rN=k8Jo!S5eGe(G0Z64^sU{Qyus zl6iOXd*a^@G7VrqMEL8kfB*F9-#>ls8sKwZBJ~)#i+~>B+Q}c%2cogztGDnP6}i=r zsvx*e2hA5JvZ&!J0BxLKtuB7j+bpyIwG>RFu(b?pRp*iY+Pc~Q=Ue~UxZf-KrA7bu zuig9q^V{3^UTXxn%d)>9wl}zQ2?gX}*mCVK=8S@RZY!ISTb6U$K-!=t$!yyl4*b=m zV_WXsHBhy?j5m;5)@_%?bN(FXH;6xuDneTw?bkH%&l}xG-PsNSi|x zcPg0+XO5h3EU^~`D$*ze+LlF7X3Ly`G&TW~4y0wpWOP3((Yjur&OFn@0Zc)R`Uj4c zXNkTkg(#WRH04$3o-M6R*`Y9bIb5f%Eb4e(8*6x~JKE_4$4AvqjmT>_D-&Qxq%qx& z!a=~gMosgC#$l^4fi=KIZ@U#<97D*Wrfbou$VWTI?XSpOMyDLhcw;^#Q}GkP#dy%x zVE6b>^z|nDbi}NlN-=6IVtPmCC3QO^W2dmK z*&rlNjmZ@NegdZ}wj?4!D}Ht~H~c^hg}$p{&QOPCk;OXx7!-l3#ZIZx4rA)UxJjPsj}9 zwH+PKjE50X$mr!~yE)k553VCKnEf7mrZIQ^@?R)2($R8KrCNk3l&&(sDg&&P)fIosFQNUjcHu9Cp&+Gp50L? z5U+OZ(ptC1)k{Zc>b=}4{?^SS*|*k}Dj5jRQi%S_ovz!*lKG1JDn$j`E~yHxEfxF1 zY#c47EOmBrZ>1l!&pq@rrXK6LZ(nj2Q5p z)VA(7Ut0b52|$Z}U-|7h?YAE9>F192>Z5?SJ^}dht=7Jy*{`7bmc)m@{rOcee|i6- zubF>)>*J>`3{TwfN#bGZa*(V}Z-Jx+ZW{(0h@5Cc)jt=+_D$$>FGFZQ~{{=jevfAXX0? zTgHu8PG7Fp7X@s#CbJ#XyH)GH_$oJsI3U(>^U zx~cGKf&lebtQ?v@JWQin(5kZ$ml-iFUXwA?tTK2?hmdZs@zpcDoyQRTEUg?_R0cH} z%y*wFqK(0&&~}2d@G@Khe}q_GVdFl<7=!M!Q}9Qn5?^!)n&zokEC!cQJkd+yN)*+j zARrPA(E}I*Xw!k?w$Q9sBw{IX&TD#bo$=LKc&rFeI-qBxk{Ye6js2R?Lbd&vu<&P$ zfw>kv`&Z>?_9xEFvyn^BDNI)|$vtRm`OCklhT4r2_{A?~*w!}fW#7tl4M0oJU)83?Jd=?vBM zH8hP&~f2OCsS`$nIb6%cd zux>ypP1Lpj3=eaIaIuKO=M8kEahNjA94c||(Ec=?Kq|8f$a7?ltI%T|o5?J6&GHy2 zo0eCVfqc93i1ro+-86O2)6ITi;}gKfeH@z%YmkFfbrN!)M>Gb$;46%?V|Nirr)Yp` z>-bDb(4K2a9JGP22E*KyM$MwVWJNZ8wV21*{kw zSFUB2BLzRuT54+n_c9qa3&>jv^JGi(vwO~fu=`Pc3|`D?a$*8D-Ac@-Ax+4jD@7Em ztyK55(>fr+gRZDP`H;$zx*%BA1T71#%yfo)0mwdQ8U&q*Z=0Q&W@(^+)nh|5L}CC= zEHydC+o2uR=g8AamZJ`Hv%Pa|_us-6e?yW`E*HkTADkYF&xC^E6tc9;^3ufUndSL= zWg++dazW21Vh;KNkTkOz*o=u)T_vdF!5@@>$(w@d^t~XfS~8@hTZi5B?&`GlR~gke zYVG^ID*>W;|9nP*kU6DzYNPGa0sVU)0*IH{dDOH%{3VfrfKhfIC zmiqG-JvJZHtT+o&U;b~7YAo!dhK-HNZs}7JX3NyYN0gi#B6#htwr@vl+28%_Z@Z-( zOu#N-4i-fE>3|vi==^Ul;Uh@)YPw|`Et)$PlC{-gYmQul9ggF9IW{A669~>ZS~n@@ zQm-84l($|VV6H-$0eGf40xQPNrl7h|A{PQlRa!^wNv5!iM&+|ITMrTv2SgdzdP&v%7;C(g^O=b(kz1S__! zGjds8>I3<>ueK2Qn{VFa>asYYU5ONhN0}xXY9XWt>#Xh?w2j({u+BqpE;R42$p?H9 zRlUtDaF{%5KXkhsc+}K3IyPJxYD03En_;V12p5c+$V^kI_7I82J+QCVwaG6E5Tz&! z)4dR7LoC}STQ_=GOhnp+ z{u$j$OGGwL$kBMyRB;V#a${{T*0E!uy|(dB;rO1QfSke^J=G+pi*3Hl4hoT{aVj2* zG0g8Z!MSjs!D>J8IB`OWVSvkmsxz`KIfsVFAR8z1Bwl*rCPS%MzqZP&pjT)Pm}T?v zS-j>cBJ-EzG?2zIQ!`O@X4_pa%GnjIa$=l+J44nR$qp0G`$M+Er?R0!!KsU!_Uln8 zT+6i18-AI(^)1E+B;9jno&{GUY#mm+sT2T^lafY4yI33#{)+@CdLkWDd|_TF=h6^N zM+%y^h~>8qX*Rhqowk7hdQe1<$zf1f_watvfc1$2GZ*RDY~2*xO^+>mYV##VTn%bAGX0Ux{fv>!T@xQ2@+_9rJyS5Dkfg@-Eh16#ag#H4QC=4T-ROXH=`Wr>DGtHv|KSNt6AsNhuO)|ze~4fXU9@0Ch^rK1(- z?yHpqsqA*27K=-qjLY6?2AD=&WG0@`Q4-ECT`tKS*C!VDSNuPc#gsrSf)Qt9NS`k|u(k*^L zJ#hNBIAvTS)_;*B8D0IgwVEX(2+qm^(LLe(?)9U~3=e6)b-m%SpC114`#*hM`K{&c z_qSee-@bnN^5u{6r$>$7z5Dd-U0;1pzJGCQEf+T50dW1&+*gW*v&+rm@U1JRZbkOg zh}`pa;R+(zTlRD`{5|G9sNn{;mP0?txf4e;%Qb$HwV&JVj0<#QFAr<#l?WEYe1K|L z2x2o5>fVLuR{SkWdn(4#)c7Q=i)-a$GVUPV8G|laDGwE#HjH9bu3QIa&-#mjYVx7$`BdbEO6{gZjNEWq@vyO6f zBOUKt%w{#2U7v7QK!EYLdV9!v*C@LLZHJM?@CASFCro}O6fTuUT;9+oJI!)}cz&=@ zOmeSp0-FoRVx{od-{?DDqnyhNzzm%n*?lkAEYJsHhIAhSI4k1cRl!pPVa>#60)q&w zmZ0%7d>}=ihA8v5NC+UusktchSC7r|!jQmn>K$fhn&@j= zoICPIYGPph+U%>EvK}RTBDKw7Z^9x){AU}h)6-$_u2O2ecqV?PCU~KJDSeKQigobQ zizm-$@A?R~*r2rKGuuj|)TI6#9;zmY{mr=teo_l1S4i~7HATFdDK<5zQQ{bisY$kW zuc4qMI{#1GN;k!v?Cc~|(C_;*&OgO=bQ)s;aatsBF`I*d%R~_38jlRbVpE znObNbCx$1rQ^ia4#K(X*Z)1=|fg7S1w?;BeC-a_Q5;=G0z~PzZNuE0SI71oDitRzO z!!w^EH;dT?-RKFA5Rb_=CcX zmybzQ=#<9lf*iR`brBQey=S0#7@xB;{QW6ML|^m>wJlB3L|H6%deTXI#ZV?J0uRDv zN~ZKB5fQtu___ZVNI&*rb^hIVbyKM8`g&@{eff~rL68x;%P-cv#@u;Bt?0Bb_`kl- zcDCFge4dJR)A}$r}~l01Y&sKQlFB!AVbU) zsyW??XhwX;w*(Oek-2azRQ-ZUzbpE;p7!sH1)o2D{O!|kT@U!7AP@YM(WK^s#b-|G zVr%s1Ek;w=kZi6A5yiI*K9uw?jaU^^;Sqp}2Q!e1_7fVF)YqTIGd4>>Kj``Aw-3)q zWW14KC-w5awW4nV^b}ADS$6&S`1b2BJ;?q2y-vy6bA2gOlv~To=^P1F!HC?;Zv;h9 zxtu|-lOI~0iD&ti8W?d#t+vrv;n}ikcum1B4b~xv)>|13vUIvcNQ5TO66#H(v!;c6 zeVd*>_weAlCF>T(#oYCehgQD#8OYb~Kfd-gh^~jeeC^*k|M>d-&q#eGqJIGQ)_-^C zyQ4YX^&7SQ>9!!356{|IqAaSK^CuJ=SU?o3t2v5?P*of&N?Va!Yz&#`sg-$~PHHnw z){)HQ-nv9o0yUjFqnExg^`)h^fi$;;H)b|q!kC{lhQwGr9(U^I2< zTCnvhj5h0ky+0TWs7uVo^nEM9$T6tWnN*faYBz~I{nq^0Ii#P?+Im?Ot(rR;!kbIP zj~A@5IW0Mmqs=O!G}{UjsYi-d-I5-@989vyP}4}RrrYRZhK61)$oD-7v$1njBYaBR zrv~;d>0wXVJgKMz@Gy^|U4uXGHZ@esjCxdG`cWFo9TOCWT`-gw&K{?0NAN1JHuo2T zop|c6=7)VXBLhyV@x-pbyX2M!q0g_03SRR>UZ8f8*zg#8?P;z1@^kIURAkw{!!$r8 zmB5l?laeeeG9IzAn3Du#7^Fz_w65WCB%XETs)1axo+Rx8|I$-sohUT$P|u!a!E4hJ zb#T$}rUKf`%9F_%L=yI=Sn+5m)wt>9v|yzfVy|{lL~fLG6$+%)o-n9B%r7}8hJ;dK zkW81ipr}j&xmx1wo}|QJ#y+#B8+0@PhbFCh<+KE*)W0|n3T+tY(J{%7s4T%#%dzY@ zmRfFm0u_^aw_zb4?jyixeIA&pP8jVsw-Od6s&&8thj@m5?gPi>0{(ZP=mOYTwy4$ozA;aVKg+x8Xm>k2bc3Yjm>4@H4~-9{b*-gGePrP zx(~WF;?7VZL@}=o=6OsU=Jt9Dz-Cp*{Bq!M*^S|6K`)KCc}e5^5T+FAX?=})eUjQM z6i@Tuyr+~J^V^Mg+&tM)>Ef+7YIe$P-s(hS<+rnP76!Z4XU?6Yvo}qb5D*>k znxV^$EDT9?I;!}ab)&w%DI)T;2<=bXsaZs-F1m}>CdL|5AiC4UQiQ4ukuHncQ?m&S zSEbwKq2GY_Wb~#LUAxiD*s=<^2_`l@v{nR;dd53E(9zQCI`nw(4TzE|pZ zC--gD2Ik}s*&J`}pE5z85NSRXLX-?%@=-ZVxiJ-&?at9!_zqC`+EBPY(p{I)rzt>S z((j)}z5Dp-qaPgV69eb^0O}O_bdVFdd-a?$ zUfhzC5QU)C_PofEVl-cKNFhrgB@)VfT?THhnrcDe>H0eMV%x60j{w{$6n*RC>xUL> zt~`BDG`7~n0?|yo%+BErTWlR#Xl z5A>0p9*P3$aE3^tAPLLO?N!Um1!23&Pf&OyFxcl`?P=7BI2TwQa1ln&r5LYEVyY5l zL0UgA0zM7M=Uxj(EX$9@kuj=k%BQ}>QLxgIGX~bWt>L|o0CaZ#&Skq{B^Es(l&{cF z0bLrw0<3P|l+mzB=S9j;RY$n!%)v}jwr-b{g|uos%^PcCkYp_ZxL z?jav)Kqd%9m9|CXlo5<@{rNIE0@E5c)bJ1{k17!1YTqV9Fb~xzLw2|YdigSC<4Y2C z(hsH@DU3|Yj_T9Of)+O0SW<|;exYZru*J{+w&WM|7|mYz?Asn!r?c8QV(B)Z>kWu! zdwW9MzHPW)^Z~gxmihT0dzh0w8PmKA;ua+CMoW|n zj6HvtkL|(|AXCQ!72acR*o+JNM%03{m<`%-4qYIE6z~LwRbZ}sZa>j9Ch5;UcDo5o zeG}Xq)zIHLvLqLi!Bwm9N+4Y;3hUeJWsu{&WIOdTl+cz3bZ^ME&-VU^kP1(E(%vZQ zhF4~KJ&Qls3MvBJ1m*8el)~(Wkn^pxj6eN4rBmzzDA8t!c8p!q{4CA`74gYZU6`jp zoIe=2p)1CtVri0c-L&jDZwlq&cuIe&PD61wcVDhN5}dvO3QOgdYok;Vi_sJ#LzUA# zz|)u8ibNfpo2_R1U_|~ccqA%;DU^rl^l&y(Y(*lBt?@XxRJr^JL(z88W$E9Qm(N~W zd+&UcAdGZ@IDD7b*=DeV5Nqu3_*OnCgtMAAEj>N;`zWmG(JdqO@NKDiy=qt9Bd{Rn z|AeBE$dir*iO)Ib*%5&D$PKG8gPA)eLM#7HZF7a8$t+w{MX0F=E%l3f9k&5^(W^0* zySb+`J4?FQeP`_`3)}Z@_1B88m9W6Dn&J;{-+z4QLAE}Gup(|^a5-VvfDrf?*<8;> zDyc0Ym(SOavH5hsw|d?7^T|vf()1M2eke#Uu@gLsT%zSIe_AtGkFC>-UC-i|W-b## zx5N@GiJ9B6*}o-3vZcy1O-Z&KfpSBjaMEZ|Lm;0PU~D5%ilf#WAc%?UB|$r`xwqa}<2b_;{$tz44@^*&CpGCalNR;u zax+^zjj)HH7L6Z%2(|y#*1vAs%O!yAOW%g*(~z%Te|+g;po@aD4{IvxijJm5g*h8q z-8a;Ra9+qBF-dd*2_cunh1l^fKd`J`tx_pzEDF;4aKX@R9BlLDk2!07kHxGjK?*0= z??Ok@_km z1qmS4m~9sXepjJ5mqN>^c`Je;`4}73d91M*)S;M2<+-x>EMDy#Bkx!-ZT!2|wXn&* zv@3x26UPK6JXH9}8&Ye++4kx(n(L_ijmU-$;>+X`9CW!6i z3r38j?HbfIU_0et94>}8@VCw&#_qG7nutsJL75L_e@&u;vGHm6{Iv}q<9yu)YVMux zHA7tdk{r^ZLm7bB!-R$4Xel)VN2Dv3fd zp%)4Isa%_AFdW;&0GfVGds6_2Z8oNk)mdpbku@PzP=39~RXxI)NU1f&nCPU%_7}OP zBg%A&4%9jd)M*3wk$S|h4l55y(Q~q428n3Lr(ueyJNh+F8InFLsW%$&v=r!cJKXWl z2A;>jtYND#@N0YQvqz6NS^Jn4i7oTG`*j)&zXL1vW11mKFPxxa8=%BwTp8EA z%Q8K}QK{X`iD3GfS&vM#W;W4cG>&uQP9Buvui0gUFFNGfO{jV4At&}{iEyQ9R&1+< zYp9r5SizvJLHBV3wUcp+TT4vw?n>YZcnF^;zKl5b>SUONtEU{n!#L28ft(Q2!KxQ4Rq*+CiBJxtiH$=677XTdd{!SW ztUT+`!6yJaH!Yhz74YNz_dh@M^POEU?2ddwnd)P$j@G=Ul)8Hh4lsUs`~JS?MgI8y zy?^i9y12zty_0*s``2y7eE`rEKu;63d`nIK*1k~kVt3+Q0qoHX0TL$@KwNdyE*lx& zopr=|j%T(MY%R&NcnfY$ChSuV*xFR_3AY^Zlt&J=YVYAE7XU5WinQlp8kJVSReLlj zx)r03NgA{*f`9$e9q>9zho}A*i$Z*7ObDk|@;rPyB0= zM*}HALU{?9V*j*N*QS7*VqsG+W3qDIEl6sK?#}`+$?@}4eQG1u*~*J-`yw%Ctr`QL z*RE<@@bsk5cRzphW4HS)_`iMm^7YFfU%&qG_V&-OZ~ype|IaU799Vk;7=E6uOAhiP zUK;X)%l@j(jqMcU*WA`lrNg;QLT8S?m@S&J&rc{k`eJSht&(4scb2~E%dX@?riTP7 z`55%1N!aqdJe9+Bm9_HJF>;2wL(?$m3a(TulPhi@`!Yui-6Jl$cj3My+)yJ3hjFnB z$4V75#UB7*K%c*ZrFSg7c2E7>qAs18;yWX43#H*m3cHD~$A0EP^ z@@X9R>6}Mrk4BwQ@SVsWT?$&JNLw48N`uhh2~C68Bzp9S%e+~f&>;h4D%#aG#qF`A z%X=-P<7oHTorix3Uc?`G4Gk@W5c`33qyzAkxHZD`)4L7rU&{fKOUaQmeC~$JX_L$h zk$6eX^b@j^0;kr;%eiBo#VDOl^yRbMl_f8AwJ(#TFXp+G%(Fd~OUM#Dn)it;3I@gr zbvT%(_=SRi@w63GI*xmaob^ITMO;(7X<`P#&WHTeK*@zDp$AEkimA@1?UEyP#c~4l z4o>wuGL@1>txpz4i?m}7%!l^tGZtkAR2N%^p+H)jWQ1?H=GEczF)24s;Bj}I&~8hy zXqqiY!l5^&IaU0b>7TYN!{<1=b&vUQ;_3)am=hz&nWmy4UuKM9M!9y%>5kso;(Jd` zoZcW6nHDU&H{DGL+F9DvW;*szZL5DL*`2u3l1%-YV4^LLs`!hnt$lw$9H6?X69!~9 zJ}udyZi~K9cDP`i`D@56Vb%cCh?SL?G5@bs5t?iYL{;u<^OJ_>{+deUj>#4JspN0U zCAfGVQ&9+W)2cYTaYiE-ZoHDfdO+q{;_gwVma$rs0BZfyuFXy17PpqjJ_4}PcA}OD zMbX_K4ovpZOi3q<45hX`m#Cig;(Z`ORm+vJtqc!kEVrw5(+N|N0z`V1E!rz?(ctEL zfWX-v0-7mF8#O!KHKv883&NHbWB9YrO-T=+=IakX-|1IqEt5D}HJybm5kG4vpDm}j z^2^&lzVz?I-hcAQ@7%X%v!y`hGIlxr0+Pti;5{kkJnn~bJaK*bZQN(e$nIx^v^uS& zAkO$ig0pR0%#AE%&tW01tDoM*5y|whkT3QY4+XV_AD^v`Ty}Z&h2TC4sG3{x+%l;$ zVm}P}{oDI)jrY?)UBAEY)^`t1^^M`a=bWsIwUWwzNR}mS@cJpxELD)&HJN9RB{slp z?V~R%atu(jgVYMfSd6EF(#u>IFttdA9*s2vj6q>-V^18tRWl7Im|N_jyXH}FJ+m(L z+m(GW+SeZX?V~OLzP9rJ^2e7y{`m6cpI`p?$Coev^M|GXKmYjFQ$T%F8vHVCcgxT5 z+%qvOQRhV^B*6I!ZkA!5Z`%!>`ox|tE}*goIXil{WHcy2j&sklrt5sjt**K1Ew@^% zdyBhR2#hY5a4zv8h(KLxb>28jcdO#^)tKpSO)LMt*-*~us4jb%e&4fmU74S+eHlPT zO7SPz!*af@A2Vo~z5ZAy=~S-q&rFC{N6Jn{wr6aY$0+f(yO@D51jcMw`#oK`r5;O! zjOCIZ<(a+ol|lw-xr@HDPfYbRuWNh74NO?mbe#qmSmjRpBA2E9OiW<=ANhMu;9{-_>T@w!b9*FqE0f=1Q0(()9< z^AH~f4GCNj##;p<+Ng|u2~P1Svsh&qAL~PO{Qo~yccSjLv21HrR^wgwt9t(zJ^R>_ zmL%6dW`Ik+APHm>&5X#%1|SHM5$h@#SMFaL$Cvyy`Z+zenRGZ(xIPr&*oGoGK=9O+ zIzK#Anv$HEKEMx5dY}AbDrDmi+YKBAwI)@Kes|QmskXN}X}fE-0otUMNUQm%ccVtqU#lhR%n)l2icDs^8-lw} zO{JR>sR)$yeHAbs-(!1aE3ZZ@QVvprLNZ8eN)3AW zs`W0&(iImpY+-N~)v>Fb4_krL`lS$PcI+CwTY2v`8QxplZ2(Qsg7NcMh8|5sh z)O0M6lvh}q=HcBH$(q!PfQ2Z6@Z;0Zk9{Zbr9bW31>yeowm-;Cy&y0-cY}oYTkFJI zWw>R2zabF!IU{xZ`XbaBpnEx3h$a1JD`gLNa={-+S2U2w!_pTN>)^gTXtA>x@)Zcz zT9a@<>B=X7VRK62XJkSXl#zVHZiaQ3N0v(kHCA*MzIXFa>p<2l+nq2g_iWHQZZ>m7 z4+Z-1M>oY-9RKt7{pZJ?1>_vySAPelox4FOFSde&I*W%U<(+R?j|HI5nnKV7cD)x_ zZOOA*R5(D~sfa$cVkIk&Z^&_+M6FSw_0-}#KeLO;8nkyc9XNJUs&8DhwE&LVVz{%H zmWbA3mAE?5z5Q1I{Q*RIHBv|F9y_lWV+snn))|jg6_K1Mu+)8(D z*|%I-lNNJ|P~EO3Y?9Ha*8MvTuy1R}OC_8iVCBp?wEa97hF)#PUPs0XR>e?>N^&qNo-#)PI-GeA!uyMe+#>AqWfgH_>}8CH93OGjV*hHc&1P=wE@r|hF386;6ralr9kkE16^_3$iZDl9;s0i+*7p_ip zV^Fd_`6$B*x2o!+uAbvYUS>xssNVJX(^dHcXcaF*2tzt$?T!X%Vz zxr36(e{X?-vEpew7^;u=y;P4K7s4^{2V7|TEEV#TKgFsdDnedvIm;&EB^g5OlRC75 z7VFHLxOR9@M#~V15v@Sst7)(FFxem5gz0#&=pPkdjqW(AL03NM6Zw$97kLh-6(fH+ zqQXnoV!IcJ7Tqsma!wdVJ#nOum5=h5w!@y?rzzGFb+F8?h&QC=d~W=xL>J_Um!g+T z6h&XQwomQ(_SUzkd!=g4zT~9eNMZSQ_PIVpc`bN42ukdhk?e+sZBK}wWh`0=A8Y)g ztD>Qy`Bs$Eu4o(X?7L{Fqnly$z%3 z?Y)ij$X}Z)FY)TiX9GW3zP6;$@HZ&Vh(9{(c0;zlC&ZfS*?*;TzJr?8+iWs{>g(dT?)1wPBY`J8BD@C z(@M#DxFvI68#Iv%o4cmi<@$y0?#P-S-1K!$3m-eTsOYLb)nXy#opH2w?qa#eY<_G> zpf5BvcTsRhi-y)b_ZMnK4JqdMT>zWT+*ZWi?i+XXdWcpyhr zcoUY(7BN#sCrF80Dp{~@S8LSO(x(6#iMCQQ(M+BqV{rL#qei+Al3krYK$^J;s+x1wf+YX1S?043~(vSM}G*w3Jacj`qT;mx8rSX+Aan@vr$5 z(~Dum*k0QhXYzn>UJUU)1nNnihJz{psK4WA9!xp>5WYkZv_|KJD7PCl@jtLsq%)~i z680jTL(2W$W=w&g5btUzJfh1#3=^j+!iQFm8u)+9e8M|vNr7O^dp?8x0YF?^1iZP= zl6WEl|9yKARWSZoX8lQ^hf7yAtD~ARl@wuVilETGrT|AlFUM^q%OvE{3XuZ)lGv3j zr>|ZyiX;$gsn>ky9__KGp)o5puPoUrTk<3d zr4HCQ(=j&Ci~pTz6YZ%ForTEOnNX$%Z7=j>NYBEWK7bw>LeY|ZtP}^k#y{( z$cuo7xM+#T%N1W!OxPqL!gBr^C&Mer3P%gGs;4emV|i0c*+%XNn`YD5=-Dc;$5|Sq ztmH7o-IXjVDIx4;7VS`Oy3|bBgl&MWD6XU>Y`+08$t?9m$L)b;vffin^eTM zz{c!cXW3;uj*(U{>Cg$HdEg5O^AbnH{< z5|HKM*IC|NtC0}ZNXF9!nDK2DTPoJdS2UtLJOhc+_4AR2o(65zsAO=cC}FO@mS|V= zz4gCd#w(ZqyYTrHE_z2nJ#06ws|)>(n)03;n)K){S@ZU8MFfU5(ecPy^Q?eUHTg#?jw4 zyI9G9xT_$3Lnf<}dra%jfw=5-8xD$wha57l)9-kDuo~8tRj%o>RBL9DUE@VBmzdT! zE3K<$rPDiJs{B|&SP5At%^iqRTm<#7p8iom5<+QA0c=p{H$CMhr24VV%{+i zHDP$QWe{?uh|6eK>HOjSk1XunFRqJU!*aeoqZ1bK{Cqw<;c4RSJ`Qm3q#(jzSJ^Pt zl2DdgTw$r*CO>xtIx{ z_abD}?WQO#b-9b2L4L3ukrEqUBmvV4oJFX-(PgUIm#P{zP4duV_Znd|bt_{4v3!BI zwMk`Nrows3Iy9VU0+UWWs+zSPAoX9WPG?eR?KRzeo2*^4)Eo4*Rl-}I>ON)-+ONUD zpv)TPacY_m;Ik)Y=aLM_zvTXo2R4TCI!BVd($J|*({JDQml?1jNr_yzbD4pj`t;`1 z*;{qBGmA}}3=BNT3>r{ANc$16jn5m45%$Z#hWd@V{Z7ua^E^{OLMUO6n=}WJ z%c6vFka?y$guUG=&~BT#q6D*B5ZBN(Z`tS_%;I|mhpMcKq@IcwV(u{MXfXmWYja50TUFtPWg&RDWFaZ1Rfsxvivl7uW zv|T9e93U^(WG!`f&kcf_&Lu?Mu(ul{$ZY%d>wkKLka+IQwsF#_!hb5RqUQMdxn)$o z_k>v2Ox-FanA>ml&ymk%*4;$~e^$+FEx4?GGJ#9=Y>g-iW_6y-IFuQDzU7Z<%W#gw z%$?UQCyK+Inv3`T*<=2<^zSr4q8t4?UEf16zklz(|89-hy17Mi&%gZspk}s)nDR6a6Qmapd2aRMbq3Vl_v2Nh(|g48WYnoq zpskXdExJ!dv6tUbE6J1C`2Emnt{Sfxt4*r63~t|PP|L+mcnnsabM&JJ{B&^b|J&PN z-TBjPKY#uG-~WBvwg2i`Sf*#+oQ3gy(*T*$YggiS8WNKGF z0SJ=jf(q|e7G0qbe5IS2468XkBWl#G>7AEa@3OEJW~6Gf)eUS_aav5>xJHl_DM)jl zZ^ALVzF#kH0IFM9kQr_bW&5(qa<#!i>^3KD-eJZIG7BaYNSGNt@7!+oi;UAMk(Gkj zM$)=5TVo?bvPuCMO2{mvP(e11+Q`!A{- zqp_t$ys#=@k~t#~A$pyB`V) zQPGSWO`PG`Yov{hV)I3CbVQD;-BK69=pLr0HmWPAO8Ti6KUn*mMMaq0s2<9bi$Zz1 zQDZ|oexes7NP0OayX`5VV7OS8nhl##XjmJLl7-l@UOu41fkODvGpwX)b!CU`2xEBo zuUIAy!*8rx+g(NaM_A~-R1ZYrDnGrj?1M7GiK^&Ca`uz{T|M>@RCY+o*gA!IO0`cP zrd|V1l5(}%PC_KtTpiTC2owtA69>fHB0EI?mQfKau1Lp$I#JsVV5unvhp*nbhkvqH zwF=&U1fHW~O(Zt;>UJD&4!ata^T(HBC zCq5x-z2*~9NApIC!Ft3)r#yAE*)DOp*o<}uB{lv3%ar1d4iWg|{`!2A2xnQa-SV;l z%as_H5IscCmBdEK*jiCFptZ)i->>!<`z{mrfRqAk@=`h-?UPQXmJ)s?+Z#@Eih)b< z&6qcXaqt*Kn(8>nO2n+2C+1cTRowonv8`x|RyN;6HDS}bBfBSUoBF(A4veHA=;m2U;w;*M$oz-aD`k&5DST+O@6ZN zWlL7s7H`TV3f*(p$hTIzI2;;-q-g-xonm2Xo2Q^vw?)1%i?jvd>qiN#4ElSreW|}G zvITFgYogg}aI7ZWU=SQ*WbJyw(-4sxESgWdd>+cKi_7vID8wwXjct`NhuruU7RloX zrPmhGELCBp%vS4LhlRK%tP4K@jg{xR$)|fXz z3u&h-twX-F`uO?f(|_83`thZCy}xa+s_LBAv$Otv8j$m{ES#;o@}3D}POT*W`f;Xt zi}j0}YMH!}?U$by;M#RW6w<2B8OV92Wm2BpBI(2a?&^b|_P+H!fS>v9S(iV*f9nA_ zdT*Dz-9RCd`bN)x0(}4a2kqI@+QPQfc#~tOl7BAeuklYjm5h!Bm-=+Y( zIZ_GW{5fArD7jM6agJZ8|DyI8D!+I+|1-v5^U zKYO}QPx<`&TekrH`0MLiH~s(hSHJ!39N_y`{)jbqvV>K4Tuc16E5jm~vjMB&VBHo& zR`b5>wGi%Pt^T1CpbOH96pq{d>NRh> z7X5oG$q%Hr^HH`kcD7p~$sW2Cwbc(MN zgtwwz26<-${bg5D;_|=RzAyH`xLhh&#l<|;zEgp^)Q~>$QtcF@ngj=(t|Q8}1{LC$ za{$TM=lmRs3XY&$g_*vQrm_(1haju3LMs>zo%w4LY{sDpiAjcQ&K5c!mIE@A@l4~p zYU%+A*}bG;fjHWB?D481Yao6GLx~a3CuoKSOqtaYaj9i%HgZcpIBBMl@taK)AP~GL zB>_P;!Gp%kiUtb1%pK<@*}V%cU>0o?HYvynCz~V3JavR9&Ce&H>Qps^tdCLpQUuqf zACT%+me9$Ndm{6&&60P8DL8HPd5!yIP~KLXzLdyCK+ritvd{ToyjMR?qu%u_Vzx#5 zP^W2PhI<(3PX4*PTGeceL4a3M!}0zG`QJzhiZ;Hmn3Rq4j_pq*N%LM*j`9?ojE-YX zPX$cTCiD~Ng?=!`TO&577ErSh_Dee5h<0WXT0 z1=ECO>l}JwWJsK_wc4~_0|vUB-~hD7$qh$8p3_80?+C967BllGcxtO@p}d(Gx=kVE zt~=r4^)#HEIG(o5SzVx;$A$yX3>~g4N@_)$KK89pvD-zbNhFK|gdcz}K%nx49zjhJ z-=2BM;_z-sP*H8SPrJL;Z;q&CccyozeC*O6PKit8=Hx^Qt6NSaK7wXNi-E5BZ_#4u z|D(lm0VST@J=5yB`_ZS~Wi>Jxy}i4X*Rn$xZwTE$|1|1Dm#LyvJGtJ+fZ4;)e6E+U zk;Y>2pME-NN`5@NM&TM7dlL=ZKOesR^XtFgeti7!t(pb>kGC&B|IeSF+yCeDr$0SP z|E;gQelGs=o3(zceP_SYDKLxj8~2rN&yP%Oud{V7ROx)bg!Zasl`Ssk4Zt=+aSwAe zX%_KS<_7w#cNfqU-yJ^v!&U41y}Jh9128*#_vKwK_|!e}Et0K}`(v|@ z?*Fl7OM3^mE2XM0OMZ235OGUQCjeG5}p6C+c{y2W3^QkAzVY;o8SW$Ay<{qgn2TaWj7^SDpf{_p;u z?|=XO-9tY;{mAdXQ!hY-nGv%$dqhq^FDzEAK6op=u5&?%wn7fM;`oK#T=2vdk(I7* z%x9xD;%16-z*&`T8PhK7S(jfdc6UCoXH|K~Pt{KNbT`XnzJv??pzH9JumX6Hh4e6S z^{O&+fc)S?eD>W#qjzKdhLFvL42Yjuwq%c+C9Iox(q=uIN{7AJ8DOTyuwf``Hh@Ck zwG&~36wCM;A3#p#Y;rco>MsAYsFoZWnz&rD=On+^PZW8y38_KVT8a}LhB?g& z;s-$+F6kM{|ej={s9NqH+>W|R3uY*p-Gl(_<#Qkxj z8wouNfryeiv0v0TYOgnImV^5c5LeN3uPD%(Z5ZqtBjh6(VUX5d<@{`)uEttLDPZq> zjb$8|uwS(3)`{1w6S$*i!A-akuHT)k2!`7JF=oIDl6RsbCTXmbi4Pg$Ucx#q$0mh@E&?KXJkpVYr|1~hpNS5IA3 zGZI%uLt{f1hHc#@(Tx)RF2HosdWY18o!)6;TVrr86ev?pI_tbRZJ}H_49j}QF?n_3Ok1E?# z9qZ&RGS-S4wI3}JqkOamG+?}&d1musWF1oNnkwNJ zBa0*sc}ycH4?*vmHRB_la|_!p%eOb(TXt3Pdt2{_S;Xe%?0Dnpu(s`5`OlyK-=F{V zOqu_D`TYNU{DSB z74^y%%oixZ8EKfrAHS;MIG%MS)Xy3I@V{4YE&aRp|Lg7Rk8jq%x6tqMf0lIrgrMq1 zlFk&q_%Eza-~ash@$aNpx*MUB_GeN%yXn@9UBJw~qQE(TZ*H{1qACEXUJeYOcD+@n zh6PRy)jZLG0nM|qp!{;B8`O>JMu>?fMt(GkC6%;LvpT$#wg0}N?3w@jhO&|B+m9b# zzxOY{miztBSNH#X{px#w9(wfFS0F|?PUOc_@%%crZW!sjxjJ=R_WE*6L0-X>i zdv^0n3Ee(|h+kv(@NV_5v4u+!YL)L^F7IBNS95y+-Bl(x^UYA_e}}c=iK~l4IbFWJ zka5+o@dG9ix&9SN>9F6lZ~eTde8b=eYOem25eX_<%lwyK1Yeo}sAcN9dZHibS_!8nH# zZsfbhIY~uk4xQhsWfgUa9F~fnf|W|28KYbp{r(!c+0OIpj>8SfEBEH+AS-&QPH&8HBKA#)fy}7%7TC zU4Q8|>jl-=Kq#SpDPo|o?krhlN~?*zc1#bTpOL(qezHnG~`ysR_fx~6ljH052L2&Y%$Lt_+3+H}?r znjH&g56AF82ZRduDP6*43YIYZ+_%y2oqASQ#hscz&rfc<`KChrYEC|xQ%UGvfRtik zzAu%%j5W_GXMw!eNUt@bZ$>aS&ZKkW|q zKHOFl*5FnSNy{z>%Q~wnUHs(&g;~?P>v^9&8|SA#SZ?wx0F#>~aoX0!zP>r~iEJ2B1!QKlwvoXG*&f&ZB?$Yu^RrN6Ah^Dy)7{<2)Irc7wTwL0KX^ zS=qX~WKB7Q^YcLI_p_y4@D>`G6;tt?Pg^)yDRtk!TY%i_-|F|9JAC@vt9}Oi?Q5q5 zp8ILJQlL4f?>m4WpF4%gltASb@~UoK@<;!m*5&_3gYI=Ic34?U=e_C|RJB8zwa2{4 z!RJ*@{JlX}?=*^3eEX-iyv3o?@hU!fk+)?To;s+S8;OE1>ymKhMqu%OcPDz_(YLNs z^e?}i0DS-Pw?F-A|Mu!I5m&6*_t_S{_|~mEB3X|p_1cD~^0ulvf_#=o%$>^Y*V7K@00s#r zAmEw63}$LUsJL$LXG^g9?zu0PNH+si(hQSBHXoE7{&>zpK%F^vw6de%rD(;ccq&(s z5^~uM4UjQ~dK~}>PgkOq1EMwCXXewqTD*n7_5VM9)?K}y>(BC7yyo%i3p)8}qQ~h) z+Oj&kD-JjFJqQ3{VEXV?b%!L&>zC#A1?4`vTA6jmIAbKLal3P4-B!SEu;?0xkq%P~ zjrg5S{n5{jJaWkl(YNY9{UqSoAi_+AM67*ozrxC?JAf;}u;D=A(IolYuqZxD470eB zW+;i0JrASx7t?&YAyL_0$taj@?qpo+AY15G9mT&kyeh9Gl*&hV`+hLV^H{ z0(_XJcn&=|l_>4vR%r60{kI05_^p~srhO8u3H6lzUUL18mnr8e7nJJI#kxnii zh1Ra`m6iajn9`}9=#^4f5k3sLNcK!o5~doap@N8qY-5PZi5XRFzw1SFQZ0xHFJpCQ zC}9?nmD(AcD6}ijqXW{ATR0d<(L;4gsuhZ}&bUy%VJ5+f<(QyxIFcTM6+irO);2~n z>R-ZVSv%xOs!tna)c)pvgb#Zn{w7^1pTe7eABiOuz!jh}fXQKBIct^K$7<1RD4E6*iZjx$9iv-E5Ot zfrOqt0U5xU6%cp+%NK+BIEFJ;jRWVYR#pbrNkgZhPjIczL&T0OUzOd7STtyN=CtPH zAV4>XFs^KeGt_S*&6~=hrXVLc4D-C|r&(tQqw=lra3X+Al;kFG@|Qs>0euPGp3ad( z!xrz8+*nOznT_m|0z&o@pwwo+6svnzGnn%fnw1ZC9$;0w60Y=iT4EjR8xPHDA*=G1 z{>x8GED4$Anl&eOxRwiVSHjBljWksKwxl1J{M-tZB!4pVV>5H0EEwK9 zS?M#Os_s{KWzvv)++JG=mos-w-T5j^S?5A~COO2cl{C1J2w}0%LY7ZfvMS!1;&vG_ zOX?9VJ-%AAv{?T6_4~)a{(O6#0DS7Qd8_$9KehVr=C_^~bnee_!N0DkrRPs`x`?)LlN zz659?ya(dzgt3`UV)JC6ej)qu1t_c{SRIM2tp7yZNd)Ty*254(#n2FAE z%J>nebAUbS$G$uM@0tH!OStymFP!f?59>$E#(8qb+CWYS6bx+)X_r@6GS5EM=aDj5 za|{!tAIl4wYY&;b2WE5`masf_&Q2Ybk5-ka%!69uq~|U}pqfgmLe4>H99|IC&no9_ zm-BvTRx=5{0D-iW6Abe!^>$8bKJ zkkdJZ>HLj}B+Yf0&*f9{fE%*X(V>LUMyd3@*!7ZdBoCodgpYtHmzcAaP3l&Eq8cIJ zb-Ys0K6EgkU;gS3)GVt)RTQV8+=y8$$xKFc-1Z0%Z^nn;N|rIy?&I2%p#5Cp; z_y{Rv4Pvj@E%ENc!-j4m&*hK$3{xGW6%F_aIjhBt^vY-tRpSZ6kkZh*9()<%c}tev zCnYnD-e>(3R2xwx;6xN^bKWT)`>Dl3`$?4S1fYqTkX|O?_kp>TVJDhDEcdPYE&i?l zx5{aaP+c?`pea&DpNBOxjGMDJrdy`W2n#fp?M~?M!7pSw4k;NG)&pkl5P?fB%P6Al zl)gBrTw!}Aqrm#^o|S9`$+P8FQ<|>M#uoXRYGAIe(luJEv{pf3&RXkX<9gWywPA)u z@GCb<0DXrZFX7^yYo|@2>`7vuAvt*Zc;sD-%S#N>Lj(zYiAs3l@>PKDAhkS&@0FA; z=qC2;u66_tZKWrI81r1hYdJ+0{p{+Ueb%aM-I)L*cZydbi__UY`PDv4jwB_#pL6)C zU9(mrl(a`b3)hZ0pmy%xmhwiwHSz10w4}og2>H+#3Q1r!to%dD+If!9Dry)~UlL%Y zEbuGMX?@54IYg5*#F|FsS+0r17^4@UQy~a|oU$CkBQ;@6xT@%c?EAvFzrOCSlD|HE z?-ws!`0oUuTY&!j^r6#$Zld>vKsP(hYy36``etB`3r=1r*C|ktxN)B5@=%s%!Wpu< zYbcGiIA%4EE8bfI=xh*G1V407$iZ#A3%bjiOve|8$+ zMxY2vaG%;0vDy)Lh9(RMIIV0B;KZ= zwDquz@IzmoE0@cy-4PzKCLJCk>k8FQXFA4G=K{)5fiW);sf>Jc+GdcmBaG0&Q3cDX zY&hDGcGlux;$#GFqI6Z7Wkp8a5{b%${9ngJMsxa@bG|_1vz82PsD>DU6OdwF*rrD} zMvAgcQiztT(ak^mjmMV%Yxh+}1qXY0gPZi)!+zdP6Hgv)$;OVX@60E1cOp4ci90_U zz3~>_2D9=k(mz>W3o%dNRLTp(iy@FHH;z>l(7s9|Gnz@G@kjIH55G9F^zRqgY?Rbq zohRXsdMe@>qvJLfAhbOtOvMt?N@jU^!)DnHTqr@s>OF(tr*n!{EnHTB=)G#CB&_yi zJOmDzKIKe&P7f=te4m-2MF>#Lnn5}FQ?t6N4c zUVr@8HcdoN>3h}qD#N7^nPXJrc*=PSk{5P@N4kQetN^6Lz5UV5C6hNy>8bZo65G6S zDjy-qcyu3}7pRm=9$#UDyUdL+GQ1cC2&E{PMLImbZ6i?rDx0&=Jv3uF$&UbTT->en z_~u$e2tLN_>^g#3HYk{26zRM1(TFI$kgQajKNReWI+jEj8RVSpzbvZ6m=W1I=j@|z z#jlkZIornfv_l0L5kC^BbybJ>b6f`9EKLD{9rzmQKdWPki#+t}7z?f*&(xT-q{JXZf7fa@@kR_|> z+1od``<{N^*$?sE3lmC?6DyXYPxH^Bj5*68RHI<+{i;y#I8WG&i@5owIp$n2ibeV) zaO7O~a4bO1Z3*+=jm1i?Yb1_!cEy6A_eyU(Yf|}(3DptYc@oE;I(X9S5IUIn)PtQ;>5_ZGW-5wJqA zyvq`^z2?-S*M^dWzLx$oT64F2UENXAPUyDTkXdB}QnQ^WQ+(OecdF~{2{S!nve+sS z)~@~6?mc~jvb~#bOF+jVK(5&bpC+pMoK6J0k;hV9l*Zg|&2v0qZNScq$ea-1)hM}| zcc(xr>HqqK0NPpTF5gjt1!Qu?cIL`{Mi%#T<>ph zF<-y9uCX&@=Tp8VNLaTFS!7fj)jCskL59{*vTrY3zndMYdRoeAWSya!Y7bbT+ni>F zuO(AK<>5bX{Q$sM0l&U?=ga-lxAOt_@HGl7tZM^ah^emm_LjtM{&(W~*;8qJ9oAZ= zQ17OiP5=t}-P}~u)iH7g^~NECj40;6Hbvz86Oh8pu5;#%Tn!(wZ7C;AXaS}$E z%DX?Km4%8$M1#?W#AOnpX0q8;& zwTzCIl~vv7R4~xHhOI`E$B7v~d9c4%*M_l<;MJa*Rj`CtLq}bV7X>A7SK8j8NV?bv zhASo)6juEKD_fzmn_HMP+Y>SvbEjUzv&#(O&bs8Y^iVfccy?EVPq2Y_HBUoabQ59s z_;120O$n7urGmWSMy7r0JvlmQ|9t8JplkQWz=ndJ63|61e;u|%$ltT`$-`1>_J zofXfHwtYGTKrNhY%U={Ni?7a^KE}`=y=Wu>s*1~&FwKtt5%rQcu_sF$Mm*N2w$0ci zpRaOfFI7`z?Nox~bSTbBq;X~A2!SrK7mB*nIhhv5naI0b6pd{8_z$9&~a&5@)v;#nY+OwwZ~rSeMV_JB6Vy+EEJ zt42h?;Vv;*k727|^dl18d^PInqXAZ(W!^9! zM?0NcGC6K$>|SfLQDq#2a|=45IGRKn>NeOAGGy(tePgU~b_)sMG);;HJnYkGNEAJe z!%OVCOd`;vXh(E9oqaUMX+@~DIjbHADexlD3j0xalF+R3Fksu8cX6~#V6eqyx~Qj; zkOWe5)DNbLHS%P~N^#!~}O9o=DrW(=(xjh1nXh$>b90Vn&)z zJ-zvETFkeq?qSx-$!$MWTx&W?K$dkmaLx{Rg6t>Xf^==`CYJr=qnyj@a2Euvmz_2w(L{1Lc9Cxb{KLIrl zaJLrJMLIyIhZOu!SuVLY-a`1FAN|i)>*D_B)}?ZxcJW2$!iK{9?JD{o9(F`TOK7+K zeE##}&;JCVgok_W1R!J-TCkAR2S+X#<8}PCwdmDT=c_QKe1l1u0Y$q~_s;nPr1eiE-?1pBr>HrYrrm*HFa2GOcFtRvt zg4D1k*tB#X6JJ;0yyt;jsa?mzX{qi=IPd*XZmwa$+d$7?1j(mi(27_wn z645irD}$lUmf-AVr^c$9>H?y6OUNw>rcH9(!;#o7c(^pDt>Vs8=KMy2{)VH+g-JJd z*1j$U`7*e{F+xDnV$K0|{+zGs?OhC#5}N=3KmbWZK~y)gYgp)2GZ^tK&1T^)A}IG) z9Z6cFWek_jxa?{~V5KjfbCHG3Im!X!4j3x1QQvOi9$q;3E-Uz_9r>j!QyyOmItm9* z&>~pFFGcF(w0MEx$OShJyj0S4$uLV#=#G4^d*T`%VN?IYaJolJdWANERocA{8HrzB z_bZL-Sy_xU#*5p76nIAbFIaO=6I%H)kswI zL1q&m(-Lqg%a1E4y~?3}Vh1s@`>ataN-QL`QeoS2AYFUU6pax%j;U&2xYY@=v_Zo~ zvo7I^FuUdpWFomXUv>8$+$b7b*s5wjw2w^YY@Evtf}E-5T^W%m8l;#TWj#Se8>erp zT8AxjfJuXQW}!7QL_oF%L>c-MStg(_9r3MY*Kz5GGdzaMDkgbA9__BmWz7rR16Y@A z(n?K09<5qssyQ=BL=WzHq;CKoH#bC?TUyLf%FF#%;*P9p+!~7Fh7hi+G}?{eE&BII zJpM7j!^dy-o@)M#ei6rHjthw+IVXRgy#o}gHxkL_xNXn)q&!= zQcotRC!X4wUxQR9JLSG04;|e#elgQfDJEM0YrEHwoLieywT04wjXWZ@MYes08o)yp zhWRs>TFJC%>9chihj-D^A~qsT(wutd0N33)dV-V=$>}9J46V|bmw~wF&ICBf#k3S*yGwBibs@Wwdi(?KR`HT)o^vxANB4>lGHEV*P9D8Rw+U zlWu?Noio#(!Y@vfcN)+Sg*__QoixvhfoRID&Q&^FIg2{L>pWA5b+_5q@^2P*gfCFn zuFL8^Iw8a%ZA{7+J?h63f%a?Q9!4HueS>pEfmxE-ITPL*%HI>e^#sxEw>{zA;%6|}S<=(zW5N$zk z!I$1XTlq29Sxi~P*+GJHfUH21*LTMAob|-m&l>n+jX1l8K|`;&#)pO7?fHod6&gTV zLDOCF@{US%snP2Lo1(X{i0%<>$TtsH%rH7xnbxOaaO4YAdJ4t`UwH_tcgE!q!H1riYhmjOhTgKXf?{zNi_1c%#)IIdPF#Bx ze+_zRusScMu5X!s$bk);8Ytqj8UHl`4hD+lUM2IYzb$#l(try3Ha#?7YkPLNP({kP zNi!6kVdFv+)q-BjB+TeF2IAUS&=1ScH2<^EAT|mOx<_dJ=(fjC1-0*jJ70^Y(QbVx zNLehL))~p2=#az>&*cMs6McjQLZhS_SQ{(f;}6p8$7hRnO%`An*X<`4No+!D#$wQ% zWVYLp02G|I$a{oNjIiH^;pl@t$LFsqliZ8zh}mlhN->@-^%M9wV!U z4ug?6LbxzoxsrL`%kdMUvNJ><_Vny2V>d?#EO^M?ufxFTjw!5;* zALgZ|PG?$a%=mmF8p9z|0&Zb(8dlGC)CVn^2U$`oRELX9$$jk9O~*?Ce=9yWrsiLy zJc6Fg^NzuQ0GOudL`QLEs9qI!>b9EBWeRrbsUZNm zEyeKVnr)44Olf2&QeV|KpqVrqI|dAG02+UGQKejxt0C>UX6^-g>c(_|GVuos4RcRq zs9y=i^D@A(4f4G9z9S}9*lA}n*FVjJZ>5<+*aHiTnGw$km?qVPD0{g(B13o;2c{YV z`%LoGu3yDdM!Q;Tog8og&qsZF_J=i2g^kKRSJ3QERrfbq7aN45ue5=zYf<|r#N<@Q z#nEko_nW*PxYzSIdMaPnAGg|tu=_wZ?&zHX?CoXDa9(flxcMkcNIg5NQCUH-^b4v4 z2{XrEYqP11#phS1w;`cR^Cn*GobK`nZ*)8lpv()kU_{Nsv9(WA@|#<%?VCQfH`*z; zi>`8uww8U{I;d$|Y0`q(W3={l?jPSV>&qhR#im*z-q6I>y^ z?;zx1ua0dRJ*J{WMGIjLsVwiD>4}Rsa(;k7vP|43Bfx{3{4HB|iF#|zrmh(d|G3)4 z{M+bq+ylwEn~t=E#!S#sCNmP>>LwTd{MBmqhoAg5`B{M^NHhe7V}|L@fU>0+;4E^9 z)+U-QmOj3LN`SR7E6!=3n^&8bx00+V$(U-d@yy^u$6vVtqllP>gB=9#Tr`iDsWR^^ z+*^=YQI&S4mXB*e+H?Wy)_&34#Z9Y|TEseMm&!xX19qGMbSdAjg!i~yrv$qmXnj9h zIyvdb1NnJNe`iW++%;sI)k3dieFSoTo-4Gd1+4J2TnkwNtLye}04T?d2ToN^9L6NY7LN8ND|v7p*A^t@)yBt zNj%fAK0731Yi!*$V+Ugl>D~=P{Y0jdWtU+C*ga-N%FeeU%g3$n6I=`b>vC1MsIW!BwI(u8#$vYs9J`U^KIoVgUKqwHSl^?9)>dJ-lZ?3j)$EQ&KEH{SJit$9!F4rb(O68 zGU;gqs`q#1Z`e0x<%$ql{pS-K;I@R%l(ABL7FKr25@{GBML;Y*Hryu!CQ6ab2Lu{0 z@Q2|Kk z6p+&As7Wj(07;4HIEm&D+aXcm#cK{|+vM<;JQZ5n`^YL^?!^Q9Bk%~`#m~ZttSIHf z0hXbLI2U`i<)KM^9V!gF77f*e7JkG^0nbC>{Y>K9e%BXL_{cwm-}2 zvITD=86o4bd%QOBM>|PX4U|S-avT*!xX9%Dy*F|y?UTB~eruv+0aRJp)u~o?tyV|Q zRpRedM5!}BgYVo}9eBq~QfZ;<9w{azJ}*#;2pRc}iOyAQ*dKjO}|IPOv;=HMZ>L>%#80bRqPM#2tpGVo*>{9jGgv65CqXZOfwTv~Xqmr>!|8!MmiX=SEr?%IeaKyql+)>bOb z2wk?irLmRR&E^=kRnSOZMPs~WTO+--8^R`V7oXFSWlaxO|D6swO{)0KqoY6k^Om4O zELYFpgVj|fH*{73jo~hq=DgMeou3eap~P?qT3np- zyhGdVvRGJwfyCM-Y&e$)*#lJwu|CghJ)A#9?A**R82JeQP zISZRit1Z?XI}Hd~MntghFiM9o6cTl#qQAr!?a?#FOZJ4N3<=ZjNbYK5{sYdV~oO}wORyX?buH9 ziY5bxPOHn#=R<~3v|FAL83b2%EuMXwt-m!CHw^iAkXn$YoGScvk^ZQ;fz;PuA%y%!e(Y6V-Ubz&O$#&Ea4=PADLN<_;<(Z*k0eJ&QPaotsP7l!$w{yv4 zK2!%aLHekqOweB>3RiP|M($y$=5$!bq);*M&U3#bC`I8{ah%gKx^&(UWhR>V`6_Kg zdA1zW#m>^N797x%NpYq_lY1npuG0auo!1n3q zkEW;FS}1H#i`+%KX7`>g__gW2ZvmR+eX{GU`rtZn<6n!mV+L=Z{6t{s?5Snjc)BJu z%+61%MNu}io2+K>B6+clWDd>5$#}#+EXhP)=ixN{ODJe9`>JUiP0WS2Z8V2i~Jr4 z%@r84a7(TaE$QY03nLY+7zQIpiICA6+9C=@qY2yA*n_rKN<3TAN%5=+Txpkc+nzHP z2CUHhd?|+CTl>7q2~i4V(s%~C2`&|(tX}SZ5we7F(zSAB5i&Ujv9{Cp!mUJID|7d2 zfJb=J#ed#UDx57x-Il&|a@c1q`PJXm=03A4yC)v5CfwcUu}D{Xi!f}}4nElxmuGYg zee)wmyW$sh}q;MZ0hpe<5BcZo#T;7l|$^L*>HG|j|9Bmc98 z%4)M`v$S%pAd=h#1Xr?arZo|+d$%Sp(GL$I*)xdx1xwEz|L`Y8J$DJ5B8G*-0$d;% zeNJHSe6s2)Y6*96baD@3Ujb}iu=>`f1zh;*w(dJ=W$!_*`ujcG_Z5+;el+B?T}rUx z_qPOwg|_bCH?ow_%Hlp@Tnw(Z_?6+w08v?)gJPPh*w$zSO4UNGg zXgI3r9aBNy4RbTU$VsJt=BoD?FhBHNY&2rKJ4kdOqk(&A+yHmq15_+lYqn$tIfAmF zr(K`mgPF+%sV=|KH9)WwtML@qLzLBnraLB>9|_i8YpQnh?I}wb5%G>D;!0d9N=HlI z{!hXvNAM_egqs{^e$w^Wv>nWa1l>fWZ#t(o1QU^`v_b@V$^HS9rF&JDLk)xZ&<*e> zDz_h4o|_>&u+fOR(qK;}Xlv+tWisg?CB_Z_!aTniZ+b*nIion@zBmhZJ^0OU{XhQb zrrH9&zdh-6P1B9>hWy24B>?m8>0s;B(Ol8e05DWhzCIvpRbd!*XMk&fyb#H;UScgt z#Th7DwY(ycen|)F)FbfG+CFmXDiCy95gq7Nt_9u+MjGwOm^R?Ya)p9>@+VjKC*o<< zgYgc5_=W8OLKz4*p0Rqyyi_a);gcAEMbW#o*hREMa@y$_WFF6e;AZJa)^&z<&=IeF zzHP4VzYL24+svb54 z9L82Auab33QBrCinLW{6iN;t;=YPmR~V;KQLy9AoGcKA@B|?xx%Qs)igbod3$5?Ra`m zO1kEiQwgIOc>tdKj_Xtbe6=)BT$s=$C|x(~B~dqT-`I58P~RpNaIl-|K*xSB739nj zhie!r?O40;17|$0kS1Y=gOifT60reeIq0GzK{ymM`7oRTkHDa8?X16A;)S`|%t{mcvN@H31 zkf5|?tBwI1RQ=)%&~y3#q+y>b0bcz-6khHNk3YUD^V!motFeWGRO}CpK<|I>iu|>h5*!^F502 zw(K=C-At14?gr{Ae!ghxiltBm(TuUBrKHB;s98=tKcwKFFYWgRS;EJ9QljJPF5MKM zb14r-a8Vu|Y4{&!?#Rk*D~?X7*Aq?gXuzi|{sbl>9j%u+tBgPlSf&*_EpwVdtLy}z zJE7XgjA)PsEJ)kMX?{9`af9L^3{RP3FJ&DNjIUeN#k7RI8u&b}B zy`mWq)tY0iBXa7^Q5cc4_pL?=;fD|3Tm64{>*t#mo$dRIX07)9*N3+~>B~Z27YqM> z1G@HgS=*NhzYGM<%o+<^Rd`O(G*X@rT*YtRWc$-BrORnQB6+VC3MtQCG*&u#1dDxU zmO4SDPGs3N8bG6upmb~M%5x906E%i18$w8}d~rkJZIk~fj(jPKrYZ)k8wLoinhxU8 z=x;|?Q;LV;ZJZhDID7kkMYOc(!#=M4N3KmYW8GiubVz2EjUSB_E)y830(y5+O=Cjg zPx^kPyCBdt21*4Om*Z9)5#WME#A6ad<^UDI^KOd3i9g!75|W+7q{DGpaL=`44Ji%X`z%G41DaJks6O2$Z*Sm8~iK-Kr z6m;ZH=p48~mA8VP2%i~7Hczog^_bQ~0|{o~Y$)rj**QRtNqWBmXq+%aef;>NWq*P1 zmd~HQ286B5)byAE$i zuZr#Z=<9f&=Hgw(JUQ7Sr*ZDjc`Jyb3zD0)NX!*%7%%%34GKIU2!@k>QoSH z{EKB^ReMcL8QK+{+*O=o8aYG*SmQu5@;gE#75hb-y2(N|6{}<__f_UIW)%Z7{)z?< z-w7Bo%VJd8wj)_f!_|dW@MKDQ~8k43izG**x zEGr&unG?X$y`sf%wm-7?vcf@*_hnuT(-VskV>!B5O4i@}AuT(+uoc*g>X)QZPIVeP20^oJfD|~9PN2XrM<|_-0sbTe)Bd7@Q zZHG{HbazzqZiAYWv4|j>s7EeJ!*=BP<^?J-7IlrD`oMxE^S=0bkR6vwau&y2RI2KVSVfESl_Xra_a3nD7Cezw~Mfr}ed4Q6OxCd=(LZSgi5iRQv?~Ay_-R`a7+JM+iFs+I2nd@)pTqX?8K_XPOrM7 z-U&b&YI#eQzJ^L?Q*3`_7ZUeURG!5YWl_?(j$OS1WVPI_I2H*O+MTY2Eurk1&{{k^ zpEX1Ou+>Qb&-<|jjN4MPC;!x%wXG&Px1^O1%Zaj;X_^wxo`0U2921@8Kn2l1eiYF_ zV{gqGVCH4Ma{#|3%<#@zYQ)H#SO4rT{k+f|lsdk*MM&x{N~@hw&-20f7H=dW+>@OMciH>W)h24@y%0zS$6zL(LQ-e7S1 zj&jlk0)Gk3UfjXa5IqJQJ#Mj~6i~jwjl9pQk$A0x5decUnH6pJ8l+uG1mJ-YDFie^EoMQp2 z$%Z*B92ng&bv!rHpQ-VL%h2g@1g&mGKy_LmnEkTEiJ%F%#5x{HQDcjD{)sG! z@cRtixgznt8sw^+H3#P*iC06Y>Ju1xG)}&6w7 z)cK|$-n!i4?=pj&k(pKjYpYC$dk4GIvp(vISWEvq?TrXYAE!rLJD)O_&Qr$TP?Tr; zb0{$au3DHgDH@!SqY~p6c(}YqTh#({0XMNPU8Ex;>M6@-NCu~}QBXO`cB?pAjvR^E z|DnwVM51@R{ZOtI;F~Vk;CXHN{>hNK_BHr|8q^u;|; zQ+P&%XcUTIz0&30Nr`(hj~7<2@QUAS=j#_~ke8fpE}_EbRXkjgD4gP=4fwO=$Wy^z zv53{{2=D4RnXn0PKv(8k}BCH?@}I(FJ&**>TxMv z5hmF)9g0WXWZV5N8cR*t)UuAnX8QGbd<;0bPws4rhEw#nQrtmx$M~9?VZqvvbBDU# z1a}-gq1#4?{_@me;m{SCgc2%*@l?rQiwan>G88xp<_&9MHPc0T z2BhgprBqdgl^)F)2_PaNOFSIqC)k*f&}I$m(g9E4MOXCcUn!Wg*&0Hpf5!E^={Epd zg@(>T&5SU>CUH8t@#l}9ZxuDG;Ohdv1qoax`W_$hT*u zR=#QyG=IlN^Nvk<|9HTgw9^ig^KPNI^8+T3$=9}byR=*^{6EK}?5^$Q?6%5tAnsHw zXnR0NGfIok7QPE}HMz33%$sZ~cUT-V0bLg307jYk6A;E3z|% zA)ZNxSXC7~-?3Lo`{Pl9^BNIR$-q?oVo|?ps|1O>s3VlP@0n`cC8~8TGt+lsz8|B7mr^A$HqL3m48z$wIlBnbmS- zpbO)D@72P_s<35+-?*GvVT(kg6&{39=!rLh47=X%3+4dN%c5=&E+cgR;-@wu@W`5DZ28-LR?by2yE0CAL1X6mK%OpO39h!63$=DeZ#hth#tP zERE2uKI!mIMnhKIW>}YB<|4up;cqq@1be87!FO(>Wb6K}dsx+hy3zVEG*FMNB5ATe z2QIRGjBGDo@Yh`3t;RXAcP`x2VM)jhM*5^|$eDxY7Q5R@N%F12=tyKz;g_28N0Jn3 z`sT^#X`KMhnS?n4{S{A5)V^AG{dpIOYmaI(WtV365DdXt^^_K-2V*mV1*Ho^%@7v* z-6hc!utGCM%eQz2dh^&U3P@LXo1UB7tx_3~KCrEWXG8mpv^;{&?IK-^^HWf(=!*GP zL&4cwOkW%v_Ecza>xnHRE(EO3kzOEluGcCgiMiiRywmhZ1ezlsPi1)%oxZ}H62*f_ zv8?h2tE- zJHkkt6@8b|z4DzV`KRK(1#kkeUm5Td3?}uQvsIU0?)E3X-OT*v{-bV;iQcX3-&=rk zg?4Li!q+OCVHu}y%qQurqo}IKblL_4S9Y+WWa-lFW_8(>$jt5`L%n)olID5*+AjBR z$gxIGq%%T2Cxw|Fq4mIISG^aKunyz76}S%p3^?agQE>wx|6$#aQMFDDGs5 z*M`FlR>Lb0!jVt?GqA6_^7CCGL)%hX>uaWZQ<&O+-tRFA{l1Fn5)4R?H9)B4xH0e72b4&-_asjQ)N*VnHukgw#D}jFlS^(E2UQ6A|XBcF_l^02K z?d{STkhM;3YCO@?dBCRA0GgI7Sp{l4Qgx|qNYFh3 z3=hdX9dwpF3>tRXe^;)^Zi&@spatI14|Of0ztFK>S9%Q@!!Zi2mjkKfJ<>3ZWK!*T zRpGysXqoaa%%w}F=R~h6ajQO4Dk=-#7!Sj!pZwGkgGw3P1BB<%E|t6uktji42%?Do zfuYJ4aqV;{rW?y-Q8EhEOHS~(R)DE?emYmZ6>_>^EdS#gmWv5=dy#K#0uf{wI_9O<%n zR)!?4#;(JMW}?mwhKFPf=2S+-)c`wCIkS{2R&&bKUI`haP8XzXMx!)7&XcZuP?e45 z6sdfab-%-;PHbh7rDEbew_%2uRqBc3`PAaIdnWpIQ_WGyyxN>(5xe{(u#t(Iw{oQk zpn9tVqoszJaR$VTj|Vp;dyrvKd1y*>frW7mvavN(HpZJsI5i7{YeZK$;ByrPL^G&j zHJDC|Dy8tLC^PJgMNyFP7Cz;(6IrFra=W;?Myw<;-fb3O+T>1 z&~lxMurR%M3g*M?-0d=Dw=}q$lA?FR*EDfxQ>H~UUP`wZ6t;QGMX&6Jds{8c*4|EM z%$2?hH%+=zw$mM7j^8%`yQ`iXd&g#V5fc|s!l92NHF&~P)JDs86~V&{upa%A69Gcn zMMyL7;@9dZAK=Tk&MEdKK&Jt%D{^HAUY;>IhSIxXx22WuRX+Z7Cg5biLZP4g-E9dA zxW&dq8!u)6TxlQX7jElf(_T0rQ#~_s>Ma>7Ju}DA^5jI=Zy6Ixv32f=m$|jNcOKlu zh1||FW$R#6Jo`>Q%IGadmHF%0=~Q{J?>h3oYy1Uq&5_Jx=yORTtq7=GyzY5HX^eh) ztm+A!+&XM1jd2}F?q>xZ%r7LezSneXstBsV%Lfj?^zv)X88{zR!y0Vt=DDkSSID#MPD$nVzW$y$1Bx$y|+$1u)7AfFTwls^_6u7foiMwMWl zHXihNWTV=6n6xmuaSx!KA6ok7zZ|?)*YjgZd!s^p1-5&!tm?b&V90g4m8G5_teY11 zC^R8+YqQE2#2xb+pFI{<%F7^drHdEkBD=x1#5Nd-f_(xjy;dGAMKNrFF9I8~iWWn5 zOzt|9S>f0c(TbN9mY5pF7atVGXsPBO%RYoR*a#HJ|-=B zb`5CBtz>90^{6Pu-IY?=-eQ zb?q_EmF=TOY*+adN3hbv1vY9~m(Km4;d?Zpq!AyMlGLBU*l7}IdL$|7@9;~*if;3b^Qb|}+Ui?K; z2+n}Y@@~|1E-;~};t{Wr6LA|-BjDta<($lP9hPdIGp3_ocnDWZyy8!I#&7Um{EpTCJS=7p|DHa@38x^-L{a_`L_&%H0 z`NQojyFe008r0327MA16n~uAx8824%;KsN)cFRQVddzxuW4{wAD&xZ-@0ZC5DM&2s8 z0?$P4hmlSJzI#eiX92B_J1^L?B)a_G%PtXG{?|4zl;1@J0UvDGHn_t9f~4;#>K%@JxOKQ}vY7t6At3}gs-;!m-Fg=Q z-Sm%e>BsymO0o++`H-?@PM%?fAU%nZAE!D10#Qg(vrq)Ds_@0oF)n;NOpfa7JbgUL zD7SKey<6+}-WfMHB-;78_oiA72Vz{5r?XCzm{xotRXd|9*5IYrAviW=7zLTKF{18PNJdZLggCO!%PSr8R36s+m`Rz%YCK@WS;|h120jSS{ z({;iUP3>7WQ>J#E1fl(qPN+oTP@s~iZZ|o4E77#}^-PGiZ4b0pZGtF^;k4DW%ixYN z@xt2i|5kGB@r>0mqPgNjObn3K*lxZ|5XzkASf!zPr{HTovZ&)Qqmr4*T-}zqg@%j^ z-V-T(13==$)=jU`Umwsl;4vpM)qw*yfgXjNWg;%auzqAtLpOb{Pb&cde*{g~0GxBK z9m2zsG?eiS#)~}Sp?Zv)YIg?X(*-;|*M839wvYv@&-6Y*Clk(f8pBEDxlIYq4c4Ut z?GvrPQK9Gmg_uF$%zv=9jN0nmu^OA}?;V62+6@x8UmOr^b(qba19oroo+IVC-SN`O zpRO#jYjZb6+cK83o{`5{BZ23`cbeD)uQCVeoVhg^6W6S}3ViIrl1DnHZHoG7Q8Z!5 z`W7C~yLM~N?@!CQR;N~_t@XF)?;7BF2cVx!@Ue;3$cPSrNC0BXre{{qj>|6LNNun@ zUPf&S$T)AwI}9Q^0EA3y&4>*s%e{QU3FAK%uJ z&ENn2byotGk_#OHJ0tT0H;b&1T(5I>FnVopc__D4UH7Roa0`zf%Bt(x0W8-!x(3K3c7YHEfoM2)BeQSF>N^(S`d!b~FB`%8VE0x8 ziO)fvY$_Vk+G(xBQink4Np1_=+WfE3(M+H(INPdPoz$>UJ!BaC_i88I+r*Z0M6(q+ zA4xuLOM+Cbzm?b6a#QO?+8eO-=w#e1BroicFBZPXr0do9zxz6%@}aLV?{mNx2)~`$ zH==eeR}lLga7#X3m0e(jFYsoFHFh_M=idg8x~G`VaNG}?Sl zpn==BX*B^v*iDi+`jAp%g0x0hQiUPG%y>CLtt!#-hGJvQ;5UqizV59z!^)OzfW{%; zCN`@gpJk0pSfzCn$mW%jfGvh+rA5i~gKF-eK{QI)pw`UWg|LV2++DE}kE_!sU_meK z;dZgOLaHhoKnaRLSd*Q^26IJ3*O>~uOho0XDa9n}F=Y5A>YBJ~vpY9b2XCS5INRsO zcF<37hbH8ic+*IAPTOvNi>za|rQ*c|U=r#8mI>_WpWqx}&)H0XV?|Btc%aYBhkAKK z-lmBY5chI*rbt_IOn2x)Jv5c3zP;n^WK4epw`__pa=k~y*|4o);_z-`2pqF+Evp@) zI)Z0II!}y6Uw<5~qw*qfX3iII+XNJAM>TvO{W?aW$)=B7)EPzGw&y@eDXI4AljB)9 z02BA(q3sb%W=un$qW*M}Fw_eh-Fp#U4$p-{Rl{VFCdd;|lfvZEbFAxr(9tcXv!+8^ zJvFq)iI=I>wyV(qC8Ca(6{ydOS`IEy$cVZ0AiE9enjeZP#>>-oN>&-^G$coh1LN*b z_eT5aSS!)5En>g3IC9M9pLv#`cpbJOA2ZscQi>Tj!KZxEYKjDOeI$ipv&m#k=y23{RM&pAwIxpE zC+rZ+j^r(SH*Rzj%)H&nG<$78^)wH!#R$aFkitN#g=`$!CToiV+ay7wz|Oqgzm=vH zGQaV~(qWTmdgI)=Y(}0V;*Ge^v_;>7jLwPaL~(;GapT%qED7#4cGhUw(in7etEnUq z-ap0tyk)Mao?O^atiIyY5b!Y6mGO-OprYP+HVCnjF(VM(NQr!>U9CReN7Z>xz02XVlBm>9*GvZIN5 z7&tI&fb5YK=$gu14K%M{nG+s}JO6Oc>mNl%mjn&NVreior&mv8D~IlxC${OfBBEiLf22#KZlaniXHKWWL84!#LCOa0i>ukSy$`v39sKR@65X^1WW zTn7C3)&EwItzE1j^MQwg@@AHqa?gSRo#gy$74CaZP%ndWP8ks~=ddUz+l4Z5_VYwd zI}=K^gtPTLk;VP{48T4B;2MAn-CLY%Fjl-dto!<13AE7ft$=;lH|Oswo4h&D z+kSt1`&)3|yVIP1k`EdbDV4HDM%7>zg$zg-Tjd!&n~}U_m41J2*XC=B4Q>a=9|?{O z8`X&8CykdYdDhl+>A9cYnCgb;Kx!5*UMH{tIV# z2o&IUq8LFXndsgKa}7orWjGE>>1MhzoIKzq8)9Q<(Z&qU zzWj=>TwFwrT?bUvW`OZ@R-ZkZO(cnq6u=!!yC&ddv3Tc&F0K@3SioK0m=R1aUTq>0 zdpH|kX8hevfUEyn3aY)t3sGKy5psyw&TT4fLOido16VFOOM|rlMBZo`X&u1fX7VIv z!@fSY#yS_0XTNTKj>{~Uo5~2|ElcO0+AeB|o6s5(P4bx4dLds2vrqMuL$hrW!>5}C z$K65Un(KgN2hB5ps;N7oqy8t?BPu^dhaYDnr%e>-n$Y@Ch;pL5W|Ep9XKlse&iWV7 z3BZ_$BfqXFk0>CI^@B%+LL0e{ph=SuZ7~c5IedjoX;L)whqG75Sn4>+i9DevXap0w zRvf1S$E?=ynBd3} zF)@sJup~Wlm}*F}VUNUojCdC6gQ|p~@v=?dsVBDA{+p7aVVS|Di-Ha{JZW2rd}Ok> z9uza{T!nl*bC!NY=dzK5QPtUOeBtcl_Ph4Dc#7v7GQ!lZN4_EFzu&uvZ0-43k z3w4n^c?d@{rfeu49)FKyo-a6qJ;8QU(czXB8lZ^Y8Kqatv*$KsH)#q>o?cqeW1$w+ z;I#y4!R-P-BKB@_ei%ZJD-4WVG{R&fe=ysHXT>2`*Nry`oDdI^8nR#xc-cCdVXb-V zme}_-7Qj&w2idgE*$ZJ-?bCBU0FeEi=5>YG7YZUw2>0 zNpUNh+QycD%A+jlLMu-j>SN}P$+nX+kj9lmN`c(gcIWIK3`pXx*yc}i#iCCOw`1z) z5m-1#iSPJq{IZv>U~UHxksvKsga^JalYeMI@Rf=7TnP}FyT z_7$jKT6gwEt6k{yPDoD~m4@Gcc?Y0pJ9ZZ3n&jrB8;?b_2DY-m1kQwXQ4K$pb||im zfOTJUq?u^OA|hN0k?_j6uw}Ic8@@gCrof*0Xz#sknmD;<=utB?e&*92M0LC;)w)^v zQCV?m1i;gZX|j%gm@3XFLlk5&v7%DcSIu^*t<%9e#Mx+?xAbT_hrc%&1hJv;p%E#cN3Py27S=@SwiV z3feEMwaLZX%wa}>D-DxFaR|Cb975wl-@OX{HmOHc1!iQVoEJ8VpiFHo9X3nV-0GG; zlhywt9u^*EN({U9*t(@Ar}qb4hwNf)4Cp3gDNuF;Srmk#k^=&Pm^NYuXL}$7I*Wc)XOl6D?{c z;?*|BiO`YEm!tzev^6mffM}47uD+|hEWP?Eo|3w!9sLA!yv+z$X?ULE!ajRwnUaNf zz%*3Q(N2Oq9(_qV!9YDwQCE%7NgC()4VslE;1oPY>u)<5+h2n=V#SL5wnzuzsMi5` zGDkX^c7|4)jt-Cfb2sPhWuPmJ44#UkJT89sSCauTu9FB8`hSG~WYqk7!(@;q+rv;8Ia;cHRX(ja%xJc zpwrY0$*kjdNO;3q>sz{yYF445M4^(Z@- zed|ebO{BNYsU0BZ-gP5Ez>Sq_@B3-`YsK7c{rt1_QlLy^PE`{XM{?g=8n>V^r|;p- z^39zaPY}(PXO`qMYI0{i@o@{|l6BL7%EF86O3^VkKqHYzn8-ymiE{!5Hik^vNM1CL zQLZQWs};GWQ9anV6e5n*)~4MXZtEIgp98LFQQIlyS%WcpVgX1V;cKCdV@c9#*(rYQ zTD$X?9y80XI5u6+cvPG2hoVYE7v%&7)mg(?jM=RvCOkE}Rn-F@{fy<;pFh8T`O^QN zpMU=K>&y3_pSukBvzE7>n{Vm66P-5!n3+0PAQpvw_SIVdz5n3w9v%7qv+vK@hXh(^E^M=5(`v#JBfVG$~o@=B*V;YJnNNCyu|k3_M;!gZtHi5 z%jK>-x(Y)z{nW@$!lt+g4Db!pLC^)KF*m&kF=`04b<)$aLm?z%0gr6!iyV07mJ{)m z6rNBDYF>$2j*D}+#=(3}OUUI|dq3u9%tX=PY7w8z?Cdwr`bfa{mYv=dDpEev*(JGm z27Ca3opA@!vx8e{HfL<0uu5Q3)~@}aq_`u6vm38}MB%Yh8#R7JGylf(Iukc072?=3 zD+NSFW2Ig-cU|zzl|!+ehC|lqj<|A~P;aV!VIe9=0F;1WNK{Q))3OXp;alDS06+jq zL_t)6bs#%LA%-X$mK6k1^Fy(y)~TPw$=@`SuC5+gKuH7nKK&dGveVY|BLrODx2H8g zJXC19+%#HZ`n{08rONc`=DP9o^Xp`09!G3tocojNZoJ8Qwc*KMSyyx37%)gsRzJ5! zwqlrS>qW;|$DA2rKfn__+c})ml+%_iCRE#69WW){L={IY5{^WoG(0lLHD;cAZMu=!Elw)hE*%gOX-|}}oEhV@ zF45DzZdTwu!gUNiwsj+WZR5dZnnb&q_KIrI6So7V<<^G8R6pJv`jB4?M|a&gJ`kZg zf#gw2kV8^qW!$|`Wyfm3pr66DOvVm7Jsg{%JO}BON^ZkW#&P3Uvc*xKbECMLLDmNx z0N|eU;<)e)pJrf_#Kj3I*2WS=$sj73hjVR7S+kw)$jz^`_kXP#u%k_ZVL}6B+pLC1 z2}*^kEIpDsP0`xfux_B?TT8?p<%eF}w4Eifjjvpu2umE_i`Z1v9aIx}OM!9D3G|SG zO^Vi&|5ekFWOhw$ry1RV4(S?a4u@56=9!@_82mX`u4kRizi~2)LaPlWn@_xTgVg{> zK)Am>)=pnNc+N09Z8PO~z|nHPnFaI4h{%j7k_Bh3xPXtzjunET9C63?neL3Bxv7`D zWH|kfpH<8yHV$jfL6+_|8vPHo7D+T5S7!JN2MtY z2o(otC$(URYm3)FATX&%!GK=p7IGd-6=!#K5l>b`MIz0iwR}Yy?&!LU_v!8Dr=7`u z{`vV!i~jd7pa1HY?OWgf`ts-BZ(o1)`!=29`mUj8#XQYj=VV~__}^d0SAKEIP?Ex<0}GATg6KYj8?tyO?X!7`d%Dj+yY=%AHD4>M!4$>@V$CBfaN zH-d^I3Aw;R$vbQXvQcC)pO%d$Pl&h85*x({>gKn+sVs4 zclN`lV2QgCk^Dr)DGQ{85hFcHaYnd#Jj^#|Ogk&$Yx-g(h%ktr7UTmOjbc3=X90IG zde|}Dm2*^76fEL-X)dp64J5$m4NWX)5gvKv{u|wNObQXQVe}Aqd#CQpBnYcb$?P82 zBr?nAD`?0R?>kTEP17+6sjbo`!lzP}PXT^Ci^s$cRt>2vEK}S7-h9!tWeVvUU=w4X zn|gC4;Mqbwy*5|I6nzP?PZCuih0X1sDP}&^uuaEPJSxPq<3*qurnyo`x(l%HC_t6` zdXR1|;k7ijE&CXCDLtvGh|%yVze$cD8{Cf|y3PladrU9Z9g~qyA5@8;WVPkOiq}d( z9Mk~&>{o(A;b}iI`H7r#d;@rBR2w3zRH}QF%&2f|3ZuKu-xT1P6kaVmPW#ATmZ+S7 z(DfhvBM;ia5AT5jHLdHYOA)PJfAWf0Z2%79)zu!VQwR2(HnZoEqmOf}?Pg>$A6|N1 zTx*(6WFwi}6Z_yD{V5r02%Z2u9!$c^_@tN|r#tC8CU*i?C+#1IDxblPq!G{15g6$; zDQ}-t<=Z_i9p+6W4SM&hqA{yAXCCm)5naTebrVnU96jp6&87iA#Hebv4enWFNUCdU zC!?xN(u1;w*}ir~+xg~>lh(|Q-g(uuRCW>AU$}1GGt1YIl6C_N$r)q_u6aV9-ee_h zR;O{$0i1~GoS27kVx0nRnuLMs+Y(Y?Ud3(Zsig}5>xKmT52?|Vgt}&l70LXubx4dR zFPo85E%*D5g+_&e;r!D|BC-ivo3|8BUXRG9Lf>>GqT0@N5xml8w^TM=wj{m(&7_^n zI(dIhgfD;-Fo;7ys0#Ge9T=oOnOx7a^NAZ16>_R`Bv(b#P238DBWl=2+z!BQc6K1~ zbv9`sOVwVAej17ZYj*F4N9wk!hg}Hk&G)T+w=^OL{J4gRFNdRrokz;^S(5(#^uDi6 z8q^w`Zw5TZeQ^zbCl_ZJKAE_gI^Se)jr%WvDM+jJPp?)F9>wYFR`=Q8J&P2Q-5Srm zf4zPAt9JmteEI(Q^Y^X+dMn`Tm-k=q-WKpyz)ueYZDnyj>27$p|2^n$#sB?(zJ33{ z@9+Qf{oDV1?=s+>u9QwM2$|wbc#>ck`$lsYOMCLf#X$t!SkJGmCtIS*#lE>`1$Wu! zHEY*=)P|w{$QxB0;1y2^nF(Y7)Un+YAz>yy|byOpPPvj{RJa8pa`%0M~ ze|%YaKZV>=Sfz8BX$b|n&x9l^|9T>;LAlY-smaU+SE^<7Cc_L2TTIg?^9P9|+Ol-m z4js_FRKSm=1I*_a7U1a*9f=y2HS(KrwQ)4dbcdpPD>T)5jWb4Uz_DV4;-U12h$d4k zRcX0965>Q+X6H>2Vy$X>nipdIz!7-6?eQ{qGIGw0qA&JRoUM3toT*oHY0!Tw%TZKy z6QM5|=>4{y`1;{O(CvpNyP^nK*9QrS&RarcmJP)wGkL3IX@@+gMaQC=TqDdjpxKdW zb0jp(Py;n8bg_nF?P3|#G+cvA(_ILLpq8e^7Q|l@n!DT)KRG#L4iZ;*YmJ)gBFRF~ zfpAyCVDImo!)FxvFAW`Iy8H%ZH4#B(xt`sbo!mLVfSzgDKy0G?L%@o58kZGD*4dj( zWC3nVPBl*2cn-y;V`JQA?_&Tf)YWKZ`y|9oLAwgfNzihP9@4JswAMQ;q0QN1m=pT% zJmS7dmbKV#h(X4;PuMavVHCl37Gbp@VuzjQGLY3Dy_fcTztIB4_plwmr##CPcEA&s42pltAG%r z(X}{hV(?8HKtRqU^!3_wgFQqwE>~9j(1m_FPvGf!vCmfto{fZ5* z;U5@AiE|*x7>x>(_PDFkjC88R)k(C~bmgL5= z3`(4H&dxB#K$@xyc!Yvhxx+cD#Up7A9;?Dib=`Sv`(@;2hXJArf`O#!1TuskeqkyT zXbE`pbER5C&H-rjFj>b*I~OD(x~8-}orx-vaqti;CY1-Sxg2i1)4T6^OtkWNlnw`4 zv8h%>E94JEx{v-)58g?uV93Kx9Z%@-M<@^+Yj<8Uf#*$^;@zU|&!64__?M;s=WlP`1o+plw?BWnFnIs+_SY}Z z2;C_z429lj8$AE>vxk5F{r$an09*rnd;h=h-~Q7ZAlV%Xsbp)QcTs1(JwUBxyLbpm z@9qq1@075?hXis+h!JLcUUt%UpL(%#GMp(hlc}ABEVQ>O2%c~FN!f1iH-k|^YwAg+;QgGgXg{UCvfWDV2CV4td` zfdS}>qN#BY|MW3ZpLgiZ4Y&V2Cj?mg60$3e1X8_g#ZLT|I_}M!iQPKrniYy&Ebtl& z_-~t|m7GCU7vo8DaI`ldSFGPyfV;l_I&w&ie2M<-GeuVxqh17@K1>P8+Y5bBtuwcX zOf(6nFK4Bra)nA$Lq8{)_t+92Y?bv_AfiSFI8u>AMs|h~!w|Nv`F1C9Tw;RYR+%z= zl9N7h(>KQ?JNuyUgi4=1@r|Ia_liM^eGi!kCNGP6_@}@3-dF3pKzlxM&vXhG1(6v= zB9L|U_5%}}nAm~gi5*5LnLC0`2XHeOsPU{AJb2=1f{eFhNxW;2CWu?8M|bf| zEij$qozG^7_C&k)MO$iio9iYOwQ9EOuGlaH)Ly zA|ovZqs1t(xUSZ|r{D-EgD;?I+p|F%lv47^f?+5XO)^q!9{{*J{&!DdQ^c-%5`jB; zAzeZ|?v9VLc~guUN}Q7VqW=2t`7T-3P}~u zX@9!4Z;a@B@Rt75xe9I~97*r=h_Gx(zzj3~8}e5%!tG>+-tz`(m^jwb)% zKHB3=BQa?pib}F^9mCy>soFL~D1n4(A~hcm6tfZS7(Rd}?Uauv5$lhJrjxuv4dc0i zbbT}m+|_Qcb}dq}ZSvcXj;i$ox~^ySb;dD3iKkf7O9);%PVP10zb%3c{!no0X(IxO zoyec=>8R3}jms&Dz8;%=(~cqe)= zF*KLL;|@px!Rx+ZimS|`P3Unk!#=Eqs&e0Z-PUcOO=C@w?i*P7Sh_G4()!2(XMUM? z!#m8dAk286u;`c)rCF&qdiEuXujZOqM=`TZ?*~p>BCnTi6U&Ou-giN2M$I3Ew3UVj zegbj@wN>-iBG9xg5Sa?O7tG;x)|$WQ3t?ub(m6b|-kotlH0Pd@K>Oi=Fcx_F7~GjR zW?R*H8wG3UuUbb5n+LnS74SUW(e(f4&u-a${%`C4-VONC1;Cf@Uw{7f^Yi=9FMqv7 z`S#b(-syJcgW3xI*ZYt6|NQ**zwbTv^ZxH|E&%?=mB7z$T^dA@v0u88`KjxQR{y=9 z;NklWU&_6I`Q!bEujhxet0hkg$&`?9y;A>omk+^PaHXrPFom?-DhXPomvX&>5?MPH zUcyOGGJ@YP|EUaUSL3p=6p^$0DS&MBoj;#HeYeOgK~m;!Zdp}Q&w|cdd^e^>b$6dX zw_MvwEkMRQtLcF7GI8UQ4X_)!>sC+BItNLSaYQ6!*qN}%kr{zNp}Bs@s+)~A&r=w4 zcJd27hL(vZp_v~mr)9xbcwBUN@Wn+vBHm>*Fsmj^VSDLAWobF|>Swd#Hp!Hd1XZP6 z0kHC919Sl{mXD65#LTJ`u}cEfET^_=1p~jJnd37@Viq6Q@0R;g=W@P#5RI8nU%HN4 zK6P=Nh%VIb^uvfv2JM12U^$mQCZ3V?G(RwOCP7pGnTN1QZ=l3hCY*`FZf@ozSY2!8 z(MrT)YIpAq%+HgP!ZNTBc0``#u3rIjnb4v5!jvzO)mu8Ecq6MUYf zm-hk;Nd8Vfsynhr#|P#?1@GKdLwl z_a;lJq8MjdvNFi+V5_+Hc{lakr)@@^|L20o3mK#bf4fIY;Huf=iCODpoTw?_wq(*? z-_W6-7&w$p=LBo2B{+;|zwuvbJTGm4*FsIDM+`Tq(D!V^LBnEKVSR;QOh_L>+4Oke zcqVsqluT!GLzGjwPca0}32`7AKsohv`^bZ5v6+h^jk`tQj_Sm12Lkh*F4v= z56WSAaX{epqL>i!=ApTiP{Q$@(npAFi_X~D)HSLTjNB6XI43^lwm3Sq5Y#_{X-2ms z(~zrgO3>NlWxSeP5J9$P13BDHKR2~q>%@#}MBs1gNMhPloZB`~fT{HOu2Mz`GMer4 zglY^PWvq2o01SSwOS4A%r=b&r(_ho0f=hF-a|k=zMv7&ehKwNHNGD$Cs4Oc0?hr-JBe~YU_W4 z!|G#$1X^-txjPXz#w~{}G9*h09e3>B3kiZUk-2O#h;BO-`hDsY)+3I8mdwnVMxsu; z^;Os{i6q6g6G=(JL8mxYHd8YxOJ1PPC$CwZuKmsh?166whoAhTdy@|^;4oh+S zDL|m_UQogma|Cjig`E%YJ!$Ete878WX=%pIxGaX%BjE9 zELr5!LUd=|8JzP_-`2Yj;Xf;!zg=FYq(7?&^F!ZCej@0M`qzHv;q34M20XheO;fT8 zrAQ9vCg$m>S{Ai!a7xi5f@H~;BAyts#N6P8GX4#E(rJxBghi-38za5(xg)PgQsa z&#)N>#US{`;N%fH??|^g+BP*>FbOL!4WKSJd)G!a%iD+Z5sa6#UL^?8#$@xBeA89* zXk4WRsid99&qff<7zrTSqW%`=I{Z%;5pzov)I=Y0sT~PX?jmiv?894-hPp59tTOOP< zxnFYd*7n5KF#I(1O3qFwz&zG0r!~Vda#W4yZ zi!@3&PfPnNtR=2h$F*F*G}?{~$T>;zDg)5Ak|AKw?2$bvkxw2tlY$f{dW|=l6dxh_ z$8`1I<(rtP9xuD5WeNxYvj1m`5Z!i~D)7pST7!)pSGa?w^=*H>1kbJs82y?a%=0;d zda{ff(TZef0$Iy%2Bih)QFVlv=G`HE={qn}{{mIl3$fyZ_Wxfx9S~4I9EbbC2H|iS z?+uO7KHIoR zn2=LK&BXCqk3vH8^qd|w%1_2R+fAEFU2zN(x(cklV=S*4Ds0FRlMvTgUDpOfEy*Kv zk~gCzVqP|(PNovvY74!8&)~W6EMbm8G<-&whtXCRW_DS zM*?~)^?^02X@p=H@I3|a_^Jm?3xVYeE%CNL<3kXJIODD-2p0dH?so2pN)#7t9M<}g zBRiXC=QMXh@(tzLk#`;o#u}WoQP=F@r1n_v#GLXV&bFP}SbrWCj+>MZ+aRQf3%m$h z2A;2JVvq@HCx&SR+0@=DfeUBM`Q~u50vm|ra>Ob?@}YRQ#vnFWPJp&lMSkN+p$533 zufq&3I0?%SnJxmHrMseyy0AgTUzMC4G1V!`Q^{-k30ln;r z2SjTdIMuiBwB3m_NejxH)^8N}$>yJbeE;-E7XW{K{?enye%ZHgz4S|^+2BiD|N7c; zPjIrM-!Fn-iojJ^kO~ zI(>N|9~@-gp6aeY^_Th(~iE-)Dzx<@yK3Oe5N{1(yDMaJopTAc~pYoH3 zRnifkyTa+);zBGA-NUCwM$t>9m`2f*y3iMB6OOF%e7I#?<<@_$2G}z5^dHq^>I)oVCRR5o-{={H)4gM z&K;{O=p@=pO0l2FLMp`ud{IG=1lxQm8y8Z&CUj0B2oDRms6ijfD5hb7Sy{JaLLCd- zVwQ-TQ)r^S=H{1SOU~D~eGs6lv<|&VXdaBaeaCjY zmgc2+7(!Nl^P%%+o{SHYu`Ox=>VH0>m!$X>Pqkj`Cca_P-v*8jts5$VEaAZ&nn#FJ z#HMYzoc|Xh?k(f*e=*Gw{&Q2;1A)_@(ai|T^xL>dg+;fWveU^A=g8}RoX)|9QeFMC z=tEwT2%5Oum;2X1rzDY-{F2-%M>XPU7m_O=S6Xd{1B~F}d|(p+{5ml_Jdwno(^l6J zVjLA~thlz5QPV;iFz*WYOhSig!kIF`a&s+nhSCHe#d$Ank1C4yBCn19(|eZG ztOK4l(R!J+g6bobc}F!;h(5GUt{p+m98>g2)nMV-h>LzicRU=AcD?*Ev)DL;U^<;+ zSz<2R8eA{&iZLX>apSI%n!6Zb-Kv#G==3lhh0N7@IiGNp(4Cw~xEiz{peEZB3Ek z>zg6xje$hNGV|!BUOb&|dcQwB0x~jwo9y9(c{okq`H|3QIkd?Y!^20Z8AUIsJU|lm zldI6oXUa97wtG&e(I;1K}$HSU+ClcPB+I(0x5#2iL2d_K_-rZj?dJZJS zzT>j*yR3!Oe6FjAhN3sSJA>5T?SD7>fA|FOf4;x>RzM#J-j4ydjyEj6T6N?Jj{yDa z?X8a%^kL%G+~@HwEZ+7R>d-YYq`mK(?+$djJS;}-gW1_VB7YmNT>)J#)C=Iu*rUlLj-b|v?# zj}3ShCZ83Fo{$n9szBf;4z=&ab5^+&lU7@Op7!f_UemIWAipNnkZ+4LC0%N5gJ~^X zQK49Cg7>C}dgaS-z*SzQQ&c-{v3CrEe3S%KeJVS`9@mm6)WTmm9F3RqWzH>{8J}1h z>a82k3?KVl4=kk6C%@+Xn+DtBj3>8U`$-y5^HraST^!RMh5Hs%xSEKmv7BJP+At^8 zn$we7NCG24Ip@=R(O2Xfo}-W6MFbd;D%~FrY8o}hV%XN@y@mm)uQMC@

KUYQzT03&{!&-#I=l}x=H*@AZ>yk z@zvI#{dXS^e1I zpPY8b8r~W$>6@m9<&dR5k`ciHuLz_gT;>_u{PGVUMN;OcP7>9 zb-y-|sJv*zDWq3z!iJE<(^_Z6^JYET+qE|yrp*X6NZF;2JXi=H{n=Ed$%gdx)R=`Q zd%EbG*pv^=?!TUj2a;yWNa+!vF1mz+Etcda?1^xy9{Oo7AZGim4_JKS&&q(t-b8n2 z1xuR;5>r2W&*rH4aV;X?KKb=&W3!C|FS; z0zrN!ENweMKs207Q~;wYCc~%;Uilu;PE*5WoLGra9E+{)C)HyP#8TU^3-L^`EA~Nt ztGR_kp*N93hrK(bKD69D;8xDS)9eO&T$!;2Mr5rNod*r+@BQNMt^vOFrJpVR{Sxp` z-}rg49)Aw$DWJX$+B)*f@KvIgsMhB_HqzQC1(|Aa%p}22|pvQ zdAI;FHgi|)s;duDx3=xZYFCL^YM;j>3)ua0s__-V-ymF}H};ceO6cFE#s+dy$_Z_4 zfA{rA@wH2A-$&U;qhoYE;X=Tw+Qn-*HjB&eUCW{uHahEQDlsyw6|;9iMb2W`K0nSk z=WD3-)Sagd96-PbzANZia`mQ$X4i(0ga^iCmx^G3TIR&sY2qVb*kcwu2T-Rq>3?TQYIoQcJo1KdveY$~*- z%=AQqt~~z<@>5KcKV2lmcH;fMY}3oIcL^f9>?Z`1_wd zIBcM2%bXt#0gD+E$k4D2$I@5KmHVT7+E4N5cvgJ*X{Rx|kd0^Jh|MkY;Yg?EfnR4m z&i6Vf-utqS-JK}GfAr#$krS$`D=Ted*y+A*ykYK za^>V?x(VNQy%lN*1;GB!%FXE`(unkHidV)K!f_2_%jJrq!{CF!h)mXf1=%cZnm{3|)xq3+)#o%+fg zUMpHHLAi`N!gvR|H$kI{pZ3k0+%P{9Klg3iPh51Oa#)-f>109T6*^sA6WLBpuy0n8 zfis;dkBNj6SIlXV>Fs=H6>TmQ78|@(m)40ywHK1?bng%R zbT><%A-uThan`&h@AE^>AfY(SI|&r~==;ilk7x98T=urG%?t*WWPNjg-)2V_(VoDGpCay?Mnhn(uTNr?b6Gyi&McIejdz&dT2YeR(C)F&-9jlLCG64FT8nrYmE?RX+%) zYrdtE#|aL7y#wEsRfYz~dqs29#1=Y`z!aN>ZB|hcrRRd`yk%HO@@?;&rIY20#th+o zDr++~L+CJ{v}f}Pjf9HYnBK8%*`pNP@>$X(s4&jCA|*1SDzlY#9w{p3(f&%U(5qO^e{VMHoM3CbJ`o?G~=II=wEIOOv) zM{!dXQm-u`8J*${kNQEkZHTObM8|Vf4C%6GYEAr zCpnNqCm^x$TkY$OpoW1Ngw;`0xYjC)@@$46UOykwufNjhqPUhv0$P0}ETom%6bG^$ zxowAe!oYjvmKVp2XvjCYXdA@!Uip^I^b>k$2p!+_RGH#L5kHCGaQ9FS9Y*=0=?K*E5zCrgzSY?6JVO!}Hir*f3!}iK?JE4ik*4oqW}Y(uBIl zbo3V&Xtc!|vx1RY57XA{wDgEG%|5ML2(aGD???Z_pzRn{%|oYRin3P;7MH#!(95~ku1D$NHb z2Y}X4Cz%7*_*pDs(k+cPzj->uY4ki;xs&QEW(Y-a0It9v(~IbZn=}(2rO|4iPA6vX zcsKd4gE@qO&?7)j?I|ceC#Y$vn)1CS@!VX4C#3uR+gqH_=iDEI*)V}SAK+v%k93A9 z0}2SgrnWe5f+BUsgA#MxNh%uqsMSQe&19iuEAXUl>hCrd6ZX1OL3}}Qf*`XM5P@um zDXw|{X^~7sEvg}|Z*wadHQL6flC+!$KkBBZt!#o7aqHP^d&|;Jd7M7Ec3WaukMwv= z0rmJu1Qkl_odCKS*CkY>K`Zc!kIa5RrRtCa3J${TcNUpN-2D1)HIf9g{uZk^qnQmj z3B~_vpQrLFXXWXy{qR#40l&H)=ng&TdDv?n|oWL8HR z+SXyYRwaOo5S7}|^xr`DyIp&ZtJG!wt>3ti{N@ac!c7MaeQ$j%49WEq*qIb-pZRit$sjwokave>60jg#4OV z40B7)Fau}h8r3d6g7dt<^oC%#Y zP{2ce#nih2Y{K(V5Om4Tn(Haer_8a#wSD5DY-snm*3 zOToB-iGxX}_mKcJMX+Jmsdy6{7ndT`7dAK*!QnsdGNviy+`6W4XHmKlmN6X&z!lT< zi*3`W_hZ7j_Zi$mcTw8RXhiLTH2Fqme|ObQJF4dbpqUhP6QAR2CvoR*UV)VzHjoyA0d3W|sU%mR zbE0R@h=oJraIP_K=H{<$2GRM|VBpla`5#a}2@&X6lyIsJ{*(T*z;)oVbgNLeL+<+H<$ z@LJvYckmObvR>YAXE-eVJ9AYg_c?=;my9sAt3y3I<-%+=E)KD!;sJq(!m zIE>BqkMzV&=R{Z8D91o2LfhD3XtM*p4OMt7yPYKocG`L*wt!{s3>}uiRjqkL<1y%f zhS1%lQ-*DxjgTfEgId=O&_FijwA4T|NOSCbIA|d|>>Vuunx{HNId+mEF=4U1mQ#U} z3K1JhSOqkV(RI2{|DCnmfqHo;2+2uEr01_AXYi@loSwSF#9*;o=Q_p@)GlRGmOZUo zU8cBczH1bqQxj9(p$D>_W-muu&$b$6dts*X>^+QU zZ<643^4dW%4d8oCN;3%EUgaB*d> z6#b7w4$MP<*r1R!{qZS`daJW1uRcR(G*m^g-sv* zQJafuv7WxBjwCZgl4MarYM5~kZ&A)&qn~X9q0jjCYF5c1U1UnL+m`5Org5s|&clFJ zZn0-QUh3Y8taejVAY$K~-x}8Pq>5%eK0lCvE<7lyjT;ih1m7uTz!upsaCs6D6yoOT zvXqqy38I&G_4B6@)_S{wortUxWUfHmb6TbqoU|RUv9&M+3WwYDqP zjycTxi52c8oEfv|o#!o^pRXt`*yx5N=sqVxT*z}D^_dk|+EfC{`ubPD{sfGseKsuH z7NZvt`DEZxPFn&hCTfK0+@ezV3(z%a+sr3`R2SCx^2mqS5D8CyYOT7R+>V>#bC} zjqaq@Gom>?0xhE1k$8sywR1^BjH0fGd;a8~@g}U|gdX!_T82!bMD#%`^Z%qp-$XM6 z`*cQ<1>mwMJuO{Y^_x1py2~Q?-T&wU(fctBJ@dBed98uGgSrLLRzajd%fztF*CiBQ zhE^YzGxvcV#zi@!1GGra0d4!W9w*?DDyZRlQ=OlK;k&cw3CB z&_ppczm3S=(xQDdSWYM}PZ~v)A#*z?H1081ET8%tBz`vt8 zX--;u^V__oeB{2~V8MVW%IKXqDp4`tpV+!;8>+#KYugSa04_9w{IHNx`G&`{kx05?-&oN7v zaX?O$lYFe!`u7EsmIowi444@9kJt^zOMSJPtyD~VH&!eAPLjE#KJ0BfzsirZ1Av)q zU7%p2VBOO?q~~UtBR+v^b!|U%dzQrwK8qA21PqW8gMzi6xT-&m^QA=sBG7YY&)ydb z1}*mjTJfctti^1XK+BSx(tp2lmGbVF_kEh}_jr957rN#&CZ!=cgDyn#2x;jF7<2?W zIkh+sT^HhgBj>li?Elx_{p8ozKi>N>*l+GZTOA?Y=xYttmCe3qysH@2EHtCYHvoXq zB}7+Ab*Jktm*`nAMzRQTdr;eE{A!Wqr_hGjg;P_o%%uKt%ZYhJao9}0-GuSr<_&KJ z!QLep8DcOqG!DyogM@`#9HxuwP^>=r`hY<;R??9BTp*L;+-WJh^nLyPQ|5m5uo-N8#`WZGk;AP@DLkf5(D9`vhGcObOLwt-SC1QNF)@&#_Q9ciL-)>Thy0whJy){?Aq4X$pY98$ z%Ial^q*AVCm-d!zsGWLf(fG(nbFMC!8PYGLCAN7oj{~&mb>_;KTpU|mk#;mY4Kne( z)~?PIOSG-p_IihRb*1wY;aSy>D(So6Y)*Un%Jtz@3%b-utEJ zN=?XZiSs=4W8GrTZC6B@k?4ZIWZBKFi<39R|*LQ^AAN{GJ);3Agv+9UXRmGZwaJU9T8@}n( z3F5X6kNn4iRnOZIG&xSwK+T8=uLsU!{y=fr_Xr-xd_O=puAcm)lbn}?Nz)S8Ql-L^ ztcPuP2`6^SrE0eY&mVYtjdv+It7<7`^Z6Rb%;d)#R17`pxq?ULX{0091&LjjQ(=em z zYe z_*gii8n+HksmErY%p3Nos1yoJL;y2WFUC3T!3c_MKQ7tx(tAS3q)f3;RG@zugu zVYco>ZK%V_?xIt9%yaJ3u7_b>Hbz&YgPX8bgK3Ldw|^`}_8^O!)hTPKYe7IyWKJ_y zy`}9d(nZLg zlQW&jb6^~?_3D2q{??cOC}cz14YIeM`T3XM<^A>Nr$7E=tN-sF@n&6~ zxKrFN0JzDG&iuZCnTq7MlIvr>TlXc}jnxi)Q(URp@%Zz+jdz6hgUTfpB`S%3R>#*^& zc{ri7#lV|cx@N5%C1h@35#*Kyi+aS4akMSA(%+!zjl?AYi}(`)WG*Z;RMl)e%ghcwS5hB_G-2Ptnr=`d-t=DLBd7o;Zw$k|5o6yN4 z?d8nRzA3CfzKtzE_YNd1B?Pb8OltdJSW@FGUbhIc2I`m7cl*;JX#|pfK^K1*+7dJL z9L&G9dx~rpr?k!*jV=~*6ga9?jFOtzR5ROjEm{@yovjtrby4`LoeV^^81GUy4x1LF zy4JtK8Yh%W6n3^eI0ag3Of+K{;BU(H8&N5EsXQu*A~`b5%4AmfkQKJVZ0wQEy4kmW zN}mni@kX^aod*Me_|7OoX6ne>WxykxOt81)Meon$?=E&-@XGgH49NRx6i>;ajQ$FP3WrysGEupM8GV(H z^eNdBaf!`j{#@qGFLPfFtl+LQzWpj163+03%oOON=r{~APjH$fAAgesLtDN2kE6q; z)RWWAujRfm{RW}hudz~J;bsG=95M(J;y&nI>xih&O zhH9ShBs<|e;tVc;ytKBn!_4`dz#=SU$)x1z1uOa{`jk^gtLZ1TL2c($Qj4$r=t97` zngg9#2K_m?gWjpUU5kXRvdoksw?#pE4X|EvE9vuC;<_C!m$onqS6f!($TEj5l-YxQ zkf(lzOpJsD=uF06PE@UPhVVrLqgr3S^d>-80$mdHZ}LpZlT5^93O}bI zl~x!iI_398oL|5E(Pe;m^@`<}HvxWpdFvO!{@geAM4qkx`Shue_C~N+SV{E}+xMky z54fhSaBNDk7ATqi+;`eXzV8e1OKERit8}T677H;qw2}bo#$qn(zmP3N%6wENWNcy7 zT_~+`jTbqT935F&kvU&>6*h@$V`j36ubVHcmq#AJim`vUz&2>=m1<%mFvxFowA3m# z!W?Jq6zNKU1}d{@QZ zg$fHv%xqbmbKbqpNlI}*002M$NklE z*?)pFde-+D(3!6nzzvl{f-?a_dD5#>GdzkvXxrqQp&BN5sD#(#yTP*6# z=;-@r=}iHx`^%fGyDQrT(5= z)YFtiLbMm7rK6Kgjq@>gOITA118!CddY2~dZ4j~9EU#p0=}P&gbmU=-#`G)($x_V; zYBTxSRlicLddQ9PUV|Ub(j#CsCy_;PZVa}s!G3>`W=>8^QgN(8y&An|GC(+O9U(khK|MD}qa=x#0??Y)2sx^{N27Bc1<-$uIW?l=7$&UgI93&& zAY-qO;v`STy6cCnhDlwi+Q}ZYNvbYvL_J4>G3vle#PB>Fr{{>6{>aiBi|WKdOio9U z?C-WJsBk`QC{Om$BbXAQ-Zo^o?o}_@fEix#pLl}ST>75MW1OZ)P`Qy~t7z0#(0gFE zpM>ye>tFSu(xjzBtrhf@kX1$TdiQZ8WNnc@&RLZ;)yJ!$I!l(?BgXR5QE5n#RnSGl z(HUNsdl<0Pqf=pjYd&(#h?^}NV7*3G)nG=f9tul*o6BDBd2Ix399*JH_+`8)C`G2W z6SaAHr?QRUB*N1$^JaKk)by6}w9Q>hE$v*Hea%p0agQc6l)&cPL++?mxm;$})(X%W zbJJ49CJmC3l`aQTdqr}7ADRU6lX88|>7)Q)K%c)&B#*?QZ_~us;YK+P^|W!Gx~{5} zj6k1`Kty0BOAz9b)UkfGwzZC^^~1y!2G6sr-5dDy!#1O*Bf7xX#`2tcKvOmoSe6$; z=D)zW4@O12PQd_QZRgCuZ!sFpA$O*LE3imL)*%QZECW7BLJ0svR(6)|v7c`Ff7+t|?ekxi z_y5+{>bFW7s-$e)!DL~LKS;Y5o3Iw>Z8@k#^Sy@pLsS z_n#y_Nf#e?p|TJ~YD{)Ff1eZO<8x&+WeIbl7or5Ws`GPH)+ixR z#Rj6+BvdrgwxtH5XO%Oix@$0_<<+6t9?LYI)})S(j*5p22uooS=YeF`lUplTK3It6 zljts2MyHZ9c|2o%@*H%!TUPiKZL%zlT^#HopuU{v?>@nirxQe3M=aruOGD~5)phB> zdbpwG7IK$wg`=KEZ6oC8=XVs9(wR3-X+j7CFVf;i0fp1Z?v}=NnR8l(_MM^P(EApt zv|4JY3vYw?_>!CObWTM!Nmph09ne8!h}7u6i3^6_M=gVakN}bzE=zO5wgSH_p8U+X-w=k}wwh z1+qgoVfGF{^;&Cug?av-Qw9^_nHvaEY}1s^dB-%AORqY}Gb1X#IdBZwcGWS@v6C=D zD$eqg8@YymR2bU-v&q-1b11oWz_l3y9s2ErA~`IgQ9Od9ExU0`^YY&Sa7|zY@L8Q^GzBWG z3&=QZi)t!2<8F>mYbfSXoz#o_9Sg-!-hx4N2H=_8&;7Lvg3*lro!+-(tH zFMDmNV$rfVQP`*MG>QrbQD!KD{!4-UV zZ}#)1OH8ryIiI=JS0uDV--U!P+s9wX; zH%*o}ZhU>}w{fitj8xkAG!S2NT16q=-D%et`7el`yXdEXONmaJcTc|uid+DA|G||& zG%3vGWkUkOY#@X@i7g!^!jSP=xz(9EV$KVVGJ|0VR|^q0%e{|b`B_WmunRdn^t3&J zo}9~`ydf#tBt&$3Pd^|>J1u%&Nt~eFyg&NoQH&AK81^iLb?bX=qW+K-tSn1s`*f8& zeVeDDA`@Lzo5A9>`p+B|rBPV>{nr*{GbUlVwr?5f$G!U{HE!_<6I}7;xQ#=3LJ`#@ zE=h9cBay?ywSj(~E`3Xp5SDB4g(dgYpMVv_dFeyTP9bvKo#N`U(Dg;iiL9+kV>L^E3RqpE86vicC;)TXcSdBxM zCFGQz5k?Yh0MCGEIc{oeiUL?BbQnlxxI{5tz;^n4T>XG#vS2opkrZRsU zl$9w2s%^)b6V1J3_HL5n-1B$}WooqBwx5=o3j^x4J^Dwosi-s_T@>botj~3hQl}Yy zNGHTRMe?G=3ayM-ZkYtMOi@wK2DAJ!pe0OGh3`(Tce78!IJOzur7vrjA4VWAs^62( zcOBsKkJ8d5f#37s;)E^_3t5dR z5{AdJ71khdT9Kxn#8q)mXcBJpJ;9jx@J!|r;~VgCJSF4FMr=k6R8;I+@?SAZOlUE?ei7Ts)fQ;8>y!rcwwCOS=4t}m>5jTWeCn!(k{s3o$xop=G3eN0K>e22K#vw)b3Us7&V9umZ{yEQk)Aa;+yd9S(m92Nqq zxV&Fz^5XHIWvNKdI!Mt zH}hK4ap8so+G(ML_?^D%oe+ES&!I)hq*E+$Bd54B@Ah9)Q19` zh|#r0P~q?-xmpg9GvZT*v%#__)7%*ig0ollnHjYXL2{;w#Ph+dno@1FAVK$_-rSs+ z$f2pgQIPSSMryD!qf2o6JimNSN(o)=zRt}NS)Vu&x8mg4CWX#MPDuS2aPJKK=$j>5ZZ=+j^qk3Ecpx&37{jwjDus6ESPtCU4a%$v z#@~PU8G|xnR{;!7X{@dOJKb>|Psvdd^1F(ZY$=eURv|s#9?1-rLr5x}E`ZE8Xl(WH%H`{r5=!{^M)E%ga}J zy>YSiozWOT$!53lmAwZ)6?MkAXs3Z=Wqs-a%kecXLiVM8b26lv-JW(3jFsO5sd7Y8 z&nO6R!(lahFyH|sEo&cu4Y6r+)u-0*mOi(oC?sAP3od6<J&ldmC+yK$XhbXvl2rt#n>UBfP^;!8FMI zdsVMr=o&YMW~R-LbR~`(%t^&nVNr_0H4aen)wQ~RerB6Q(6L1+GYRiSMP>CR$;>^Q zSvIx~BqYsfPaEh9L&yZNEtP^`MobUWk{nwn#K}1Z8L16br2mhpJ3(w4S)wfJLAHPQ z{ukS2OLpJ$0(2Eb0*TBRUJN+^1gYf7m^?N`YCHGLKdFsO@%kW8c&NmEk%@PaQMHX+ zAS7%dcEy1Dl0X6{%J5ZgNQP2AhnC{~#&W6hwXf#nD_8O+6McCuNp!<&cSSCi3#gzL zR*`jKw=Iz!p-#75%J~2Y(RDtf!I}*mx%$z7j^Svo3OoN~7lSZX(l)s@si%d?HI1Uc ztd{w`=ed_eAg?j71QE_naOqmA2=jgpuwUIwi$otH`M=vHq^Ud!2XtbDjAC3O}vz<36Y_zU0|Nuk!MQWRydk%z+dt(A`Y zh+~}grw-D5Hx1Pb`D$Lk=EGHCv?V$dwZr(BQs1ljdB2JW?omrdpy1N2U^t(scqmCI zDRFFDe;Hf09RE6+T{QDTg<8hc!SaYOs)8O+rH+7djpfB1U%jBJDvWb*A%DW6`}Nw@ zf3;?orDu{#hxLGz(56TG7eEzD>}aRe=r`^BGS161MSiR{9#7IcA*Ii9k@ly1wcq+# zRL}7n+%~A`(+PIAC$b%9r=QSCCvxoQ?41cmHN_yKGrSn35 z-RdJfu3So#dD@=UyOGF8?f@k^5&CMIRo7Ry%OZ2n%vxkCy2z$xdJMbIhrrDh?jkYl z`Mrqi&4k0xpdkC~9Ty-z>#!npF(~HUTimlA$i4W6BW~-)ts7FYeERYu8RmT70e)-$ z^{vAmA1Vm`IajNST`R}Der7i6-+TT3=l#8s;u_&nI zB$ab^RZjhQitK1Xm8;Ygu;#k0UwghJEBeZD@hU0$j&jHTU2*ElD|6?5eqR4KZp_$L zqRBunGWnut^s`8bu!!8x?)hQUeMwpt0@D_V;p!wROpqiJPbWo<^KxL!ISgJJtQWxI4FI=LMMmqR$N-0=zADn+e=I9vNLscg==eU4#L z=d|1@;fm-wan>*WdI+JXgZc!Ze$nGzm3tnp0h5qrx{PPRmevLeAI!{Iix1H%?aw1B z3Q;E^m45O2dceB=V!#?H5dnkRef!$4%U&unhFMeF5X1Jis!}oyDG-dbSQW{{zA+*t z>xHF{{1IH?Dtn#)P&bngK@VS)5i+1o74YEJgX>f|iEDT-%!RZU(o)*X=Oks4wQPWO zVS|S36O)RpmI~*)Bqg@hf1nCeoo^^ArBhg1~cbrRi#FSz^JE}(CuJ`^@`Ne-rX&6-$qjZ}JqU}39UzsblbtQcuNo~*E0ko83 zk4)@r!q>f)9lNI@4R*y983&X)$d<*B_OVbWfi-v}Cn-gEqhpp|>@W10hVHyW?{hvB zS3jc%vRzsCueq4p&S~}|<&5DO>5~;CpG=S+#tPU`GkeEL)Zgm5MCPN3lTd%jmmAkW z8Y4$^cup+WwS{&^{V>WWTaYvSxI5;Pkn(VK=S^x_oUG3MtpE!k-&pDdb1-zo%B}`lYrdla5`KFZwWWUqj}^~#xREs zB!-i14L`O9p{|TW83PBOvd6m#)eTK;T(I#-=#rMaj4k_NN3Or`Y^tAYH8#%mvg`(_ zXsosBFUv73QDCN`mpn;8C-LU+k*gpDDI`*xg#sgbM;0c|=r$)NZ0XIe@7 z%&SoLG*vgQxn1|Nr6enAP4t@)O?R0xgMxNlAjg8Z3A`GvPfHuE&)?F-NmbcR!J>Mg zMwFhL>WA-~ncQV&`)<~7VqRN56Jxq)sy0$xYHw**f z5NIafD34jLiV8z!4kUReKU_zvKAANIDZ66sAL|z5w7r3tr`Vqyna__c7~0kkTG{{U zKmWp$H{O4L`|{IF3r*^cKMeNeZkHppn27B)Y2w?*9mn2j!I@q=UBxS3{DiXXi{;H(6Ip?}g9067;5d8Lc zgMBPdx+PaAvqyl+#uZ*=tGcBc@3MWVc+p~DLh3l%CRQ2_R@j@ zNR~)j63-4Mq@ zdr3Mkz_hO4as8lp)-R;`Z1dR>1!b~-V3omr(8uIdhAVF>$J~$>t}K0i9YBut$0q(h zH7GYKNCkaUuArHSj_(w+jE!r{T@yk?HKE;>Z#TZtVN&DX8ahQEZ?gUI_=b?0BAAeq zEDez28{tL`tD0&O=ia;0`2 z|9hX4io9N#U)!hB*95j1f|5G53#;68g=%lwSm+%SEy>u01FFYSDihQC-0(1eazmR+ zu;)n~A+!(E;&mixpKog7FxD#WENC5T4xw-?A_|(hy{4ONhNnttQfm<9BKr9sWbdH{ zH@QT=4(TP1BkGBxs%l=E8P$n9iQE1PO;U>Bu5l5Irs7PgF!~Pc%BQ1K295f5|6;+) zhW0p;Y{STKXxodNnyFkZGa_TN6kF2>+u2GjZW4m(F27D=s z%Gt;TVn`ZMy5Q_A>|yCDqcK*o@`X1~Uw1m^dhy={f+HHlDSo zl-;gk5bZS^+nEq!J35mGCU@dP>S^8ZN{w^E#vR&QF` zZ7oZxJX&zD7#SLzD!S1X;p_}XWT{CyEwPd1uWkG%r!$!AU*o=L>i~;1PR+TctAT%dkK4h>gN`KReMl+{4_Mr3r zrR+<8Ekv{A{sObBU4Ap##}0HLWCTwjtOa6D4B$=ElQN14jY%peOtOCZ>CAKwYpW{pz>&)S+WeO8Oj*N6ZR8}|V4MRlbp7)4oUoh}XgzKvkvz6b6(+UE%maQ@Drw#?S8`S)@8p>0rtdBJHJ zm(`MIj1~{P?wO9(9v^*1kenP)KFyT+SMGscPHx}opHYFyM!z7`X-`a9Qd&(e1>%Uk z*S}j51z(yxlB?<8319vmWv!P%$41@X7bd3xZx{WtJ;F|^Nf{f3X5FfDE|E7bpS!wQ zB4s2CNg|H@`da2V7vU3MkrN3dGC4U#O6QJIzSQCDcW7f4FX}bR|ichd2dEBUG^Fu=Zfu8 z&rHrW4!d&e#s+gm|&rt?ylH^u$`3#Omi|ZRLM0?JU6edyx^dENFgGdpf2IjeU8BI%v<72+TdR*UK~jCHl;;YO0moE-f5J}VXiVK5>Gyo{^$-hGT8`^+O zdS0}bR=o?gM(H+~eSF^tqrw=ysnz`qiyI3hcX5xf22Rl)24nJxK7?lVd)C z?0>=w4n3ToiaGy$pEIT8=DXss9IZ>G)^5%+%W&4gnqSyX4tJR67~h`V88GvBQNYZ2 zQJUvQoimaQ2td)C8(B#NA5HDczlk|LwV}_HzZDDNygJ?Sp-@ZbwUW5GxVCv!XNi!| z$3QYI{nhFQy&zVVyrG(I($88Vw5`V_X(u|R$~z_-oOO}8QHy3mFC99W$o9jGA#S;J zRaY?d{Y9}@&BmyU7i*CT7WpJ*#T8VIx%S3RLaLbT3P@qc%^9@V7WcJFenCwEnN;yl zw6h{EMG$bdd(?t7=N*)sYk^+XzX%{6Y_f^twJ8du5mW1mY*1hd^ff>lU-^1!-drJV z*f2OCz>N=QE$4aVf!7p?cMNr>j=i@yq5&Pd&ko1gJ8k*V2xWyzUbKW@IB)P1pE~2O zD61i}rOzWqBW_Js)KPYaAj$O&i=FhlzFyeq3=m?yrftLf`FO&GC9OO>nCpljz>~zQ z1*zC_#o{BntP{Y;V@``$q~Ifo6%3cuNFfMHw>msf6nnu36Va-;@m&YDEmiX zDNQI(hl`=7Wz_)BqnSiVO>i~N5H6%OfK=V=Wt*sUYu=(_Hj&HgktMFbV@bq#CW}sv z(c$*-$s%3=mK$Zq;p&ia6adjYmOH6_wx#-c-%@y}0_+$bgZ%(Cn3w`OSyd=lHo7I( zM0Q>l@HLxiKSc6hzi$cQc4qKo`6@}YNryOV8R2p`iZkJ`@b_qwdu)-;8ypHvy9Fcw zTuhdyr*c*wuC!S6%(wsI#)j^ACanMmqs3T)>aw->PR;kPe}3et-*5e&FE^?tI_2!^ zZ8fl$X+Xa{uQp9R-xBs!0sm*9*(-|d=)UPc6bAA0x81GxV?@9G0O)tZ`^;HbD)Q^B znj4)ACtMj)C4goGhi65<^=ux9PU`@$Q`^=)_xQ0SeZLHY^cZ;OCaXAwZ=q{ z$sMd$iq{ql7dR#v)nb~sM@8nPKY9x{k=2(v^{wo_d1xt8^raYMKyeEv~yb~$Ik1Gtz;?=bt#!a713s6V(dT!+ewNiiE* ztmxAT=1GSzag1txpgmf8p=yQH@22=C75A<2D_H#dT;O){kRN6orln3@`qP&_nyM*L z0CX|h0MxIrv`+6oSZ6NX(16t|kL?qHkd}^;3fv{ntPCgNkKnnn;j)?}t1JY3S@b$< zM(H}jkDb=`o(m0A^Jj;BIau_-%Cab3Wn31=-WlH;>U7;in?v~p&)pvr+``m>tE~;U zXAin?O2Y&Litiq$bltRTrm9q5rdEn>y`;~S!nz0~hzc)Pb0)|}14Sz`#)PtLi~D+K z%i7QH^(lVHdzx546FB}Zx`|>=m}-w(@aK-G#S2I035n2LzEp8qTWjW-ILZHE)Cgnv zV$bsvSsouql_VL&v5q_ya}bSz%kWL)lfT)p>O>yH03`&nebn=yf`Qa*i@3Jsy%lO9 z>9n!As2Hce?u$}Z{VoAJ_U>n6bN+1v7qw*7WtW=5nYhY2@~X;~&Q4p!G@PYeJhWL6 zW+ytv!7YPB#?!g96fNJ3uO>m}8B32m1QQ7w76BWHlV^KMy!nc-mOHP&4P&hpyf)1S z3Tae(uK>l6=IaRA6oQ95QIC$9RuV4+QC3(R@w05wPC8Aqv8GB1UUDWG3kcqA_3Isn z=p=z0!bx8ikRK88AxqGTJpZcQv7K%@;p|+5Sd2@U;)BNdIVfXZxq=$AU~ce#q*sNy z)yyM!h(kR&<@ewdUDfimk=Qk6pYl6>P1L-HOB^LiY| zrsdMJDmP$I1u~iTwFcBFHM&`9Y;_K$V;!DxO;3Ok3LJVSC#-UkJDQ1W!sI>4$>|8{ zoww9M>@G)Cy9;H4U_Mc2u{C6^yyvgu)I_EBp_!5!IjhP-pNJPk#O|mxew}s8;Vf*h z={`u!$xeoTZzQ1MJTXnR@Qz$5pIVn~kdgzJ^FvjmZdGNp?4&9&pyAJh0h*^5y=sjg ze<=qMiu6z4Qwj_Gk+y-+`=T@1E=#zXk0Z(3L-X1-NDpQUAoFDgM%RbtLzaLtf4Iq- zTibhDmPoYnc|Cy}+c-bUYPqOr;cKQad=lR%vXQ3&wZ#_H6aTC4Bt|A02KC&w#UF9c z{GkK3Wc>5?t@WATU?eS7r;-czl4!DjAHrC1^&ECh35=A}9R1=iJ2#5`d$q8tBkzFxN`iVlN4XLv=ADQ1Yu@w#WC zcE7~M&txYz&9iQ7Rac`H0SU9$>4UvuArQy8LZoDoWWS}2bZ;!L;*tR|{f*&lIRy}= zRh+X-j`M)Ib01JKR)v6~OsErJ*eHb(?8)9xpeQ(r%~( zL$GsRmGf@dbFV(K>59!qHs1SkKqPau%eZ8b?A;(&DpA!4(k)mJWmzr#9Dl23lDnkI z1N@&t9m=JP572z|-2BE_yqO#f>yz`?a~mTCbS?KH?E|KEJ@t_R`e-v9_aq{aku@YF zrM?bhHCITFdr7Yxm-|W8>$nE>e1r_n-UTKrnu53b-_gjkOFd&m3h|749uMZrNgnRc zSKPFJ9&W|cO|ftn0wVgQwQeCxbZrHPdzGPd3s^CFttwa#2R!0bbkn%;eaS#a`vxBJV^ z8*sWH+kbs$M~QrLfhL;Gi4yM}1eucMaoNeXog!D@j7Hz?-fgsH)!bY23}Q;QB`pr# zvyNf42<6a9l{YC@#I2DNJNAcL{k;*7r}MUJCz|wR_2=vb!m&>;kDh z;SRi8a3Wr#Dxq?|dm^2llNJ1xZOdpxi0#81Q7j5VPFnYHZooj08C^_*Cwt?KPsCM` z>If0*Ml@G}T7f(_`hez-k*pY7bKXXy85Cg>P*M&e&$nxXY`uKG7)l znVvT9her}_Zl-mcRmag>pVwPd(6ftmx+^ycRyq)kV9gZWHOW|7BN#5(5h`o#gc$mW zsxq7lN2C0d*1Kq7K_(^+C`^hKGPYN)niDNVVtjgL6g+&A*gqZZHZsAJh@_f~<{~SX zA==q#A&;NPe=ybhCespb!5E6PZ&*OVjH#+NTZUSAA!Oh-0}%frZ;M^A^vG4wm|apD zp6G8dK!`aS>)UupTQ_Oy-5sx3a`ZAG+KimO4f13{bBni7cH1rOZERi>W=dx0bYW1L zQeSp}zbbIkUttC+tGc6dx-Ks}3~4w^Qw}p3%j-a@A}ea{1r>~EbS*?hM}$i)x3T-O zbpT>SG3SgYp9RG^j~w*rZ^!99%P{}BsE)FUd28)L&iA+y&61%qTMrZyd-XFut=M3O zJ))$X+To3jKbDER!R#!#p78NALH*OQuYG~oWk4+rX})@B)zgDOP5wP+|Niy!|Lq}> zpbEw&wlFPe)P^Nnb^0Pa(_3dVZv%v$@NKPn29sG=_KUZ5N{5cx)m5 z^Y@>h`=UjZ<->%ZtIkhJTky-vLLl(v(w}eArjtvHInd)bT?2gj(LZPFLZhBs9pQDA`lC@}+Q7!yk04!!A zRsABy8F|A#arp_O;iXsKk*KaGSUjEM*O8vD5m%r7$mlu1`!iWSm7mdN$qOG}MG?w$!q z2NwkS_7%PMqjp~G}()i~|L0M`Nro!ewm5(0qQik|ScE}Zyqmr$2RrGK>3$Q>{^ z3sk%fww#&#=0Dr-F7maTVZ2 z*XEkw!7wnKK_c%Y>)B4+xQnhPQ&^;ND;F(;G{j}DtHH9Mm>7EcUwuuJ#U_>B{7JVX z)q38=w0v907j{xDoNF!&l$i=tl#y;TS(60i8SPCd%q*PJE25j9imHn`qjN0NJZa7P zAh>$#+8i@qb8D;*nc|Rg#Tq({tOjk$=RbyY^4OC!ueNGJ#Yl(h6sNk<%mJVs&aH^-{S#^)}*#Q9p7CVw2E=m(Mpd)zz>J~>2Exbjy*swtT z8FFWpBhG|IAKB0bX|%(#3f`C+;=DU&IO6FkYF5-^-Ei>0B{qzrb=fj390EF4zDtmL z#hofLYop6mXHn@Gz1rg;FcH#5b!MEz^LArGLk~Z%O*<_%=H5`mNwZ{n*CQ7Bcv$@R zk6Yv{P3;2%eZp1N_KLy8Hy+keDh4o3B0LcIV%};mF)ja$I|hqUaOwKjIWe|5t-Pn= zgd{Aoxq;eFxsqUSITtR3VA}#uSn})?TwQ$9Kv$@3B01&^&Egnisb!+unPsF6 z-3X1*N4%bxo)-&3rJeXQQ7|)Wb0RQTL|< zQ0Wu-zNQOLF6px=)7z%F5a{g;(qHlNVHg_P`yOyCn7CwuSVkY+A!CLffvar}Ggs~D z`F=&H4*|Zvz5RImRpG4_(+qJ9(A1G#RzAV+il9kP%1^Mz#@!n%cA*5i&;~EJr?F!{ zRAUAWM<5O09a`vBZXq*wLKT`bk;`6|l`*nLdbd|h$-I;pl_5!7IxiptmFVEw6TdxZ zST&3-A*=_A_at9aZTQz(Q(H|BQ+zVHSElDTsDXlMlv1p!hc!K06H(_HaFgsWTc-E{ozXFyxdTK)GwfV&ddBQCBMUT|gw|MMr&lgF07 z@(k&6r+&3>8<(zo5UB5v`2Pca4&ZV-m@>Hk0rBzmJ*P1bSi4=?6kNj$3-e0=7M(=K z7_F`$MxrE9&V5$?{>tj*KpuC|*7{#yo~*!C-3FhR(z@gZ9Dc8%MQ&5v{^@G}q}PI{ z?-Kk<0wMH*S+us;ZL0SBv+mm7?K^RE%e?8f=Xjo=`Y=QqJBvEx|9QEK5ftOCK<{S}ee(XJ3ryx$h*rU2v|OGw>X>JFS9k)wvQw7+55X`WNfLVG8Cn7_}_jdRL1)jhN& z+u#!2h)dH5oh9XJcgEdUSDq@?nt*}9d2q^~#K%AFVo<~I3>Luv#!xIpSQAYPpIJsH}@*qLZaeMR-wF<9ZThtu`FfIS`j^9EM zO=65XNDXII9-9Lt4|DahoKk{z2%6&n?#PO(V2x67eFUj#TU{-=cIOrdjl9=Ze2af9 zRoo~K7{aOuYloCbPCpUB;ZZkUy+#ab z0@)#=c!snA$;ZibOp|77QwY^v(<8J)Lt=kvSnQ^XT!~F!c#|Dxa&A62R@L(oWTT8wRPK`G3P3Db% zHFV_4DRpRjArUTd>ii5@b`BLnzF2+?kVG4H^h>Udk5Nat4y_|MrgGbQZj8|u*PG!zVJhJC z%m=ecbgH?WWvG#aGI@I;1dfUjTw*FPfY z8sHaCWI?~)(-YRs9m$M3<{oi2rKGv990amQIg*@>8+Y86o>@i1RS~Gt=XYP%jh7u9 z*nT)wRU|Ppc*ea?j(orRzGeAp- z(bZ^*BM%>q~oCnV%m5zrMdrlu)Wo)xR*0^5o_=2T9fV?Fosb; zNpf(d7R+imqfw>|Pmqx|;UEBqo-Qk37p1{Xb{0v_IWbe~iFt|ZnkhT)ZRfW{BkD5* znp(l}pts(JX2cn2{t%YAmKQz!5#8o3OaDMpo+nxcWMcvDr(s&Q2}>=Li=$FRyno6j z86y4kJ_M++4m@|JEbFH%b|WoU`Bm3CoCo%H@4st+v}d#*x!6+i=38lvE-qU^p%>hpPfIGMX1Fw-vX3wbg~Z)ufUNKJ(8 zAaBT25!wyy28-6{jZ0>* z>;_)M+D*j|>?pL`LSjTokFyn@3{B3!Ih|%_h#2*kywg`L;acSoZ}dl=1W{ITt)d9; zNqN8`x>^Ef0CD%O=25w-L!9P>KZB}pGsey#Ax?_vFs+lT=#IEywL%h^c$B2C!PI)3 zwbHYPF>5_SHODn6+g+$aTyRoYjN&q~#FEkJI5is@s#e1l^}R`rzPMMF2t$OU;IDD} zs;l+aRNZJxI1fdCkt7snOcIziqlqzl4+n9x?ZMA*Jg5)#bcHg2$3GCQ4|qDTtm7$@Vw86c4yrK(um%R+Wlv?}?>wGxj_+jj zg5TLJiNX;XM01W6aP6T|OGMgt`P&)MQ;PAl!#Y~zn<{lO=)jrM(#-V?MIgv*gK=+YUo z;m|Q3xIna!DDB)~2&bQvD5IKvfiB^=bMd*`N}UE}o(aRivmr=Z`sC@>i-`=GDH;Y_ z?NLP^2k;hx{hI#gmj0iA^eoWVkMAwjy0^cqKpBVZH_P?z;REQteERn8!B?J2up!BS z-a?@oJ<@yzZ`uuD))|{A1+JC@gY|(jO;SgaxV6DAFy;fM2Pbj3?eVkyR+g~!sy263 z*m0zfuG-`;Bl-_KQ$M*;GI3PKDP!I2HA?O_vj^pKLyygbJ@+j) z){6`uX~&W?a(GhG(t6T)Zz>^qb&1L?FzfZdf4ZdX|1*60*{{O)2+*Hu`kx!F2Lv)X z7|RYKNt=;eu=8aIcR{~#GTXaZTMYezs#k@s1QaGCEI!vUjOVl*4`e-}p1CACpo`g( zO;o7TWp8C~h3ZoUUo1&-UARY8ic0cc>!l>lb}?1NPd%WIIQTh)r`rXA$0S0(){n>S z*PdBPsb_(btiO~hwe;Xp%sFSPYU#e^Qu2g02|nqt5;nMf%YEI69h5AII52WBK$PWN zk=X}wdfZY0X7>=kCd$t^WO&4<82PhX(pQ3*vFiv$-;!IW&8THp_8BU*(}A7+tHzWRUCN-8ZVc z$&?qhM+-pZXY}C^n|z2G1K#T$&SqP6irYC@gK5c3O6B)i->I#$Xtxrd>vJXZ7bp6c z4N=S;xus$!6Q)iNkfPE;E3zFD4a0n%8_sCb@E(LbO3? zLEC&qCQ^X4q1oujb?I-jTceYWnsZ_r+s-}e_a_(P3f_2kltZ1x)V6B0+Z-@JFeQM` zpY~bo0(m)tj-B{{Y&%(;s2i#)CTp0hkF0(6v8mmW&g0{W>+wE6PX;bg3zYbtrs~4i zo=*q|1Z?21O?kvs;4Ln3N0zo4_KN=zgS$rYk(3%L8j<#!1fw?uj40704{KRlV9L4t z45()wZVLw*X;j5laH9Rx4O$AM-fAQ2I2kEkd6q0qSdhAAaXyFdl}U$lcEo#_v@_Qc zF%H&L5nG}z_qKW(QRij)VT&Xd%Q>VtV}z*9F(;6@_706#a%G)!vag{%fxRPtXk$Z- zd!X8%SPDgG?mfhHwNo(hj=lzzhLakhaoV)5f~P6Cuoxw#uH30zh^EAF!?EMUH7QacstKy1bsTvQ2eI^3 zvypObI50V?SV)NNJjc{60Mr!<&r|QiszRuTu z_nNxt7!DKX0^#%OHpB6PPZee|K-U}n4RR9;&?kMFm{xA;_s<*0TZYVzx^JQ7lqPQD zJ0{ln!l-rsua6JElKJuF7Yr%tWVzvnBeK6^onSQO_-_FH`{572_S5G8zrOsxFCV|k z6RUoX36e|6 zNK{lNGGg@3V@(57*0maYL?Yl{M-iEL@Pq>`xl|Llcf&xX#&xeh1Th+DV_r+hCwIw75>?9ck zM92WP#cwhG)~&=U-2%@FcbQ!6gwtF?e{$+C`s}`0-UBLqpro^SQr-GI1tAGb#jI7F z_bU~NkJ>QUvlbS2{MeGsNQpAGLgh;S+;V=xJ*Zu^>+&zIC( zJ(3zLryh6-n+oQ%!?!t1f7@WCTl6tW-s9;~(YO$`TX9u280d*n^VU#7P)-n<_>zD~ zZlNy9Vo~MP=6xnvqt!imyWz3WYfg-N{_;0($)#ers|dJFK!V^aRlF$&sj=ga4Nklg zlIVNMQOJ2JD8P^S7-qqyj6vLt#e|L9aQ|D9Tb<3^_H+9DM*sjo07*naR5s-mC?KFC zEq2m$ha*#p(nVX}~cjf2E=A{H%DO zJ5hzYi?X@8ghI5f<46YVRdcN)!>8v_HFZKF*1Slpa4%HVNe<8u4q>FP6G1Y0;8{{u zAtfJ+A-b6**5co!in8%z_Z|D4Y_Wd?JJj2)B~oha^6}_xr!PALBdipZZU99#Ya)^A zLIF@zeBBAT=lH12byXPr%;x2#bl1sNi6cmVLzv7q7`D5tC@XcbNj)74|3mQ%55`N$ z8!9cQ0j{+pk5FY2pq(*aZy$->i_4*NB8FY<$m7lVG*mC z{##b&GpYRTdp|z;Yh!yuEif6=GyUKCR_NESKi+ zk#2fRmjaoK-wQ92E5)w+t!Jx^V@<_SHjI1X0Cf%GFKIi%_KKx25i)N zNrjBUd-|Tt;2Rsaa1*~MaM4`Ffw=uGEfa8=Iboxm&nNoX6`4huIly;zv!l*-Nds=r zO!V67zdQ8*_-Nnl(?FYAvv&4&oZ&<=^#l-^{qBKLA_h%f{T*z(HJ!tzdIc zz%wASJ?rd4#f&K8Qf-UdmZ+8;lu+(+VhMQwLO{L0kEi>c*)9gY-!HWGxlxTS9d3(B zc+%M4qO;?LOdZ48fgfcWlU{O-CtRq*m%!wf=qngQ`dF&8udabc5^$kwv<;GpNk$^U zNPfNHEbTLqGNP>O*8K|Qt-b0%5< zfAP_WCeIkj)JQg-3PDM9gA*S(o$!>EOX8Bo$dJZn;5(&`f8l`Tba2rne}of0@Od>b z`%sv|NXj*%X>&^F8}xy7Lq|gvGzC#<(PwmFa{r>>Qhy^+Dp)9Q3~SR9OkUn4>oSVp z5CO<^`*z$xB9lWVU+vmzXf6vz1kst-OfDMs*Tk6i8P63bl=%MI$V@#nAgw-LxVBGW ze^m=Z5)q+ETWTub{A)TSXftT&EPOMDM1B367=0w0H4P#SER!cqor45_82EbH}9 z6hM}uT@z3>N7(5j2WjV_j;oK7W(B!uIU09wkU~A`+W4$<4fQ+XQ1l>0{{Uq&s$lVe z^8MPyBK5~4U*OEsFI@d5MXbf%7)t5|8%5FcnvWW|$9@Th~1C^Lp*%qI|> zId29`_hrH&JM)v}MBJyBGur+-fp_==qAG7R7!WS_&fFvg~+?NZP+A2}?-gOru*;-4f%q zs{Gbn^1{63MABOowBYkSK>s0rXX`8Lt?IhuX`HZ3{P_91&pK3+*@L5tNs>jg#fr?DdQ4N;eO7smiPUB!oXF_2yF$EzazfhV- zENZ4uG}zckx#es?n#Wv}3{xU%Y>Pc_Z%KiBK6ve&d#R>&F4)=Pr=phLl!+&Au9(!z zJz!|zSI4zF%nsP=8y)6&rG#B516a4n-us5`zMktFx*pKa47m4LZU1|Db8}$sTej({ zCMSO{g8dGU|F+iaHD2j+&EgC60OofQk$wJtE!&P&UxeB?Q@a`j-*?+>_t`Da3WLrBn7=eAIJhdWZ9t6_1txyN=a=JcOR)YAkrm9>eW= z-1@7|nj;vibs=EyE=!);IW_Ep#5AB~_1-ixWsx%F74GU8Ua`gz3Q6`r*nP-Kq zCoDhQ{!e3{AAbANuhw$u*Uvw9XIAo@Wksck8HtM_*hXh^7wxD<%46#jfA9PzKuF<< zRMO67g!3wZ-7*)R$tAzen3Hl)7)!n45y^Csv3_tc=x#IF#GHc(i7O(GScJOw@Am)K zx4Q<|!#{a5xN!=d6qy)dzZd4}+o!)Ln=1w1u$-g1Z1ZGbxPqPw3v->f-&Kle=wjLm z7E;q%M%VP@kwT8F(so@(Hs(JHy%|A2hx{es-aEb8Oht`#EZi{Tt8$q^BNnn?FuXkN zotA+p)MMx8wqRCg+^IEbu|7a}BhqNAi*^{sI0ls*-y2Iz(Z&OmK@w-aJ8Z{?RcDpQGSudOG%CNkSq85UesvURJr#Lm_BE9D~u(=p_5x2pC zJOUC8LXXESy=A#N`IT2Fi^j5W)%ILc%j$AlC=EcG9n211y<@doOFmVC?N=S|9ysncTT0 zQ@CvC9a3qsdF(witH!Xr)jp-FLITvD8=X{8j(Xm3GI5Cq-KqSan!n*af3T~xnbZuU z#&RIBPygcd1gMFWAkKhgh1KC62CH2%sLKcFXdi=8{czBxkJ_|VQIF+XLNJpZsX~8H ztA9k33&~ut28=*oO?xs1hyBM)3C%kNDZL1dW=BzD^%GU=8nH~$II&EMSsHcsUI)Vv zvr);N#uXYj+!H?f{J4*bF9s!sVKuOveRqM=2GC|SWoO#h5yck2PkJAQnxQ^8Bjn1? z*9qIz#}`F#Z2Q+#Jv3~pYp}CO+BEhiHjc2ZDzc6cWao|GNZv2N2Ra= zkG{SKa1^x9FEFk%f?DF_{)2=pr)BkGIH@gkxTf_@ORMi+zJCAKH2{z0#H?se_|^38 zA9%7TVZZwyVISvGZuD`0S_^HDlJ{YNF)w0U1es^`ze4N9e>64MKs#3ZKz28}FE?~c z+hWw(W-jaq#ht6_Y(0okahiH|Pnl%{*VF-u*K<7*X~PoRuO|qk>0m3pe6UwduTpfy zONK0+a(w4lcFd@WVjXu!B-TRaOerKlgS)!zL)`}PTytD%3WjhL_Wcx3cj9{-zlVQ% zSl`}LKzcua1)}v9>Q0O+65jIdR{x*>*K+*xpRNS_p3;!Tmi74-umLuW`!C5Y4ZFE) z#1XLcON`)|^Ap=m{cgAS?;2bKP}B=?s#(HXpWt;%=ML3q;&uk zA1I_}aJeam#(@mH(PIpEuW%VIzWUhe`}>#oufKnN>wgz~{@yit-^TCjEZpj;35#;+ zFyBrLeTIUJB~?RebD0(V+`)n2u+?h69t^PC;av+`4D6xpyyt#vp3dSw>S!r*?$Q!8+-a+r|&}F?f+b9bmZaag=>r}r%433rpJ<~4te%% zdJb#%h)w++@`|gv%z+hVpZYKkL~{2s+;fjuFBv3A6{!)FXjWktLAlT&A^6W}`EnSp zM)q*P=9R^Ne&EXZX2%3RqOy&!i6a!1cYCqeE{_{zfE>vZGJmUz(8UhCc;|_ZRAApw z{1RADOqJ~ChS3zup<2M(RjY;WM36GYj+||XYu{-%MqCci`oJ2sL6nb%Z1@5iil~m4 z(%cHN!)gmzoN^s-6W-pw}n3>60=qv2)P{I^vO~3X~n< znteg$a|3A{EMe%39U^8_NYLs#bnJWs(hvjNV6LSPLn}PF&Onl8yn9-ww{+#OXCo{D zKy?k$#lGzncQpO9AFkTeA+i`j<7&X^o?$v)3(Fqbd{{|Jl9U5d=ZDS%A$pGc=TA*Pd3K!=nB$}N1fx6 zVi$S>^sKb%u#&mclT$Ib*r1*d<2Jg6oUab|i+}-=9*f*+h{~)!aU|e%myHXIxY7M_bytL)Z*pBdS>L zx9F%-(fpr81J14#t05Qqs>QMBoXMC)7k!@h3J()9XwXr_5Lk?Z=k+(6b+Fn3Kz-YC#kvojR$wgt3IC(BV zwQ@20LSx?&r)d~*9J4yVS%uWkN|l456~suYfs=`=4fTTUyc*gu>4Q71TsXo=dH5N2frQ9WvaE~KG#~gnQv~>1bbUj-N=OxiJM%Rvm?6Y$;a-}67A_7gpOEXPwiEEm)Q(xNMa0&j6F zr+kT}eb-n13D=iT-H&$xz)<$iuRaaiPqF{*`@}(OoDQ-K^K0Y%r`l#%72u?4m-I>R z%Nriw^3jd!0olqAXCbHM($?axI(+8PmEIDZtG7NqBhTiO94J8-)b2!EtW4kexh$P@ zS+_-J)J0+8SgA@KsKY)ASfMDzL?w$M>_mC5*Kx6$ZR;-XGT`IO55KGL>a>Zk#OzDKmE+h-(HzB3 zirqIYX!~yqTkZRV0;llLuZ!Rom&uHaeh^YtvHQ8j(jRbgI$jcX!^ZGfYBvbw%6qVC zsm|ueHY;Etlo0MX-xrlxqZ`##6B3xZTO=*H5|=l+M1U*J86OP?bi>X^hv!;=lEWR+ zXo{E%>Xbz=oA~J#u8<_gp{=7clh?nSCREw@8q$+#^u$2P+MTfRI$RFbNUPkOV5$Mb z)Bf7WKVPawx*DVZuta&Jc-C|(slVPwJ(5{D@Hvo{R99Ke*zB$cLSqac7Q<*E<#SHW z<1;6oEYy=`IG-ksS9#*jm>b__D!OJ~pPj}N@AG-mtS!4@<`Y}vf`y8Sc>%uhkTnF~ z2a1+|>wG|;aRk8Z-JXAyOUQU6B>>URhj$$B{8fY59=J-j09cz=XupM}rxp$I`K)bA z+DH40PgfS$d?@0j{>X-cxP~;& zhOkYY{7hWk9`1_U8(9~XYS7Bek>)2*QT-dRNh4G2ylJEM+%UgC zU1|&*lN^{>IJPA3RBdX&6|fCfo#UXlwg&|lk2bV`dR51oMDApACLPHNL`A`?37@R? zxWfCmln(qaB#)Q~7BJXUQAgSX;xGU@m&i3F$nmu5JN5!fBg>N-i!lVq83D_F^>q5kRH?wX~&WFKN6jS-w-S4eSmO%^_9bkB4<-fts*nv$)myBQy z6>qU(j=vnru}>=xiJ@mH@=W^{{d?`}m0$k%)<5uS+1CYtkb>SiBw13#udVjKe*W>L z-hPs0zTMKGd4$;o*=={fLjLo^H->v~XsN}HmP9R5TS~saeMw;AYC0@pB6?1fQhfp! z3NoMkPX3Exqh|GqC|148I~NQM_4%;vJp}&mSa*w@_ZAW>(ga3FnHam1UOK3#Q|`!# zaG;Fm1@K9i2&+e40V3rH2UC1hZ`K<;q;0eIsE~GU9A#+K7EFwcxe2v}emDE?5uh%U zs_7>?yDvvW!mWk@ZOi6dx|cwBGTvgn%K*3J_Z@(eF440hR2l5QcYXf$QL6Yy3rqE; z#=;s^_i+V_Z|ferqA31*9;~cz8Q>Cv=8&_QL<uBvK<;OXWpiEa_2nrFdmdlmFSF-w6_5AYBr zI|4w!FCHD+-mDL!>J6w!I6s zDq0y=`{)}kb7=*OqTD{)m!eQ4-m^0|xRksW^zE6&lMRcWu?c%J&aiAq291Qj2BA}l z#>TtaAesAi5oP%1ao1Uzy#OOmEQlMMdf_0dV2M{);_QExmBTrL)Ko|AvZa)o9*e)R zCFR-jNppdDjXfvt@DsU;Xr;`vJGDC%XjU@UND+>)1`ed!w4^)lMM;|Yaw_9(172eU zRXblx=QNzWajaZ3V|X9o#+ZZ{@yMuZf0dw0+|F0N=(u!y+&OfdS4D23Zf#OB_nuE9 zDk=PFM1&kt6#cc0B5p&VlST^sz!*JEv$H#toB%;|_?>YE@d_H`5$me#&8-y@fHa1^ z3=PDHra;foD3Dz$t08do370!uQ^1yj=b7r-sG#rxPz?p75X1Xgqj%-i1WimR#VrIVd2& zN+aY+UDK8b)-h{_ItysexS1GLt?shL1_(b!=hT_8pd$@cC;?uz!Xh;UK@MQeF;3MB zi0UNTNq1R*(Z+s2v^%BaKS!=4JKrwv6Os7Nt3?>wH6OF+>*^)OZdZyoCG<&F2fRJ~3@q35KDUx6XZix|5)xFky`X`vW zQ4a6Z#izeyIn|v-L=9BOHBo#Ea~r=s^r!0SgmKJ_4JCMKgVMf3Jk}#|ukx=Om_`J# z>2x-pX(>8bG%p^wUh2m(Sy&%~k)tGRj!!(079;KuyP5s= z?QTHQNA6Qb)7tCROid!F9BQ?YA34txP_xm9&P@3SXuTMv39?)9|l-^tlObQ*iigqIk z$#p5hFF8;z3}_>q3CnyAyjBy9CZF4r5v0Oj+M;%Fpwb6OqpaY1-QES^a-&(f{Rp`_}y~1`4T_ zRiv<3n8y3({oQi2!Mq0ddY|VpqH_mY6y~aMc7^Arb+K+OZgADFqD%PO@0L?6ini5x zOne}lB%eeGr4zx}yECuOfMy)ZIBn%rC!Ib6M8ha=Jx>66Zbpnoy3UUqePS$APEo`8 z6*P%Mhpd?Uh$g@npIpE^nGuB!`czt2TiXZHDtwn)^HwU;(yF#6G(5O6D@3xVxHs8X znQD6b@IeBelM}lygnWDZBt7a!UGmL<+0X#X=E{L_0g&EUoL1NCk?;5 z)`O;t{7W|~*wuwlKY;b!xYbC(_R;k1;BL1|q(lu{#?6v}y&PK)SKlA2=8c`mr^Ubh zZON@-_e=S#yyy`dMvbhrXz4$1lrAps3>q2LHsAvxyBf11>#G*{T`=k4wz{(m zsYz{zbPUFqv7ITIWjNzAd6^_G<|v~~`@(@C-TMYWlVnuBHcd)x)63RM5MAvW93{k6H-A$dTJz4gYN?LS&q=njud8y z)~?tlz80cRxZ#7Ja%arLtbw)+GQUgD>Z6NY3qwhajQOd*5t<$i{aO5qLLL$wQ=fiE zilnB zTu{U+wKJP-ai_un6q%oLV)3e=&$-l|50dPp)2QSf#Z?iL^3LMSgomD}y{hbJBsfqA z#2<`GP6ox{kbyO;T4Wu; z_Hg5cp7a=2wli!+El6sy4Z=7^hp7U#dgTg=V@9yoD}gvWp-p%_%!i>ju@D?gj4DT2 zAz3d2t3S72M|Fm-y&b{Bhk;^MtE^0gx{V0VMEFNG)81+y#+C?sHCkBmhH0W(Y<=n8QJ1G!&c@$}P-T+F+ z3@}%*t#o^*JAhSe0NB0k{*PA=gLDCvJ_BHAV@caX+c}4{8CCL#pA9Lp$%v&l+ar-q zG4`-l*CX9GetY}e532llb4B+*mjRytDUeos>s(3YmffmY2-C33qTLMMtzU||9=H|U zy0xIi<7Y#|+{#jd?dO2rzWkV<%aZPlc5Tr5J>TT@)~^+cjeT0fYE8m}Q0w~w_qQ(= zs??;&c$6QS^kfCaZfKh+qURwCoXr58wtj)Lbz!$x^Fk;7H}Z1N!%p%d=Qf9Q)!D}s zEtk6i-;Z83@b}HQ08_N}Ss~Befw%>kEC`4&?zPLFoRB5$C5_%C8pBy7_{oncqZ$u> zb+dn0yYJ?;rtqvL4E(m^6xK_nOC32>RUdGb4t0X|^}okqX=G5w%kiNtQ}v#r|Lg7J zfA63F_wl=b(U1?wiSGT+KV7!vv{35c{X6Ha{`X+mT>u1tO{|Tf6TdpzBgIK5RLtuR zK?`&J1!)fDCN@NFxy%!yZS@|WdO@AU7jCNy{*wqUosqU*O^u34^$-*@I<3+z_D;4M z-ms91r%R+`z@D6dwaXnf>d0vFNf`v%S|TZ!F_jwGUa#c|NOu z^-sD!^rk}1vrE>a0M_Ii>D!TX%>oP zx&N;T#``Z{<4#9H)06Le{Haj^8#lyN9P(ZYPwPM{rY0;Aywo{Hux{x2HUpW2fX@rizPJ+jfy*`krN zOC6e_mStK&Tg{CI{ZP_?sBd(`kiSm^7J~@I&7HIiTON1n41;P*rK!*xNrmfot&Z&7ZJV><0SohpdNk|_? zMpY`vj!M0=i>hG5@>CAAxhDU`R@Oi%rUu@Ra_EM5xb{a^)U~IwJB+*;Y0keeILxH8 zJ^W_EoC0`R!98J3OxuuD-s$5J`8E+b{-Css8waYRj9=GfWi-`R!p`kZ@9Jf%8n?f! zCgLy+&Ne7cn>BDr$ssal(h;)7V31jG6ABGQ_kNYI4rYCj1G|FC!s-G>fQU)W)g&ao zteYS*b8q9X>H*q^F4H)Rf+VSj;4nzENo-%S^+eG49~m21!WqNI`S@yu`&tpmT|f+f zC`ZETni4ChW8FE%%Zn5dgWi^alRfQv&GId=+_-A>|46Gj^WcJALn^lJ)k+A_+-G2nU`28n z=bL1)c@S8M+Ura!0krg}c5=yHX`m9x)MWr=OwYJrNwZpV>9Gk#ECamwBFAgC{%(2D zn!lz0_kP3jdw-3q)><`3C9_)YOGD#ovrqPQ`u?*YeR=m3aysAr@*pF9r@+dFtGOXs z;O8C#sCOr1 z1A}im_56(O_H$%>G>JY;O`63hlHHJ4ysTuHUC@u|LGs0HZU&7B9-dSajrXgH~Tl<-^2T!x@>yO=VK&$c``% zaR)LvXfd&8+MqYdfXz%J3_I;v z6SN}IX$|e<3Vi==2RF)jEt&%uf-)|%yA*2-c@y&QW%3()B1w69^y~f8dmXCB+)1tF zFxMV3e>&h}z^j1$8M4!vNk*Omkb%-c*k)PLNLu)Zw_;n30#px*X^GpezNk%z8%)hm zsnuPkCH6deB<)}ImgBv2bwGQe@1;Va_iS+5M#IdTo;i73s$cD^MK+LwKb|a4z=A*| zD>!J9(6Y0ARL|iVyjJt5Cg3<2h(2l_n06Ie&?i(9Sc;R?t~}9MNXm#o*rky^&6m<~O)t9kTHQMI_CT?tO^_S*kZ24S2G&m2+ zqzSU--ljfZPaUg@yg8pFEe)$?s*`*BKl}AKX^zA_6$%+B#GO$l2B;0axp`OX< zpAV(LY%UB>1&}LF))0+o<2%ExFz|0|=}2>L{w|tTbQ7i8XdVfC-nzQLFsdY3fHtJK zftc5ILSY)HXz+GO6`@h!a~YJJUL@BT)kc`Kb!s9G6e*H(UWlPesfl)GY*{{LgJQQo zy5CSfhA|R7-#Ac9=1`sjIupFZ;gH|+JXFlr@srtbXby&tb0&x@qQ>u9a^aoFaQV|f zMD&wIkzr&G0imW@&5dNwkPWvrs-iJy0*4+5-jwU zLUD7Db7Z-yy~W616ltr46;&07v~>=H3lb^_MzO=Ngh0c;?M*`{8mg;4X%m!4*4Db@ zWV7a8*pK$k^u#gp{nXjBzdxioiHrutEj}&|)>U&Uy-Yn@chdDyTOW(}TQT3hb^);c zJvh=JgE$VUZdmP4GBnzL?YACl@ql70*(S}VP2{p=w`FzeevY=)knzYU10A;lI76#2 zJuXx;mc3Op2PekL)jHDC3q8kTmE&;;^)hTuB%$P;O3S;t=D{2peMq72tZZ3a+h$;9SuM-AR_@EYL|QATu23X z(^eJZg43T<1wI3v)k$Q75h)YJESLnN^5HCV;i9QRqA>{!Y&YJw(iELu;=-NM(<;(5 zMd!ZfP1Fa02UH~znNHDDRohXQewpv<%&kXl+!NBSfHgAxXW)PnOopfFff8E$Q( z-s-<{&9xiB$y>}UKeqjx~ z9uuiQUK;DUt^=&hSjybttr*4gcZUIyKyq8?hOSz^r8R@u&^R_<>KjVy+v`4q0-J!e ze-5wnWK3%efcDyu(nD6i`xq-7)u(-xzsr1A|H;pH)N_6t8^Y8dd`_W)He46B8sw=OKPfOudcKMJ*_bD%?c!_LrN%!cWKByY$(O>-vg zY6o>~`h~MYjsY}zu#OzPrs`66vkTA7G;2lvl|tK|LzNl90CdQstFS`~Dqq6S5gHE3 zEP?=O|GM#Up+>Lmc?+Vs`C_zSZ3^>_K zCaME6UFU$uE1CqFE=0METad3Q|5(KwP#&B5n$b?i5_zXpoU*AFjO9}Wp!z$}N#O2U z+Zm;`oIb)8(_K*aRgqbuXGTqeY|}1Lpb(*)vFsvd-`UttO-1Q^!tn$+;wh%Ss$|qZ zvO;IO*J_9E)d6wL$^7$_hk6bWc_ZRF!_t7;^vdcb%HGN%zXmh*X6=$pnq>qZ+Jzwi zO`1H#mz4G`<>rFo#E8dLqL&~*{WIg>Gs6I58)T(uohG1k0tawyn=c6r(OX~pdn zUC_FL)V0q#67+`%Q0Tay6i+K;P}6It5$WNL2sMpp`U_IrWIskt=gt^|7;ARGacHYx z#{z)4KON+mQ$~i5kk^!aiA>oI}(GH=O&=jn$~uVh|!XMX}$pf(NZH?xV6T59d>G84cZKOhwpCO z*KY{LHL2^Bbsl<&`5?WF*kDBiGQ5arxaqX&syy%0LZl^j zp91s+fPDba^Wz>JmtE;efLBj^_gDJ+1v6bk7Psy5rB+u{+0rcSy(w=29Fpq07_JlJ zI2@Ir9QZK9H1EW=oORsN-;&$s0K4;UIiZJJ{4s?Ff^^nfoPlcYZm?VfuaVuX3{&eN zjOVe1V%;mo@Qm6KpFvxpGJaEaXoMw;MRKC$8S42y$putQyh?^X z0$E3EQtai9OH7F~0)yf=Plc=gygNS`5Tqe04El2^m?5AyIm|UVq+^>8xmp9=J@W{%9ju=dBosH-&%NI1Z8QDuQQbP;qi9PQG(Smb7RxIB2! z)pQp&?)Yu2M3ShP6(EqrJ}-U%=&aFRhsFp$lF~xLvX@aVBr`v#4klaH`TDq>R83D>MhZ zGAOId7BN2-b(;k*MhFPNHE-7LL@>vY$o^D149bLF#Vy=w=$-I+jZ8y*-{8{{_psp*$jAhRp0EShDPte2*&*~%_@gv%rD8?X0tPZQVIOir zNL_7(4XJ`KXbazqgx$BHC(Q725^2;d&Z9pal}}p&D?Sp?Dz+0z4ecXQfVnfRcRX9F zKdKH9(nensdqWEg0{+wzVkCPBg9$AO1rW$tO9bKVt+>_wm3yi+rU@`bif)uzv-;}h za#SZODTs)h4Z%odH9Fv=t?}U37sXQ0n78n2en^v%jeZh3+QCB31^5DD zgbK~f_00pq&;2taUhI+UgfV6Sp%Zb9C}L+3x=x$;_V__-NuOM~muywpP30O=mjat! zSLg2kwARPLbk%M!u-I(XWl$_7-6b~+wbc6k>DRATGOc<)KAJIm{yGRt;rMkI@Zjw| zI5V%3#lvJYq(65Pdp^PHF?$TgtqOcBx-OcG*7s)0Hf#T9_1`T4p9px4 zp7A6s%{VTt;zHz5T<#1zha(KiIm~jr*c3MpGx!j^OStxZ4p8`94D9y5#c?_clMZj* z_SpNr&THxatJQxm55~X)0Mqk(!&&Bf9BAJU_Jc4Uuy2LB5e@8sWdi41sH6dpU1BUo z7c5y8iq~cpz6I5CNm z=8dMd@N!OR@}7*0>KC#Y(3425-TPTzNK;=(5(10B=_BB;?w zpR0Q6=a_WZd`|bdd*{I{mP>)$9fBH!#=0zkXWDTv9>8sBFV(`*01-#((wr(!p>WD+o};B@x0M=1v6rDRMJ&|>K{M;J zNs5#w|DUNlakgx^>N{Umx9+W~H%mwYF&H7h%m$1x?!Y)4c1QTn@37k);h=5frV#>3 zPj9U0-WvMz{pG1g?|be(duOhBt;}4RdpNU5aQPVV;PY1X8`+|th{qm)np7}&DbiYK z4vy-4lB_eIGdKmn9FgR0-RDd(Td^}-+mi7LP# zHxdx}6s{sw!Vfn0L@_C5q{}hftTRdNM=O#J!iZ{HxfQr3t9X??RAFsG0GR`FQZID2k{C{yDqlFa49X4V1r9ejW4+z+I;5* zN5xV#mBLYanrV-foxocgu6oi%V({w$sEk`hljmZIjCc{h1Hpl4`3b<(I5?LO1iCnYUJ&s z>l=gh-JPyob0&U$H(x~u&YE^-5F(rnI{ikLDZ~7M)VAvoh1FBt)=7FC!kzbDiD7_9k>BET0@Yl1nObqb~ATJbS!@im^8pv#)Ilr$PueOw;n@)RB=L=ehAOCXPM3#=8Y z-PoKIQ!3P%FTCQ%Lae|>lN=0yj*({26ihAZZ@CA|z{Gp=6t|EOOQXb#}5De|O~ZZd-bU;}Yh zp2hg6l~r8xTHTWx+kEGQ^at+f?Icsq3dNxY>C;%?lN%Gh5!=yeW@zySnn#RXmuQX} zZK`I`*(uBVTg3LOyH`?Sc{zavj7;I~?E?=qzh_b)Yj{``J%0H1+O5OF!#uOO49WV- zkx9dy9T-}`bd%3{s6)<;LU;U^0G0X+<=!Z7Al-3fQk;y$$pAK?ys&E8Epk?pz=5PT z`n9;N*KK($N`&LZJwBhSk-88zo+C%x;lz@`jBV{i?J$7~bx}P2+xr0uIb*wk56$KT|C&%H$Q3Ewowyw@-NUk@Y16z(!wR`p&?>!nc97FZ zNWIzfG@gLcb$sGR<@`{I8V=0riSKBK80mJT=|xg5WwB_eP15r@PDZF<40HT zJ^12_FO$=SGGwoM&8q=Kf$N-6L?$sV>0V4HU4lR;tF<)-5in+pu!C65rG&We#91NG z5D!k!2sb(=G;2gEMuAKQXR#K*%F;dGCc~T)a7`+3|5e*oc?4!*$P~*KiF?8-xx?CG zKS(sf#l7*DdV;erG7)c~e3%m?IV==lGa|4T+lPN$FT!=b@O&%og63&{eRt&sXS1NC zcmStpGkOPo>}mpQhNmbUoyQ`V)^HsLS)bz|?b;O0F2xo2BJ5bMG^d6(GZbT~tBp3` zB;sV6<*Ias7Kv69@Wtp@X_hle=9*(BR;YMZ;IG5UcWA%!3WP?~)(#%uJkrGOp6jPA zrfPfLKZv;3@S&o$XwmO|{2tprefHGSTk7VXxLz1^T+s?HHLJ>4bqGjP57ms)?t~65 z8FRu1d89iE)XS)h@JL4p#5Z7zHK(YZW`{|Ti=Wg}lPD`2_lm+Ay<=8IMf%Et8|Y@Y zK_%z7#S{+g>5w4##5q*DT+Y%Ycz|QF-56F7Tozb;9GT^)U#)FMNcb4}8eyal{nOPd zKb#dNXrn>~7|)CGF6fy&!7iGw0RblQXjXCX5=(je?A2kK?+A#aaZvouvel=-Ock!W zBi13bl5DeH6V@?Xr29X+sau<__rmPhaLRx9qit5X5OS}_vVU3Ca!Luj zw7|chE>p>V(O- z=nz2^?1Qgx(Oip5;)Kgy;DH_=a7|lx7aTO^&rF&$QI4& z^AKn#Jal8531bAcEh!PgBa+{k>Cz=zNU+r9O)FA=i~ptm-reNs?LEvP|CiRK0TOL^ z8L(T?lnc7)|K+P!mj6uvUYZOz$c}6{hO}wy&w8$*^P9Z?_Td{pp8Dw8t7dw=kHYSI z^GWWqrHfBflGOgM6$p< z6I8{cy00SZv_LB7nTs`LbkvulyHxB4j~}@y=Jl0~Um>l_WA0sNB^z^^$zJZY#fm7U zL`!{SJbmZPQ(_Ka_Qa&EwKoUw$|ti6!>L<<7^1gwvXSJnfGU3Jk^G&UR-&xBVv9BG zMMQz@Bs#ZCBfYlB-Ocj;<^cEZ-CTK4HOuQB0NSisk4$^k_r@nT-#&l+ z=GmJEUFYd{onOCrecvnaWvxC4GPD$Nn2?zvlJ-8}ZOwV;vPho06K_C?OUkxo3v z{Iz*wa{$xq5}~7BZcBBPb77$~hWQsKdMk}+ieI?QlZp#It-N$VfdRJA8A`_(r2QcXA z4)bhy%-E%i{_dUIp>u(DwrTi0<_FCQB%a!QKy<)fke#o1g3-S6#EWv1-aB|_`wdN1R4k1Br8Z% zS6RK<5vXnki%+9Y=T&8C=mNY;fyp_b(kHP1(O4* zs)CwC`HzAXA(O;sVow3D)SVCtFMNSQYukPjcZ7-K)#b`c&|$O+F=!u926Nyqg{ z>^1USS+h^9pa&Jhu6ekML7WZ>tz+~+7+k&18Nk(!uA z?r5uy64CQXa67(;9H%1!dO-R=%p;pnqjN|&Rv`6^vRJ8k1Qq8@bEr?Maxj%ggpNQR z?iYy@HF~Y9ouDGRkt5|sNcB5oIGjl)1uC=rp{6JiE?bAN3_n@pV#7pf*f<7Lli-X* zx)cs4%jGb!y2Fu!H7ZIfPM9ifY2t9E>I!J}Q1O@mhgQIEdIVU=w(8%)vV#(={pXY@ zW+HmyEe=W4&=_@fmWp$|@mb=z9~0OVLRUDrH?Nuzc<)Nq5yhu*C$0aB73Y>?SRlbw zI~6)qFe!L)S{QK}$_zeDudPK%Yg+a*CM^W}hDHTnWS6s^&NP{lSd1-%vc)lOM!c9V zZ#$ejE>2OWl;wS|=b6o&Y(ynswMQdTFK;P*vW#YD_$gydy8e-YKnku%pQ4i(MM4@+ zUtjeH5stO*)iMLyQ>!OmbC#Qb-1lGhf1{h}7NFz7959>3vhjjYZpUh-w4~on;difJ zx(IMv387@JB#Uhyhnc4wv23CTa$kF;f$YvtfO?OlY>#yj1*JnzmvBR_EOFrqAwu}? zz!!lE`HL^(Lub&_57WN=&p;h#7=LVp70_bPmNz#EDPAL>UNKO#=J;R{xKL7IC~pEl z&8r7@S0(_jA3yX2yU*)a5BbKsOb@&a6T{wmDPY8EGR%S}xFkCfG0*JLj#fbJ8L;ar z_It-@*}tr@{2|e^ihY9HSmR7yx(9tTfC3@+r(=^JICf~_$R*X(;AKzV&bbM|{d<1K zOLDIPdiXc(?>)G7bqxg!A_uEUnx7Hisa?P^i@SgIs*82BF$MB&&}%Q-y?^h{q(YKN zT*j?GO4#-D&914SS<&$}9lJTcAN&vuz}oPPk;`P|kxdt0zY>6%5G7LB+0S+2<(MER z`d{bQ#S3u!H@j1|PA`+K#TB*1M+FIJi5(RO3EAJ-#%@9Z$a2f`vgPJH3xqsbToj*6 z+%geDZ4Bt%Dhuc3{9WqEP^P%OGAiU@_T5Ml(=zsB~;qqH9FVSC86cPmBxcGX8g=-f=x`vtDH^1~Jv)V*gKu&wDL5 z5HN`vEy$GVT9Nier2eW*{S2D!(>C=lu0C|A_V(j%efZ?V_rCi4 z^V?@%y?uQ1OW*$Zv(G+z@#2LW`7}oH{rv9D)7w{KyI8_}QK|=;tnCUbHeYnbWG?Wf z(;L4kR(*1)38{>@x!kGmgymvVDmW<`1$8(!9JUUIH3yI1VHhxbP4!IVJPL7(UxZUL zgG|g?3%GLZ;utakWF&Zl76E0sMugEsColUa&{47|gjy_R||m{sbZ7>4DbWaiAe=mNZ`$Z4q(#hE9C#weU5O4bqI z!9-+@{g_ejO0*7#sUbu+dVO81c=PhTM>ijS^bw?|Uw>tB?)uRqPo;eM)mOK-FK5HZ zrp}x>CYPRf-p?~Ravn;$4vCTcsh_?_qeCIzfG;{u%U7Iwsg`1pY!MpMNHOOqL}y-( z;`oho`~!c7%2dOhMOKMH&3H@(CODMADnykKD5SW~vBXHM8cwvv0Q@>A1&VE{8bbBH zFYLo}oSZAkO%y)N`>r-8M=wan;V|=u?y2y&8SHE=6NcWR$^$K>z7%&$ zi|Wl~v{Ja<>@LqP&1O`?5~3s95R_bY@JdO?m&JDjF5uI)ij2%~QFojRZ~7{l%%hk9 zSTts@O*Zn)H+GqFB)$p6M#T+~OB6R;5`Q1ygv9>hv|Tiuf9v8gtv8)KHH?(8U*}4w zeDhDQ`)v9D=GtR$H*SR8GXOml;O2eOGln4=a-3Qel($*zchAp@UjF0YB(KzH0XyrC z*Z8gDY-Mukosrk5f`Spa6?)UWdq!5R2`2{Y{3AeaIROcX_)vO zFJ<%YU?F23=%%2q+hp?Ed)y|w$Oo?lKDvMZS#mVfW92(ne#{Ww-g|^)jkOfCgg9}+ zJyQ>=dvITlX9j4x*ySUk;w3CuhbpfqV7qRC4!`DOOTWl}ysQ&Y&a;OuZ8J(ISwt13 zZP~igI*nqVP&hQI)k30$!=X|ZI~m^vNPa7Ks4;dx=5Ug_FxqKhFsIHi@u`&oGh6ex zL@m`N5!t_X!qD}d+~<*FhB!pPX4vB4NDO;oAe|aKRF70s%4t_MlS_%(IwAXMNrcN( zh%qR`XEZ7rXrEx*G0NYV?IQZP5CPaIW;9|>o&-xYR3=__cR>RQuvZT1IHTNSP(VN` z>VnGTWPt^DdH?zZ3;urmTd4P#(4A;8;nibwu(r_r*W@6BNPZ~XVHQcqbmfS7`uMU! zsxb(GfcRc-O#PyF!cT!n)VpSr(_gD2@Z@zs11(v4CzpEu^}TQX!neQkOTTdY{Ob>{ zUjN!Je)9MSzxes*Up#;Q;%+iOyS@AQmtTMS6J;nDczw-_ zSui)MSL$FE&Bg$VMNpY<__Ug_Lw6LpCM<1qjD&-Z`c^wFs_L+4SP&*d^@rF(k9f7` zKr0PC6uO^W;t&84t-5HEGFPKBtZcM=P>%}%W&H%0L+Bzup)G^#Cs*jaKVU$Xa!sy=u`aH-NY zy7^=_N|VW`Y(q%~o?6Ksi7C;^Wxm)jgb=31;f3G3utQ?@xm1lh5a3{=JP8PpM6BCA zqGp#59zDEz!s@4x-%y$|32=mYKHv!8#i<1`a6*nj@Tm)^OD z6G`gK>M4*rij%iLev89Y&I3hhHn}hk2Ex>+HXEmZhk;P6?x3(Li1wA*s}1Wy3Yme2 zH|$O`Tq+|Jty5#$*1oWoZagD5cjXoIU@f1w5uK`~PA>%+cbB>QAg z1{X|VimE#sxVnl9$;yUz`2ri=VF4_a80zuqA+55)60ZszhqJ$!#4g#-Znv~mM8ad1 zI?(L9`VgQb(8xHxh@(f3etyFU|7fmgkTY6G7u_8x`X?HZO;zZLwo|kh<`>ta2BylW zR~Y;x8ugVN&QlXmDC^4rggx5B&d^ntI+_6Z-KhS`Fi{q(pbt@`GDOI#YL2d*2Ni{J z@0!6b9drf3Gg+EN%f@dWND|Pq0Z-6AQ%;y#wOCo?qI@_yiqp!HwqsEv#}wbay+uhm z(ehgZQsY6H*)o7SuIY$cipo!r+XBT}i=|Xtq(OD`v&7NKzOC3b7~3DmHNc{f2%2qe zLygv*w0bgA3^;bPO^!OwE;7LDfLbRrxDu|Rl5|O#h7d1774l>*_$PHGK^y^d!4NWE z30F(t-rUp=H(2%eU{P!TE&=35TOdGT_2f}Hu@^%(S)I`g=N<#FjAv1t5lKqBpd7_A z(K3Wwtnn<-Yf@QVT2#_dzAROeT-JN`5_;gdb>5x8lVQ@f(tSdu94B{V#RS`)u9z`Z z_Z(cZrYe#pGe;Rt3iXfpy2_a<_C5y^j~$M~+y;}liBsZ%Ab-hayPx5KdpxAVV@~e1 z?rD>x2zFb{B7sCrdoI&B*?*JH+c$myLqh!erCi*;5_5pJCIFOj{}1+AaDkLblb;om zT>M6QOS9x%VK>&&9`6LwaD2+Pd8g3f93DZ{ZR4JUmx}onTd=62fJtkXUfx>EKp-!7 zl;>04C*GHiv1wNCel^A(xlBv?>9>9#w>BEpeC~$$!np~9!J75Dlj8Q(z21~hyypM4 zj4NYcF~G?#e80W1cCL0VzN8R&ihSQh!BKf-I1~;vF};Z*Gv#&CU;aB_(?*qqTR;M# zSQRv+V`>gzd1z*_*>Od;;{=>DqOvH;Fd438eqBnEQ;~uF^;;I%&rG1-+b(FR6Mxi> zsPW3`u|a$ElOdy}c;kDZx(lZplZ*7DS|QMeVb@L5iP_ym<;+20!IX)|YyIZ|$b?^P zX#s@aNTg)^k38+`lhUwJs2NO8vyg1RYmm~FWt|2hx@49?&QHJi z>K}gg*^fW{?C*Z^vw!^T%NMtIsPrhgZnX8gBWD5-mjbNQA^Ny)(sB_lX@Y6n@k9@O zYS4lvND$qBR5o5jE0lCvWi-B_6^-;$sENiC!QfZcoXvqx!M94^$T$?3q>m3n2pqB3 zQ(Mt12*7bqD2c~rH7us@yj}4+T4R@y3z~dW?Sg7ImLP%tP;RfdfGd_ z@Yq*Q4;qQARGf^qq7w&`AmLOU`J5~~XL|;1zu>tKZj!w(4*s$yS7C%e2mOIbeGh z(wQwsQ3brq@pxqAe3W^2*sk&Csr$B1tG@%%!A)bnCeK8UsTS`u-aMC#vaoawK z?66L6)f_Pb>{alIupEj7UWpUpfr$2nGg&%LA59#|=&!kCfeCS8luCgf;ogyG1fT|( z4B#&k_0_nh(E7+MTGgy}q>kFS&Q7pZ!%gC;8NCvO1@)z-AiOB8WO78}SR@neELvKg zL};R>PAvT$e)G+xC{3e?3d(JP)kkzhcr+*P7&}uOL7##4wbtG$L*mW=s>VBwM|o(q z<&nj6qV#>3;}uBN5M~4mh+DYt0f0s$x}7S%ZOw6j;I0G>GtVjLf09hRjvP#8d#ujRC_`f%T}ebsgfjRtBbk}yNU74T!K)FW zK|ytb;X8XV6Ze{lXnxDFVo{rm`t&?(LP)ypXUg>LiQ(oP2~vVUPlzESyWwwkNYorW zpIj7za0W>uWslTtc-+cAxFH;pWR15X8^ZP5@A_4;Dp@xN=Gb{R|1?vR^ZV@HpKkx@ z8bH&x?u^MHSg^ImS(4M7CcQ1qb}w(&`kAev+aK}N&cjFXM8Q~*M$8PU1&t_jXobn? zWqb`e%)vU?&2#k-@ws?=v6NHuGP6uYv_YVZ9&#l~<{>AKU=iZP4};)GDiOvZ%g(P= ztTs~Chu8*Wlx@vX#zM))4$v!zOE2b6XLmD|HK&qI=IVwSpS@{^7L3krGL$w}(s7sm zmIKL<%Kl%xdgDS$pEqxwbx^+EJ0L{1V1qe>8j_7@*dLcBB7-@X08XcsEWBiZ1yELJ z8D`-gDVN~yasU-}n~C)&xy|X$bp=^QC$txG(CnB3^qy^qLYUhy%4iuQe{?Ov#etsi zzJFuiO@Q6^(#C{Ry(pp6i`F^*ix#FN^0EVg+0)Mk*yeUFAiE%jzk1TI4&^(St8c-xTd9$ zk4``101+WD!(+_Avw$7WT^&6Kt>Y#O$E>wy`78q4%Hs}eW*IH3sws0ylgoUb(U1S{|M|Cn|I?rU{MiflX|JDc;*}}-V=(As$Wxfk%pD|^-2_gQr1hl;$O0_* z0Morg21xeAt`~}AYD9!`7~0}Hy6pj1J5H+xxUzRuF>6`DKD%Wm3J6m#8Xf}C$sH*S zGW(Gev!ra$n+7wiDpFoNR(asrdjBaAlS80t^u{%a)N=~kk*`o16%Xq1YS1YYk%ZeP zqH9o>^Sm18=qFWBsg=o!lQF7wZiqQbq+SF|`9Q{g-D~^~zVD8BRohqyRUVTA>FdH} z$d1*-C9X+RBhTOV<_?9dEdR~hCpQni^U+5?_-DWT&wu#C?|%Hz<44zT?rynIQ`1s@ z`svUA@n=8%;^~X8pTGF|mrozQdimW?KK$sDk3awFEBC9t_vDGIXV33U4eF26A<1Hp zF#c@X>l#baDMFgunK)19&@^7dQ8d*RsJ~c&n8#$yI!q3#ocesMSCu&l^E3G8z;|8uSLTw&k9A*bld=GbmOpcl@H=qKuF+0?H z)1key^%!qxNdPuq)0o7-2bwG$U@#(eZne<%8vU0OajLhxq&6jAsFr>D^GSmwt6j!A@eo+ctI}Dsn|MaIsNd}nWtpuGPFQ#j+?Z(CFa4z-#x zv=Q#8;Va9Mj}rJ)GoczH9kh!fzBw4_U<-cjBN6I+Vx&+{`OZke1ina}nLVxT)2yw# zes&q4YdCo^51iy8e)*gV67>{INP=xS!?JFDak5OOX=iB-4x-o+lMauXRlv9>u#dA&!rtfn$jmqQPy8srBoZa78^6qHdg7h z05wuMAeJs>DESCNv6H7fsFS?8d2s#k7e09Wo!|TIKltIV|NT#Y`d|Lr|NQTN`}d{+uiU@ZlQddq)<}%p zzn<#iW^s|@k}}&9A^kv;&RMM^fmgL!J4FG6D1abzXcr;`ibi!^Z$(i!SdIXNAMSmdoc_w&ssMGhXS`-^g1#{tF>FY<}gc~#Squ(Loy zD2>3F`x;4vai&moF}`5?ibb5lcjkQY>nql_Rad;@t^vAsx;B=r6Yz8m7>sS?w+v(uYCXafBQFn z`!|05m%sPj58r!o@9yR8i{~CVxqJEYD|h#Q`T7037vKMdZ~e+I-Tdsc&+ondyGIZ2 zeem%oPrEDV#Ru;_e);;I`R5*_&mu?uluVo>_ZlX-i@?gJgwBT*E_XdFL?(CkTl{6^ zY%_oR&gj{9V|sq?G}yGwo;c2j=eR(emj`qKIL9Lt#S=E6cii__2mDV63DdcKec@On z0~At>M%cCst~eoolfgKTWd~yt7~LGt2BnpePD#75*Bi%h`{87X8p$AMzll^kRzGSJ zWmpLS{#Y+64YLuPP*`<93g0_S*K=^ZTdk6+*UgV8@xv_6p&tUl9pEFW@ykk|rePo= z>M_VWqahb=<7H?}YqUUU$#<*B4RhVnz+BrpSiTSAjIlWTK)1~-lQZKv$C#3eZ5KV* zKeLuV0EnS=>%QeWhYsnt_GD337G6?944pm#{-aDDT~Z5qr)!^KQkzW1&PJK61fh}= zmUK+Ue4oJXP7103j;pknE2~Ulh}&lm1t-P_jX=y9Bs5@wqj!kbn2<433%C?J@OQe8I&SdEQtu;)ih2t!W53DU~PU)EC{TMfUX(MMTIQT5$i*$_# z{ieiLAKj_3T$)(%UyPS?GiGw;V0q6!t^Ip-lI8zruikw9`t{RSZ+u>u6;Oel%J$=s z?lB??;b&J|$!+-((E-iaNkm}M7rN6chuKJ*G)Z1$1Q--h@7B6~d+&Kd*C%1q^0W(F zdreTNCF133?oe`#!aG2B(V(?7kty|m@8Q*Zj~;&T=;8Y}SMPg?;`NnVS__~IMm{XN zQuOfkgNJu}@oSc&i+tZq;Pu^ouTs_Gx^z=nGyk%!+9Qv2b9q)ETHRb-%VnQoVv%h! z%8quiTP*Sv=}aJs^K{65C>1!-n|A;!|=psq2Bo< z=zD(9;U0c@1Wu7zuTcI&WvHXDjY=cu($u{-pS*wbZ~pWT|Knf& z#d{C${q)B_{_^Xm9#@d{d$;ei=eIAsA=DkMkIZnr?bj?-gXu9$mU(n@{mb9}_}hQ> zM?d_P@BN>D{Wt&PM?dji#s?1{`7hU+;aO+Q6CyDagb53PoeHXVRXK}K`N3C;5jJJA zc_uD4PUOBLF%AKbm#=N32V_~(k>Xv=+*_+62coubtc~^zlW;syj~g?@3L$<-pzN7g zTr{?4uPE(or-=;!F4)Qr$~jXI^1dd$eD&y zjF+$|>75kK{{ie6aG|?)6W2m1?d&@PQ*-H!2*D#2yiR6cX7CO4urGkBRaHCT^+gw2 ztz(QLYW1^?G~NgHt@j`Q^B?};&;R&e{LXLw;FAxYn0LSU`s<(l=+mcPeew0z9`Swp z#g|`x@zqxnk$XM+*ktp?-R%dS33w#w_g<=}9>*iYGx2$A<)rvX{S24debUD_sXLpF zc0mC2hsy;izC&6`?^pEkG_p`chp~gx29I5~u%Tk9ztn(N_JAe7*aQz|>le z3MRcsI!RPB4Yg*lNfRf3l{j28>L9jBiVZV3Rv7Kb9nQ1ip$a@*pv***zvGpU)C@m< z@D-0?V>k{gnJ%Oyo2I%3TRC0;E?ml0lCA%7QnqD3h%wmVP@|f^2OJ*3>n5GVF#*3~KyGzt#VY?_?H`nq*95uTJxFdfA zN3{Kf8-=L15hv@+LW@=60XxzMDFtXSD??1dY;j1_D>1r-rJMt#5z8@bXPB{R4aDKa zK9ek)R6OPluC}6fxx>UEehsM3qoscvhK9Qh>LDnov~0DsWkd1c*6N6d!P=-KJitnz zBsXcA7R>h;WK;nChti;`veOHbgW3t9toI6Q`91pGA<#m9cL8|`L3!J*yPSkVDnU+k z4jKCtbb174LkBDxU2x1Aw75vZVk>!9kT7*vZOV$<7W9%5uv#USlM$5h(GxiF7$O70 zu!NU3EUmk1Z`QSOBy-wfh2WFBZrB>d|SvR zLm5|6_h=`IU1A{|Qu6@!HtW?p-*}6&pAT(55HHLIpRN;U5`MG%?^d5zuV1`*_59WA zr?2mx-o5_1gEtm$Iz$`rtSmjZ>8lMy0 za}OuT&OLv8@42P^AkB@fI^TQjZkz&6r(tI$K^*s(!mga%cb49tm^t z!#6Ct;xSWlaW|QqKdcV~M=8wjRq%R!X;VY05#z$mx@5BY<306&0?zuS{=Q}~Au3>+ zq0xo3PU~G33U&RAXvdt=1q$|IknJ;2S4$Da3&okXD9j%GyPLsuhHEla)^O5{JpSOZ z?0mHR#e2f=R_0-kz29icH}X)bm3nwlaE(i}^xGhB_Q-Dw|JKpF+wGO$!LIwzOeXdE zm)5G|6@Tlp7=Z)GP)0%0WG760CDCkbn^+lA{ag;PFjqDZwOJHmIT)zmBNA|=7t%nB zz$Rm_-@AVBSAYJe|LNcV<-OOpKmCUvfAQ7VCgfi~yM5uoru#Df2hWdgtF%&vkGJrjDJ5Iw_HH5~YKpumE#1=;b@(bCIQvAcVuY0~qZ)*q6g; zQYSwhtGfhH;aPo73vBbQegX}C z-(tIGTEgrr@kLJ%!NqVE8nX-6qvQCGb#Ptt9C)M?aS_PT>1DeJxta?=LRWKyLG*B) zGMC1bZqx6md$47x*?qwguB}FN5q?LGm1=kD%ohL33XX>P)T*%)+*qlm zepMf15UQKIWX{CEZEc9onJ)GI4U?R@dy*SX+`dfU@vl?07&>Q--uhoyxd_7vRp<{ z5}j;Wn`~c}-;F-+^8S9Txmi-VTe01i1@J|Co00EavcH^0ym0kN-^sp3^n56@%MeKP zVV!K8DKk-_3r9N3E1mh`AZwR?Kr2_aWL}3FedLoqflzP|PI_fGSn>&^^`3n%VGR!y z;QFADq1d*-FY=svSxYlQv7x&LE?G!li7!zhkRdnEyIBv(NdbXOX*4sHC%Fyg-VIxN zVe`YQZieZn@Xjb@Gc zb4liyFNk=|DNxow7h0sNZTCgWOio=b&oRZwohn6DCQI^=l&W8K53=8Tq1!BCsBDht zot|#=1{`GX#(neF48fHOmn^hY_bvkT6WV^)cp{3XelS-CX!(D6A#U9UTK3;8*pugD3=9Q`LYy zZ&IkX@LvxV#X!0P0+ISXrs+&#)qRiU<@so)SlA=dIt&6Wq7dh~F^F~OsuoVoXOIgOMd~tQKZWt+>i!@9&TsyQ zfBP2??!EfEzyHyvKl}9d_VtUq`!C*Jy=)!U-E0r9?>&Bbck7o+Zf@Rll@w~0yzhx< zKM?j3(dRFozkKohz4spf(Qp3hcfa-F|MB0t6yWEB-4Vn-le3O4krKaXNtDj*Y1r2S zw@R)f9~dW_DOOt_RL$%&+=zs{-pMxuI)}u&eGj7u;#Car;>xZ>Xn3?lCM*NV zMP6+og9Ue-t*%bU&92&|H&VI=aUco-rSFW@?$?UC*c=U1}r)M z(`PSVzIf%Cs8>3}dwyjyK}3#bK?F&|l~D4y@RP@81TU{&KYe)n+{AR_&a6^AI@$oy6}S)hVlrkQ=7>b z5u}++GgDap^ZbDfPg9Thkv?uwDw$V0qYnwtK~91KW!qRLQfI?hG1zEE>d=m7+nD=4 zcw^M;w*DufDju<7qV13g&QY8PG?)l?KCtNoEC~Nc!jNOQGEp+zh_p&Gen|#CIuMeuDA69}= z!SPZ3C^ctI&q;e2$HW9+X!;(V&SJA-awL0IH9CEZgiYUX^P)9xLh+lK(~?Pe;S9Yq zD$?hBRLB8)gO!6(L)1b1*6a^h`-z;?qz6&wpln@Yo-SpoNq{tPbrrgkx;9nQ*jZzF zU553{lNuWqH>Th!DjRfbrYzLZxr&9t}LnbNwCC!4p@vCdG zZQX_=KR3?TCgTTa5{Kh#vWtyTquK0o*IQp+yITbH%mB~@?I~4zqQH`bG@@P8fUW`T zuukz_hC0YTJE%A?8<#1n9-$ zO#sSd-D8*vInx3GkVu7|FV}W;4<-PXT_PhrO_0TWT!?^MlGXLBH*!*&tMyrD)P_&1 z>7XF$DV;pj101y-5^NpSgzC600o3CYTqyo~Jmt*dOy{E+gU3-Uom3JHPPJd;j%se)O3Xueal} z=%l0({)C}R2lOLrO%Cx~d@55&RpAhxuQ`?di>-=M_mEK3AxJ^SWTEDnZ8hvAs`)4= zpxG-c1aQp8s{RDN${N&Vkc>xCXa7*v!XGoHO3Oaz*s_e)G5f{lEG1AN=z7 zUcY?#`A>dgo&PJV{w9p3#X7*Z_n+UseddRBdU4hRO`|5+sGXX%Mk1S=x+tm0nhLO0qH6BWe=rM|R5=A2-tfgFt9rR!V?C(Mp+zL+@k=cPH zMqv__vyDLA(V{?Ee7O}ow-jS>JG6#Ipb0lSVC4c?ZE! zn}8{`ztoQFzK?{&{ib!VWYj>nrQHr|QDdH9jCdj68J||OqG4gK?=0u1Ndt8JIa55%p#eGiCwbI-W+LJ!};(oH#XRST61=Gp-SKwV@7+q>-N}lez(N zCW*OXf6YWVtLwMZp@>Da!N?&9l|B@go%=&B`P0Wjn~tMw*}CQ)vDw~xJHvKgjdeBr%3T|y z4c?9DcLp^Fh|`J~Syi7?tYq7O+JL`igk)0z;Q{{^VikD_mv1y5umNQwz#x|UY4}JZt1cIFCRfg7Z(I$H$ zwOYz!iJvpC-njP?f)3X+0$~f*V=;@4M5Zk7)&^wWMCsM<0rc3b-)?+(|K`3MV6J)# zkhj>cugF=80Vd0a-Q;7j|JAd*SI@2b-@TG{FZC~l>UNLXM;RWGjOzuzgHD*Pz%()m z(tR8!qX|^~N#X^239ofO2)Q50Xd#p&X9-S_|S!<(BAuCL#_xq0%)FCtt17g6F%S61W{ zL!KKo*F}ruZ{J+Mxw(0L|J9=hkG%`%ja4oCUV-|eDV(d{G^so5)7zo;Naccq+*7dq zMen@J2|1!hG|dxwq0F_8ZF#a@rLi!$fX|k3$Br#Zea5r~D@Qf~D3;dT4iYO-O8aft z6;O7zxMPXJL_h@pe1R{B@N06uN51f7RHcPcdDq;qs1lfMbUI??Y?Q$t;e;xc^05X* zfU^I7t`9<9k$cpBCv$c~1TUIz`iBlAy4AFj9e^IIrd@Wfiy&1Et?rKa`qrRIZm2us zXkX6O&DB>n*iwp>P}K&lXnJl|gmmXmKMvTcfA{~~onL?J!A0E)$QVVTR(k0*aX*eB z;94*>X8z8424S{ItkjxQ9cP8vzwZna(Ta0GCz;-<>i)ZQ)59(1H+Sz}Kl)dH`X`@! z^3jie^benY_T{q|8m1ery6xxT^^^Noe%2%Ha_^O&!`|zvJo=E0k$UJ3IS&wDUAaT( zB`>_Xdp_g7F%$l?-}~XWK6>(>|Ht3`__L=kUg-(+3i^-4B2)bhd)7V8LyIs!6L;Kp z)J%))69^tuUVT@57d{~dlV{FGln7M8rWu&|Lhl1-hmH>YgPfUj_gVB;Mz7PCM<$E1 zDXn}nU1~2_bv6}b*FcJzAfuLfeWO(4Q<#`@EP};m5)58x^@VG33q5TST1OkA$}rK7 zMlFnq&iQF>fRsdZTi?-`({PzVhY?#0>W>`WnDEv1D$<(5KL3fJ6r%x8A}9DlYkT)K zfJg0i+*I#O7t?JKyUu%+j`-&pd7g&wf;R13y#Z!62*68$$V;f!GyjVf;nwpLCUzT`&RY)Hd+Lw!FmavdR_WO*8~vBc_GScxe#V3X73>lz0TdMTUR0dQn?O@&l>WxTGm~h zo$#i5O&@V7E%Q?hl>OXfpJ}oWCDdFFO&;=8qo{Vl{oQGy=v;0`Nk)@gII8TOI%X+S z#Ca|IY&-YyVznZT{mm)p*3HHrk(TIxZ}6 z<(K@teZ_R7>AR_?(DojtjF!4dQuNe!F0EY z*xpfrSVia^@qT}O^}k-f?j?`48Yi??9!%<3%`Jw-VvK4#7}K<7IN8UNJc)C0)%uhQ z1|G9)n3Oh1WTjA!t&1FLt$?ICF_$G-G9KdY%W)u7n$(sY3bvxeQA01Wl0;X*YY4%G zS}Rup(oxK_S%jO>S!xlLw60@HK?{Yk(}L?h3CzSSEkIFG6IIorYjjNjTG!VAcVHUO zV!ulPMYw}K5^!$uiU*F_zDs|optGsa5hhKyFp}Dsp2va`mt3`jiCl-wT)~RbVl2G9 z4k2@<8!BtzXP^(W1{MAq58AJP<@>+*+rRmPyB9B>x#j=P6m><~O9=7s7amCS)T*lCQiL_lLFA@=ezz|l=cO{+5z z7G6f1-C-YjqBI#6J)8tE1;bVWOGORKOMYBAzW4Uy$2Y(Kn?L-kfAi zegw&7Ww~~WxTC>3igVv869B~2UFpPe`c7;BQBQ!QhT|{~jftOJZG)KPQ*RWeAjJMo z)sd9_RTW-wQiEzF`~zhjia1qzVcW@ov%_eqPcnj(A2~}dP&4gFC3V0@=Zg+2nL=vn zVj8&immmd7f=7A+93H3SP>bn_nu~NJpIGqqXUA!iH#9rU5C;?bNHBUk zpDY(yrzv}pBVDz#JbMhB88s@9{T~EkWg|xFV@PdWm8ZDAp7^!~U;5SXGg2M1!8O-u zDKHUQxXt8_i#!a>Z;0ERDBFjk&ai?;2+Eoq^*m|U89b5e6?It{3tTLB*JMsH z#0s4OIF)4dp6xRX*XaGX0MbBg0&piPT2-9x=@K>A*$R!=a-RlN19U~TuS$qKSE%m5 z5L@5ga--WZdCK&YC2GV-tPc)FlGUd*z+7wE;PGFXff!LDZ!RIjIIQ|@Stxuvk! zmY3L(TSExmmhQ9s`Kk$kXn)_EaB;;2N6?T)fZ6;^{neuf_n$oUuy1=a0dJ`jqP?}E z%T|*AOhxX547=v`^n%3`qu3~xLfYwo^PP)Pp)I$shp7Pvxwx1?Psdux zBwKaazH3;`3w1hq@ihDkGJ|E3J{QmW&IHACeWV$jR#ChdcDfKiyP_chLmp!OC^q^n zxm|_p3V_=fT1Vf)|GTGwyYIi*l&ZYK$XuHb8q_*4nx`^qhx+!!2CmL*a^)t-r-xvx zL_HBkr4CYXHdI5v6&)wA3wsx4^WehfmFrZ$_s@U(;}72Z^3$Kacxg35f8rXD2>{3| z0&~s7n@3RVj^BlBQg0?)`Sl}@!KiuL8`)jjc;MHXOoCiFbLY2i=9j+p;msfX@X6!< z_rL%BPru4g1gMip`Mf-Xu6uT_V9``!|s|t+Z&JtO&?}(4V zXh&-yHM5LERM_J}DwmEK{YE-dD&yVyz(SJOc%PlKv@0m^C)nfzI$>A1xE{sHF=-N~ z4oq>Z3iUK511d?HG(oKL>_SPYEVr5|(JnUc;6skcc6E72l1x%W<1;mnEwR8hdWAtj z6O+tPQv|c`cTm9uwGJx*WBPC!IA&{>(cxoYVTfOE?%(_9$>V?h$A9#f|LV_v;ggS^ zeg65+KmFMkUwq}2vd>?*&tJddjeD*@ynl85h@;;;@J_53e&5uZ+8e{Fs9T#dnjvaA zMbrp414j+MhS|!~TI|+tO@SR2d=#-opVF+mI9P7E` zH}ZCcL1!o4hBzI3;@Fd9>+Fq0$4^d39r&zjm5! z4ocdjhHs+9Bn=O(qyJ=G#oe8b#~nMQ2)ct+c}t4{~h_v3T1uu~4YjN|*Jfb(QP$#kd043K(C+64%Me zFp+PLw5M|W9bSu2y^DV{08f{)9SIm@yooUu={LUX33L`^GG+hXLI2<`Eqc2;V*_VH zD+OWH)-uqnP|GdF@+-@TzUwg)Y&N%O-5oy`{9U^23c%L>t&MrTpYlv+E3F$<;-#5l zgExW;g650ki&ZX4^x2Gf$8af?9zL2KIrTY2{SD7V9Sl7qUFXSVi8p8 z#<&ZO!h_+<+?5!?D&?`BsBH1pHjbF`aAYy4h&JY)n*+!=xUL}M=;S{B-2|X6S-)RI zhvGwrcM8Qg%W$|zowVKhFXKNygX)bad#k2QOj+aE0K6V4hUKhNAkmG!tr2JNjFH`% zI@V>)AABG642#=1t1dQ3n;aDt$Vw{_ zKt5puae|F7Y2?=kY<}BIc6Fm_MA7M=5-NkyO&+8y(HWp_PRK}yv%8Cj%_GtcCzUgC zK?F}{EOyrtTk>Zvdrfi+|Bg2eXfmLXb=P1wa_6FSXe4liT{_xjCFcNnK!(2?g^rst zJvnW2o&CN+TnZb|XayS#3ab_n$hSo$%V*}?um@h=eEZ{%y_1nZ&z?T>qR*FiZ}tEH z*&k5XLhiNvzGd2aU@JhgjLy(i4H0n+oMoeTb)$RMI2^t4@UMqivW{qZ^9vt5`J>6-rj#KvzrtNm-Q#ugqvb%iv3i|KA?jLY}yq#1xHStFGdDTG6l4T zpwM>A7k2dHyS{$0!>FS@X=cYj&Nf!38n+^`o-rzr7U9S8E4m1WSjb?KB&p8^yP}Km zPFBm;vm6C8W$?;7IJm<;Sf?zk>iVp*$VuG+Rdgbz)5e$R7h>rg-SD}peau4Lw*Fr6 za)edXl3>GN;)B5kk?C2ETK|y2Am5LFyA}w;FmZG5?f1U*(O><=U;N8I`~&ysfAQ(h zzWUs>75Uzm$-U}{N977FYoTT3hbq7o-V&LER_6cL`Pc$`vbO4Rf> zz5t!1llK^BcD;0n8Rt07_tFQ@(HZDW8@*X%iXVF&nFAf&*>1VV(RDORWlaEtvA6q) zG`CWgf9;~1-2cYIfr8O*$!zW>lU(PT#Ts`@-D;)eypqzn;mv(3=~BZfhv~j{V~CN2 zu9peG5i_VdH(vxu+c_z#XI3T0e$Dr@_8M;{OwpMS9VsF@@$WrE-N@nJ@M_r8&OtVN z@!esrs%1Kv1c!{%>JtON2DN&Vaa55@zhT|DzpdK%iO25uxAxB}ELZlwXU>{7HPCOg zuUC2}M%?=+LH&7&=JJH5^YI9ECNhVX9D?;aFcKrg^7%itn8 z7lZFQzQukMfL_*Q63`gfgSRooo`t%!bL9#NoyfP5KC`;&-*Bqxq@D9hHit!YmS!FZ znd}(9s%l^}8=uS8!K5|wEyCH;Isw)uj7!mq2zLPOZ$@7&ur-i+iK3|fbPC&vKo0Vr zlVT&0*^RmOdZxH~yN6T<aMe2$hc?kl&fff^ z$M0r$KlIzLeZB1En|rN(?Kn{o&N@s{CGB7JvFq!jCqwg457R*#btM@!Wb#((? z2(mz4zO)$VQD}S-zhR4l%+tU?PISrAN%k&iRg%e@=zG?=-}<%cf4_Tv9(w4xPj^CV zvqi9t9g2u7LH5YTaE7J^+tf**-0zDOV_XL60XwTB&H}@JhCQb$iZ;?M2)1gM#e$B{ z{_^Aw?C?x}{|SHbOf(V2p5cOx)dhLb*yFOlrT)G2-SW1AB*SnL8tY4-ds4kto0bJf z4gE-)!#ZqBfD_L9c6{!x43aGTQwZfYz@ry*UL1bn$9Iv`fPMU>GP(s{jDbcktnn-` z8_1SxhLw0N-`1eUDWj3mX+pQLgHIfi-6i9#b26kEghg$yaEdwSQ2@kZLwB+UXo*ta z-1EM=i$1Shd348o+5fpF5Lj0Unzt7ZJUH&hJji8;Z7Q-|o=9Mtk#z2y=##WCN>(7c z#Y6{HyKJB75St4Cs?9dRv|qK$iTCgSv+w`Xcfa+?%coy?4WrjDKD~Wy0#LiZ_6mO! z`+j>)w;<*6FaY#IMyCJh;iE@SBDH`_T1zL{%z>Wy(amF~yMFoNDLYst=i-kZU;pl} zfB%yY-~X@w>+k>Z%NL2`$zW#4iFT0FCfO)T#za>N@N7HEZ35AkO6w0fu8Lfbc?XV0 z<QpIMjIncw*z)6f$BpQz@Il?7%#bYyz9O;$u!9-rcP-(SrYp_02 z!UW`G_|aLEt?!MQ&M-dFAzR-_40uEKh}6Qwa3C%Syk?% z;_1_$e){?6Up#Gl_VshqVNvw(;iD&yo;(pq&rCOu58N&J_NIpQ&}D$Gom$FDG;OZm z&*-sygTZO`zK@UeU~aSYi(o(h`YD%`pb)X(No?wcGzE5^zH4>UItMV|<_tyCEFF?s zTOL%Llb-uVZY)=&ku}3qEb{B+ z2707J5J?_C*o_^IeJYJ$nuL{xiN0jw9j_#v?t$StG0?C42$Tj4Ipc+&5!-@4rH3y1 zvzGFQ-5Ih4SN68*4@X{b9bH*oHGyHKQ5 z!{p^K#1^_k?DdhN##l^`8e5*|f4*kj3Nt;9HM%ap;H9`} z$ypDrPr$UxhxsVlK)j6Y_1%Ci^6a*NewAx-b~l0qVft`!TgH#c>2hY7RGg?MiR{&% zE%sXqZ5m*SwC4dN96iGh*C&jlcV4f}6p0e)cYi6eEdx3Tr)Ge+YgOiL`ibJ;sG1bT ze@{J-3#5p=Oalm3wPL8FYNx0WLUvZvda@s@_*~zAV|lYxf4BXV`n#RkAXN4*NHa^j zpz1FK+n<@lwW3hot^3u6u)%XWQhCErC_i)|9drEA9G6!`1Z7yIDE0I-HQC zUpG7B505fCs>@$O zk#Qym#6WnkE#3}m-5Aax;Bn*CKzxD#$!rDQYA@Ala^oUlpq=*-3J4$iEJ3M7$#_I8 zL|-6tY5!=~$)x5*fhs#6o!JwdQ)13U;N~@wbC|xj{~`inf87t%#Y7vGy9K!zF6kG{ zT>&WL@7167?jozlc7aYe(gOxe46DY9+*HYlSS9-;!XZw@;L&K&pj+G9z(fa))D)F7 zUw}jsCLH;fD|J|L_O@Ow4}q<=4;r2J7vs7tcM8uV=hA zvt`&d1ih!^te!X4C(6||0%shP1hK3Gw5+wOuD7P__go}<``FI~zkI3DxHag73s&08 zuYC7g7IgkEj|G46+}haQ4wM;_ws<9Q>eYi3s7zImb&?{Ph!UMD$XGOl6v9mBWaCMI z{*`q!`UeH6OuKBQ+DOmWvl1TTgEgq09*hcFVy$|~8Vp3oM#%v`3^@hPfi z#PW~&fDZFWCuBVK1)?h89RwPYbA(`8_aExI+UKyI8HcjZO5a$mDp z1UsBm*`oGVM;A8XgEnNuGt_KRt*k(+)+UOz{pNOp zX<_NAxT)Vd1`%@44}?8==(p;cT#yIM1QiJB?W8R01PvV)FRABKZK@)={=XyU!SmQS z6BQ~M;1X^JB?g%z%*AY67fLzmRCSaZb<~jsJU2R=R2i0ibI(NxFsT=Gu)MqTz@4Vq z{FsC=#y3g{JarP{G=%)40nxSU@>&Tc~Q)lgLk}%|?>go&z0Q|i}hJLKH(rJT$cGdaBnOT4@BI_K;ao|u@SvVwG z!mqrq0mo*?2^kjKO(|4rw5PoQK`)H?FP$;*Pe><#z3Az5~u;k&K)|XmVSLDfmh^2x*#ES|3r_s z%3Y^^dQEs*IHD*?@_!6A4(xkssSNbYRHwvL&(JnKpB{kJ9cyTakf9}58%iU3=w_o4 za}X}3N1>4-2A?9*S?R$$2@(zb=^#fTT`|Jd2=^}I-+I3X%H)2mw!ev#U>9E?PtHt~ z)<9x$`Rm8D$Z7)Nor|v}ucl&tcelHlh-0+u%^r0YByWY^+g*(4%u}w9wB8cb%kg!e>`*vP#+iyjxAm=dX7~tzH)y zK{%#UV&(=QO|ffNjo8_Eo}B|ymF#_**vzSBH;jz~W2~7JkJ-C{Q5&qII!Rey#xAK2bh)M6{E^lZpxB#S`2=iTH>NYcg18QGJwU4Ei)@+QCRzfb*kay1St z^twrDwKd2r!4{0hwwao>PKzJmBwlbR&3p08Sp9mJ?B7lQA6)z8uPYyS|9jzS2Epz` zVtyX)#UiIBTo%fMnz3Il%60Inxr;mFbaF zbHB&)v>kf25FrO-`9gWbh(w%>O015Ln;JcEOc}(o80hit=$r0^pSJ-nkkr%)Y@-vY z=`|ST3hH7~WN0jETk`i>r)C081M0W48A}OuOSBnyO&FbX7t2Jhsiq3zv;a`h9v_}O zGrf{7af{Z{rDUotsDjQ7MYd{?mS)qV+H4emF7S=p8+TBA|9jti`TXTqU-|*o7hgYp z;b-`6y|3)%p`pbBhyD{}%`PvCN*nIhO2}-Lwti6T>-27JZj51W7J7U0$Patm-99(h ze`5ms;BnVIy2yu$yMb=M{lTL@_~EZU_?y4`yU$;e(uzV2f5KKro3x=S4_ZT60}g^wq1e%lx)#d88N*cNI0C^T2F|4ZW`e zHuKkxYEr;;)^6=%KAfsvC`Q%Dx(HAVms4#0vbR>*sqAFiMQmT zVEihnI1i{l?1q$xP>ORjzp1H(IP*83P3<~}C{`YoKm;KC7&%@wggUvYp0%9JQgaRBFBifTyFC80P=pQ^$0b%M-(qkIUC2Ld=Flsgh3nw>HyB&Y4e&ezpr zpAsRRogk`0fsgJjYz4Lh`|#z<-ug{P>HSjm3@=^OW7cVQBWQJlP9GIwIM6p&DgwfMUGlw=1GSurP0;dTlS z1f2qL)xkb`0ml!}{ybmPTh%sszm z406!LZvDOyPN&peyv+f8%muoUH~54{|J<{*>o`5$F`wgKOkR1J0p8OwCC5Iw2*6+Y zp{^`#0ByQe+p0iG0D;+pPhJEvT1`F8`c@T*qaOvAJW2}`z+){JA98mBD`LN(QR^~D z^qYN=gm8PL%}EY6GWTrCG2gO(k`#i*I8qD8-3xYTtBG=C+%A%Lbkcdj!l^<@k4$K* z-Le8VdrFDq8A@sb@7h=jrDR;fCR-}{bUY)b#Z;(}8So^lHQ~xxtv;cUKqiBvbMpb- zZ~;y(V+~VdnkN!hGhw2wwFI2gU@)mBI0_C-V2=ANi7xvu^?&ci%YS+RpjZ9u8da%L zjw+_;P`cBiah64OQ}taj>WarMD7pQ~`&IfKw72~Nz^;nCeC3|l%!1TO{iE-83?Zh= zXo6g*NQIhj?BYOLXC9x+9Fi8$+XJOcD%BvlMVi`;3@Nv?YSoR0XUq1AazCg|SK6Di zmA~dhG#Xfo<09?eU47J>Yjzz`(Kv1kJv8%K2Jkt?$rg=234Eq#M09{;4)s4buViBJ zMFy;wMp>(yDktnYaL$a|_{Q7IZHvzsINg@K6D*0MacJeGf}qpH!*6zr)Xt((kk!Br z5+jVhP$DVhOv_(Qgt{VVI@Fb&bK2U!E_(L`F<{lZuRi|p!w*0B;OW;-t^a$L z(iH$5om+k6XDFn0{>24NR_G#UEx#C7^14$C3VS(QY}_Z9*;TU~6ZO%lD819IzBZLL zQvh=Ni{J9T++Tn5*FXBvXHVbynXvmexRSbdFF~tcb;k6O!&vc}XT|f%IzA%ip)|>q znF{l)2__^bJalkE+n@*4LBLyS2tE&oF`kF1GoA|U*nWnO)&=VSaQPycS?d?Yo@Bhgk{N8`}kN&hl{bxV^@#kNB*{{Mqd)ANpzWw0-qwW}MhWzqn zqx9-Ww}w6F0pR-gm#=PKKYwnpKeM!J_+JQ8CwYy5YI;snK41-+5?Bz-?NNoiOHcQj z$A0kq>&r&buQW|I{$a3=K+dkV&S%EiTKaKHM+(h^5lzMMl8x((Pf zBn>klsYAi>Az4QIDKJ2ojK;|7+E5{$>;rXn#B?g14DLA%R_Rp>o>dM8=2ud};Zu!w zICjg_H)x0L+cirHZaKIAipFw=RP?kJ<(G!%*dxE<`jRv=BEU78lC=#-`%aF#${d5v zXAGxsR_2twaB>w&ILC7Rd?|xahVL52!oBseym-GGDSD8d0SK#@-rRQ=2Wk_0ps;gy zS;n`qR&k#hyDI>#F_M+Lnow%DaM+!uo;_W?Vc#k&M=?9Wc+jIyO_Me=Im}qju>h#f z2{<}M6XC*R5Y=o|t<8CKQ7TLcV&>PSgKy)sA`b8SPjUgp@_@4c$%VWT#v>K52O zW=A1Go6_Ko`-#9(@VWZRevi{51ZeV_^aLQV3k!3c6~xmr&ox2q49yG|e%gsTZtA$@ zd+kG~?Q1dnHfVCTYNj--y7JQ83`3RyPTGg2XzUXHTvmjXXvr=n>8R;fK83+mRkO%6 z-oh}*C*%zo-naZ8l-@DYyFPEalg(q?O9%37Li|{AR`D5AJ^W;9gzb z(+Bo%-$Zk)`&m9I0>DX8>U7J*%@oD_>goWVRgPwkV-EXYE@1@s``@eqs z{`s4CU%h_&{Pnx%JMHo429%BPn^$<9=zdSlG{8$$^neA4u&=Sm_uSB4Z3)1M0fwVf z=9?yb5RgKiO+v`=O%T&r6AfHpIsBrDLq{N@-~|eQsUT(mn$B-;;F$Gz`X_} zA-SZeGeZ#)To)$Mo_or3avalF38742%m(+`pAkR!QOCvAp*$_V#OI4h+kNZo!jp4q+X4hZ4JO_ zfVBaT7XkBvR~iQdzaG$P-T#6061V>BWp(+*&sn-FijW!-v)7&&@Z9J9?rAl>twFi@ zg>QfS{lEX~5C7^v{9m7b`Re@8VND-xW}RaVUJx(GNJ(XZ^ec2`A|9CG5L}jIrxQ5& zsnl8bl*e)h?9~Dp`@iIAw?}+2F=!*p8 z1Zp}2RVnCJG}$=ao>A|ctjH8Gmwfy2qx*mQ{ontm|LvdG#{THjPe1?6FZ{l8gYkoE zeC)dis7Tmjz3ZXtHBB;8O0_6-ipyHK4?lYP^uzbBdQs4e7q4ISfLP7Asiq24OOy>0 zxb+Ao56bM>akGxDG_g)Y1U{X6v=0p^iHFhKE9N#i#Pm@K5J1OzH!%atHlUcE>d^l-N(MEVmOs$ljFzW{$ zmD`p%xXRHv-*K`I2kpa%@VnR#OlNT!ysL2w`OX&x9k(fbNE8Qg-0dcd)L{!lDa!~;J|#H zUkz(zYWgf7UiT2^jpHE!9}AY%vNt~M`7?t&-APMLX3Bl^CAvE*4VAiVSTr~yUbJ*_ z>4rIon@S{rvxLBworo@BGFhHEj_%VN4@p!wdNg65P{x(HrlgdxoWW#Rsb#a8N ztqi45R7mfI2HDyA=DJiCJ!)?Gpc*1n$+j5`)de*w-C*dH)cMpD>uimGeNMv9j@^RY zNHcl8^S7^XgEHr~9sj>zrpoPSCau!*uRI}IdM!4?2?=>jr zzW?s|dD>k--Tl93e>WC)K5bp;@n8l~JoseD>&2)3$wJ$=A4 zHw<`Gm4cK7+I3V&XJcVn{{ERp^l4V{U3z>XUwt{*ftlNb6sFb8Y@5&;*9Lg23%rMC zOx&myki==6v4tjN*i25&j&f^BIvuuXIm{9Wq6(Mn z>PI|Fs@|$gF=2p;gw$m@Non63s^7o+`JekvpkKV`XSl=I3V+i~iy7gm@yKl*lyy0f zt$U$G(|xN|dqHzSdSqC`s`l63ww(3&al9Yy(&;NNaeVx+#^cS+i+&@lU;KMp=+zFq z`1{B~IK-AAJex{2m0w{lV}5<3In?dW0W-`oo|8AxQ} zym$ZOPdR39jQ!GMe33fehn2(tzO_l09at4#e z4!X@TnU_^q+z#b2DjN{xj7!zndi=w!)3WfFF}6FmE6J8*kwF$7+;tAKdnlCY3+;(W z+=Lo&m{ugsA|BB3Kp%?AV0*YJBwbrgLZz^2rz))}*kQbYEv21|4rd_@BrJK1G4B+K z9B58gcZx|Zn}l(6pl#?(lo{vYRjG0g6Kp%V#4J%>od_*! zF&AcY`(P4xgpxq^Rlpg(vA8|BK!8K#@p3RmopAt~oeidX*xSESZt zYq8d*JcQSMt3HkxuLHFruIQQ9skQ8CwZTS$jSeebTctQ4^dcf@ek7-lN4lE;^dn^r z(D-NyO=D2g?U>6~42U|$pqe8F>l1REg~ayKnXWybsdN~$Q*be(OvE*R3g5lg``6+v zC`u=ZiJf!a7pPvh4AjovyI5A^J>S)9?>;K3Upjg<~K zKlU}x(m#%Ytd7VfAw&0jd9BO()dKdrG!Q!(Lo=z+aRCD=GAP2#VBHpcSU6}PE)9x{ z-uJ)feLr~I{icn``^BR@53rm6)7TadsPQVe8i&phstB?AwzTLmx9)&?)6G9`>my&j zy8Ha*&6hXtzUYy-n|IG{-n{7fa^^CnZDqC5f6=wy6!VLCeqOCH+BFl%=VHHo@y|os zGx`j*Dx8w(8p9l9KwNwXCxaE3ODxzMbft7iOBQM?0k&myQ4n}{wChRRXynVJIX*(p zho4vN%MQV|00L%r;)1P~gN1)+w3<5<YhS3#yk^}ENz`xZfrz% zjjb1}Rgc`5K28{4HmE#uVQ@aVB};_DI#tu!wCEr(!R`3KQxh0WjT!)gW^NS}L@T+H^zyAEQ zk3aeL@Bj9%zk2y^{{2s$y}$d!@@??5G=9h6(`||7W|^aX|3zn9IBTw=<_c0ZTd-^U zz77{AVhkYpGNO`Zo>??v&DTZcAbLR+j>y|ry;f%Aj`jzL+Ldu8q62cdY8V_IAR$bv zF(#Vuya1xzNocQBu{#c{uFHzBGFP!ft8ln5sMJvyPm)mwSqA=0wBu3AD_*KIx=_WJ znYuq?!!VagXM>0+6g8Q8$mG0z)o&pF=^y;wU;Od+^YEuX`tgr``dL5m+iRct6}cB* zzqsQSP`xqg$%l`hK79Pt&0|%VH?OM|-BH`^KbpgS-=kK{YyM22OqJv8sgehJpGJ*d zs#``F!aeQRDJmD!Wr2T)(K*C{+G_tqUaU(3 z8HqCdF%Gw7rpt+62-}X{1Z~58MeOpu{_n)+!BGBHP|xw7ZNy!GqaDOJNU}LN1rAk3 zC%W%M$Mwh^hZ0T{oZ#_mjA0qa0 z6790lJom3jK$Epe(# zK4{92NCgM#PC|=|?ns)E5)_|A?u^JCN5(>!YyO(C)X7+E+GRPdMOqp~o(2bPe=3Sz z8Y+7sv^B`oP#sSvN0G75+G}GIGV>{^PcrIm6x1wWx1?&(k_IJHds~l}3-nN4_wdx! z_1$8W2l8geg0(>>QC7}Vw?qAm${mKakPUGdVxUIn z7!HZ3(zy1YVG#!`7j^B1hQ_tLKw8{rSFt!WuvkUuP+=_R4RujkZ160TXbDV~4r$Z{ zmR|DH{Xgf|zP$C{J$pTvyMZ@-X<_`AI4|iHN2ZeGv};V3N9qf$1r&Q8_U$g9R}J^? ze$~+b<=ba3-+uYZT|hk(oHETWxOag@klAM8*7!^}c|Jv3eviaR*VMP;tj~=(40KAc zKr9O=LEeXL!dMrTqPzdcvP{P661qNyPZt^)rju03bUd~5VmPJ=i7<8492VAXB$Dki zo^18tOoltSPA^D9*zDniThf$~*ACiMg%~X@I~Id{pXhwwzT0WUk84wfC6vreEM*0f zvi5%Z@8TV9wULkzF9tHwxkI%n1P87za=H^F7ltzXTVqvPJeJYHs4#}XaNlBo@bb0X zPFLmCor6=P^%*xx6BN5t$3Nza!fM+=C^0^B<%aeVM3P>Nr6>9mdh@uir^$zcutYihff&5}`JoWOg%Na#_Q&F;N@^`JKZNA|d8`Q_IyzkdFm4?pTU z!0UhUSAYG}=Wp*ndiTLY-h*O)({5Jam%uYs+3FZym5@>UK9(B#C0~Xcyth-g%4@u#kO>r4y8CiGEhfB zBrA(;Cx>vPrC>iT29ZDz$7R4sXgmCHIAT{kIek}8u)ryQt20-V8<(MNMx1L$1)Ktb zu63D}gi=-U_0tdT{rMmL{$KvXKm6eRn;-wpPk;K^=bwG~RX@vh$In4Ly16M3Uft{7 zubyst{^G&wcaI+T*p!PrT|9dH_-=3TFT7s%>ZyJOV85l~I%98-de9wccYE1(zw%oM zO|gL)po5x%pk79+Nvv_a^XAp1Sa~cmU8XlfcY2H=hu#?(n@+}V<{+cwRf1(GIe#fT z7p~T^G}Dwg9FZ(8lrGJwcC^ZYKRVy-*l|SAoYUi~QIG}#4kgfUfp=-r*TuSn`wNK4aAM^ZtQK8I^|sj8ktxdXVJo;H)*rjXG|T*mhn!laSd772CZ zj{Tva%L=lc7VRL74Fpl1)@~NU9eX~a-jbyS@mo?~+#eG}X|A}^5fd{qH$?Rk_Z z4uP%Ym9$vwxbb}kxBgngtXnLl`SWyWok`2Qg{BpYMJinC4O4V+9p(&qkV!f!5h!mU zuaYIiYmt#b+|I$}#{KnY?ecW0R;6*9RjNTtV>$BbA>!YS&o%OTf|cl3Y|m)K^l}~i zKuz|f;kxqeDjtS*-IN1dI<`|TvqZRtHxl?2E4|L8pGhzr@5=Q3H`?)$;4x2%rS29`U4$sTRJLls6~t_m*5)1tNfjl=IHseBZc}L7d8lf4r4zQX zpH<=X4#P>XvR5qhdD}xLZUpX1Yy-TxXw#ao)NA8hU2oiY#MMbBpfqh3uTF}j4oYqC z&!D{&sF8n*@vLPe!Yyh~7Zt2S4+(oQD&0 zc?W4gNtJlwJEB$LR_$HfYZFdmZ+XGeuZt#mwvaC;Y87y1DnIR-{fS*;W#-FV{O=QU zWfD2xWq4JvV|C)p0rE{Ln`{oo(a-?Swm4z4pYdS<;*XySzr0f;6qFM=ZmkiMF5N_9 zxHU#~57sN_ulC%%L>g6_q=>5+s`&5|C)SUp&H@o?lIn2^ zLw0f5*43V@eF|3C8xB(Y&f`Z9o<4bcE+!AKN6So{lbU_+*9v=nx10WZVavluj~_g2 z^1CQ(Q9*-t`XQ1sP?6WnK$WuIqPZCw@+2m-ybWye?R$!)$T|7#i ziMXKbi~YV`#Jzp{$>WEA@kihPi$D2(Z`}Uzr$7GDkAC$0Ae^C9(?D! zA2nxmS-QqVYw&@)AK$#Gx9q*0MbDPz`URwiZ=Q6qsNbOO;or9}`mNxvzs?X@DjTfZ zRFe+ajBYF>{Okr^yJCf;>r8Xi(Vhinr3#?iRs^aW)gk3sf$Kxjn zdE2sgv^GW((1a+X{5iesu_7mhBE*gBM0AE9^$f5fxT+5>2W=pS<^VXRDfhCY(ud)} z(_YP@S(K@maVC)QYYb5qg$&ODZaSmv=-3qGEc_c$^+XoSxV&j+EEQbZdT{h>{1IK~r*O zeC9niD>4=sI^_CNL*(`B@#qD?>b%+st}TbT4C7xZ7iA{mCaIb)tA&zO(;wu>$<%FuW*t#TZqz$(k?7Q2?dc#)! zHwM$|Fk(u~hxz5)HMAyfz2sKE;|sWY13d|uU`HL$QG%}dn++;v8oH%sROZv&8o0q^ z3SdCmIxu5Vyo({t+hWD5P(8>EEEf+|s4(fF=Z1Z)wKQ(V@H7+xm^*L)zJlf@Wi~UD1g;4l`%~Ivlnr3+%X`c=!!oVf(HT~I|(#$3>vw$nC-byFl5<4#j{>wcBit$X)TzjWUdz%M)o@c!#P z1JH3(0#BHnD+lIy2{NraACpW|%}%+zW9fYt2f1(ihG5mN4mr^DgCp`(m4U8mtG;_} zn=t6@#QDp`tSs4qnYzx=9YGn3@uG>3@mA#<{j>ezQ;Rg_);#0r@n`v_9w@%;U;NDoI&en@8*0?6IWHxBYfuRWCBl}?1VR> zOIpqX%hjuBj7HgR8n$xb3gz3@SfbheJ7CV6?QFg*u~#!R$Z@6L9=aXbum;Qsn8gLH zFw5p_w;?Mo;`1;*#%K?oc|rzSu+AVA<_{J)@>%CP!QeE$vwNR!T< znb9x6m>cK@K=b`64<9h z{lW>Wyvb_J6Gtc9uoUHqK7K73p0+_gU*kISkMXjXGk?Iy=A>)PiMe2(WYXl+aI8Qi zw`O6cBmLll90T$%vrsSim8g$1Q(Sgdq2W|f<{X@o)VdOu3;@2M>%KrT~@v8#t4Tj5JS)|wgiDU z-f>Bkp=|5Q-KGz|ksD~+5zT#_4OL^p{(8G%7o6*z*z~wCx(2LnzH0d?qjwQzqyMg^ zxM5dwKJM{SS?lU-A)1vO6=~MSL)R@fM%oYSF#CAjwQ-oImnJVPNy~xxB&?QuEpV84 zB-d)#nFd5h*wM7TkrE8jlg5Ij?u7C5H+gL?)R@(pY#enB4L20*^#{FN1C(t9gdvCM z^&5$3A09M0s=wb&WNW;%plQ~vEEOxg_{=+`36Hbd3JAL3?kYtsN5(BY3+O%;_Pkp& zYO@>Dq?CE@f(RDjqWP8qG)gviHy%2Ja6$TD>cutiY2X2|EwWY zc4N9Q*1|W0KQ}nJs??A!-zBta;?zaEyREUd(%;a(=YU%(d-3Mo%btpBSyU@xGHq+@ za*h5s>^}p3vrprGIfnmd(A~j-wNPz1qS=}Pxrt)DP%;ZOY7XE}nm(SH{z zTW#2Uumg5WV+LC7jU_|J3`_oEtY(ph_~s==5lF+gX=N6)UHWv8$=E0smEBo;-N_@o%3&SdRhya$nc9=K54o&`=-<#6U3XlJFPad^0?dxV}m?AKl2w?^9o0L`nT$plf={w4P^4-uNb z>6~=4vc&k=%VqmnVF*MMMlp%6B-AcRsGZ6LlQTAd^Zs?Z#oCpuzH}Fl^%EAjAMQ zoh4pCe3wu|TSYmHDKOVgjS5hRNP&+T-D0ezfkx8k$1BcDh8Qh0qW}&YFNV1Dt~()r z|9Ai1pZwlGc=!6{4|?>@aj(M_Yb<=t4=oGXRR0A?D*!z^Tz&u zX|4-FX5h6Kb*0S|cf-$vm)#8Z;-wj8tw@VkuS4ZN0?h`^&7xt|o~(yq_shi6?=%7M zQlii7alYrKI8FC5G?OC;EA^p}dPq7C$dV9^qN>+?kd+V> zB6XEK#}yzSN07i5Y$|7D-615g+8_Cd*h<}b@N}><;G8Rv$0AqW= zXZTqHE{W|724uxT&tMU=lgh_?U1s6jsNfQSkFan~Sb-f6f;yOVR%Kd$i!%_STbgNm z!JAwA*Ws;L1qXEORL=fS_oOx$iuI|$;t`+vo;EQ4%3BEP-}0DX)OLX;(ySO+<5C}k4+bB z+|%XHjSP)~D4(F5$u;1308eB@9q|DyvDXRV=3E~N{?g?Zz? zno?tn73ri5qqNSuJ$dEgCT!Xrv$aO-{Xo}`XOTug>X=+lG&3=Kn;RsmrWiUC7&@ME zcmGcoMM8h5Hqd<$!VDPYMO3j^|4lM=pS((qCNT0ZLa+;?A;c@TIbvw zQM2F-Chn(smT<{@clPTi-Q2b<2cM}xj^039r4lTlwL62ZFy3|vTt}14l-_C9khNWs zsKwGOsIIPX_B2NN-`;1tWX zwyBqXjrEJ&T$c`({$Z_QNE-p2h|~0l!+DnNa&D&;si6*Z>AEF)E~=76xoOmH@x5us zNU0eU8DrTQe}_q-xaTBwFDBRsfs*>07At!ig`{!EM4cN_qQD`>)xzAe>V#tj`ihv& z3Jsj5r8sb%^Kth8J$m@n^JgA4o9NODy-a!A*0ti%>#sPk#M-{dYRlYk(MwutdW!jC zoS)*V1Ihf&ElbYQgOIbkJKNB)p{nJOe0=ZzM<0JXaQ*yKcgMba^&$Z`FQ2`B@$6TB z@!PN7{ObSwfAkY!y?rV@7rJ_aP8bKy^w>e)474QSxb|E1okb4$l$J89R`%GkGP5G0 z5?zK`pHzNaRF>sTg(E@|0D?e$zh{*}grk!SLMcl>W1yYjD(3+2s3Pq!5dY+_*|<$3 zFoN4nU8wX5abkJ%@&Mv-s2!D_{rHz(fx4vnKK~d+F{SYd*--*4lH?LE?(vpXeL*{d z!#2P67KFe18^87^fAHUQlh%)Z_`{}!kDq?nI}LiSNh{aAT`ymK^zqa0eD}j$O6q1P zeNl`*{P5$JKI@8Cj|_8M3b=FU;iE^79(0?~cVEAL_4U`!pMUYC%F}IR?g_eC1Hja) z95deVv8G^S1&3RqqpSw`^eWES#Q#Z8E#_i8)w3){Xhpg6olw zVOV8B!eag}n`x^$J85t@K6WN^l(TDx3~W20(7{k0{^Pk!5V2Hx6peJCR!Zwtn|6d? zfO0Zu73UR`R_YNe5&hXKb5xZ}#EDmoCP3!d+ybGSh_TFp7cYjlosTJnW*6K_-~R>r z;-cW7Xo_dGDHbtF9WNJ+`%$$;OT$JY3 zHLlJbq2Hn1bO&#qF30yYUry8^;FL-qDAC-)B`8?dRWa9O*LAf5?MGx^bq%bkZ>c&G z{ZH=%+_r}V&gibyIXy+ov}DG(*rY|=;<;g78L6`FfJ!o>ywp2VS5}O1kv~Vn%R;lL1jXyY9S^bpqp*IQVBGYk+#8>r|!jQ+P)7yyVo9rKSSFZ_xYyG@J zfxYqCr3;VMUg1|>A4poqE3wEeA}whf&N6v73%6HEt~hD}^00Mti-DC4^8r7#IbI0F zuxJyv1;qwaA+A-|%wbb2*U+K+XH4{pIe}4$VugeK!jROl*kFVcdhFy)*ED+ZT+ZA) zhsgy?zB(#&RzNli408;MK2BWFr$fF+8Bn8kTjNGC(~lXYubZHnK<>(EE=Vh&D zUPJ%hsM0W}Mg3T2%qnjalAJr~#;>LaZw;`)rzo$E)xsRy6yI?k%X>aXsm<0}v+Yyx zwJ&Ayv&$3e9GlrEll4^Az|sWVCVx~>&zPaK?G|jbE`mZ@XzR4Q`>vlakF?A4sl`Gq ztkmLHMi3+=<8DIILU12L?_TZQ`{Pyh4mPn4AY=PKaZ3){wu@KrRQix)pV&25J&n0h zxZprjBcKE~Zx4YYvCz`EPoN8QDr}?DZZb*HK9dpO!XqQooDf6FjFLa9wA};KPj!D_ zdTyRVe?BcMto5qAkG1kjF zOr$I_qmZejEf-4fsIuBKd(Ks5Fs?oEdtF(vaOkaHMbt5X7A~f zUM*V(2|K>rz4#%*5 z+|`wZZof46^MrhXvL%T35B+#Y*LHfXSf+c^^4Qad?|$=FzL)X;`M>!OpLN0Z?qk)5 zD&4OpteG#DmX#TEtorC?PvMZxJbiz&;#L0+K}SaJQAY|-o+cLZvWHfVfR z(`1TgS{9PZ3MuA}T%*ySAW%Fzull>8PLPkuknIp2Kntz>nygL;=v&R*jJWgcqN*wq zmC$IPVC@42hg=i11hq^y+GQW>Y+fCx_}>zCP?cs8Ew=BKt7cH4W~-1YnnXk?TEqgS zVawn9jo_R}AK(IdaT^7HNcuWx#d)SYGmAAaafxJuW}Ks6D)R^#c19}!re{ib_X>o~J;O;+G_Az4Cj z!IlSd#&8=~6p|XmJkvvnim+n7%NUx(CZ&~2ue6&&v@C1JO?QT^ZYM%GR4C0|Bwu~% zh}Y5BuC*^Npw1+LJBfSVTZscK_tpq5A645=Iz3II(svi|A{V>Ig)Hp_t{YF?0mXcQXyT-8`^ z2y%WuqvO>dz4-6u!A;#1NN*n21Gy#OWw!=3Ty$@c7PB%05)6lZft4v)HX*s`o9wh1 z;V}wx9Y>v_41&c4x6Fm{ZBUejQJ9lhQI$>OmyJ4~)e6aPBDwtZV%P!H_H1B-kYM{Z zl~BEjO9vZ~tg^1-S#G2pJm=D^*3PN;4@)~{Yq=xdEc)*S7_EUl^d7t( znk*WCi`ixs@e5wp0h;FdVe=+?{Q`Qo)dnD%gLNjdu$-#>&;Asnb}_^P1(P6Rbc)oJ zQ3yl?RJ2q$pDA~!BF&(yG=1GhX9Y)iDkY}-i?~~N?)FIQy+Doc1=6oV+K!>2z;K<|V{$jX#T}lUSYsHwp7AuW6 z13W&>Dhv@jUrh$gU_2zlX?!;o7?b^~;WH!E!fAL~2iEAJmb|8>p>_j|rYqv`+zVk$ zkN2@ZO^!*VF*{ELyMDrW6>;|F;jW6rws-i{aD*;lb?_kovf5sWW+_dFx;f;1tNT6W z*eE~MnvLaOF`k-Hvqq{I9M_mv(oI-(Tt%9(WfUXjj=CS?e$%(x75|QmZUtv-)lgp7 zoUXnJd}(t=aSh#cWc2@ZE1XUqLkl7a+Y;$&PJPL}yRR_USl1$*5>0#xPa(i-Nm~DC zr9*89(zPQMyS~M9aS%WTUG;?U1bKTCv)ed6i8yDgI84e8f_Z$-q#&1%AAwu;cSWwz zgkR&>Dg>0|)QnX3BG)_C3a`QsUTTUbpttZ}@V4fR)CHsl5#cJ()XMxNSR2P1gJ+N) z1wb_PQ?mkn0u}IflwHslNS$P1U_@K{aypHXdeLEuX%JA#GuVJV$?e)o#efd6ma4>9 zmXkrwL;V_tpbEDv7ax5u|8hmh_9Gn?4`)ehO}M#{dF_;)=nfMGIceJYIRj!_G_RPA_>hFTe<1<-P;z+dQIr- z-U-uvNx$*S-+OiQ>RWS9;2xl%b@0hBmiE1<)iFKi4GPprKUcYP(pq~;-Cx(f}5|*?&xE)UFF*hhn=zt`)p4Nl6 zMoeRt(uK{mijAr;#fG@7^$jVGmDj1)s*FADa4os>j`(_C+n3FpN!h-P!m`7L{t{Nf zx4dLv26Kr(hCYc8xJo2-z%>%x9^w-$OcmRg5C8Tj; z&lq$CpydotF?rzFq6et6HAm9p$X}_W*2=4bA?IlKFqW+x<%~+h z7s=cgnhkr>SvG7w)XW0PH|F!UpGo9{Rx^u?5*N6dPgbnANkj9%6WZvtD94d`3B_iMPRocX#-s=&15(>Q2vAZ?km%RK%5<+%b7zyEWg74|{%=rjIV$7j|Ne#O_+#5aS{UV37T8YK zZ%Utjc6`X*wyDjX4vax(hH*5Q&@jw_abH~!a~Dmeb5_g4tC;{DAk5n*U?rqmpJWbh zB5Kl*#^cyQpymbpd2;*cR|9kLF?;KH)gFV>L$+3M^Z{x}Q4MwBAxpJ)qtu4~9m8~# zOQf-8I)++PMf>V)mRAjWt(EGa$xR_f3T1pMpm9(7)Y`l2xs&mh*iaoCsn6!w%K2jWYWDMxxKNyuDB zQ3`A58D3yfFsd}LmT18{V|0P?wti)6Q=c|79JNGn0LUS9>V)C(Q(^19Vtnjf68T6I zwUfOZtXBwiY3bp^EZGg1T_QEY31YRgs?$<~r-XZitDkrcPx)=Iy0n5CN}OnG`__Z_Ec(#kltzV&;F0QB0jgSzxiQ=om&<{iqMWUa~Q zC9r<~7#}UfB+ck0iZbTbEVLyk%m_iGk%#F`_2TDFO)#CC71Lw z_n5CRqE*6_6gK*;$*w?2l&2qw{SSXGIxY1PF&g}16pIONUvQ|F&Ly9)I?NDfmiQRg zzk7W5{crxAzx|K@>`#8>7k}~P^JhQ!!QcGghkx_@W!+sr4e{~ArysSH-z$K!W1Vam z=Pa}Hs$_5ftQ*Y2T|6%#QB{Q=2XMXaQLhOCLMVRF3p_r!_vX!$$B*xP>s#+$-+bNs zI)t+57Ev(KvC~Ht4x4gO z72acfWnWf`q`$Eumx!~Gc_b#_1W~%A5vqfZq*-C54oE6?3U^zkns5okvzzKx&>kie zaVbDD4Yv0MSOW5>S~cZmr|VT(iD_KYJcm{lq1#gxi$YmZM%BwgW7${QGfbH_2@ z8Vtf`n?4n$erm&jjH0Yp)43#sOmTD)zwM* zM0)L4i1+daHR2D$zS<=%6G$#p05mY?#UVpIQl0XOMs%9cMQ&+qP>$iZQDil~Wh%?R z(@Df8I}q{i7_5ac{nU2to=@s%%JSFN87+TVziPOFa$&VKYTadjO*dS{=`v1FJ-PLN zBbULAf7=8y7bDRjCW4#%EE}LrDi%H}2Mf(E@G%hwg<0L*dHUog9q}5JbW82vo`5aG2{dMHiV~_nePae=qs*$VY+dj$gqhBE#6A0lqe5R4E=y3Qa;m zkh%D5MjUe2s}^=z^7&K5ttH>}mAbCD*Kp8RoXqy(+&R;l3td)7sqPdK=`eL05fAs$ zp-vmvbv3qYYC5d0rldJOeg{Uo^kK-jY{zWmm=zx}(v^{c7#zyHS{zwD|^I)PuK z5=)j*cRSG&9j%J1QUuavE`-Yy;inAs``_P^u&7f}XE+SiE$bs>1Hdy-%!&)zWK3>c zRG$jD6=UXz4JPez<10>)A~`elorI1PlS+c4G~uPxRg88kqhftioL1uF0zW&f2p!-r z4vxxIF^w0sCXhs>kKFc{PKlFtic4QeA9BT=cemD~yYK$aFaOei{}+Guw|?n+J*)lW zPk;36tLJw=c>Kw?9zXr$yAL0K*v&h&JdIpPy?6id{f8gjx!WVZy&UNIilNp-+ zcYqY!0bCNjI{|D|l@(SRIF5-CpW8L_xOv9HxB zm3fa@OLx9<0&4kam~`^XOE!gCyX!7yC=F}#as6Kg^BN)-KfGDeOOOh@CYz0$Tcg&H z)$6(u$y0p$ovB8gE(kzZd@)kZq`H;qE8=d<2F={Th7{@x;@JYf;4B%gIG34g8bc<= z{lrm7EuC(iO@(L|j?NltY>@Cm5b|4!uc9Td0mAa&U4!PB)w;kBik|Yi8G&h)tXtwRplwjsvZmF56zspK246P{J)FbTUe_U3^m=y|Y2}Y5x z24u-*Q~}Oxs&pze?mu$?rcB|&BL%z4nS@SK@s?|hl1!*Uzj09eB3KafTEQT^Et+@G zX;0V-j=&~YqX0wFj7H7LE&pHNhNycvS4fk`Kdjo`03pSJtC8)C=Ax8OHfyAaz)lFi z;Gi0@< z(C@0!-QKeHt37s=2wJ(hq7neJB zQx2tN&nmNv-Bmg+3LJZNjFLC=)G@4JkxPsND)mtc$)d_z4+f;1;u+856mxz6g=;PC zK^W03Ck$ozIk{UC_?sETHAIz?QG8S*IA9?TJtcs48!tb=2gqlrpDkZR-%M2&uUwwpB$hVn=Z z^Q<*?9m^yqq_fU69`+K)itL(cp%lf@W_a59Ua@$t0v~N5ICF>yNZpQ73W$LKVoStG z4pnD$?tCC8*50-J{$4Kwz1Nkln}!dsd&TXV99IUK0Q88!D>VW$oW>NQpD|zJv8%*2 zGHN;`OfAxqRiivVFl2Ny>@gq25#D0u$1!UE4G(E+2Vxro;>$b)9mOp%y246hZcJ}{ zzY%1E{;e1pljuo&G)So9k-Y zI`A&-rU4#2sEsl4xV2Ku=$uV#1f)(7+RbTa8SR=xtr8ASup>AAGY@go2txkNKIv5_ z>W-}-=lL!9>75eZZAh>TOnAJ2p@C|_oEBlKh8-Vj#`WVHVjegNPSbjix4o;jTQb-8 zGI?w{hG`ZJdQ=_;G0`^F!sjYAFTs}Bayi8yy8l+F80 zCRmK^3p4&zwjy*OaR1>3!U#=;EVa_<8jNIgB_qyp@Dp|DMA~FI=3;dy5Xr!B*XIoM z@?HVy%`@-r^s-KCzx`TPSH0;ooy_vi!hch&Gg?hO4tp4|6g{5g@lLLKlkteC-!I9M z*Lh&5`wNn(wS9c=u2(Sn34#+Z3xZQ;2AL9|Ly?@ePCIcwNxBgwDlRj}Lh@~|4W+!L#@4y2DR_5bSsz@1>zWS)93X7f`6_0pz zkef1c)-4iB=~%j3@d2y%_o?|ja{zIt!UQVYwAI42JC6W%R3(uG>gLGx@>=HG~1=PmVzDP zpeJjC%y}7Quk>;+$f6s~F%`}oMcNi^MTd$}L(*x~>qbSh2@h4B(zKtLDgv$ASAQC_S52Niy8p+&_h0}0-}?2}H!q$& zfBv#p#=d#`yn+AS2OoX-=!1ukK6vmbzwDhCZ#sVT_+fF7U3!Gf%cMSd(6yfD&%bVM zv&bq=zJA#YKVLq6@}x{qYy0aub9TQ0_T?9i{u#{Ol)cEUA6%@Z)aw*K){tsm=J46! z_-M#y6P3FyO|p1)xi#bL$NF8$MzcXdM3-ITUSzWRaSyYneo|9EY4Zh(WlGY;^5hF9 z>0)F;h0?aZ_^7{YTYBUpPy|d{I?PG@cJhyN_$E;oJ#@G!M$$oE8k`2p8ly?XZEi;$wFU;wV#gRizxd z#yM2!W9Or^B6g!WXdQyNc8lo48Zf$RR7+c_skvRzEq>IMN{?ZV+Cj*Zs6JvzY=zq) z8~p*2F;32{FOJrTlIFip+bJ~hW6z+c@zClnd*BN79z(4e>m3qBR_h-RT2U0*{V0kb z!>(f0Hw9+;CPnd@P3Sr@ZJA8CCbwkfCVH0C@`#C9SE=QLTAajLaYIt{aW9%@s&gI{ zwnVY$%ySs?80$}_@%@gZ_U4Uc5nA6{^)x)K+eluEHAa>OFkzqHPq1IsCWX<}N3DQ? z$l1F?qm=%)Q_Y1eHIh8@CmM8EDH~-6RYS*zZFxBOLKIBYD+19NMcx+EdaA6uRMx(- z6O0`u#z_t7@Ar(S35BKc8i+>Kp5A_W^QJdD_1IHWxL))6x+M*-8#4MIz$2bbS~w#y zyRCCbd?S%{)}Y%QuLEl*iUC5VfK$$XX$H6BU#)ov>azoi*=IDO>*iA%tda+OJ!fBw= zA>*A_`gjR!vn=Z6KqE@A#jpq=_LWVR&~7n!&I$_sOt(n`zbf_U-rdLj-kkN>JU(aB z4KQCkC;E%Y-HzVO#Lz$POFY^;;VfU$RS#3rQ_VI(t-MoqC{rwrN-a`?X@;0S*Jyn@ zTu$T}Tb6=)9&acx#dm{Dg>CT>RKD2n_^Fi8yI@81VNd!X)dCa`^ed3BMnQNCA+DF? zwK^*A2Aex*w>C$Go<*QEv07x#67!1788w9hl#aF(XxJLWd6)LF?XIxfvM3{B1CP>3 znie>N6JNTl+yLO2ZHMQCZl~;8GOIE}l|4q*Yc_>aPP@44Dq_%9Jo>s{*FV6N&5Pwmv4z!axW=SQ(;emmt862ubUnp_vZG4EktR%{klZh(3v|5XY_V<71zj*sE{_6k!^sCpspJ@h7wPPvuNNlNilu1Mi8lg@9B`U!*SD}-WoGSrG z@m~U_eXF>YaMr(U!QH-wDdwqXg%Wu=-ZA`bWfJf`6=IB#|vESW@-L z-0$7Hj~?Ft(?9%!KmX%D>;{Y2?1SIT zQCh)znTSc%xM*2Or=VHLRyf>7j-s9J|H8Etg=4qSyiJHoO;#t5EBof~eL=L1l4MUx z4EmmzZl#;c9p(^tdzonoYJR!*>4euup#FESo4u7U;ymhCQ&=Mb;ClYZI z59C$f%5_#JZzQeNnx!#Qp*hz7f+j|WNBXda)pb=}zHBI2!&>8J0`Rg~+fA=Re%&rtvzbBUS`&7fwS+o@DtpXPeQ`hL#5yexRuH7g<^)sCGnN2bRK zR2(&u4U~+p-DDHG&qIRsiKJ71GxfjcRB)Ndbnmyfi9~0 zZ-uYo9{kOA-7OLywN!5x>~8ykUrCw{My23q$pjRcuFgbZ^VNTx8|)y-#2HE}Qc_)S z4=wa?w%4=rd=%RlE%P+Cw)~uIS2en5t2c7{MiTjlp+_$^_-6^p5z6zqlII##N8z94 zqb)HlR7Xz0rd?XjG==G!xho~~nK}z3G}bh)GwZepfw-YcqNmrCExS$Gg-BlofGJp( zah(X_H0fLmQ0*@-ti`R6II{r>VTvD%QjD|ay3Qo`e~vn9B7IaZd=2(x%*lz$c0msF zEOi>0!);szK{|6G6?)}Bq+?eVsW^_<@LutqelgpB@J#fdCQw6kt8FR zxV=kDR72G8AL|Gk;`Ku09^C9#fxTFvz-j8|@=F{!?Q|E#Mb2%j&8eXQRv^V07ww6^ zY;y1Z+XwHP?Y7w5L%^nyU43}-y654ZdzijE7;j$R`|^vQ`?>G^;qU(4SO3fZ_8*_U z>cx51C1OM+O2ak3@ZN9TCSthPw*0Ml70o!p(nc=Vge<=F}* zkM$K9s&<~7>r$y;L#Se-DdrB)$aZOr@ujde#s|JyY>=M&N#KX{MBpS1xB&GEf)eqP zrS%v3!NOPZ@=C5*Im{-m!{y9sL{x6kxZD#J7WK^Sy7}hO{X2i~4}RyL{MkQD$Dj7o zzWvzmi+;+t_cOii_u-m7chUPzuK=z4>z-F0e_!7h{?K$<&scu91AXh`PkKLScmHRO zf~5|!UjTUa?8|QSDY54Vbiv*I(66JvzUj%xT}!>@1E7>~J7VLt@IfcF}1-mQtZ3X*g=Cqk-Tej#cMQ(&QU`F*?QQ zr&1F>UB?tG89b4g{G+2;mV!~o%WxnZAWz9#M2b$LS=La_aMG4F^qqBr0bGNK^td8C zj2tG5rpcm~W(x*5)$W653Gl^ju;S4R$z!GDM3qyoU9o1ob$B(NHK-JDt9mz4HjXf; zga?u^*DxM1`vQxwpMk8j&gC?Qa0U3B&xb{LXNm=zDjg#j`(p;oG+QvGz$s^+8jkQG(Y)J65)antE*rp|}Ol+G=A>M(Wm27((gYjX$Jk8~g3x zUYHd^8rPVyvoBDv<_2B`y=9Ad_t#+^im}azsOgdi;uVoyq53)F@roLD)9q-7oyJE$ zXNk&xm}=vivMpd-3~15cryqB_dENV+x;*OTKTQDEvvR!_Soq92ER&g7#*Gr4uv;c$ zz0UUf65VNLtmiEwjB_Dmc`=S`agH*Ngmg+vE3Pb6eb@-@O@sdT?gDD{-;)Uqr398` zq{ds`?e2zk>#W(ldvEHFwGNal@HQ+P`6WNbiIHc3&NvzhNBlZh<*ts6yO=5~R_|yp z42|qlWsiED!W->*hJlB|&;~|!t1Fy$#@1=^7ngL4M3vNvXfw0k8`IEV^psl}2G3y41*(O` zTM`BSiuToEyVSnbCcw|Hryll_)<*yN|6q$nc4V!B&w45KJZj4e zjJ~_fqp@P9x==42RRjT&VY+}(l9V!Ma#r75v7I5mXGJ_Ou2vd#&vr>8Toq&6;1s>v z7<(ZRh!C+u?cEMglxYz_(MtJ>E-RG(gR<4^aWcX)edW(`QAz<#*p}F*sX+0}O2H^y zB8#;u5KS`3PGOZO`O|-Hl9a#B(C~+CV0{N_5dxLUhj?vF8g6f|F+)4T~cO z(W!x*MaSmu)tc7J8|a~2(I_J-coUiemgCD5om$xykBtVtYSD^BKVDh1w4!_G(Zik| zN@Zhx2H7SC8B3GWTa=Sd>oAPL&iou>1Wg{=?!0^04d8Fz-1OS!+#c~gzd_REx8D(Y zQ|tQGmtX$M_kR8le&;t{{PTbJldoUCfAHj7y9r9hzjnmXW;RNVMIDQ-S?GJ#Q0KB4 z>qVY!QK&^S`5{NK9||dwI$8EHgb<0IZ4q&CrYmx2m+*N&VzylLM^%|Jmvze*OGteW z1Ag-8|BiRJ5Da( z8(wk)kI!x_6%8)g4>_z8e3@=mzYQTIbh*`Ir$nHt*m(^^=g9hE`%%Z299w+Db$wam z!&HfJj!KuJRT@1?wPIxtj6*Fll32W?KHeQK08`H7DVi($Mj7BkhLjHwJR zEBDYJj*h1e)D`a(S9}25m%}BWw6Ty>)Kpb4jWuK&E=Qz&KifDoi9XjW6~c)w#i{pX z)#df8RXrG!nn``v1EUL7{ASkZ4>JH~oGcKV6MlnFgApxIrUC)*GgkC@w$iK}#7n++ zYhKtVV$P7O47JclA>1J-`f2OTsly$V+W4q*>w-y&X#IVuwIr)8xI_^1N*`DZeAUPu zH1OCpk9z#QLtrD1Yw>SZ%N`j-faLZ@fpdOiXB-dUAq#GU0mF-`Yt%K!b7p6OEaJ7s z(VvPQ=VQ8E=!{R`r%Pz9g{3FI))dvv$I=Vsdp$_KzT35FZapwRzU$Jbb&h@qu@}6$ z`KQ$lqgcaRK|eXnU00<3S3MI zywmXJsiKR=9IgBAvE22B$HXogx0S6)4uBW)XIo&vSz2Xd(jHBTCwfkS3X7*=fajyn z8Jnu{PA^L8t8q+EKDy5(_p#ylO&$GJAy#w-U?n=@(74flM$1I*TxR}K5?2#KS~wge zv&fs(Ql?nxW{Bi8iUr8*$pF34@S9fHmn6&wnT{pV+uW`*6-pm>LS_b-FQV8dj^&>* z_F^f{9>>)EZ33`)xT^ro$Vw~?lZ|0ae;Rwyf_BvoH8NTCmHD?Jp&2@dXv4J~5nlkA zj56Y8qrA22rcZgyaq+gc!J{B&^W%vJqM0}Nbj?6bjReG-=41Vuo8T#8_m&yBsJT(l zt*t6Yhu8}|$N2Zn3Oitr%3Zm_5;;iFYCvF%el2tbVFuWm#hJhy`qW>qFlBR#cp7Tl z0w!hqjCQsqF;g1|tQ(Hb(W>OW4{e1I96IsRXDzIJG2|2kB`|!~~_Frq&D)u5mbLH2f8_73u)ZApAg1LIaS=ke4W*oGI zLZ{+sW-Ftp+oV$i&6a9sR$doXGx*?Mj~(}x(3`hkJo~CP@o)dq_kQ;`U;VRx{qMhg z^U4g*T93B1Qff9uX+0x5mdGAUACOsP#`(rTOsQ)?3a=95GwZIJ@syHdLC!JpO1cVu z9ZX`H?D|)e^Y{Q(d~Kl*EPmCTu`u$o`D8YCN(lK8P)Zfw_==i{5fp`MInb0?3GLcI z;Gj(!T3SRhrsOOU`IM1BA+6UEPK!1-x3YEb?d!k&%fIkX{`}8Adhp;!pMLtqvuB@u z{?+Htp1Wb^gI@CUr0%bNuj}5u_w0VZ72Qt(G$DBW=*iO$Tk`L{*&jWA{PREmKp=Py zt!qU_Jvp`YwAQzte)#mM|0h5B$&YWo{`%(Yo1QU$(yM{)-!E|X`;P2#>P(tYHKewh zmwEyHso3p$q2QsGn#`&ligVUdp4XUfN9AFroil94;K-PDn5OKRMrN;#;9pe4X#r>K zC1~Urml$oip~Zl99)Z;3vYm8}G)|*ew1bjzCs7rpq!hqY1UvoAK{=NfI;9pd*1Yw0 zZNZOTAgtNAWlPl5!E}g#qfK6lGD5l1w_9sB1txHm%c7);1Ridc6uDCwf(y0`^rzt3%n8J8egZ&s zJZuUfYC14)eIt+xbPjP&B-(MdW^*9aSAvC8EsgM@}b zH1Qtf;M8@Ts$5gpS&3|TxjI6RGvY_<;SUt;H;_pd^|MZ{aY#p^8iwzTq|&c{d=6?j zhBMs78S7#$zjDUb)QQBd<7tE~9v&iBu+N8xzV3fYlQk zPt`%j?Yg8e{1qvI6?`YMfpEq0yp_>f@i+Zg=$qy{nb~FDIxn~V_v(U|Z+raL=MM$)K8bGu`4M4;C7E|h9dl8U#?rp&`^1!EHaoE768A6{MFJF#L#?rrsJF#ax z_qx#}*X$$w1(>#J?&Dc8&}=KlMqsw%tFp?86mCD)s~(4qHi$Ye($#db6fmoW1436w<|Ber!%Ep=vT>~XKJXP$ zrz)SC-XA>gD_$R3^cnoS9s+gM&JWjWq(}<~O|qwZ5>|I^_laifRdt{_N;t99@-ly6 z(?2g8kTokThMP26e0MLKD7(BuUl=P7r$AE4nsQWAsu``xXK%iVcTjs8plMGCsW^<_ z_#aM&5CetJCN07&RP8qbaJOV=N?As<7{_VXrxKIDde|`JBEG*WM|R07G|l-LN3f>e zv`sylMA&ZsE7fjivQAhcr!=V1tqQ1f=@`kWO1#2i2XIsrgOHvJuk6I*oWwr`bm4ja?n>_}Vh%L1yqBbtS?E&7mFJ8Rp=IyMIm#P z$gO;)E2dJaGcyV1jlzS-B=AMA$S^g|n)lv5eERg0hY#+*eEHRjUN!Uli~r{bKm3L7 zz5A_S{iSY0`IrChKYB*=-s2kOV)a$*pIg}S2RT<*c9lFCJJa3^bbNI-N&;TS5 zW7?N?=A3En2c#_J&&du@jAL-iScw$7`h5Cnwq#xmkw^|#6{Z@~TgiJQ@w=Zq{U84F zFaFlA{L&Bp`maC#?DLy07T>%7?eG4=!-pTWQdJ$Re`)&oq*pfW*T3)e^O(Kd^Wj53 zFA?jWXX^c)U%!0z)w8Yxba!m^rM9#Oe;+=4RITZy=k;%&eDbaI`1gT%yewu0lfRq z5IKu=_|+Nv9`iBL_{@G+;FOYM3%Gd9&6U6jcV#(1IYRJ`Td+B1yzIsTMVZ0k8Pte7 z!leWG_)5i0M*-a!9qto*%8@D4w?Bn_26H7#f#u4^)2V>lxJg0msg(9g4@RxS*>pQt zD!C1i3ydSk*xc8+JKk2Y3EO${={7$4ZHf3!v+-U&0Ec^n04bF_1<8WassF0sFmQ?KEn6eY7>+dxjYz>mlLsjI%*%gi+Hsn@GoI&i0hIII=Xkh!SYgWawDTW&9< zo{52kjc%f&%G79?8g*mOgC3u2q~9oZzx8B2rB#0eO~cJ@``(GG02Ir9BF$h!li2Ov zZ@PUft-83*4=uMQX=dt7?O3bl5(@f09Ypy5QFW)!mn`{x*L(f-cKhDBnnh?qLRj)H zd9>Toc#(`8fz6)_k9YtCLEta}=7F&R6Sg739?i^Mdf&S*ulFZUUqPL7>QrTZ^Sfu} zH?y*;(q>#a5|uQ9=utKvTd8G7sYOkTNb4tu^U5i&I%m zYVm`7JjcgOy#{(sZf0+$zFgTL*@G7#+#JpVzCuZPLyOxv4p1M(hFIKyeP*=wqxD?y zY(%D9NVDt!D-sRf$^Q>ai*J^G+=Ow(r}%&dSM`3(Bm$v(eVifLuEX4BIA+gG93Y=lVGnL@I71h2()^xI^=chr9fg|wZ^<6WE5Ae z>JjpG_g>b+hP+=wV}i;&h*1H`6nHicRlf>@VlSh_Fp;OsIfSIOV+*jQ`GpD%Y~ zL!PJap7Vq!!ScD&q&9vlIO1XgNwIkL867VKqI|ilRn>!;bM6Vt#;}Hkqj{Qw#sIlx zT&r2*1%y$^d<+uU4y-Qfa^DKzFuAETFtQ61FO1elC2oK7$->N5$ zmz=>;m$QU8-YAjgl=(PDHA&IJxx;O;yG%&QmA=%E)sl2QGp1tV$9nc{@KWQ_7MJ!H zmDNJ!7v;qJGU6Uwm2Pz^CvbW;o|qRcf9;tfc~yR8x08GgADp*FC@;y$m#-{qL!6`Y zC+z@ge0DXVRzcyB##N4r@%LU1?H-jY%XLKmmKTDC-ZMho=Be&TwFpuvEgJBYLgCdC z`fF%5Qo6-S(R*WoVWGyrO!3<{uZd<236s6?da##nVfx^n;|JHj^W^I#5~ z^EZF}SBzW!<^S{NuhgV#SETSwiR{#DEp*kq%cp9Tx+O|MslzrL#9g{Zekh#5Q>QT} z_pel-lsWWe(09n}9JF1)w2o^}1~bD(c8skJnsPHWt3xvsPOXeQi4vk!C6mHk)=ETY zk}NCSm2B3^9|FqGRhE-dD-hWu+J#Djlp6An0KwK|ue`iG29MnT?Sre^|M2hq=r{k$ zZ#???tEW$%K7aAbn9nkwyLTVlxw`L~+_M+YpFMXU@QwQqJ~Qg`VE;RJA6(tJqnYFq z`otGcpFV&2^7Ts>#5G-EHavg&?9KH@j~{;e@WCe!^m}R)xBKMb!`zKr8l2a!YkHxW!fqo=i1=FVHwhwpK3qqD%DWC z&&uVg!+o@wuB*JMekuN`?!c#Nsj2t;ha=p6z{@SW-_jjLb%pp)?u6#Lj1_WD#9*)Wz3FnrE>LvivZfUw9ms2l^OgjLE-99g;T8NNP@}>RAk*>7~`z zd|TGh2nFVuF6)S<5D9TI-PIf9Z}UdSF;!BMIGTWfaU0_84vtjV(O3ztHc;`XRzEHn zIyg#`EjIzA;dH__9d+8Tq8~-mz%a`w$VwoDvP8mEco>2mwU`~dfMh2MNmWdc`&D3t z6gdRQ%AewJW%a_H*9)H>c~x@*1E|#H(H;5d!+)5WPHF*KkFmb0@N66$l6hE7@lhNi zW=qAjqA1I)GF@%wM~P}q@Hx$u7R%+1>2WKbpW$@e3iquk;-@+{-`AQrPutlF`%QUFi1d4W@1Gf$J{2!q5n@M4^ReSDv*S z#4duNUW?Y%7~zUjySBK(JX+H!S&7_3fe|ygYTonsk^^>n1o%%zrLGP(&LEsC1D zhOVk8ryu4Tz7~fSg|{}WgwmFtDG_Sj_w>w|a7z>l11^A6Mc4y}DW-vvODUkW5D}Ji z>i8%|E~76nga^;sfwPw-E^nWYqETxw5fl~NGe@pud)J#oy_(JA#NC}yO0QMi60oFA z!HK9?l@q#JnRHw13nVN5psr)EcM9p7f`ej-N z^Mj1Mlq5fiu&2lTS<0y{87LjcxWaJL9`G4#=DK+G3~A?+=kSm8`#u+A=f3;HzxwNP|IdH_g>`c8dx0S}9Zi0Wop6(K)2_oAB!dQcl$}1Q;bcLOU4T~X zTU2Wy@`dtfhNk*!)@!h*CsXJk1|)~K7b7ralL67Y?1zZjAvi`K`r(GT5u;CPyYDt zycXp@c?zEgUwAy>hc~YuKlOOKFU$v35v=&Ref9p@bGcXVzj^-B|25ZsJj~ohp|My! zt%vh;C-07Mx#`yH7cXBtefG?#z`ehEJD!sNcqq`-)l02~-iO*$XmbbEyfol5MiUyWjkoE9$q?$ z#tg9-!fKSGp$k$Z0wF7pqwwfP07l|S5T`LW6~nPeq)0_wty{Q+(RD-3FQ+6&m39n^ zIz7_RR2rff7)RQcdb`3hJ)EPYpK0Vz);Mp@zV$e@+}K@cnTmW?n+~JS7H-FPRL4J_ zr4MwBquzrshdYdM+Z1Wc`vwSU8!1u>XHFrlCvo{y^sgP%i<#c?xuNG2UvAql!0i7y z^IycNg{hA&`ZBdSrPhyX8&gR-MlS$@V8wx73YrMYDdTkM(v*q0c9~dBqkmJpo(>e4 zPV2)pF9I?G=vEGy^)F3xYWc15?8eZVWLWbgp{EIPYvN7eWq`P8Ifp)6(0OK#sYGfM zx{#apj%q9v3yjBjQ%|r@FtCk4s$mX+PJam+chZw3H59CfNUGmLxirj+vd6A$5bWTQ z1OqI`8V=`H88LLj+9mm5x-p~OUOXxEzuh=3d)Rd;U?YGSfqSBk@wny0(;klEk%Cr- zV$>JzZE}2Y>62CG#Ywy?N^sJF>RgL4$HgG!JMVco1zn2OGmdj1xFLyk6GV^5(S9+9 z3Y7j?;aZo_8RAS-SJSW(lE_erg5;Pe$rN7dETRpzQU(C-AUEIJlzbsR<2NrKTqaYP z6i(GC!^pQcEq^j#D5*-zyU=o|0)otK(w?Ho_~q7%{_QUwlIQeQN1K#at=cs{SnHsK zm_Ik}{*J<0+xr{uT9s!Z`mTo<2;OLEf*V=zm5d`A*&|?ebO$lMO8Drjcg!F)-t#m zoD`hhM(r?ZTK07&cz>5>5MpT+`MXl z(wdMiFKNwO3XFj^^rRO3clW2BAw5elFY@r=ckbS~dU@~OvuBSi5s`cQ>K=!9{OIwg zpMUp?8Xai3I63_V^<14L7M74iszB68bwFIfbrxWR^ z$sJP?Kg4m(VZ4qf0@`3`a2krkZsZTkqt{6kRo(FQH<|cVbN^Pl-oEzX4}SQ!{`5zG>}t%< ze)h8`ChgyT@Pg($-TQRM%KldVbU)M;cVlfHWBBJyz<2K6)3;%%{-V``J_hGCp;cqw zM3m-ExmT|9a0V7q0pGrQ?NwoS@7#I+$$gK0>hjT+lhw%NXc(R3r+&mV55YqS%icF- zR}*E)JC9yWGt*51=UA$V^OWtELaXKJpoRVrL-)~;Ow#c*-oKNe`Pjf`CR_2PF-Q4^GPT@JhrvPPH|B} z48pNbOh`Q(l{AXqDk(#SVWJ#MRO}?dc)WkI4F?+O7c+l>=?FRu<1CFQLa@Vi_@7!N z#Z39ZPWI+HQ%c4-Rg;`j;yEr9rsPolh}@uA+Gr!o0-NZ{QJw3l^d*^cEx>}1Cq9?QCq!vN6qaiFQc6bJH){$ zZ=a+cK@>lAa5}p>xwt;N?K@4zpko8F$zR78@9d?wafqr8`C$GiHEK&4hG>G0Ar=U{ zW`kf3AzR@%lAi9m4brHCjPaRS9k7e?EK%lBmOrSIBnz^ z2lRw((SJ9cQrnS7m-%tgk+4LVmm-VyQs3%d(v`}9d=1s)&45^FXe3;cT|{Q>QlFqJ zuoc*Xd+Pwe3yr*^CB4#4`@Nns1@xTzwbzv`p16LauAp~Rd4I4A8wpN5MT+K$dNk_w zHx?>2OC;dPj2pFN#Y0+wv__*a8XgcSvFx-zgHa|9vC9r5O4-3EnRp>3SnG)E&2bkV zKNyp?!quM4I`t%T?B`=8ZwwQhVuUsCZiLoaEM1l-S`__ngvJ(4qh^mbyxc0M%JM$Q{oC|} z>^dWxxFAyw(w>Zw)Nt5j^#ElX&Df9UOg|+|8)AId4}3)^h3ID|f6P zYRje?)D1@7#V>+s>`-HOq!+hw#XX~}=e0&&t#AdPnsMF4zfshj7wbYoXgxJvJHkyP zmYv^3iqH*4DphqPLW(hVyzR+pH#XZ(*=zWcj>^EY1p^MCc_%a`x3->qtmI=baFJ9wo+fZPj$0HwUT>WY$9{nnDwz| zen<23S`Z?n`u2XO{{;}v)qqPDt|O@=P0p-`k01f2xN?0f2V#m`7EWMJ8`ZE{n%Iwa zfM>T%rE425w96m^fWedsf$Y-K!^Pde$J}~x6l=xM^5QM+`QQHC-~KQD>`!ig zc=wYZe<}KZ{_?fuZ4W>F{PvxD76P&Jy?ggHc2@U1^`_qIw?29JnY+HLu&-WrPtyF3 z!E_5o=W8oSd83D8zt%Ho_tJrxFwaqb{pzKHsyzdG?X*pk`>LT}b!-e@6p*|&Odd&7 za<*itV#{r-n=;n?h{U3qyjWHm;u4Cc{um@Gtt4p9}iN)Mi4Vn~s!~ z2qV2{sgao~myCE_Skxn0d1O*dq=T1{dsimkoHlaVLCPFx#T~?AI25c-Nh&tHXj5k8 z*Yd%&R(S>;pjb)q$)=+`$BGxHbgi;5MYpS~i1u6sdydD_1ntm4-#)g9voIY?)ELWw z4#A?>Shj^ErwIeENF{nRM-E1C5G#Ap&D=jjAat>%!qcm30pnk0qQ?|vG;o=Q6fkvg zI6aI)L-R2{4@&P!`A$oRfVB_oWAa45rUASmkU60qH(L+v!gudNymF!Z>h9g!_dF+| z0l>Kg+60M#Kyham$O}I^sDn&w44D6&7*_r0O%nu*W32WfIq?zb8lYJ8ElSWq6Bj1o z5fuy5g(1NUn^p^xQG8h_r36&H&K6nJ!JtEw@gUS%BRaDRDKysidJx2Qvw*EtTCjiy zRDymJK0=;URouu;MkY6Pm_(O$wHR@{qjdZ`i%hZvk59H>PPHHb<&~eUHxmpp>S+@o#j}I-?LQGv;jmYLL#gY9s z=s0V;S5X_0RAuwhG1}h-GjYPh7uBQ%Jh;@oYEw0Vwb4N4R-=wavKP(aagrpZbv>NS z(jS`j2v>v3XkFw?K)OCLdh37O-tF5hyjVplpB!C978EvHoE@T&nozH0E9M&8-3lC@ zh7btD24}9FGBX$5(Vqj=-#EeSu6f~xj(gVVhuha~c0cDLy|j!3*j#f1iY$i~o{Pze z>Y!6pM2T=3_LfqT?8wAkilnCQf1t-+e*L!9KjLiuF1YOG9J;bLtd(pk#7M_0WWMS{ z^ors)tZ^oCiGvb}1BeB((oWM&O$5rCKI2=QU=;)5O9`_&&E{#UCLTcNvo-o$L6xO{ zhOCYA%}u8CQ+R%v7kAL#>PM!sDGqHSJKfHHfpbxPm9F@^5Zn@r94N0S;;)S?66=6p z%Z7M5GI}Cw$Dv2HC$11`6ScP|R91rHJQ_KwIL#f?d_5e6d9*NJymcGn6@T^mx)-|M zx$ETsA%%cio2npa3TxK){ky9wUW_No8uRR0ta}E8E^9Gj^}5Sq@2*=8&Pr6i`rjhB z=1y+iG`0?(g${`8l=_xW%C)nEH(|MLHO+`}npTCL1>*^M?+XS=B{IV7j({+7e- z=lBw(?t=V@=XM>j9X)TL<=o2)BeRzNO z=FQ*x&A;}K|KXq9xpm_wKmHG2KYHSpHZ%Qf@YSQo@83VFd4KoK?JMsq#pkVu4?p|t z^Y7lf{|RARsU`32Tc&Fc>-H7I8_;Yh7ZsBw4CzPHkLrv%eClTQ+vto-B*M%dk5v=MVDf2C&IfZ4G z^w0f5IK3!7^!x8PtgDSus4gFvqEe-#^i&smh-06PrGxAfP# z%iSk)QZh%fzwRb*uSt_*X)%@(24GJvp1#i_1!z zm2L?MYTd)Mmvu)6=52~Y!I^YD;M#n-2k5&B^1`GfV;3Y;#ez@YP zpmw`Din@G7>*^egY&0;0#<;huxPCCUYOpO@tYaS>X#h*j@1U@ng(4ABgqFZ`IohEp zWQd5D7v+w@bE$Q~-`u@@rKO^%dQwf~{?>6-yeC;(P=5P{{;?aSybkf+{rh*Xx^b!t z05z5i_!s?e1kg-KVkI|}#slN-&8+r~&G`X=VQHiCH6oI(s zEm*(y%~cbQjY*40x?iH^Ic11nda+&vPpgSp%!&JSHQ8s+41K3H2%MET(ZYvoG(>DuTfVX&>TP^Pr% ztSU4E-+X6nOQX1V-9J+7+w(6|oUOrlp;${O><#4Hl524ny$hp@a>pD3#fS=3Eorp9 zdKW8a?7{G%Q!A%QOMu+kYXz_aV}MBX^fE%MO!x$X^b-uuMF>3rL@(Sx8Wr@6StK3f z9$u3#wx>xtF)BxKF=HfHP*x8!D$4<<;f#cYCqFnkzwRmV*wu&d-NmaL04<4JbLHt+ zn>qEj`)5%hz8X%5{FDRCbj1fDxI1%Ymt;fg)6ulOG^vzcwEQPt?$;EbQV?PO$luf$RE0r=m zMEOQo>~kc*>(|H+J(q5XhP_idmk4&0Lo?oH_-N%I$rA6Vjk5;xc0-oLu_N5AvK|NKw?{wH^D|MN-VX*Z^-1fS@G7Xg^AI~8iRsW=3Rc>&}VU$dB2A_N={l9G zZ1GW15H8?%EIoxMz}ZUhg@UnWr^Kn3^3OA<27sI-y`ZjuvsIpsE^`_ccU)~NQxQjs zkhpd{BpeT3E>i430p;asz(;`hAg=7X?<3(73cKq9hvPzY2^!~&)qeKIVC7S%%C z<@8len34UZVaKuDw=f*(Vx8zh)#PDTlL&da9NtG3zT3eT!dyYAFi-701|=?*1F&c& zcC{#F)NVQcstnZ{AH&8cvkfQ$c8&5%V zO<-MH$kq?H?*Go6mH^!|4DiSVlOZ;}u9k9B{%4taoq98R{wFq^xZhHdY_x8(_#~+( zeO?BrP@BkWYpBfhN7VtR%qcWUnJwD|LD~?d7b+Z)1a=*OHViFwQ1TP11|r*lYoULF zZi2BFJewl21n8bC8wLPXf!Dlzk+kPd3akg%%ZWG zc{^3z&AH3L>6i&FD~;izmN=|H!1-COQpKPdh1W#uFeAD(LUakqM@)KsYGH+CZeIaS4Z@GKR_BnM3ovhgY;(e**B z&&n=9GGD!x zyqy{}^AI@Ai*ay*^b2LGi*HloXYRj&M57CVdOFT7brU&}I?m@+Tuv&$O&_n33yo$R+YlqJg?lmX?q?Qai_C&0(FiRyxHdBDkb(w6CySbyr z1{`3kUNyX0DwS<;Bn>pK{3?AmX$QSM%t+cKjp(T5khFx&m9hiTu`j5QMskdugBUMO z^9VrxY9yT2?oiV=@9y5d{kMPnZ~o&y`_s?v-~IAuKYR4($v|eHy3TYOVjPZlZbq{DqP2gNKbPJR9npm#=m?M?3F>Aut}3 zXWOKNM?Li-VAl-Rw7A8IPr0!3=8did3i^a?)$hpSgb@>IhMJq8^igj|OO5X}QEB)X z8{M>#naSLv^@^f6QPj0kYUI*(cHw z<>Vz{P7xP5QdX2-!U2OpbC&H+3V#B4(DxY7%AkYuijJXaV|WjrL*zWv6`Zv?ui)@6 zOUiun;OVTqwODywGgBpn4VJhKfEL0rWbO7TK~p%nB6)bL?C8r?TE=SO!ou(@M4UDt zu2tjb`nPj?SM6$jGx_O5zZ6Z4^z=5SP?M-sf3pangjuzfP_;je= zYAs8?+6`q>@QjeaLk~0t@EkdcS&v?t{-v!^N)Y-A^;<|V9bjJrQvb0Ww5oc<(Ag1F z&$*P%^-}T@Fhpqt>XYP?eCZg3s z1dxF=+jCw@rD_w^))C*f%)(O-g;%ji8R9(tXlj2_9TVsMO3dAWhz8*CU)-pxotSBT zzLBrSRg)%Q{ zY>2anaXd`D|>i}s*T1#`uuOB^oGw3oU<+~;w%nO-Yc&mW-LL!$zG#~ z6C~%}Xht`Mqz@bm{@c50ATvN@u;PhDOcqRhQcTt8d>ElvqlIoE%0LLVi3a41toBO*gHSD4^xC8he<&^Azxu=- zNmpD4>4XbR-Ka(-4-UwgYPxiCE~F%TlC%)W2AlS*r9^Af){gC#Pt&?CM2Y`I`tQ0N zLS?HASy)OTKIhz8rs^kLN1e5VH$GS)dCk1OXj8L!*Xv_nKKuIXFMsyp<(oUVuV3AM zL#q3?Z~x9;|BaWg-~RJ|{eQpFa9FiM+`1xOFfDUbTgUUn7*FZRX@4vVCUI4j+*ZeR&g6A&A647fA%Mz+`IGj zmtTJU=+U!hFJ5_}jZ26xU%mV0$tR!Q_}+Ja{?5I7uFmPiT%*xictFsL=dT_;ed0yG z`T=#oJcY+Kd0*RPxN+~^Jsu0nTXGn(*@V}x$!*fBF`owoc{_ctr+Mq~|5kK0I;DE! z`PKHLMqI~qEYIWTwuIBFQ8aN0KY4hHNW`YNMEY5CO!kgaK`mFhf-Oa-Et$fT^)lhP zRJyb?bHJS0rOTTF%RskI<6^MorR9+Fx}ZCl0cM2!eaIu~$Zz^*NjMJ3#SsQpCkF)L zevqwv!j`D$q2+k?j2zd%56SJA)`$ADnX6&ytRyRA4dguhjb__00w^nr@geIaF7K8=L$i&rC}`* z&IzhJ?mY1GH+gWBvf@8l7n~6fYTTf)5(ii1!B;>6b}l{30YBv~49vH?NHq0Rp3F4nB6ze~}XHE#_D{Dr-wcyFep&NyI6M-Eds#g*e`%9M{t>1D0=$W|Jx;?$-6K>jqWEyxVqje61G}V9SF#SK zAgGDV!k6Oz{ue>Aen`w+H9?>KU^qGX!^RAH>U?toQc6#Btzp1)#!Xe_qSOIRZt;Md zE^jsd6!|r+&d`P!f#seeO#N~pnZcb%ePe+1>}mikdIf*e5-R+sm38Mp?3nHI{gf8F zs$12iYYv6TI_fPe64-A5kXrgqMgFR?tVkcJbw%?}UaE*>O!8BzZ)nA*3y1;SZw4w_cXauK6fa!wGq6Dth(X zXa>^|ulpb4JSM7nX1`Q=j;Fs#T{e8FE7{wUsx41KDAktNFpbQivq;6VDN9=zN{FH< zj`MVjy$l?|TF>KnDF)kg*fui$k<3+#l4*4j|Dyl1cCZUdC;H!FL2U^pRncu&T4+`I z3!d2J^{3{h1Z%oIqo8GQx3HRD0+}pWi7IKSI&pRpXjTK$5W}u1437c2ch7yI2%FTv zGS!+M7T9qj>Vybatxe40nUoTgHg2h%m=^0sx=>J+OLwD~Dm0JRt~_D@S`*LT+uik!m_3HH(Uwq{bVFS;*_dfaTJD+^= z>8Cn0)@R~W$XTylv&e^~T_kX+*v%WSTIB!wwehQ9eYg0$+uXkkH5#v`;Ht?AnAECy zfT3;DtYO}uH8Hc8t=NUoQIM1L>kmv4hePq5EGbpSSwbIO z6o718Y%cMfX^%088$qDwwSSLje#8QCK-Y{MTTVWU5Ow6lYt_^I>hd6Tu-3a|tc z%Q%lnNdr6emrTy*KlnHv1^dw}xliKA1P;QV{v)pZnn9wT8luk6KdyA`(r5j0+G>b% zue`*8os3HDucwd%SOU~%TW;*eD6J=Ny~JFIO`{zjK_6^9sLm~5kg%;2L6wQZmf5&p z>2p@)^O1O6*AUx^hOeDAc&HL&_5QFsEu2yT&w#5F5S_{AAaa|sB@SOyaE~o%@fXp{ zaR<8QSRcj$=y1(|6>qCcz(oz)R)%QEbhAC=pm05tOd*Lsd7_Se(FJKTX;PRMv!Jy{ z%Qr>V0w2v}qX}s1)7M8Ml<;YZdJgdv;VWptec$z?8bao`3f7|Ca1>Yc6e2w>)$`C+ z#it9JEpVLgHMveXkf#NFbKRV0;vJ>eO&u@H0yD8NrP4_ex{s|=3Z}JH@mksx)hTYb zt@TYezH%~fPkSUt*;(V_0Ta_n_^U7oTr`*KD$ng?r_{n435%jq6QaZ2-#Amai~ba7 zUAi9TFzvh+ZmkJ4RtIF_v{OMF6@=}AF}dPU{MpO5Vxb6t4A5qb6Z^NCp(#yZSw~JH zh0a~kD>A>Jn##1DD^DFjc8Kvjb=Iv5#Nom#p84~Ds(V^2;4}QWVVQ?sh^xH2)qr8_ z=V44B!lJZ2`@h+G*6*>QG>y0U(WUXAEzYoS3g>@%H&i)Av?DhuleFSW3;m^01Ez95 z&Bc*2LFXc?J2x&b0i*b?2VSZzgpkHU&(4kGG+3RTHq%(2&VwpqNmppKOb`hI5x$M)8_0^otVOgE@A34C8 zxZcLE7Ji5^cRgTzj=+);&#BHgR$>bAmu0?*@Ax-g~+W zG^W@+wJ}dh3v#kld<45KWsSEIHC;UcxCYFOhI^at-m@Y+ZGflFR)xP!l|D+rWJ@R0 zn=#AvUiBqcRRX-mH>Mq>+<)*8(yzaI^W?>gd+*@~b;E+VrO-KSKa{o%#&NA{NbQPNHnqcbG}2@p0@ zDD#v|BdskPf5cQ(>ZE~0wq41zm@okil+&%^jgiCA2t>5gFeuvRusQ`^@($7gWzIt` zmF{GYx}pQeYl5f$kOIk|+%b+Y{^9+v{NgYEqkr(D&mTT`{MFYlpFQ`+uP?uT{PNAa zPd#S!-Y0^4gDuw8p`6D(cgwM8zw`^g>@MJLoVs=EKHvurcmj@?uqgx4 z{;OBFukQTxr$4dc-<2VkZw%F>(~Oa~#b7ncd9sf2Mz{ZRn5v|aAcl*)X(aDo9Z1}b zYxB%5Xy?)DPHp=dCku?&tW5h$jdgrFD(D8Uv@7DTfhmcMe8%;E97)`wOc(JYvSq@sWt~nxMF;CKQaNE#8ur4mIZ56vL3)OmoAG_sxU_x;3^fi zivU`l)4@bw8TQj?JK(3taf*|F(w8?R{7unGd~6|WCL;UXt@d1N4j6l}Q|s9k{&2T} zK3r?E&?NITagG~enFNR%G!u@5-z7zcDd!8gnFL>kOTyC_=p2U z)+r@SJPuT6KKwBXalGT=cF90sVK?Sy{6M@&}j-& zK%}h%(lX-N=0N^@vld4~Ip-Sd&>a7+d>mI5R%T3Bbz$mqL~7{+o4@fxfXygKahRt2 zgd~7qvjlTAD==LR^UTT+x;e|5yF+)E+O=650O(}rhf4S~fra7ILo{(vG9Ia+sDJnS zw%n&pD(^t^h=wiO>8Y^Q((ox~QHvgW?WUKjm`rK?G8w3lWzk?qEkJmx-^$)_J0<~@ zSMyVTvWYR{Q2$(DhkP;y?@-*JY#$tCB1+GV$3fF};kHXg0KVGp8lJrpAw=~kr5A|l zDmK-64hq_&7y7e#BLpd7o~tGN%-_XH7^Bo`6bu^0{6*T}Q&9N=kDi0c=vup){;VE@ z`&(a7lem74eIpv;Oab`-A;~{dLn-(<`zio7&z_|XX{AB9yFywIURtRQ*loSX4DusF^J9kWRw2RXDrdXvKaU*J~ zJ(m?*yD(yD!rNQl-1d6W9BA`}M@{K`9++zudQ^w12{NEfQvs&bu-9b7OFwxrGY9HV zsGZ+YM=Duzb~`TOv`baieP(VOW(&bBdI&Y03P%m&CSQd{bo(fTCtFu(agn`Vn#9%%_I2de@KU-6VBP&>588ne?b=|=Dz$N#$$v_O*k!q=FJ6#KE0v!Gmgtiy zf%455n-(C(bKVUTckCJP-5YvGG*8K%SX;QEewh1jfk(Fs(={hgiX*X8HaM2BSCiBt zDawt?l%0CnqBbS#Td4K^{fn2cy;-)|q+9OLzpG|=N~NdDK=R3bw+nyx`Om-mXFvLb z|MI{2AOGFY9$&w9pT;Sm`WJ#(wmxCA;TUnifRY}GVm1G$1uEuevj0BP6lr}|V4G9$-CHq`6El+VBIMp-|AfQw? z4>&;ODNs5*eo!r`^PSDpM?*0XBV68v?uyMES!YhQa)&(VkdB!egXouynPCcNM^(TJ z(*EFC&F$VlonlGVpctPF6(6TZiX&{Ygerlfs&!w9{$<45lFP3r{HTf18<%P3B5#_|~r7j^! zoK)EqN;@@l!&8P|)Q^@ftcf*YKP*P74QX(|=Tgd5C@rUecncLR$3x)BN)=lIw6p{n z5)@?|~&xRaqNttM2d=pp;Y)ewr_SIx#JNpHPcCM zW^ApAX9(6t^ftz0OQo9;%sH}@Z+a-vM@oTF)8f}O+wT3XrG_)iX*Mnjh(CBi=xcBP zefPn)n^2pS#j7nO@XlFrGGQm_(F%OIL*|{iWbJ>^6Gi41gLK6>o;tQEF)>R}meCBK zafBYrawQf@GadDlr&NvSILOu7ux9MUkp$guhEZU$C^7;uWjD1?Gvp#&fn*U*EM&f6 zw#zB@g|$JKjbz0#_J>&3sUjQ|#<&mYDIi%Kq}af)Ov$GKWv;Ok5lk`H!OW8oGS;Hz zo~skb0i-m{a8O;Sn%H2r+fp~SLp`-p&`7OJ)M^J3Bej)X8psJ+Vkl)tJHRy5&80D^ z=>8O&)`4LV%xnoAnky7niA^5GvMjtuH~v%8{7|t(0yiD*ojr*IxgxHiJNLsPMV`=kzQBIi-viHLUgN)@Fd*b5_y^86x@VwCT2 zvWF}oF=+5+=&f<*t`xv*JtkoqE0A+F2YU6b;^7B?M5v~e6OR|~EWV!uYycq8-$j6i z0qp1w{i6TNkf1?A(NY_9Rq@b~U@XR-^m?Df(MZ3$ZZ$Wed!QF0Q&-YsNAd#kvBXA7 z*08rg#X#rQ%{zDRKX~9-1}0y5xz#^J*G}~q$8OI(F4rtCRJ9Fn-?e`Ka8FmWCbK1?rSkCpMESH_q3^Sx3c>xrucM>)CVu7xVoX{@6 z_Qt_LLpL-F1vHMdfHAo>55iPUaB~|QCpl`)u-bsNeq$jn!yGE9=`KofBXNp}i6xf) zvFs_F#h@JXN4T9DbsUvKnCD*m{&zn8C;!=>{Eff->p%O+Pt5$gi}&&Km&TG09zM9b z_aF{$c}m4SSFWA*EFda)^QId`zPWBC&(p8Iy7uMQyiXsYcf0rC{=NG=!As(Lc$)VB za81t#dh+ZU=kywfPd>THEAHRD`|#nzr%xU|d-}vQh1LI09zS~Z;+Z>u38$`5Q-(=l z`QEf}su?}bOO8hqV4_bWIiv}YH?9Cs3Vl?tqcnI<@~AG=@BpcG!RF>D>}$ih-TV`X0VQ6k}zvr=m07?Fg_FuwW~{V(m6whFaT{+@{}9@ zVcY2-agkH6$l>~6S!)ZhW*3&#^_*854@r25F*Vl~jNpjEv^+)kU(SvWh547f!Bz}* zCJXNcDPD*ai>LfF9@}vjZ)cYrmu*InOxn z!d>!*?~s|2mKlhjNycIb{yH8tBj5LCgREKmbWZK~#VOP}F`* zI_|7w&Qa%;ca5!MOj&h@vXAXoK5>pVD=tb1gT}tDqM6H!FUg5`mQk2Bc6?@c%9%*8 z8`Znnjgq539|3PxyDXX0Qh{p?WU*%THVGz7#8Tn#hv=?B$Q>M& z8f<3{eCq?GAkq0gkt_|<>*4bTa0xNkPT|f1Av}!_N$Z>_x(T-avr)y;#|mm^5vHLw z|2i#OOMv!7$;J#p7QVl`X2wD!VBW0zs1HB(wscakaw2D-rpZK`jR5RLI$O;Uj-sC6 zif$%fxC<``*f;fZfLy|tAY%j(XEYIt z_jy{3FB2!sLoNp?0Z#bzJ`UI7DwL$P2mwu16I$b$A!1=ID222D-h%yBJQuL5#9&IS zomDtou>u#n6v0#ZqW`Veih8w`bVB@niA#F-ne7UIcBa|a4Hu?>F$UkUMGR(^Y|~*G zg!$&$Q!}!~aUC-e{Y*!ANEzAP!HOtI&P4X#!wbvMFJ5+6afEQ*>WLrcIsd)0;CuF{Ls& zQ|Y1^osue>r=2Y5u;C4}SPtkG}fyr$7F4HQ@D|_jm3-xaC65jVmjx?%e;xP1l9?cf9fC%8MVw>@Qz= z6PVX9Sa0d`srt%c`lF#f4Vo0eGMb55yHMlHa&Roz*CzT_r>#PFJ2nFrRMB@ z?V7QIH@-c2^1|&RKls59-n{5%^hLtSS-NDqaXRpij0I3m zynq{vg)S9H>LG-7F^F^0CGT$zeOO}l}p--eqwkTvWF+3V3&r0cnBgERI!+sGyNW>vT>U7 z8@m(Md0kfU{e8643Ws;#4kk>PBYBaV(h|leCZY%ofTJSv9R%zkpeWe+184~l(DX93 zhJnsg-RTK~8E|?FWL9*XFLoT{^0c1^>?4nHG_2?ZbKxug zn%SY9q65fnvo7z#X=$qhHCZWr>Wa8C-@2>{ukj>UYo`lqbXAI{cX;Tlef(D&wFTfy zJ)d#oxIQx~{g6Qq&pscw^HIraiC0(|#aH<(UUw?BMZD0{|}r{^q6I z#kvk42q0;AP#6!X>)0m2X^YiRa4-#LgOiYhv?+O>@WxB=gmVv_bf;yWaq*Ywt$1

98;%;GrlC~FQ12{#&n&c(=Eg(Wvks^Prsk7$91^a| zd))OXP=B_qO!Syy!W?A7m$@$6^c*W?GY2aUr1BE#Q&hXEDd$R%Y#Z=pkIkrLg{su( zwM)B;u*1XIY72u~T?(PE)~NqW4ah^4(z&U?{^pzWX0n0qgYQFZ{Lc39rVaa|9suZM zn#8YRi_>Jy_tN-;B4}1A|Km6ObtM-mO2&@bYL+ii!XI$?(rb-2d~US|$~}J4f$JQV zI$hDgqg{nyhN{oV-@JD!~Ny-D_E$ZT-*gyDYJk zILTtg)VRUVw4I)&^LUC-R4_W*W#vato_Q#X#!6OdO1CqXf?fY@=~)6+FqI}iZC(#h zGIx9P%`0B=jUchQbM4-}yT)O!U%TbA$$ruQ{3ceiP8UMEZP;Q;Y@`6shHP}u)P`^E zm)4?YuBnZ#;1=#9X32WMzJk!#FJChE%U5QuUwNFGp@s!QFC^Z&`OA0j{k`A&p-}a| z|DXQ_s>Vu+tBR?(pQ@f<({(O-E{rDfQ43kgn+5{Ohe6Nb=g&LKR!1LoZ?yvWgFLtn z>7iAz2sY&(ZUyYrz>1}j-Sl5tWm4>@pL}V3kJ5Kycd=Tm#ohQAu7NG7+84_hB4Z8) zi=?xl5^LYMM)S#?n}6`bzxhYM|GPr-FTVJh*`(*M-oENRUe}(#c{pM*mee^`8`)0P-sc%cG_z}do9STWGRPrigh;bOt~Dr+JFW3AQCkd+nJr}=Z1l!u1>DHdmk z(*#O9MwZrDY{g(bg`}3=2R@`n0jQ*l12q+u88^`hOK^z#blNV6k~%sdJE$BCbrlQ! zwySfgQ03(F{#cQ;M4CXqp;j3o40#8K&V}MBenMfe8mXiUZf&+xMn|-5oGPv+${)_y zf-*wqFnn=!)Osm4!f-%&THp)p(a(Az%uTDKV|V0_X4t`3yOQB#WZ=5s*&v3&i1}o9 zl!PGS+KHr211{x)dxvm#L+|<@x%r1ys|I~g5KfuOMYf*59F8WeI&3hc9{~7T=F8sVm)r$;b!|4sP%3w#Hbf* zO>3dgbz@esaWzh3xifs7T&E=zVvJJy96&W8p2Ad^2X8%RY$@yvTFFGm)l^#2BnQc# zPJNvM;a4hD5X?5;lC1_mcC%sEG<1Yr zj+)pB30LN8R>?-l4|=ToirJv zu)pBaEBdhJ4WdORZ)_8q23a&b4x_rF?dvQIemaao#V155p&UY#g~^jNp6Ao2<^}?Y zg3-)y^vF~lF)&Gm-^Bjb4t#8@@_vLPq_9{X`=NQ|v{dqV|l2eg7cZ=q);wHypg zcL_j37z^q$ecX)cQmYCJ<$GU53!isIV@nW=`wRVBi`YE?z=Z8*&Cjx^chUb0maXue zmd^?83&Ed@;N0hmI&m+Eno6Bw%49Pwr+stQYpSs+hmT)R&1{nEzllPYD3%c*#y7}wB z@*n^4AO7C$o7eyRCtp2#_Wa4SmqjTzZ#{kV#2vt{*17_4|Ndv6eDaCs)SamyxlQwP z%bT}8dGJ8jWnlOE^;-))-3%bY=Z;tR9)9OLpBlQEmwNu<=~rKRTI-iT{pn8)2QYsd zSL2KI4UZJLYYcJk?pMzqd#J0SO2a}HpLhwq>Tv3WnwK(;nEo!QFy$=uECv9EnrpQW z9Mz<;X_H|nx@N0d$tttdInNA%1rss_V3*e?b`GK}Ssz`SIir;;@3=(egXk+`c{@Lp zV)vklQrke`TIvjVyhJ)_ARVO%Jd~?_VH|Nuad|--u6fh4?edq{2rL>qC$u#E$|R_~ z3^DkW;bbvE@+9V|i5DXAlCBQQgg?X`91eJ~zT_jB7#r<$87H=d`tfVS-vq4qki}~# z!+#D2a{yDIf(g#kgNGy$V^x_83sJ;&-m@U%;0|5oSNvNER_TVQQtM?8ssSrdQxjeq z8!5B{kNX|@?|fc32|C*5AHEA?fyKd@^8s#pLJzzpvcg#S+#M`{JwFQiA-@M`aCt_JJAmwR>nrHk&RGkpLLhf?>(JsEFh|X=5OtSh?)Y6iwI_ms458kR3ia=}{ z?mQ$cL-EJ#S)j)dN!@^R+KT^31y=zmM;Ti8sHNL6C{(w}PVF0{jVYihuG0RHXQR}~ zkQ9)hLpEzC;;Nk-mXm>(W1yho(toT-t#z)iB6oU-F#XZIX^Rcib$GuPm) zHVm>Q-UR1boawMMWhWZTesJ+ubqz4}89P})wk75frOP!uGsT$0KcU4~j0<(T4!>AG z6zz|d zyC87aqvfZzIA&*a9!(zeQi`O2! zLz+eRN`C))|Izn;@JlB5??3$X-u+Lwl|}sC>h=8PEB(j)Pd*`*Fxxw9%-9>(d0RSi z^Fb~A2X64{l}j_0e~RXB{@mxE|NQsAk1h;PpFDZ=)mKj*KYsf7(fjQ80cm=2;{b!` zAQg2DXzh)ss3mo>R3WlV8t*HnNL6X8{2H7(!#3SbFPZ=_dDF`Dq;3zxtdWJ%??!FI ze{hvFoY^a&2y%^#GE)qeX_>tiCVf_5KWYB?$Awo)&oUn|g)W4nQPBa1a1cijX+y|T zBpmR;ks=cL5K}5JyU8Ze>G@4Dxx^rb;iIlf$#ZhIGVoie38!OJ#4f6}`en>*M`t2R zi(Dm!WJiz;#SyIIfo$31a7j?ER(s_uG(!Y>`Bi9SAeVHAC&EN7@(jt+aj-LxHo(p= zWjfrpL`RqEw{%!*h^s{%Rz$g&=O?Q#5i1XXqbgT(e+Rq{c)W}U2rD}D5G8qBReA`b zad?RgQzkD;2HvBTE64j`MB6#kWK7T!A;A7L8$dbm{xdTv|xU`o(LqdYT{QA zF3hy-r#ljFYSZ@gj>5oXBb8pTs9v+D?VOI%8?Rpz24>56F01`}TTWYjppn276eb)H z6UkYl-i&4y)GveD5W}wCJPG=yA_C!5t;kD`0ZXwZ=@9BWlCSZ}$Bc-|H@)26%P1>Fz*Lp6z9l*I9dmRq;uDrF$^^k{QfO0l=#EZU zl~~zQVkUe?5n&Bw+mspTXf=b)mnQk>#$!GD3K*T#95@q46f)9PG@)tGd3aCJN32ZJ zWX^mA<+wqLt84Pja;^QyV1g}=5aZ#=nGSakU@k*OVf}A>xN3i8p#rX`m^Rq0` zY54qtn;}wEg}nwjr=>cXf{}a}fuqNmj2slnZNFxDDbiB|;^QEDWHBn3#Oh9b)JWKh zCHZV)SKp#r&U{!uDA4isI3S4zNLZk=eUYSnq~A5>j088&iZwi|(ba}@0&~&-$a$3q z{Ed`PIBnJ+4-Eza;7)egD~W7rDHL|ew5=3UPp*_WwSjs0=C#o57ry)Xq&-Cj8h6y! z@x>KraSDo5=B+K8`lY`=t9ne#chj?x4@Pfzy@1K%UUiasHGBRj(mhcrFsz!Y90#73 z`WP~#6f(tpV}Mz_gQgfjnNv_iQ=@6sf?7IcwYq~>ugx^m$kn?Kubw`A@{>RR55N3N zzxsE7|F{0@|KWdmF67rbnHi*H_MU$MTlw2nt zp9`JmQcOc74fPNwUdG8#4rT+pt_VeIV?!is%tA_PLU)ko4v>x`VqX^-h7?XqhyHr% z@tKm2kwV~`Ka6%LYc9H)^!?91{e$26?cexMe&xy6Uw`%0SG3@*Uia?br5=m@Z```_ z5LHHSLFn1DmzsIf*GXNwZkk_D_41`K+x^zhXxSJ5Tdv}~wDi8k_l-?04Z3sJ(7>G` z-@JVB^6A?*^lN#~cRu~>I}aZU`oI3l=RkauZBez=;5vkRDyP_v?+2I=|U(+OOUK>N0T^{ zP80=?;3s3~V(fxVA?#D<%E!D0mG_KpN95G^Z{qeqzdqo;V9XT!k0e&Yt~0` zIF+)DGbKv9>}5a{s<9E0%RvYRS6Ms+akfaIypuB1-p<0*t%e78rP?YP7ySpGzf3P_ zNV_8)VD2k(UHJhF(ox-Ke>=8c;cz$v9WuE{*TMkuAI!l@hl^R_47L^qj2H4CepDR` z%0wZ~V=}1Aj``y`QNnQCf3OFzXeqi(R9D>p4j28eTQ1~2eM>|NSoiP^_XxUJtgC4v zaqDhlYU;OJ*SdZba=ovDB&f^xCm*L8>KpZ7&SGJ6D&7(oOZSG%()Gfd< z)O=?lwdSgIGA(;`9ks{hoe@A1(_4zh*`oNKGvh#)+LGArv8%9NkODvzC9p!D6&tf) z>ri*6+@)uocC{z^p(&0|npVA?NqTRwa0+icA{EC5E}TFr3p1C*syWWQv?b!1sSKU6 zjDmFCAfEBTW|b!yalKeY}f?TtIFS1C^ECWM(eX_I^*~Ktv~$-RBZ0 zhX`f2+Zv~I)^+c3yj{`4wlO{_)|w~0ASWql3<5{WYMcy*^0vb$feTj(mr+f+Bvmfs zFmb4JtxKC3D^qA$t^giVrr{n`0Ug18U*FKqUaoRWr|O`HsCk_wL?&ga8n4~ z0gR2^v=bkNvuMXBPP0$;i#P%7riw|B=u-x%r=5I+{KrYO7+&CthIJfoZ@e0(7??ak`9UU;62`5EsHzV^+tr%zwMc=~sK z_lI9UdiGEM*Zr|S zpi#{{ED>*;G!LC-CQw|(#TYo=yn6A2&p-W>zxSgb{*Ax#_VtUezWl;O((Bjnp1t(0 zE?T@R^nd>NsVQ_Lfp^bdG&Z<>mu_9XzGwCSm8Yz_GwSA@>vs`Bs{W|aTuXQ8r}=kJ zXI0|%RlU`R_oVh_0R50_0$)FUW=LUy7H6>})*Zmo0ley7k=Nb~&~>ex9O{}LqSh|u zq1lFrWX}g{ROa4EUM)Y4qmz7NJrdQHnjV&$I;s;ja?7cPG$|0=C1JJhXs$jduO6N- zSFy6bA1q2$X=lOu8=FLWj!OH$GL!Qnok~iOvy7p>c+|?^>7b4$a)mztoD1W>T=@>oAsQAUpr=AR&HO2xH?I7Wj$gLJ4K+yfCgA+#z9(y*wZxeDItEzJS#0!=rVMCX>#~*9E zx|7fAWnykHfa_4}*=vQ&$%)%Ed3b1AcwEGlo}q_D);S}PG?laZ4hDQ%BVG{^7mc5};94HU zTYjT;HBeE$XiVo>A6*_`%M#L=6w!m;(8D#ZZZ_UDZH59Wo$V*yE`RHs853|!l$w60o#?Q!* z22+w>e}V*Zh|6HVldc|_6;|C)l?qd-}yt~V$?RFxjLNR~H=h@BgpR!n%2 za?rA`Pl9&jLt%@`{47n&u$>wl!)pr^?TLW#vB$w`OTdLcyzM5Tv{c|!4Uu~Q$`32H zhaQAeJ_hM=gVazyA{;eXIIMH$Q1l=nxYUwJaICM8yl#KkjtWvQZL{@}Frtna(LxT2 zAy1c^PxVD6A4F-Mv8FuahkA~iX@)(bNwwd%5NRh}Dj(D|FwO_b?kan&O19O)a8z6E z4F|T`p(TC_lghq&$oW!;0F{+JH5cc(L2B%&S+CJspeM!q>Z;A^43MfR6ga`9G};P6OZfh)M~{T1$%L@mQ`4=? zN&u|bbTg%(Q$)(FU;p@P4_)*8N@BY;)s!@)TKgoaQGsCnb}OxN=32Kd1WfFh!h|0d z7=e1ufW2uGFUPti#=q_Lb8l~Wi(0irS(q`0XSodI_F2!z)YiZEh@|3o&&a5Oe24q$ zm!(Tk<%0xam+pv&nq(&FQeCS~!(8{$k;AIfOu67dLz(q+Ux=EE&pGehyf1ja?Ky<+ zZauvFqu>AC-~U^`_3q7!ufF)%lc$f>!Y5B(KJR6{W~knJ=;z&g4=C`hS8BX-G3S;y zyNUbd89%Nvwvz@Ng&BBH_R}B(?&=aYQbDllxl>$F``j|u9x$PDKKI>uA zUiI~jyMsOY^VN%&76O^~`RqHNzIk=~<@4U;P-R|ahllWU5?+|*(~87gm5}t5>^5xZ z>A0ajR7Z+WDk@_>=XefO%}{pDMK0xenNS5!U zKje-W!x3O&O*Fecp&Q7OiE|+qmJqGAEHV7$4%7ZBNa9PEwJa$gVc=Z@Jp3Um?+_ej z=%%BQuW${A!dYdEMKkRj=xB!^z_l*<_nM_tC2dNXMN-kWK_q2GFK#g9g?gAJ7IrGH z%&f3O&C!Kw;IPqW)GmdVRddC^^m(Lu7UEgpwL0{bLBQW)-%{NFEthb+;*l$y%XFwxr_uJ{Ob4C zrB^UbEEONkJtb#))8IIxP#pwpai0R!h@!mpcFgUfYP0@5*}&5eHWnzb%?{}S#;`U7 zXxl-~mr52a3U@KXa4EMwN1NYke;bHIFLpGxKu*uJ3V^@xc8#(JqU$fZMvf4@Yn78H z>ou=E$IG0#U^J}N@&3jdaIQD!V7hRJU__KRuhlh)@=fsXlgE+Ayxu(FV^1j$Qi!L* zq?i+;#Y-Y+Qv6I^il@X+dp*K}s>~_Jtd+12njZRMfP&(t)tbT+DCK&wm1xk-)f~{v ztg#95B#B_8IaU^yI~&G?v%a#2L+BR2B^0>$pKJCnQG(2TZq5GaCs9x+gk>8A!2-DW z%rnw69a*9~Umbo*jPX4#DajbPdQe;enIB4H3+56oyYVw|B#`~kgt!8F$(X@PT>ZaH zo!M_(NtT~UCdp(nlgXVDCDkHTb#--Dch?N2XNL{g^I*e&s^1J4@WZ?q7}$Vq*aP&e zHMLGjq9pE2GLyM*@%R1CrK&HPnfJzW_Be4OZrr#jB~bv49t4u|mPun$UzX>Gwn;ck zIgzPiU?Ne&^FSiqsNHBw@$)T3QmG{fA9kd!6cO(ze<=VUI0`?3Z?cuCfU-)E1t!TL zG%5}Yo~Lvi1S-%Xq=?&Y`XL#2Mz?@PH<~Z{gG7=aIOZ&{i(gLVTWT9L8qK{gd$2NP z6`%+8s}w(40AfK-aS^x)0y`rtpY6eV3HO}9v;k90f~|`+K&Q8`FbUS>@x(m&EvF#F z&3=|dEQI)sy_t#7)T!g(#%!f;l&$~LCJE`Nn_(oV_)7!Q0qWy)N+7XHV=z(>$(kU` zl9GfFr_RD&7$ttXy*s-YxV`VvGIuZCY8NT%X(Dqf0zoSfgNNAUmIPfQnOeX00kzb} zA;PxSLINaakYux$936$V_)yN*3`T|NXnSXKnTn;|bKI z)*3t1aMz%lx`feTAP;J8D(Zzy{I^&^(`zrUUS18jjjQbMd7U@+b9s~j4}rX#0YR?`q3IIb zWOIQX_hA@}_z33%>`+-6JwYT7ey;#J>FOL^T;s zKL6~~Kl+1Tv-s)a@_amIv!3hmcs62-26m&pxm)5^%H6{G5Ays7L+YKVOD#KrC|i{EZ+te9%&V*z$2awTOrr~G^!hv zXju|8qv{%UObE?Hs4tuM*E9c&`U6L$qGEhfOja*QJpox?Iy`Gg6CCzJiev_m7l_0p z6Raxkv>8b-F}TwE1Q`;_FYqNU3=7Lf5pcf`niPr`kOzrKdJ0NSP?F%`@F`4_O8b>= zU_Jm4o64{R6>Z6!JirRbqA&GiRqDbsj0npVhzm9ons(k44IYXsOO8mpka1Chm0&Y< zxAxx(jZ>2zz(Wv?K?j0lLb3=6-X)Of2jHxJlwe%GOb)K`0i%$hCAhwW_;Rvy^XSX(GG?fM9Ii)Km;r zk1SZHrWGv8Z)#UXzJsPZ>UeAPZuh5P40~t+KMmHg50azMZL|kgG7L0kA z5yqjIT;)?vz%a`XGfkf0N9irIu#`Cq+>;aUaTdVv!9!;lcrO*avjaN)gKN_vjeeU3 zqw~d+dRw0#c#;qtEYuTM8%j&0#Q04xEKqrZ9O}phqQ&9jE1C6ay9|=e&5RI?7pi-< zdSKeQ(RtzMnR{}gUZpvRkPylO7VSVAWv88!-U5Lou{(*M{Z7AWi-bMhZbNR!(=yqmmG1B z;!B^Lev=sODJ0qaI7foASI!9PRFKh=!y42n-xQMout>iNB*h^0L}c;R{6iv_R)W@e z(UeFH8<=z00xXwSfC7+(wwfd+A^$GkXSbl@jkljupI=jfM$fCtpB90WkDwxbZtA%VG>gMOZvij=k%g z0Hx)kSxggw#e+}?BQ2OPWCShOV2x+V@*!l%U!aLvG?D^BHPdqXootd!gtCZG!8>K6 zs&O*V>KweLtCMivmzR6nz|pnWVW!c7%yTfgD;YFdp@f-5u@a`ara0=# zoe^@eUCVR8443-3`jLcOC&$TSoq|WC2lbu`x|BdmRXP-j5D<)vMjkS(%&9TBx*U(k z%oghhbHh|OKv6}Ky295I6+}>bfpfA`mWkvVgwogum01l=+-R#tG&FUcS1~C>x$%a8 zI=Wq2%A|-+@h-c8a(gt*TKfcBEJFPy4n~p$t4yOK-#ZqZ^I$9U%3zt!2yD680;0v6 znzxdg|eE0IzoBuuECe~eUgZO5)(p&2=$=zPLc3pdGt=prj&%6#V zadDk?dpw!2a)n2)kODjzy|!1t%0hUJmfnUZ8QL9MIq%I~USkV=9(WxS0k8JDi-Q5r zB;bY>{rn;Teb%|f@)&OkAxgq-?RRk%BjP0qI21(1a(ZtK^K=9D(OO~Z$G5Fvj%`|* zuRmik{Q@)Vb0fCloXsF)#FM?iS-j^#h(&frBJN*hXMVVHfJApkj6GZHu73H&FT0)g z#n~}0PG(#FZ1*|2UsytU^p~&()vIejT}5$rC3A-RG__p1S`{PZZ4A7-7rUYs!>y~~ zh+!8w6L$9GHZD#1_O{*erU5&I?(aV(*1tGE<-uPT{iBiLDdV48FTCnU3 zo5B{n?V-ZU=8(6BuJAl^=`$sE_8L0w0Yf%JTb!~smfJX*eQebqtOgz7>n&+x&@FHx z4h`8HODZC6saB|P)fi}4FNu6SUL~_-kJ@m0fdT32r}w` z#{yyBK0|LHQ4N1y23wIT+^Q57@MKy9DWu9Pz~BP_esbm96}gJj?zwD5eCFm?fb}Ob zNtWTXl|ZIs{ZgtaAtr3wTVUj3ON@YwKBNK83|;soiNoDiD8jjAHMs`^0$8yhe9|T< zf;3J^!J;=cL|#Oxs=%>-lvUC2MH}Kb*Ptj0L=aOGZS&DWh3}MG`c;_m9NM8%5;#vM z1!q>Hg8pAnVuGOvwWJXmD3o4B;*kPzxq_sQ8nu=-751BU%Q+b!0+}cwMn{>$KGlAN zJ`k30;d~o^(bo`o6VlsF>Knj-M#t=w z%xuvGHHN*}y@xsyn|?LWp$rgoa%^d_v|=aW1_{hE5gb5;{iWjNRzE(JJEwR|qkmi< zWFZkHi4dAE%Evnq^_UC$hA?y%gdn$c6VYj+AOgu4B;+U|C!iqBr`^11><sg_k+lF7pDqN9x2ob*!Q$jKK++UF&Qpn#$pDdMIIVLE20^^%cY5VarGsoLm)Fe2&F)(PunLtvaJ zfnW^NX`{9R`*dPM0~~-0UJRDB8eAbvgo`TKK~eit=4wPaK`^y!M~>W(s`YFVeqnwQ zt)zz#xexGE&rCBU)9opB2!~SoO5sFZES#~b{R=;QDKVNa`X#tl{Y*ncipB#SMAZ;L zNU9MAg5TERn%XJ=Yhn-`0+A?+z%x)${h$P9K7gFc2+(4RkN;q|xFaE*3-AyGqy~yp z{XjAVHVf^*n<|BLLVwA3owb!5^LgC>^~|b}seWaEf@;3dOFaDKN1fz=WZQ)WsI zijYI~4R&5xRX#;?^@V=k*>j-JiGzFsP77crvZu0zWIHtm!Rsu#aLeFcg94rQwlE6^ z+_=N(38z9#Lkci8ZYR|Q%U{^BGfDV-*|5B5zDlcR- zsMa;_xy2zH0bLLK>svgud(*;7krM7fyDlKWLBAT8h~dGpkzi38jT#u*uI-r@$F@5i zPFT5vkQUp3T%sD4@px%i5j+H;b)X|_zB%Jt6}mK2Vr}IiMB-g>ZlJ78RVQ5Gm6>s; zp(!+0+O*r3SC>o_;DU#@v&*aD&CcCuG(LEH^2uk9{^C#n{2#yi?$zPxTE82WFx5Zi z<+UB(>yE!BWL;&}2$3$IiE9FSmq8!cXNG%A>7*h z%-acINxVL}zWUz0>2#OzHob1&jaC;IS66$y*%`*9 z6P$B%3?*2?lfH*H{@yLkZriJ90WW3;Hs6{pv?lGjxviy5$0^SG&#uqUAqSb_2G{$2 z=hWurhgX-}yx3gtqeh6DO{VM!G-baa-ZY~HA_H9x=NN@8E^e206&LItqk2^@HNM0Z znuqp6N^9;~Q1;~-j_I@|w1Sz~XOL!Fw0W4E29ko<4Hw#3F0E(_#N=s=XvEfqAA=gz zV}b`sv;?kfe^@n0J=eff1OtsHlqnK`Z7&R5PNl^&HO@5^kTt;K`89d-I4oxSu-lRx z%1fkJ=|HhD5LXfXpCIAOB+rE^(jW_N_?Lci^$Ad=6wC%(Xd}g{)}G64F3WQfc0;q5 zbpnwb;7xf7F*A{l!JBy&@r!?17o%b*7=QE1KFZXeWylYaAt}ly#w_r_ z!FU6-icn2Pny(Zl37@E?kmHSCxb>DT2bJyAsyni3Iaig3!sSG}wS&WKi%V-sx6~)u zxgSoIm5i7z{uo7Mqhi!`*yCFwq(pKo#tAqC5)8!vV8E_kv}vtcm?my$c( zTF}U|03{)^3UfsgZ8CkLgvbd6h*Y9)YTzxM(8I~3LnlFgu}{NgZx`IK=Y7q5Gas%D z^cDFrDg>9^0g(X{DNYmBMrol+1EV1R(!00>MgVZ`--F(*c-OJv@?Zo~;2fIyAW0GV zZbueA@F6x;gOo+7dKrFGLM>BJiVVwys<=$yfIg!p*>S|>eChncY+)iqv<6OXgnh}y ztP52;2K}KH3#I$2-WlZ<3gf#JlzHtd==e$lL>fX?Qx8T>Mx1yWHo%Eb$(?VuZ%?lkvzN2j9-F8iv5QOhYv!Pia-d# zo<-PEOXw24Dh9cjF+AtNv_-_EmjW*g!L$qSa!wlQ>F^bni6n7G#ZY7WO}gE0cnn<) zF=Q$R0U$bSY8)vCO9|E~$+i9uHA1=LRxBcGez?m=*p=}B%8oM%Zy7Vl19c%CfSN9_ zQG4hk+y^QxdS)qYZs$vBg}c(l?h=zLyOp%6Xl{RB+)a7*-taF7j-k_W`H=dr(ad)z-^Y<^p0&!hrEqy)LrR}=dNLfH86kq=YRH} z|I2@!POn$GeKw8ZsRsga!Ue_=wAR@K9WCD8aL3@b)t+-Iy|aQ}VqYAhVF)$wqN8A% z-%$P@PP?5|rOLZ?5ebTGE%L$(kNT;`DfHz5j<_e!&YIFHTQcy~DVOI{;Th#x?ic&wxUjGDDrK(fRHD zVP~bc%nKHf@xnrHt+nr(+YStlbNq;Pj2o+ZG)O@=(| z%O*d_q_xoJ9SwK4ZW1(hUnbrYhO%fm++A5k5K9XTlTa3;0(wRCYh$C^OUEA85FZ|y zZ_TMX$QZdcgymFQU_yn_7S~!lpST{)9Ah_l)`xIaZdQc?~8wKaQzN)S?RAWc^AtoRNW@(GY!pXG*Fw+_~MKSnmIZzq0TF6{VS084vBv-3W zS@S(oG~dD#s;yA^b8a#JEFt!aqtf=MEbgj{Q7XggHEq@63*Ty2Hl#=i@}0{FCRhX= zCrdUsw1%dN3|ZO|esS)f9@g^&L=6k!k*^R>X3i{C^-?O3K`Bt62r2225=lA`Vnv8B zUXFDyN{Nv*17g`s7rE2w7v2u?PeR zkmZ3Ap-dr$y^>yJCJE#gEcq>dM4VHfiiH|QCYo4HsBnJ*H5*y^180#O2@VNdzfFa3 zW?!4ycuZ+%*rjU|fRQZ$PK_aqhnV~|nd%`%USZ9!%ys06(x58T0G6Qy1p!4VsPtb- z;6U~jB*>|bT#WP#naBn!=gfdO2B3q-o#InNp8JSEunGebH}eUZnA3zEe5UY>&(n|Ak5@U5zj30HLh4_>Fc(#! zW^xX57(=YV{kkL%8$fl@Mi~R=C96UvsL+vCBI5;+)+P(wO(HU+V|6S?Lxn(#XfiA} zcd@rYnL{L{C|Qk$XkRtDgnZ0Eaw&mM95-v_{s2-f(G;+Di-97kv3r|YIR!+4mZY?4&qxeNEy+Ac6}pJwOmI%-6j?xkNV2#VvQ98&m$oGqK0;KyVO$ACVqoU;aRy21+q%XGXosQdmk1NnC*7LLUQuew;1PYQ5D+N`_%V%t!Q+rQJ3} zfe$g>5Z>#Rp#$g6yg27K+zGW=`!KuIMcL|GbEI)=r=bu!BsK;>(fm^=5`2&|)$}%0 zHk(sLrAQ?vW2#)rZIV@xVkLm)ky*hL=IZ~5!xSD|P*!+V<)Iq1Fz;fHO^ISifL6>j zc$>X%^o5-s=jaSmK}aPJydVBAZvw#oxx*RRcrS7#&m}-ED#-|hh!7RahJ$FbSf5~z?wElAkv2QqFe?x4%gc*SZ~d?S-M{(joA3Vb z@BZFfwTo?5jdRBUW^roVTbQ4}B!K35QnbWAIXrBzy}3v^IoA>`VxZ zJrTn40$s(KnUdTQWt`}aT4<34G*E#C5L$1wGahq?!|kK#wKNz;BY4%@+tKAktJCgm zt|Lze5BTl?06+jqL_t)T!p&~1um8oL|LOkY{nNuY?E86id~|Vj#p-`-kS8E`>K7|! zsKgdJBr+>;b~<2Su(rOj|KuqqesML#P*5xPM_g=Wot4$qE~?ft?(Uc8SGv7EJrd&+ z_Olya4j2_N!aylA*8DIs@D&r6C~K{^hAmteC3bBF!qVt8&p2z7Xyh1^AwPg9;J4C4?C8`+ zrvJ%7xoSHANlnUFO#-PE(B#-KY(k{jihwC7zbr?2K7zEF2r(#AC%IJQFSrHnzhr6l zGGN3dfCg42r6S}}IJGS&SS#lf81hWL04N`1HU;bz9%?HL$)k^?EGH5vD?$gPrwu1b zFfZi<9%v6D6*6gvg|-Y*Z|35}DU_0NLJdeYVhBc+g3k~CDXKC6V7mBqK?Yk?5rH59 z65>Qm9Z02o$W)3#F+0?FW}gc*Rq17+<;-$mNE(SbFUTgzI+(PKl!1|e5l9GokPP`% zTng+5t)c|DXgKi(iOw=`nZluf6CISuY#-HvPfU@L3E+oM$!bWZlGFo4bV0J2mlfQdT9_x5h?vX;Uq8U&>9I*jTCYGhRm8s z+(knzWTeNJ2B_q%Ne2c16Y-_vGxW29+>xhJSwX}y0Asd$YK*LV33j;(gOKaf2L?r6e}f38|7mmi?G(3<3X}0 zgKwlLILDAwbO4|h>91PGlo3X39Yl{9=x$drjFsIg3zLm2Ki5@V4tA%p7# z>_X{Con}Q2tkAl*#@XJa|Rn8S~FB@{7G;YjvAClWHMu!pcA20ZJJS< z_rFvm^+{2?6a+lRK1nR@k0HQ3&x0Hu;Y?{gKpmThNO+o{BnJy56 z|I(#a@+VTcA{FfcBt4X76fZz4w;l%;$X7aVRYjuYOSb00t)OTaLBLm`1ZL!Ab);hZ z2R)J!5^jVjQePQZh(U_rj>9jBw4U&5J)Gi}dfU9})I9Ucl}i#dSP$&PT=no1ILTFh zl5D%w08PO%982Sl-IksAgdr$WYk;lVBQ2y}4k465u|x~Lu>-G~1DdU(A|V(?<=#X{ z_tKj1FQ5_+DCNU*gy>FUK3vv_usMl|KVX!@!<$K=y}9Qh)cZf z;{nP%Hjn>+HvP%sDh!_w&e&EG^a!7qQt^gkM16gKcl7Q}ufM*#_4q&ihkyUW>zAjO zgZ{=gK*p?t!2!2g%tOOVmxc4~MP5k7N*2ZdORXJ87z=G40=r;ee1 zVKicBI(Nr;Z`B=lhCF^Sn_P{rnLgjy*n^i;LSzJf=Gt@?67&*KGaipZ?hwUwlD?JcNwN?H zHFS+qh}A*FBO3^EbF2I4@&4}K6K49;W1&y>1?5d`DjHj1aoF(kQe$&@pAF5~uN*+* zl>*zh+b(``NP+wZqv3G4x_W+kG8|k`5*cw92YR^2%ZtOn4FnXyI^HxyZCoFRwJUiJ zS{s)dU`k*No|N}VTrtnUV|p`fqIwNZqqdDOPSYo8IbM}(seJO%68Hr;mXq?;M$AyY zSXg7~aO(UYTn7|i+O5t-*oX~;I_Js=eX(oL9-3W4lvGiPf5ZtuWJnuDumR1NlmgCF zFH%k{6gANkV;bu{{2>M|tzlypk>CXM0M>SBR*0$7LKS2KPWxz(7U8%rAlodrHQw=_^B-$fs!d8d=m?4m+hCdCu+ znf{@eR$#+cZNzn4+ot)?A$m_EHdLEyq)P!Ht7ar&@9IBXSvgL^5`K~aLmaM}qt!z{ zb84>twX{I^V8$>esdhc`V~<)i$Wp1*(J9rfWYH7^FR>GT8qChUXVhUD@gj>}b8zpQ zJF!egaqR}6hJmyk=5P{>PzkbJ$?5>WDIIQ;A`cFR7f4%+kr~gO^NLcT!Y3tgkv_;k zScS9KS3;ckOwp1-zWzY5K_(~ECMkbL9?9_1!zwvN7F-C2ELYq($0VBhq}hwR^S6`h*QMcg=d^nBANnA;880vfk4a-@0KT`b@pHR(J1*x#Eb*9#L`tm zSe6M265T55KvAj?0mddAg&rBG*fb{~ZklL;6BUFSXJK|&7@-t7%o`X_dEZ|iJSb)r zIV1W9M;eiA@vlPFtV)z~3Sx26ZCfH$UF5bB1bdCWl3>L;69d()47o@UP_YC!5;-QT za3RcO7e#6}{E-6~m6!|Z1E_XO#={Y>LXxxwS^?)&@rNad_0Rz_(GE+Qc@eho)U51D z2v-_WwM5b=p@=;&5~^w#WgBrrBpSPTvxh@2H~<5^p}F{=Nh&TFaYrDh&6(BkJ^-o6 zihl;RuCR6qts=1#7L2A9{R>q>Jh; zQaC(08I8wlOv)0N+O%j#Dno_pg2NQrw_d2QSvm19EohDy)-@u8-wuul)?o~pK+X*j z8p|Gy-RTZu+4l1>9j*j;$IU>rqjzJXpY3~i+?2pMFNg(t9=k9G<(>-zgh(1;xOfK1 zdr#H&WVA}K-Dcm-m5%S38VrDXKbyWge7pO2f3>&%^G~1u_RC-Y_22wYo)=~v{mnfa z@v!gkGBZZ5PM?lhn$lffoRH14s~0Eiiql0dcmDMig*q_H45d;Ec=G>Q-eakYe-QBY!X=`KsW;|hco{hEM`Q@P9TH4>;fo@hh zZLY69+uy$$3{FoD*~EV~8eN>9Kl=O^zxw4bAb1?e%*NC0FpEFDo^j)j)i03Ji;sc0TqhC4=O929jKlG%!O) za~0C{7Z$^YgDWVhAOOBaAE^3bp^1>JoF+#SxVFJpN!g@2C98=^FgX_bJwhr)%FUs$ zokR+W19%ehKvW=DU1Tt>Aj7cLDKD{tQhb(pPYsa50g1Xrc@1J=b1H%Mx26Awk8&*82WS|Fu!Z0saex(6AC1fH><;O~VAPZ=gY#LH7b=!emMC6V^ zOh14nf^G+nbqnx8mKq7P_y^dq^=gT+co0Dlm9;04sboh0p!RuLI-fjqmK+tSBb5i0 z5b_{7Mr5Gm{peUMFBHKR6p97OAjqL$%1V#)Aq#$-9m#TFH#VT>vwLtQk$#gvlDSdM zUNWVTHL7@P?&bQ=f%7$wTbnW_%i5{@;Edz`K;{*Tp4h(Me2(094?ssM)K_9!7^kJ; zCY1zSpYrf5e{$_XabkRti%kGTe#AvaTlUa4FZmRvf-UClu$#e$DN++?Xq+l)j5AxR z0GY1it_eSJzi=pK#AXdcZiHjpWVr1to3TTtQKMsGe}HA9dISe#g)ac`f@vAVzQcf> z6d@>ulOJaUF$m9F!-Q-;AV@qzil6#22(^d-G_#0=`rc5=KP9}TY%KyxbfG3-BV!7% zD^mjHFzq-Gp2wyQHT=2u1 z+H$!AXtZ;aDOZ-DMXCm9^w8&**#!)Q%L0X{CFrtAaY!6Vu>@fOCn+opaK4$sLZv>I zW&2V)HV5w$aH%8#mmjZ0Vf|n?0{~L^aH7ltGZ?wA5<$!QuX}xGXO+s;?Ag-_xgo{fu)K&Ue9^VR?#^>EW zcyy9M!sQ2Ql#xfM6SU`Y4m=BmXMihxuGI~noI#3G@*>$5QifccuhtY|ikM8id~$x? zf4m=Ca_@a{Rc!&NZJDr+p!kn05mZJ-2q-hbbO^KQ1c=%t8&o>=j}LXN7(lA!Aa?n5 zoXr3`rpSvUaqk- z&dSR4m=!*w_1-GCVhNF1e6lvWT57K{0Ql;gZ(kps-Lm^76Cb=-jpZ|_20|yJMW4%U zUGp-u`JCv@-|5`VuD6)f zpSyp3aB$h}+^=@uy?gg`XPtGW-FBO$Nz=)2eZAjmb*?VYd5;;x4rmy6+732WmZu~9 znInA!302RYKIwLQ+%%xGVKEL5MNE0P15eofB_XzJK(W<8og6{92%C6#|tW^r{df0SPN)ctCmDvN+CKQ zXU25wY|7X%OPe316QS^xR$#v9v|bw#=QaV5SHaj<%#%X4A|pM@ga(S~7^||ag}SLv zinLo?SU|LyST|_7rZtO3@IhhSLToG6qOJ6h5(p_S0w9H0OUb?XrB>V?CNvG-6f4lh zND7b}0#jB*(q*B@F#v4KWT&bnJgN3gs;($EX#+`tnG0v0P_jg%BjL(9LypKu5P_Dt zE|eIsl7lGHQ=1?|pm@)?2t#Ty1WfZV0ze6l|K>|aX?B1lk%9_G#>CKwi8RXaZ*5B> zo<>;GBBPK%iZT>iAPV(;ts*F3DPReqr*@=OSRd+3f|X%d|AYmAAUAqNpco|6KRuyE zV=E2edclTTc@Lxz2s$ppfb^C;P2wU91r?--eCUez@*+NoKA34x=t7zt2T6TSM6InqcEU<4IkBD`3GF$>5jqQ?&&U zC+xoO)3wk|K|};z9&K}^EC#ATmex*tqios0|Wsl0G%WG9!mL*WB*R`sG z#Fu~tnpI?2Q==Yrc*d6?lF*QIqG>P#5bBi+s*QlChOo;CQzuM8os|_imW^~Z6$xN9 z5%vJi%(WV8QZhRLB4gl7Gt#4|BuY$>)LAiZR0RvL&A11-E#I?Ipd3oSHTjOSao8eu z877T0)ROlM7{s&rmO+N$f)N~7b}F7b#bgTnMpf+EV<$y4LUsEdx)v%~m=TdwYDXdh zlLjX`<-=EyS}et_6qd+Q$VfstJJw_A12GOOZCT3f8Zbmfh5$jXr^8%SA^>Fw;>3sO zVJa{!$f=}pIN#|I%f(AXrjFcTo(glm;2Det0ueaS9Rgb%ZKE;ar!QhALT_0Tt0PMIw>vXRC$9IDitbC;B1`;!7rCfKVg-Xk3AYS+a%a zlQAHZ8;mh!6$Hh=s6{C#$T3Y^N=pb_n3f7sp7XGPvHGVzr50kHxR5Wcz&|b&gZwHD zO(Qv)kVc`fP1A6amUBmjR*6Og7vzGH6cvh7!L7K2u?7HcR}Lsbe~cG!JOFADnDx;Y zC=qM8;j6?S9}4w=l}}HkF=X0pn3(jWMH+IG>PgDE)zb9R_^gkt@+F{!x#9Tw?eX!8 zXHPMw_(I4+1l}--=o_*`8#*Kc_Nb~nFg_aui}VS{xWpUY@M!Mjn#p{$3Yn0klpO2( zBsIPgzOYm8iB2cnC14Lw;(ppRjsgKOmjVVZnOUcdw1PL%!A5X0K)Pq4(f#cdi)JW1 zVQYPFHZZ3SLHK4HH+L59?pC{O!@uGl{Bz?ck01TnZ~y48|K@+3u~Z#vXDdwB(zA~a zZ)+6Sw8w5cofQ^oU0j@9&u&i#7oE-;D}_39UA79{*<44o*OP1Z%wJt)>rV%Y_zF4- z9&k*zqF@9z6+%(1iN&{nX%w#tel z9%tR#-t?*3>*2TGU$OXx=LtU8?r(0a?W}Kd+vtYvZf5LT!~+KFZ0LD29dEAhGc>1u#=Kcf3(@! zh+;mtjCOf=V75FRUk#}Oh+eP1!46Bqt)rtuo(Hh2z!0NZfTxXVCR$`!yNb6V>CP;w z)gkRTjyWh8j)xc+=>Q>LGijQQ4aVN-{XqjTE2dAP?Uk~8wBLkWq<06P+DD%+Bg zG+$N-^XU#b6R6CJD0#nkQ59s4x@y#-;E#9ay6R%ZLNDPIy+w8~3Y2`ikEBY1=%fq= zB!=sFG6z^*VnYv@rAQX|4tz62#xI(;e4v_$XvnmD1t21kEVBwVBSk6n25kpey3&QbR6iVl+U_&_1SK?-L5O_Nm zY#;@r84h47m_|nhUv(r~syhylucgR&;S`kg7p!JYTQ5$RG56Y8-RFh0y ze-#kF0nJxwK^FM~o3i2y=u!wGkel0Nn#@IsRO_*V)JY6w%;eI_MjvL&Yh^hjo&=$p zoQX!TK%%slx~e#rDI;WHcoj;Z%$EWwyX0%}RwbTRpdbJ>C5<6TmK}+Cj~Cmt9vMUy zwkC3|#tr%w%TV<_)+iH(pa?JlGA>OsusU5Hce`2jNbt|DpNah05{_6b!KP9XkCF#i zR6yI!;dCsU!Q~+UsG`qNTcG7VJU1;~QO2wWuf_@Y0O60FTbb=qr1W+XrwxzzID5%} zhkZ5(CKO?a6Den5Ud|N|^iU}UskL z0LSW(FlRB_xQvihjuEj6#FAiS#tsr#$^^?y*%+WQLCpY**ovqwW-q(Ii`0NYf52q? z5(-~}y1QlWgg`U4$v3EFi#zb{S1oHqR&>Th{5(nkJ1oL-FyJZ;(-A|$IMr8zO^ol1 zzu|^8G*~JaU_&dV0Lcl5->{;LNs1O^nlm*W;c%9JvLg>T)&RmRT=L>4p8eoGM(jMo zbf~kIDj_JSX;15Q>@{$HLL%Ho{4p<;3DI&2(H#Cj21n7VJ&<8=#UvST$Kx%lkRO#r zt64;k3;_#X5kqOC_HdbT8)sN1I+n=l9t0dYXl(hztk7>-ZunMm$Grs#|fw(awJ0$M9pHCIYCpL=1Cc=k`hQ#Ao^%Sdjl^Tq;L@cP>Y0U zm0$Hz-HUX+5Nna(=ZD(Zfh|3j0bQ60ra?|JMd80T2tC%8l=_k-s6eW6Cm+`Fj3&TA zC58k*!$~^rkc#L?Imw!^WWu8Q0+KnxVT#IH7&g&-S&2+UXF`*zz#@N?TE$ide$ZLhR_MkH&CLpeKia=Ti9bCWB89 zvE#lq?pXJK4H6s$jSk_u2hqYJ!L$T0nVrXT*^6eQ-+TVi2X`+I4o@yvmvp@_W)UJw zYoLw>dVkx&Cmo%hvRh84*B=Z=hbQMO(dn#QGZX)4=aZoC^13lxCtilRpVgHWs2vP0 zm)b`=J3BOVl(yFEvJi%-ceT5U3-U(t+}+99Nx$1YK0V>}1H5l*ZLP}<(95f zSzqgZ{LzQI+Z)4+0nZd~_WOvKGK`3rCO_wqh-=@RHooCC0?bF;4cIAl#-5=(_HoNd z;P&o^~$CT?r9{i$Oun=&>C$Gkh<_<$zM$}Se*_-+C23Ct$v=jVpz z?yBc(SllQEu1e`7FYoRz&Q52OF}mPAY>0({#b`40!2(r=-qVcVo31U&?k4Ssu_9hS zwx?-2cvj<+2?H&fOPQrcs*Tt!QIv8R3GLXxfFd@*pPCB^W>AGXW||LY*5}+t$SDU^ z07fBf88}*WJgHAof&kW?7ttFgQWF~=zCxAKq_(sb5%$$8 zC<%%o12y16y3xLZrL1Jj0hJ!=h6o^qc?GKs>=zOw13;{)=&%1)m16iXmhtbQlC(>t zEK;R`FCd*V7riK0(-nqw_>kU& zpxTF)Q)Lx}d3KG|c_<(7a?vCS#K94838y#`IR}s2#$<>oQ$4IGL}WpEa^X=fCBv?o z9Al=eIc74}NYfr!9DGsTc^L_^sm7oiZZ%as8dF#;-e=;nA0E_l<&XQr#z_K1szj@r z9X8-AwL**qV16edUpPl5V-{xgCBBX^YJze+u(TT-r|c}SO)xdd(iq85k?0(Y(-WqJ2fjrkYhffsjyRW3)CPCHoeT*y zmlRgez5z9S66WaZh+4rzIr_+eTwtXzaJa$YiXDi>2XGnw1pN^)Uu6l3}8XmphLz zDUG(2{GdLrI4`SH?bIx}K)n6~*+VXeV;r7%bKw^|;`v4$?*fql0B^YjlY|Q{tf`~e zNlYl>jJ}E#skM%zhMR0N<|#xdd;w`dSROXSzC(QFHU^Y2c?4xKL=;?;5l9hTBYNAT zT&Ys|eV&eq3Pq5nZEv-rNWdfDVRjUMjrb46kZdt?=i}cr7Zc@w;gl;7; zM9m@OM0XmH_6I;RG=(Y`^0_9<1}80|mO#-;2rY-&07FPt%SV-^h(%RVQk9}j+;&lH zOSjVkBr0R2tgC>TX0ktG1f~V}N=eRvq4+%^gBBs*Qm11RRIp1WEM+OYEdiOaltEN5 zgx(zrz!PlPJfzVIk)bl+nooqC9C?xkne{YvQh*DnYwQTD+FMZ@=Ln#xq#7_ccU+r) z^X}-9_?Y=`_Em%e-n`3P3!@$^xx^E$ISpyjqb{X~1G@k{6&PN!)`Z8Lojaxh5GdbG z=2(O78pxP7(Z3sa`n)f~%}W*76J~M6A-X);aG(gZ&Jl3~l=rtVh;?}o_e<xp@b+(i_xHHmFFyZt>DwRQ z9iQ=1T;40pKxdWbxQJ7)E>4b52bUwbW2gXxvx`HvZB<@3vz>mAg)ggJUU7B)FW>#h zV))IqJ_~yw8((a=eRb)h5bkx*f+yDYJbC)$axi%FmQ6O>EW|VlUw-oZc05^b-HpbVw>(-qnP;TIw#dxR zvm*cHt5@iNcXFMcoXloh`;Q-ewEy%N6c^{*Zk$c8Vg2pfcW;l5FMvC7`OA5jM>oCCcuZHRKUFL3$;ZI&?v~3Z+g;-aWeh09>vIQ%G2t~ zXPH5MeI=Xpgj7JHYSp!@PB*?%NLIk_S_YbR4-bH(PkhuQ(M}nxsb_GalcQvOh9CfG zO-Za1PSeyPk&qxP@&H4g6sGJ$@z5A-g_E*zCdIcEg^Y3QW^@%6pzZ3pB(2Ixq4fX+ zf;4s`MpG=H$U{ahtQo*RQNv|Yf=m|_@~Tn+po-j>p-dXEQkS}Z@d3ZRk*DcNLJ}op znzZ+YEG#?{fFym*Nk~meOLNGB+LJOFQkbci%m%1=1wxoq7hB+>!9cKd(uC9G2YcX= z;zPjrGKp8)Vl1K?RZ}j^1iUpH_;TV=s<8WQ@1vbS4&~!f$p_Qx+>FMXS7uo?vCG%=9Ofi$g zrEOiWV%$-LmgQA_$biZO7qAu~Sx8J)aCjMRVwxt~gGmO!3e8U<(t?%0wnIr!+T~}j z4`fl(y>52c=vWK}5X#&@*nrhw+!couF@$8A*u5;B zioT5}AQ>tUP4%kUS1M>GMuh^vM$n|I2I$=!8X+(cvwJ5|GS}>~%*O>DJiN=sG73rt z`HG0ci?hPjsEkV0aW6P9jHSJ5S#yu`Q%GQfQBzMY|z04S9Ee_SilcQ*@YYgkqm;exsXFo zW0ZPQK|_vzPW?MzG2HhdIi~+z4dGMZW@uiHMeIezt*~=MaAh(?gFsT?h15=%1Ta8o z5f>RMXR;8S_$>olpddJsk1G2fybH~+K0~`y3;Ezs7a}s34^YCJ$&la>As_&V{A0n? zhfqn-x>H)#6@iB(IhBSwAdl?aa|4E{`8W@h(rOhv;3X5PQvg%~D)=A*pKX(rK=2DT zL`gAiJrg0qull4sltvO%E?G(y90d|GvvQ%yPf6JEl64}${7Ojp2}0;Zk1R&FJ8=+u8GtMK-r}TBA8@=P~eoHi(%Qaa{-tJ zjGqb{(w-%0Sm5n=G-4SL4Zyj6tiRA?%+2G%yoZauZrbdpxvcGmIZTaPq!kc)pUhci zXNN(<6T1+4cf+H><1wI^k7ppEGsGRuvz(oY?7Mk_S?_$g42g$`nZ-xI(0|Ehl=t)B zy?p(hxM?cJ_Z(bh|__8-B<%yL?!)un^5W#czV`F`drwe!p{1=~n%Ep~n*Vn)L`KO2P z4!3u={_?kf_-JdLjdxf=bAHNVn{j`=M`7M^fOxUhv(r=e5n5=S4FpYj zz$$y5A6R+zl+nV`$;q3;gc&6p^imv+5)@$t^~F82h6 zS4Zd;F`{xhI9!zXX^9~)1IG_6vtpZn-w8l2FGJ2G*vd+;x3$I6q&}!GF3utB@a>z| zCnwV}cTQOA8$x6HC^|9j9!z9GlaCELg2Xi%w38CU)LLU`>6f&vSaU3qx0T)U z_Bw8;QijqRC)Ke^|7d9T!4J({1Wjv**^{mPgI?EXP8Dt7m=YLd*f^`MzS+nRTl9-| z08aBmqs1N$7?LgXY_(RjVVUL zQEKQZSfY@LrVRLWOA0p#JqMnokVDUBU1PFP985NPjt41GvqVN!lO<4jP{EMFY%rHR z!wy8tHZ(v(W1 zC=~S!Amu<2avp$664xQh0^}>b(wkQR3xS-OtmedP8VU>?&pC|Hq~2V(;#;_!hZyEA zlVPUiV8bf;d!J!hN~ad*lt^7+Si7Oy)O-rBwaE+OFeU}*I4Bo*;!F`ND$+o=>Pw*n zs#i8f=_W0b!g7=CmBx}(G0QX6DD0J&p(DD>^ngmVQ9y4du^QDADY^q6xsjA)+X5V? zJAW-a31yjOF^i89x@mmXT?P6Xb)|$H%`jskeg%!~+8#^Wuum-wML~qVl+wQ8!=4_X z^cVJ5JcGm|Q_hJLWDwf&!kKue$z&Fln3<-};=VP@MGVAUF=EiqT~0>DJ#%ZiPKD_kjS6GTZkb)DvM*XFm+PIu}f;FDw1-yk^nOyb&OeZ0dW+M z%+aZ9-ZGSfT6woP6{2WFj`?5*u_6La-$39ZBSh?pKZ%f8*u(q~T=_P8qkJT!vH%t} z+F-a2s%)~U6ucYYK2IF-P_~0+h^d+H=iQ^+2Vg3R=$eH%q1ZNg8@5IiyHhf zbd3E-7?g`HW@}vxiG$Ld(-mX%%0OPl<2TqvgcuV>s#a3rjzd5e#Mfe+nUS)vHLAVY z9%V&4K#FEk9DRf~1#6N8{1F$Cxvo|Sf-wnhoimaiiZ~aame>@CI5aJarBaGf5}PKW z#7c%J`F5lVEa{^Nyb4dIfn$>r4fPuTgNS0n;^rSUSxZD!Fu&oBI)XL+r3(7BfTENF zNJ*4$Nx;awq{b`^lXZoB__ccsXsln6s3N5FQ*`W@OZdVGO$w+k161H*K!PDjk~zl= zxWpSVWsra++=D{$#Em#4m_i&BD=2kE)0UG6M%bqEsS}f{=_{8|C5RfIa{^Uxke1Tkw9mFQ^MkXq zG3}>CI6IM*g#}iYGa17ZsJx2>cGzy9o#^TXE+ z*%?#iuF05@LT6`_DSu|sr{fXtHA6;hL4gtmS69a;XJ6k=2)HLVcmM1E{J(B@m92%| z9iN=?=3VbS@Ku66y`C2e!t)A_xE=99_{UZ_`x%VG2eaj^}9E3*j5QW zGsT0S6S&^rj-Kub>DlOQSk6Q_pG9xMolD~q;-$2`7_jZh))TM7)5$y&od=F-EZRvO z`V+k(wu(;SwJLIxSfr(t zWzq<(8lX0_cs84vgqRROz%Fd%2SrrqntHZ4i+G5K9|UDXvw%!A&64W`FES$ZR4lY< zwk4Cs-4@BJKw#1$tay{42mlGOy(3PgkW&lm77Z)R3U(81pZ*qE(&nM}%0YTVR^;os zc_ag=@%%&%&L^42(NyG07g_GWbX|cV2#7Qo3-TA~g&>)g2qjbSqYcW1G(m*h;+}Xf zR89fNuZ}OY1cW^gS=9%F!+X+PsEFv^u7%b_cVcr0|A@_YBvX8fva)kXlV0-4wNDN^ zv@9r+Kms35)vx?hRtnL3X8a(ZO2M5Rxd_-)Zpg8`Y;tK_5We^lo%-}wcH}a-vJno& zS6Oll1h}V^$vO$rTu)h`%qI>u=*&?70EJn_pp9yjk(2=+en^24W(}ktw*#-$rT_pG zN|i)0Mg0mx1^Q<@PNWpU(kaq!8NK=MIsIRu*W(zoQ5Hx}(kF}~fI$u>WttZBbbC5G z-JVw-r0Lkmfgq)C*DA}QU9BWo;e}XC&D{|$w|HiNRj9$GBJD_#j#NiM{2GiBNhn(R z=MTc9PKgRqnv8LqBV-G}BIF0R;s=IBbOLTjnov-iL37@cj~h|A5)LO6`}m>8OyEhYM2yhRMG7A{GE)sY~7C`K)62nY;`GbOl~H;wrfEk@Z=01rT*5~*BrV=KA`V)SVaJe|mv9AyNDcelE*3aWz;OaR zGbs~!fCzKcR(UdvKQg1a*sbg^Sam3>c}kWQWkOn|QvJq}LpE#!8VG8hGZb5k1Vl)x zRa~EGl@##W>SU<(q$>pkBqIJTn4)!3C}U{SSwb?=BVyH=$5aE=7ED3&(`uWRkHVlQ zJeUWbXb_rHG#4T-sA;dHNr=Sqg`x%7(2fmgP(Z8#RbwG)l1q#RNaD|#Z2Gbn1_3Or=RsXen5M=)^UFLl+u;L61KEp>aea_RpkG8ja-4)&g!1@_h_4n5|+M65PwHPkW^Yk!X*~$5U$#qo! z?&xTEI=8X8H5`t9_YeR0?86VZy>NVfF=p{knhJ9^xF+IvxPXh3c79S!Qk1>%Yo+Lv2v)jS!%-vq|q%Q*nUnk}(1(q>digMfG z<*S$YTsLFo({6w9#jl<}e|~&;c<}Paqr<}~FIZsxEU#4N?mth&t9}fJ;V+GmC%YX_ zVm?TL3_Fb&J~!I5ah)%2l35DM1dET_mreagOS{OwjNoN975Cbvo$7_2L9guF2A7KV@enml1kx|(|%5&kk z#!ViQTMTd-R4ijvplhT3=-Gx^8!Py+!;AqkXIuz*MguoP~H z^E+_(so;x}@ljSvZR!^tiUZ5?+6>@RfKzoB$_td4!IWH)LWa&FP(&;WfGZcPP()1D z;V;fB2;L*oHY_DymgG}Ji`^?>^6e#GrRYjF-NS<9{2qlSRD^1EGbtn+DViS%Gc^@C z)AI#j0RUiaC57}K>S)D9RVKkirgjBF!3s6Y3pZp* zPw^AknWd6FUGD*G|^VTGr12}hzVE=^Moq!@g}JY_)UQz{pn@#h$m zZ>g3Pb6~+ugA^(kEJ||flMY(-YdsWD6O@)&U`8v**HD#Gd{b4`-LXKOE}q5xw@cnq zW=N9-!7Y}q$iA^xGdCY)SrJjlCh@WlZ37b4*}g-K%MP(VbN}3t&xtRQtZkXRNt0og zfT>R}V@zjj%wHKAJb5LG=qY(}z!B!ltW1$l{y3;Z9;Y^eCRP6E(^R{MBLOb>B2;wJ zSM7P<+yMaZ5aqWPkx3Pxpo7vgmhckxGk);aL2H4RjB;e>D5M9bESZ&H|1p5-naIN+ z9h^CO@=UJSLdtR;4!AH4Z-ChdG8U#i=_hc%X`YZtidd@Q=ucAHMgr8dn4HMIl81k6 zQ{Bc~BAJpcH6Q`(e8>*HRDv=tQ?m+FfJ{;d4MFZR46#@Q77R8ikXZO98ggm~$fr4r5$R%vWT+~2N zQLn&p1vDXY7}yf`3mwHEZF!VMC%{lwi3HxD0;u6uS|U>cAu{D6dL{QjhnGPV33w6P zd`Oj;fDBu*rtk&gIak8>zq0O5*}(GRqQCcuFr6fBO-K%Y`^HfY42#-vqsBt3=RpA- zWWt6f$lBdT2M^ z79l8Q4LXj7F%lEuEbL({Fk|Zl{5~y%I|{svb}%?UIXT?i?6W+E6+->>bzH#B;ms8< zsh!+@|NW1|sgL%boesxmgCUVU`_!;v2YOFVj=PJ!Pd<9WQ?{%b^nG3|u(_V_Qn1CF z`=bli)!>(w9_?=Y#h?6^QNh=*UowzcnD6ZEZ13#tcQ>|=-yJc;|JlbM1CT{NJkEPL zTxE4TD}SDU_#qpR{_x|!!UGSCxdafVbOrZ35pm8+nZbb3Pu{Y-$ROe1?ZIkyg#}JL zE6n53JZbRu-5bUUA3S-&P-Aari#EyfdL*;8xyk9~#>V04Deo3$z0EDp3@k3A*Q@L4 z$>0@hpDs?X{{E|Pd8}Y?IY#DlF1eo{T%WN3>G9shp)_)^H4lhFX-iRPr))8J}C z2mhRg08ch7zwImWD5+%o2a!~fF6Si zv{UbMaUOdYpskFyoj@Qo2qiZsa8mI~(xr=mZnA7e{Fex*5~6XN2t#gnMgwq zQaNN5IVB`TW4Y!hNCoD|LiW`YnGJgyoruiQ#ZoG&6l+2xadTK)EW+hGgqqs$qf`UI zB4*~J50-8bke(xXsV6*BlOH#oy3`B?OPkPek~|WqVgWv8rE=j55~I@OiAI8gE{auT zsVhZ>OZ+4WBVqaxnUbdI6zL%Y_!J6fbOd^L0EtTpTTV$kHq2MluzJ zF69EH-}b^7%?=}i2;t?npK`IVA%o%y@=ztQ;`1V* z0JWqj=eYCKYl2Y2MyMhw>6$^Tf<)W8lqh`Z0Aqj_k3u1VGWHKt!}C~yK9z4sK*Cmh zH~>UBut|#feurjD$ZO~`?5QviDC`oUru^ch($nfSrK5Yj;Nu8feF)V?PUf4tcFnc zJTC~PWD4De!J`7iH@*{z&X@HR*aCg<{Jih((XMo+nS8P5vX(rd3A8Vr@Q0|;n1f`| zW-C_m=9i?IOvD8^)aWywQ}x5+*oJkHu?{Iu+F(CQDqKo`1>&ldgVq_;sy!{WX~$xg zm~i}4Yzt7ej|?R7z36gX0|_lr!p%ksC9XuvsyBj;$^ZNMir}}vH4cjgqqr6GT_AmnEVI=GASh7AFn1~ld)S_fOq%tee8mja&Hb{Ae zF(Us}CX0JLripQL_y>fS6@Ij8O^$H|5xx@%27rtKoMk4m=Yd~c2NT2bT($<+`tz{v zQmflx)|Yek`DC8^nsqhoV869}bbPqIxA)P9&whN#Iu;h#ECQQnM0q~|);(dd%6n0D^6hd*1O(!l2KR#v*>tMS3xqf-L#;beQ8 zy)swN&Q9OFJ=odaWQWaPe*W{x?ZWpzz9HCUNemvGM}hBWV?3H0-OSJJ@NQnV1iimy zK7!YHJ$IqP!{-@IY#AGQ@5UypD({odO4 z))qdA+byWv8<4h6x0s5*juOGN?l^^RW0BMDkD=E(Br=6Huzui7YjB zpj!{Gyo9PmIp@j~YjK5lYm^m$ zMuq6dFalU)n3|slN=m9g=A^(Ilu%#xNR8W;T+>8IpUvgOlS zVN3YQ_E!M;wU0<|!8IuuLJ}-VM8_lvYJmhn;6@drb1R~vgApA6$|uA*g$LAdMi6Hn zT5xu%pgSk5Y?#_XTsTS1j$#U@=s-d^T6`=dl0u$p4#7=73SLo?DviMMlQ7H3fzo_g z1B}r}z0<)2Ac94xf-dl!n`2fb(OZZ=2U3+fO5$^{{kkxU4`m^-=oFU)<sI z*eW7Ei%|>y-KK_-fK~X#GGN!+EygQUAl8yy)mp;=;3x*_V+t`7tw&ju$iYy4mKqem zAiP;a-_M#RZvwyrF%@_X*cle^gR#@!BxnGJhd=_+H7S%M#mQ$P+{gssAwYtDVt;~S zzolHU0Zyvq4>)0gbjigki}oT~G!~**6eT`SLN+Yek~KD+E$MOyI~Gq3jodbGwX$-} z#M>zfmHlD~V^BK^XAD5-kedPat{Q|OfQOl60I+n!{(6h-`O#XW^PaI?Bb#fB861>y z3}6jHCCnLV-g&bFw5?iv2$}H~H@vFV_*6;(?+Q3{8bzW3LR0{GdAoplQUh~CD`H3! zpbji>ghfG;1urlyYr4(4pu8GUIFU8NMO3EXj7sucK$XBuZn_RltjW)>(j)~KhTx?M z&UF2n#haO%FwCE;-p!5v(aAd~`s~Gv z>*46&@Exv1Tfe8h%%1M=&u-_4=C`*tyWN%HaQNYq-51XufA{UTpMUbvm%sdcIvO4y z9)0rSdAHYPi<*nkXn$>+!Oz#<{djmX;B5eGGW5fb-;d7EMwjOwK7GPsrl*hhXLF1H z_|>;pSHr!nEnWu9I{xw1#Xa{XaLc^CxHUh!afeeT!}01Hn~w?E_jY!AMDHsrbPR`w zhfK!*=2yS`qhJ4ee0?<<&9jgGS6_ckX#eiygx3^sM_^;E&(jEuLY_Z+3Ns8Y*jS7v$?U!JCE6bl^FijtC#pC`Vl7f@f|#t&4UV<6MK6S*K_BDn*sDs^iZgq$%o05 zfdn)D-A;FnnJcY``?1`}NH+&xsyRnIpzHp();AwNdVI=8MMsB_%_AUtkDjn!>hbZ> z>B&3ZyKJLi>uW~U0AN6$zm6ZvcURw24KF>OQzUqG=1rKxV7%doBNnB7*KC*lV_F;C zp&?zGftK!}$)^ES6Y{($MOxZyoQf&foXyxeg0`Y9P7hByO$8i$^JPPibCC{Rh?UX` ztiYdeJmCYO4@advP?vHcL23yf zLJCmLh|`d!kU>g72`Kc2F3UxXNh>m7*uR8bksS1xA`u9#aBzhf1FWTozq?0b`C_Gdx94aHFqX-Iz#Aw{f(CVcmLa1@Y=_@cOQh3w$GLy^B4`)h^2vBDh zA{r{>kFML%km6%fiQBF8B8ZsyH0%Khp=fzLhYA*ge>sqk5($ErOb({5A2`#fEMXCu zu}3r^HTg|JTsgfL?t)Z4T$T|=CDN1PWTiLpI^YBi8wMLfmngogY+Xfo8bp2Uz$_bK zmS!{obE%aojg*BVmquCVJi0PX#_Ty2sZb&w(8!Md080%{%LxU1YFM

gFdXBbf-%;5GvQM*znEK1$^> zP z3ZF;^k|Uh(?pM@oj0Se?M~?Jd+L>2MG-hT&Kp_(c8%;%#AU|JPke6G)1FZ(R;|FoL zk#`mkmQkUZfJPB%&`p(+?jh0pF3lX6FHhC^s7LW zXLC{?QM{U0Bzzn?gl6?5;bEC1Z45%F;}O&2+@?r6e`%Ib2dhORr-rpnsEsIPLZ9LR z-A=*_k7f;NX)elMb4VfEL)x)ji@l3P!VL#HZZptxt0KM`2B>*f8o;hd6lCwltd=!q+1BqY~2g zlmj+Y57*L?GhujczU|Nxu`7Ro!C(N=u%RX-2o(Yy+<9s}1rS2TA&3S*sE#Ir%p9+R zN|eAx_MC+r(m9RH&8%`R8jKx83Dsw)UgHgn*L#^ZRmwZ`6Erf1=->NCdzzNCQH;Mx zz=1>RDCJn2{Boj3BNHY_vkQ(@AdWNX=NC)S4{)!BQd@LUNRk7ZIuNRH7gN zb5mlTEi>6@XJw_oL{NqviJ|AFPS87*5(5J&-lt|Duq>IdeW?O>{!C-@%QB#6EgK;0 zDj|c(e1-zd^xwHSgljmdp)PKc{c_2|UGl9Lf<^2V+ZT3gb~j}=&xJX*-EZ@miuaC( zPKjl z_<12N4+Jn`*<_i1p5>MQo7vUb`Qh35kXY_^y1?vT%N7dF{k?ezAaJY_Oy zd_6uqe22`APmT{>zv5-U{nZt|78W}{ygFcDfIGOq^=XHl%?%_81a_=un1yF~ytm7O zrGNgXfB5#hui=Gz2;PUq#1@G1btomxL3dm z*7LIy1{*jsdqX=LeVs-Wg3@HmxajD>z-QrVU{?ciuW|$L- zMJ<_dQfUZCr1cd@LZ=962~w4xB}@ywWPr^IM%>|8XQi7oIpP+mhoYmd-K3~+DCpS} z^3t1fB`_i`dY~{tf>lD1DS@Z7rR#ytpa%|hataoIONyRT%MT>VF4gkOHv%oX;;zYq z@QedgjXVWp1fU3&znwlX3ZRrxi66?q4ZUSYv0GhkNiA8jddYgQzN{_3*naTx@{6^W z)KY6`bys&~RnE-hJQE;5f~0@{eJ<1Oi;Hv5iDA#NV@I4gaR>}#9kAzZ7;&5Da*pev zK?;s0h|n~`9o1oSG*OZ?K$&CVYW+|Np{Tl)idu2vgydZTQgT64lm=^pB_~Ba(X}w0 z9|k3I6J6;7LXBTy&F|Mp}w1fiFTG3X49JkAuY)jr>h~#uChR8PVg^ z|4>z@GDZ{4JoJ&AeXJr)Azw9N9ni6kyf}?Ws^(dn?eDw=loS4Uuhg}2mbfqu zLEoBbl%l{^f#gG5wKB~D3+W;DiH+I05@O~XdkG_XCRJg< zOv$^Bf%wQQCpsFwl*!(d;(-Y;V{C0p@0rR*)k?9Re|4|Fw@(=z038|gkxfQ zsYJt+siadJxNbIU#;ceq>}NIbvnv5TrR&C0xO+ZMZCNs1XhSWuMWijK)($Ot|70`?tnSw%S7crs;r*l4(aV*flELIwzFi^dvl!^-Q2gI@IYzNTz9d zFoA*25Vboq%P=R8mB1DTlLtMmEFzgr4yv6cvLppz^kz?)QmrwuNbESG>R}G&E5MP3 z#y(O9WQvN5l~ABJzUao!SQ^lkni{xD#s0(ylE4XWwZ}{{2A37?C;u{+tJ;ah|76QW z9*g5Sry+u}5Ty=a4GD>x+TA~6mk@c=0{N}-C<#g;Y8Ur9bcaUceq(huqTjI7)DeTxsA^)Kb_LS-Z=iGIo;$yHjb z^ns}D>BIBe-e|jSx8wEAK)H$8Bz?=pdVnO2bi=9%kPRxFON zQGk%}z(VqtE~n&_#<%O}uS@8;E#uo`?I}{RUX~^jgsOdd=FN$8Qc#zqK^| zeswoj2c}S=+dB zMtJSfUN=3iUA^|zH($DgvbDLs|NPBIH?A2Y5x9MXzW?EeOQQUf8+_wL-*0V4`s!eh z#zp>mBa3z_W)IW0{CPzbTifSe_M*3kbY~wYPiyGW+R=6UbmIc!AMah>*kBTF0j;&W z(UFHf9=5o9sB1Gap@Nf}Cu|aB36=(?E(7#7H;s$CV^hmVGg?eHBh3-iWqM#r1Yj+e z(Fm{RFT<@1nkK^(lm@OVhdwhTnD$nxZik=+WQ+4)8jBIuw3mid(b^Ql^D~KaoUWim z9jjQ{34fNC$kU74kk~z5O`<|m)xj5O(YQ20IguCiGJ92mp-O4*P$@2JX}|gIX-_5P z!POy$8B7OtK$Qbt-Aj<+iusN%Wx^OjO>@wccd!ElspB8 zzX1=c&THTS)LVYkt0^ZNZ1MT^1mOWJ#RRd_emKtVoGubtWsaVmQ+|I_k`$(v&9Fw8 zLkWK3v7C{E15n`_rpr@6A;+QW9SKu@cpEynCfN!jXJm%E#KRkod@`vp#k8t;#AJ$a ztzvSI*#wwwGl|Hq%La&iPMz8&6JdI<0@YRy8(|kDlIMDud!L?gM2aIq7^2F#p5Y|G z;Ix2X6$jK95=__@y9-KD$@IKoW2EA$id3DkPiHbTYcWG<;VJGEI?C9YJwQL=4TpJ#AmSXGy^qV_OMyG7^zB5e;UwY;azA<%SEWG z1#ibDhm)Pr=JbMN0+G`Pu7wSk!~CMT$?JfGibUk{aS~!W6>n*|6GReBVc$L!6vg2_%JDhm(k@OuD!9DgHAK8kci%YT1U5!VsuDKWevJk0!l+*#hd@p zy`yhdAgO|cLn`>8n$b#O<=6?lQ8Mdt87VA<)JzQmG&)T+i$c=fh&lBIQg+QaC9X7( zLZc>~EMeU^W}Y?D^Rqo~m2bDp9NVACac`9waKXl6S{x%HGbl%H+G|KbACI;C7ADuE zlrKC^X8@1Sh%^|Zs!IJrCv^!;`b`C~wg`Hpt;xEEp<3AhBQe9?Z`4Z9vkZwy%aWi2 z86U2bXE=y-p*%CFSQ!nC<^>O2%XpJv699iTx~2v~&&*nzQDestWPD5SL~}?-S+jOG zK3P?eo2BkAoXp5+0O@%VL~jVGYm6c_n^HzEYc+me?&}k{mk`Hu)GwZ6oa}IeFiaz0 z)S6pOlKne@vHnuQY*F(Ddu(l`14PNF&Cg$l2?bdKk&aFl36O4zAZBZDlY^ z3s&`)#;-aq>xQI3?ayZ@s&d&Ss-)E9<}ibv=;BeLuY?%zi_64>dOTvdNxY1B=o}`^ zE5bpL0bk@W){=x`Ci;h|;7)sRJ**&OfJf=_>9-3dhP+DF?1qi1Kq?of3)l+sM?A}s*AH(0aDMCL=`-)u=deuvu|CTfhdl|m<_&g=_BN;n zvSy1DDq2Py!6YSCI(IyUdUmE4|IcIy*VuHyw3(O?n_B|sAy)#eZ;gz%Cys1bKeJSk zf{J-eYRwAv7tbx8&%>UvCI0DCI@R61J&ou3^=sCm+mpv#2hYfQSaxT7$H@Bj+)871 zYv<3P*m@WbrMkmlV!;aXyLa#7|J0eY{P?XOT9e}`f`h}o2M=yP@N(NX-okGF;m{)f zvu97+^JiDw`SzF%KN~IF{NeetCwn&FUq9|`W9GY`dHvSqO9m}&0&x6h`<{9{`SFu8 z=P$TCdBSrL^m};e!r6cJkN=^l`itLx{^r%b2_h)s``A&BEMVvSh4W7=aM|5GePZMM z*|PxddyCh>fsH$f`Mw=OufOlHhig}_n_;)f=j*+Fy0D3-x%q$j&;Hro^GBxNuU@}S zfchY-dLBPLVO zoH%{jy#Glnj(Qo}Q>&|WTP0grJ*2s`+ z#8sw!-57jUa z6}9<3RqCS*;TSSaG_+X z#dueKl1hmNt*5rv`8%V3GZ^cKHuG@zv)lJgWBa2D(1eDw%|Z`b4c?d<+t!^f;Ic6v z2ufKvsMPS6#H1(n_zr(_W$9VbNJ)O^VO}HVV!H)F)PtNEIvbwe@xBs5KLs8DI0Un3 zS7PTuNpeLAvgf>OD5txdDk!Vx3%bgca=qXsAw^>=nHFirh=iKaaEC**>SPN~Ku#Vy zYX$0WN+3WpnAJx_s3W7wtE@^3hfR84yTma1a}4p~qhGN?IsRohiA}Jgka0*gVdFG@*m1;EG3HTj`H>k)UositMHGz1-|mIzT0pKl;hUF&TEGZ;Dn>^j zLQN_TB_s(AV!XPnp&tYv>PM~aaawO!*c1R{F3OJr0>gpQGUd!Ehz52eCB;UZNgFt2 zMza~)py5%_)Pqs1tF-mj0rDYlY~0~idak1zBMm@j1d)(O;B?RUyi`>9$r=jxNhOmH zuuq5*{DguOMhdx?#?*zK7JTU}CAJ3(mR4mK*f~u=M$$PtKo*NgnB-MLk{IdBDK-A5 zs9Gu)-SzcZj`AunWI`K+cUGndST&1a14?PYp@5-h+{ojSqj4}R#>pi8*b2P#>Mn*o zR|!*kSVqQk9F8hFxJOjZsvdNh6%X%lKN@qLrNmtKA{T>6NTX4yS%Lal#zK3LM!%zN z4qE9bb26ZvES|=1Rhx@SHo(`SPH9q8ld^ch0U4 zjvlc=&$$cdjYT}x`sfjWZ!cavf9CWKsXlwUr?{H{R-V{f=jxR!SKhm%z@8m#_V+K} zxIq>kpA{byUwha;dwTlRX_wx*B}RYZJz<^)e%^%iiMtz*pB>zPY@7SGFfxt(a<4m( zo+tk9$&-z>wTl-o)rOB9yY>Ce-6u~x(&Hi4bEmg2ofuYaZ(iS>l&8<0y|SC<`r4(-SAO;OEe@>6C&dR3A3u8X z;FC{2w3(CjTNXdf8v~x{<#x7DU%Pttv(J8ha};NJZHyX)7kUAlDn=;8ju`}f<$ zr)P_eTsDoe%>8p?kG;*{!NW7>&i&-m&uGZ#$o@$1F8-4fq|)`1$FE(z`u_Fn9%3Ly z50O86^!Wa(){HH~JRGyR4oK>3}@)z7C5ywtP!EtlFy_$pd{Z6zdCt5HU$-*Fde z4}>YRA|7#781z9`F~p9;NNr|luX|q5%w^4dxSpCZHE`5lh>6ZB{{KId0##JKE4lnO zmleS|=0S+@3J#QrDSv6i3d|PD_0_8a(Yt`H!V>lkh6Flp3Qu~Tl{E8W3;vOk5>q*T zQh(i%vw9wICngwS#A8YG8~XuShz;$Khtx@S&EGh#B$7s4%9J`@Yj8@4?S+5}S1cqI zbeWuxpv>5#nl7ru0y9^I9DJ6xfdhI{f$1`N{;)Wv${6wrr#4U1?IuSNMRUw#WhpOJ zTXETx8&CtUBS^r9Nvs^rokmUw`@yODLvcjG1U^{p(Hh9WHLqCZ==2{it`;Q&5qkr! z`kjqfN1OU1UsCRO$#GU;pAur;G2kIu2pa*95j#{5!l@sbm0l!-3Y7Wpq))1i6i|}N zU|<6?MNn@`fRTb}NBK>)WdTj6BO^Ig%O@;gEZt2@K4yh2yG)k@3GH(yT|WLIXW?fhP4#1A6ghCgMmW3zRPURz?Z* zEGQ&~Ay9+TwPN5!ue0(i9dyO=IgLB0YVLAOV?flHhmxc8`3Y)>@RcuLmT5%R{0+uM zQdudEax}v&+XS#;r9d5VQk*Qr3}ZOeV12q_*Ktrm=2ywW1YpOu7ridQT-+xtg)+BM z7A~pMl<1+gx37=6J7gJ293|vw3=tD&-3hbEUyxUE^o{SsR?IPjyXe!?6-y?s1vv~go%ZDCWFI@obF7jmxmg;Ca~pO+0hd2 z<#zC&g++R@OO%0d-Y?b}e;267S}W{AG8!y%&Q$kbK{f(c8}Qcp$0 z)_*gFtM(5@$P>IK8h<1 z&jYz8fQ~5zo~v?}g8fuiZoVU;0?%+%*<4@qZi?rR?mgbvJbS?abdxpJ3Y#sUDvy^7 z$2QE#+sif!%Mz1F3m%O=z4*)B&V!fsle7zujd4VLhO5KnThlD=dh)kk2X@@Lx0zPv z-Z;sW+&ithBY={<7tg$l;NE6K$x88-_-`~1z zssHOE$HmuHKk>B3)@B0DJ(a`He)5T}ej3fZc>Ti<-<>{n>Vqqn z-~Q=8v~8$y(=>;7tO!V zO^-FJn<(yAzy9^_e)l`?*gbddg82K}o8Q`j)V&l!HIRDj1;P&=NMAp(`KQ16Q~MWv z`Q_KF;ggR)YO9~4$A9?dD^C^+{SDu&n6jz=#_LUksC#$r+NIQ_|FsL}JqTbU367!&fGDc;0@7%im{r6r>)-xNP zPhbZ8%HgZMmj_5?K=r-HPcR{>1fMxG%8o5ubd$+~qT?1(JbUzT&+`M8in6)!)J%Bn zIBTDFq+c`IJDSsOKaW^RLaY-f7`1xcosJ{;8{?GPN2>t4hIveC0Ew0^+hp zJ11ct!UuKED^QYx;1iN@NO1jFrwpH}2FTCy33W6rNt3-~tqLp{!;moe@?a)~IQLm- z4roP&n_+!U-{I+^O8r%1A#|0yU?5U0KcI;r9VyyJVT;ulfBz01f#%f0IHeM@BcLJMnQo))T!!&Ucn7z0*l?(qfH4+#-PbS9egm zK|C|5db2vQaGR@~p$>adDf&5ZzY*btl+71|9%d{N!$3_UeTXN965_-)9Dj43cn7|snN$M-P=p;8g{9p zJ{1yP0hY#J%XfQE-q6=sng}|C5`je-x%dgu;00^oWe^~Rbm5M+1|fJA@SnxZm`d1| zeirXA`0`_^6ak`tz0#bRs>PB%tBpDaw~^EYTB^S8)i40r^I*+jz)aYJyDkvH`idS> z=$T)OzI1E4#Tg0I17@&^Oz2`xcdfQOmE zCp5O`IZ=_$=!9+-B_==uD1fVkzh(HXR87VoFJ1uSV$m3GeB={jrmT8@i+&vwsO)IP zY?Afqr-60}Q89Fcu8I{I++C-O04|@wSu>sPAQA#LoVWl8pV|;E=tzd0Dh-?!TU5NF zcp{MhkL(B-I77&U*^xJYpo9bkOK>ZCaiT6^=0Q70UEIz|tS;=w?&^z1Qu8S={*k@P z514clusDD#PW`5d!B84n2+vgg>zBtoAGxA;QE{ul;;42o1>vF{{x<`cy5tShnLhig zeGHl`Cz>p~t}36>2v-(#ie{k8dDIVaRqTI7G1!_~s2fnXaLFkLJbnseco<8d$3_lQ zF15)hSFOf@scNA9h$CTSOiPWLYpb0Y*Nnh9w^eQ>LQ)Cf;2KpVy{takFwxDR4B7`D z*~y}@#8EQd^4UQMFjBgl>%<_t%T=24DRap$D{$j7DjS%w-gtta>y9^rt|2t+16~s= zsk0TAdBL$`1YW8*z~rh09Ns;*9T_ z|AZ7CbOk1`s{9CeDka##UuGKe;LGu2_I3B9?CzucLiCMOXOA2;aZX@v5eeVTlx(a9 z0QT+5z8Vlk1Qc&xy*+$s{r(XP+!v#blfD{SPYl?%hm+-Mdy2HD2&8J=Vno6*oQ+E$^Xx<&n(Rzx28|J4kUV6{=Hy%k9YbH|*scqiByXWE3_uhMN=k(dV zy%(N1Wn$LM2s&^5@PqY2Hn(}?`P1(odj!Esb*of}%gc{nzJ9y4ZM?8^a_iLHd-uf4 z$2VUk;v+|F*P~a!J1=Zwdvo;N%IEq~b?m5Fd96}t?m<0IWE}(68mL98#iyS*KY8MWHB7f|e)rw0!+-TJ|JAR5{+YPe*v3nIKmWrQ5BCn9?CyF1 z;QoWh@Xn1mFCpId#sI5l>}8Z02qC;uz`VhcV|tf$-ofIfyLWXuSFc|4UbFj_A-#TO z`yes>4>M3(~F7F#vn`;E{352OoY&4z?%y z_U28(aj#?JP;NuA)2A-IcfmWz zyc*1%6O!cIJn8%F$&<$q?ms&6));^WoBcOeVNBttb+|9yvvD$C z8&U3hg80>o-D6gqX&h{hZj^RX%Njdhnx-6vnNzCO%5s+V&))zBT4vRQGhpUtn)_5= zy0e;4g^=ZM9E)-R(r)?-x&jiBQ0!5P63jA2e4?<#T3}7VWRssNFX;<;F}{%PHvl!? z0C$mZBNS66os27*lMeOd(+Naiu%<>Nha~@#Aj6$s7*49@uf+L8$MTK7y0Dx^L{wq~ z5V*xx4CHtU^j6+I_WlbPB zz*#)ejQ_4rQ!BngMT9Gc`U$vysrss6Vo5`g>&dRzBCg7JQN|E6=i!#V^tM3pcG;=3 z2m#E} zQ%NQP>Sq8t0~SNW@o-Ua2zEG4Xbd=YgH-**30FK!ogoUeEY56qZE(5auG1?KjMW5h z)-kvP78ikr8m<8iaI8s8Cs7p+7K#PaNJBEt>6nBJFWI=1yLfxT^rbY+ zxbiX_hp|2r`QCTEpeU1d$XN=h4bQB*$m)&dWFEDU#&2uF}p z!{AI4W#8o_9>3yiP(}h&kq#FnPm!rD#GEI0YkRZ;Uh8)nN+?_a8iD}$AT{HVFj7wf zL7luJ$PCm4h;tm10!)o;u}I8H7s62V=<9Y}EJLq4T19rspYOmNMJTc;ei>U4a`rq% zO-9Kto_uEJshGVMtvl_0P-17HxVYwj%U;k zY8aw%x|5&{Dde$iE#-0^hG+zh-B^y7ioqZABfDVG|BQw+wTZw%F9aV0lH!}%hF17E zB6I+=)Rjj~GUGE~3kykb3RPClNX21A_3GN?xE_+Mnc%f?e(3xSfioZg0Q`==O9xOB z$}9CWz&KzK(6Q(y;rPplMotdMkdjsascEHxY1Ae7nW*L+p4Pm5bg#utM|o<+b&8z!ce^KKts(z1xq!x6RCp zgC~~02z&28dg6wK$M(5pJepj$=I7wW#>o@wULxjbRA8Zt>RQmLlUsL>+$YAw$ z*xIlVe#;ZXTC`Vb?LD^vnCA+J8k|Q@9(ym?v%TFfzPa^e-va?}4__P{*?-a8lKH9i zHSGQ2>#q!EgwxCGw_YAT@T~2f`}f~-t7LEAbHlf9-MM_}ise0b?tX2)-(LP|ee>N- zn;M-yefGkI_kQw|Psz#BpmCb!pKO}++>I18xtDVO!i8V|`j4TweaC%{+uK{G-uvj% z^Sxbj`7ccTKenOKQ@XWM$g>lochwCDcqi+$PHyrA+oncz012v>qsAj9&2!qq9 zsFEu`Ujd(mo@u|`q{JU%QaCza9m`Bxbt-ds zl{A*y25%xpOF;F`4-TV z-x4Ii347m2aCEr{%5g&fD1lOB7R1p|(ilCiv^qn8p+jm#R7f2WO?ydX2*~VAteWr> z54F%jIFq!q;FkCIfRs+kSNJayFZhh$WSL2VJ+X1I{^6Q#&x=$J*ALskfuR7Uj}2}6 zy08=v5ObJ@06k9z4s~UPJ!<8f4aP}{aVX@n^bpNe6{JVQu+Wb|vg4R9gfs1i_zN@iL>z%%Rccn1T>^i~(jAJI)&6i84bG z6DG1DYmR7-nvwEo8(X6zMWS9)RWo$p#o#%BUjsR5iSa5NN3GOV5ZaM=VLaYG0)mhwm* zN@XmmX#};X!O^0|fjhZs4eXJFsx_R3@=sXYc^w(fnG29zOW+Njt za?l#X3wvV3oV3VT#FIu@au=~y&buFHCCjc~U7$>X;WePaSRK<)ABg94#gzgJuVqC> z*MnyGjg%3^B;jNrXt3#z7O8Qdzuqs``>CUrZ*_w;`4`l9us|P zU64I#2V{(Wj3eG3=8&r!MU5T)lL0UqrjNcj@v~ z9J!r*@xq0tyU#rQ>cIvRdIz&Z=%uq)h5o#zsQT>1Gh17mY{@vwqY_d?`ZgBY_0j=+ zW_Tt9-Sl`RzQ1zey=PB%zy9W1ZS>69b7;PE@80&QGvD6);j=f#u3UZJ9Ql`De`PI` z6*$%$Zyw)z^!S;F9IV)X`gHHq&c#nZ{n`81uHU_L=fT4V;&0m@aV8J$-xHl{g4eEH z|MaInb6>_L{R*7-4D$K&dLKK7uAO*$@7_J>Vs&>I+1w|f}aPG@I#_MY!~8<-ig?d@&ib*tpr)5kA8CE}Km)wf4%?qej< z(lAj#@f=tq>cwj9lC<~i$-{eh_jh+0Jek^dI^)gjbNN|u4MC%sE%Qo~QoCCI z^3JQ8WlE*fy?s-Z;B?D%60Mg4&}zY)LN_G zzgJL|MZf`^N{J~{Rg>m2nv=rd966S>0IQ&EmsMX)mH(ItRAI~F$ML3c{sPgQEHbJS zU6`1gj70_9ep{WS;g!x-O9$$&R97p%Ib=qvzbJjDkEAu%L0-Ynmxrb6I2uKE$|c2I zAd92|Qb*|2U>8xjn+kC6e(K!^=vatz6C0AF{^;8Ni-TejE&b5?k(P+${Fed1APrv~ zFEDktbu~cZN4n%8Z6Pna9v-H6!AJ~EFcF2-OJoukF(!|wNHzl}q9&#Maw0KdLpl)? z@L>q@h;RC;sU4~!gK9aTWI)x)lFOD7(fX+ug!4(N;kA0mF88URV z6G)m!%Jr(og^Ovc-N?uE%fJ~lF;oYi>_$;|?IeyX54O^9Cb@pn>F}N2I;kRp{rEUi z+^X!{T^3N9v=Vqsr`8h(QLdL;ZN0D6`H)O=x_TLN{}XW*cgl+X1VC^<{?)hVo~Cef z{BEII4x}qYAt2GC|68V+AnmEDPD%ISk0x1%klM+VM9 zL_v@yYiveVZHOqxmR*mUQ+n`2H4`0ZgQAI)p*5^S&ls??s0xTJkI%S%PiOy~Il-bIg#yO@NGe z;7!xP#EdGNCvCcO#tx&+Mc9$;n^3cB(B~?>u}A6%6yhSHCgMLP=CVBBphZ~#$5uM2 zPGzUApDb%?mlHn6be7(OUd>Tndj*x6W5o3oQ~>wJCbfFwTz!23qAzou8wz1+P1eB9 z7Lzw9jCU1PXCouGR6&@5#?PoF*5a-Dk=}7^Vv68KiptT69{B9_=r4g60 z{uo@M9O2bV1zS^)4?EcfS=rU)c$LgDfu)O~rUFSwLbLFp9m$2pe=?VX75T|FVLDNJ zA$UobH+HGOyS$im9+b*LHh&Ac(xxnc&QUv0h_9emE&8h$Kqeax;^`r&N6jtiG4MRT z@pjhjYsRxlgQ{XLK zCrL~sO;sG#t2ak_=N6+OT|V~k)%Mn=l{x0}?X^ia?86flr`Ol5(f{`QAAa%6UtYL) z>E&<#_NyPS zs1H84K_x%jy``?lr`v1j*sy~oZ+9aq@z}=+mlO78!`azl&3=gH$9N4&WWDFa#`>qHh|l)J}Y6 z4qUNS;gaT~Ym4ZiTtba5dKPM@bV!EEfGSs3`7)j8MD&J(&cz@ucf~+ON?T;Kx(L1r zkGh}O#MsmZAKd92Hr`%s?t-xZZ>x38)T%7fP>q0U>C8KG7y@&bHSn##8he$ zB-X1F19a)nS<6Evp+uK>sDvNCV*}$D(@0_n%9;&*^ZFcVPLe~>YSQ}D7|@B3Bx!n5 zp^X@#;JB2H+696a}FQjj%M^T>28w zHK7e(QOY!=fQBDqMMzzaK_rLhLtgxHjd4x2b!h=zsxOrTpe{D!#i0;DeE`!ULh1q) zLPrzot51~?CL?OKuSC!mDcv%d`i>b?M=_m(fXyS&K_;TbUePm1Dj3_1XNHpql@=!p zlB@=P;iFKi(3^wuo$%u@Tj(`PVXrIihkiNm;><`N8HOy81r&aI8tl>8Sax+sXbI_p zHM8=zib92wKo$0z{NFbM=te*hu#JD3BDT-k+KF~uvur8!{-S$$NCRVfi^vHxRV>KI zJ|Y=y!z$4f{gYKYeUvTPPmQ$JK7dzy9A{z5=#*IULjTxvgBGSL#b;3tFcB2L@n20R z$a5Hx)UUD=nDUqMAR>WIL8wyY1I99qC|YAZCNf1(7|6ceAsl67!(2n>O+3HD&mUx01DOopQ3zO4O!H90^ z4NhXrcaJs%udVCL&!L|{fWI`cfEwCH z=-H0yH8xdBua)B5ENXC)O76jbGRCKobdh9BfP$EDR5HTU%_DWvA7PDGaHic)E(F;N zdB<7RWM(w6Gb9JYNk?$X>N_bV41sD4hUOS5G1%r;IcDTEW$IGo;teRGj50V0p-q%9 zLSP;oWfQ(V>lHq@nL>ACDjDVuzJ1aN8}0<$%8xA#O_iyc$q8vhsjDpX!_brs{$veD z%8f<_yh9bsh=C6Mgvn9GxKEU07Uis9VYZ0?o(bj6&k2Bolj0j9d+=9qHgmLjh}@#% zOG_cOB`1rs9PTiX@ulZuYaZKSi_&2=`R_q$PvWY*_hQ$Jqr`&UIeYV_F~FMn1*>}Y z6L3v|Z+XQ!?2g|?qK9H^t3{59`}g;s+Hc0boEI*fzkTP{ci(+y=y3MTSxe5%SpV^# z{DUvP`S$BCzc_#4BJR9b_k;I8@VxQOAAUGEcztT;<*)zfkL}7}X2lNmo&YvuZK=-X z8}HMXT{}JP$}OYTYMdnBZNsCYd8>FXoIPiDpTIQHtJf~Qf8~mmONaZrrl+p*&;S5H z07*naRKbsKZYUf31dSa#9C@gA?fCmwE`Rvphu#42>fnVT4Ucf`>SgO^e)#^o_uqei z$NdQF^|!o=%kZh;!)yzF$hmWpZioEx)1P?t@Ad1~zrFdrMsK{ZwRI@s|HuF6&n(Na z>(7gqFE3xYcKOOxZNm`g?Q1>75rd8yer;a7aOG#8{o>K%r+@o*zqJDY#s}B$-nTyL zxs6w`Y-;Ds*|S7M4xHxsix+B*y0_f?8v~4|yU#6@I!DUqE-(piG5hF~v*&is>Sawt zcxkjyW#i=btW*_|o5ODbqx(6=X?7j;=OJv;qvg|{#m4NJ9eVc~Uk%y7%){LoWbLXE zcU_wYwcU4k`*z>{Q68gy{&erj69WKGbRT*7QftelvJ7ov)!@gNr>%@-XWx0F@+Wm# z!m2e)tLeNZx>&xGX6>7uInmQ#B^!fjgMd7_O44!vy8k<_7yDtx6K~ zR1IeNm4GN09EsS?r%lFGkMt>JV$CUe1;J0Ka@`9?rqii!?PbU)C;u1kGNw2rVQfTI zT5!TWiBpn)oieep2u$rr^96ePr%slLVN4;HS$9YL6%1^Z9ToqP=_w3BA5*bJ01_;1*XGIcqleLRO)YRX`I*#)kh4 zS(}}~ehFiD5kH|(6s)KRX(gpab2FuDR;3y#vhFcO*B?g1l<$0)!bQ9-TuXG7IHC~P z43zM4*doC~C#S$p359&ZGl+pnRR|-20h)ATT-zmRlAR4G**-7oriuum}8rSiA!6 zO!FV}cAZ)L14Ln{{;ib_;yu>V5}@|bJFua4aX0E-9Y6H;mDh(SUT>I{^IC-FQ|r3* zxI;T$Mkkqb34QZk3DtE8{jYRXQ*%l?T8;=Nxv4(m$GEMAJNdLQHk+Z0BvOqoA_dJ* zk@1Ynh~E%pClhS@`!TrLz6LWm?fE45(V$-h zDv-#r=y;=xv1>^qJKr(n$O0MXpW z$>W=wcK*rlu__;KQIxE8wbIVyR=yyaYg+U$LXR#pBcVJd^oNY)itL-#UpsCS8@u0} z+Szf-OJfCriPeAsFmiXplMT0ktd{^-3*`R5OF{cvoB5pZ;;CCdJhJQmM<0Cf!G|AN zu5Wg)e%-IQ z)7iEM#mz=O++zjPt-jh=KmO76YrpvH)1A|&to=9Z zZ(+>OKl}OL{q}Fa|KWSL9lR3@8vF6v?|)r!<7;McD951fAN}KhdgI28|Ly+3|V6BT??zx))X|?+JfdgZFt;6HI*HoUJL801A9;FB;Vhp zasf^P2l6{dBhRT%mR>XQ0Y#~rjHCr(nVp5ee-!t}6)irwoaIv?_8W+bX?kOM<9L=b zHTL0Tg0_ke&IQ2`7`?T%GEgzFpbKni_>!#TRP4~luWUbM#ApPTG^O6Lo=jzoAf~?w zR~WED{>{V8?@BX?JT9va*j)KZFsgtZ3iT%Xi%v z-=`DGV-Aic{lFUn3seUAR0{dHF~37f(wMVJki>BOQD%T0>)hmZF2=9Zo05LV{MybN z0H5jBtLZvAJWl6DHrNs4S?O?BMK6)!it!j=#LX#sVTGj<23*_imawVUh z>FaH)@O5wEN>Q?~2#g}30hf4dbAZ7)&=!c%9i`YsH$pm(_L!<*Q5~r5n73&1?U9$q zUR(Wv518st-@fUknPykKMa=DhCZ$YNdrQml!wo~5-UHT-dCj8+uAP>!IE;+s5^8NS&F zVtrfmZ`dYWwq=KQ*DZ?}j2ks@#FV5M1Ax!w-3PF$*x(?;t~ZI9j_m=M-d?lY~FR4jOxjD6})tv<%F+)e;7)8?<%b_0z!0|7zpFuN3Xq` zknk|nig?utDRFW7#X(JM4n0+gAv#QfVR$FDFsF8ihqUxk?kLy+iRE-?+AAXKnEa%% zK2cp;seU#6EC7v6I%a&t+OBN3Vc=`*y31fL^_R_H5Vko6VD3-`%`vp8N89@7ccQ z_TAfBkOd_7?mqba7hhPgZq*8MyqVT+Hnyn0eC@gcf_*LRYVSdSz5Q>#_~Oe46T*6( zu*lOO;o;+_`-g}3ANSU+0?k8HF>i&Zw)ouM*!<|m2OoX#A)Y*#dgaQs)2Gka6PiPO zxxZ`gR1dt82J1LH^mbVyI^5se+uGQ)HK^ADFdIAh{QisIZ*86W)h~V}%Ki46ugtJ> zQ6hL={KM}6nCwqJ`3dLo#3*TRN$*`zIpm$ z|K!1|C%bzO?%es!pZo)__@yn;zsG&=-@mUnv2zj@-H>r^M;|M-EK>y8t>^xZA153u zc`OIA1M9`}=e*?i>uQ!0m31!bN3;TRkImlk?N ztxCAA%|%bS19vJo8+TKp|7n$h@)x9FO}kwHL{V*CQO96{+DV71NdTg}Nq$#&#m%6o zmQ~k8diGB4;R4$UT$*m z89tn)+q_4e!Gctl#e+R*SpYD8GMde(Wf=CRd+U>Q@>8+Vy`%}(7LGYpTT)-uP+%X|NMPBBim81M2}5$!kC`tm1VJO<`XvV10DeA9U;;xm7pws-^lzYp2KVmkgp5gsY^Ni@ z#kN35&WLKvHc=U{=~7M0Aqpy+eZ|LOZ!w9&;bKUhp;($5dB!b_MIRdpie^k6PNMV~ zjnn{RJEVL{)Art%)Pt$b>V}BLnq5gugyayrz8xlZ!Fv{yit5*9{>&Gdw;bQH}! z^*6daG>HG_T{`5zklbxZZSkqg(_3e{scjUP6v8riW4O2-0a5A~9WF*#73|diDzL~( z0AJGJs1I6^KBA$~c?m>)d3`6HN991FHmDG1&hY{D1 zXfFj1#G4t8uz&+Lzu_|y8vNsU!O{$oA0|wpu2UkCRws#)1XIO8EC-dOMQAL*$d4xP zShDOw)%?wHRRO0cW%QF8m@m8{6CKn_HG~=mFiBH*xwobl(?dpd& zJ`i$?@Xws{qU4ieRt4OjeRFtVYkN~S^p{z_d9!i+*b5KQ?mqXLFcaj45TblbUqsm7 z-u!Fx-M{&hKcXe?BOBJtZXeoH=*HE{-`{)Sof7LOwx2wEN`D?u7Dr8|8g%*E<9oN+`|b-bOt|N+g2hE1!$Ufa*Jd>$X{10;%^=s77spV$oy*9o-B!$*LVaWdGU?zB@k{ZQ`(Ef&qOzgO zOLhTOZZeF@1p1vr>@u(w3Q@qPxi2iwX~e)flvX#o`V46?t9sd$So#5?o=@G&H18zO zqZg8`x`hG4-;Y{`$fc(UHg>=pQe5?DSmo-V7NNy7O^%eE46-3X*p_3fKO4HH=TJ=l zSRFXiMSSMq2MI8X&kSM+7IzmrPM1z-8Bd}|eU-8zz&de-ucEN7UE*;5%9UIvGia7y zv;rb)s9t46(@0Cs+N&26|Bj!`<@%cmwik>U@en5~xssX>07gGOo|H+jLR%PEemv+U7Gq^5e{bu=*w z0CW;C&Y?8d-;&|xeK{B;NsPXs0wu9iXVFBAUHWaG_A*P+Jsp4J2|km@L{P&>M2PW? zI+PY~&Ha#N=-55&ZUGnrtnP0wkFnLa<>rfN(t1QB-ncIUXjBu`BaPIQkY`9mq>!ZH zgcL$lXR=WAYt?|;-tC70MS5BYi>_rTVgQL?(XB^&3%l_Emb>Z%+=dv5H0D~6j@kAfRp0W3AL*s0i3kX5LjQr>E_Bn zRjsISF^V`xM1f8aT@GbXD;?a0%-9HO2~ME|MIoRvnbZ`iyb^qXA<@ME05VVX!e zjuJMPcUkZCRSNot7(R>Me>P7 z9u%FZnqur#tB{09sW7-?8{g6AhbwWUxx~YkkW>2`NdOnVSY99_Hy;pl0~bwN)@_UPU^r%LfMkCmRtyeu9pzO?l9V)*ppwJzUZI&_CV|S7HD?5cS*|ER|RTtr=@o0e?ADE-;}Um1PwLGA|pdv_2@iK7OJMCgY2m?XcBeshtYD8bAx3VTzuL_gKFPZHr7eIlD|+*CE%Ox#-{Xq~gv>E7XQR2S;8Va9){G zvP4p_JDclBXKvoTW%bC(?PD98w&3(`tdnQXym#e_(w^xw-7AdVIC7!~oRqT@%;CB< z|HkcJ{PlMCjn(dCTgyG_DFZVgo&q>9Kg^QME(;#bRokFu_xYY#6Xz3-RZl|BFBOwClh7_y5ik zAeL%3Osj|yvqi*;nqT3y#QHnu*t zue*@aXM8|@r*ajQu2`3I*0IEQO;rv3UWrTFon~EWb=9gQ>jKb!o?v`HXA6GmxrK^= zvG|HXP#YPvT87Uwn&n5%f~+Egr|bC!J|9+kEo`Y84wU36N*t8$>Q$}GK9Q?UE*Epm zaZQeclgNMf0j$K4$^?!%+NeNOq;scr0R^Of>nKbQwv|JP_5aGCb~z)=AXjA|1j=c^ z)7m={M=h05R&Z7f6-B;9A5(mo0S6znU57V>j{8+&>BZ}vi)tijez_xtQbFvnkiZU( z4$~91g6l^vBeIF zgHx;qSv@XFhW9wvDyMtJzv{?)Z6JhIh)0q!Xpv};2MtUn1^8Dd#6V0%DB&mTMN>{e zP#;n_`N38z+LfTRHplc#Q<`qzfHLGRpD-?ja=(qz>Vl6QD%~vdd<2o$5QN1>yoP`5 z0X=B)m8u$Q1R_JjVuT%y$3PW^btjqavH`fkY9LZf)y#rvGE`QWK++9%hLsE^5%z4c z7!YGF2s4l~qDq|U&P#F{`(~2yFrQT0=z&ZU&3H0U=Sy6vIlW~xah3ui5r@pL!D;T( z>|TLna;?sy#Hb<-E-6{oCgcEviD_~fb1R`2J1SlE$`^hvC?ier5D(@b55st6mh>Ug zd1{$xzo=~b%CT!q%Yy+jNC+n>bql@~wDbH%Yl)?&=0P6U!J8?F#qK6dF(a{}rxX9V zh77f#XrdV1I!0SgBOoi8S3X(FNfwJssB+CTHBYn;a|-I(uXLA7%^`m>Dm}x{Vo`D? z|CU(Kq-(aN>qbz1se*Z0m;7EhBrQOrThc_E#MDZD`4r07s38ur3e=7Qhy~=X7aLjp zY}}Wo0*(M@af{7@3|~nR6HiOa>PizJ?3Gf=g4dR{joyd0F(wcfZ6w@`E&}q7gRYBo z9RCp@HMROr14{dbc`O7VNFbu4ex>tp$u+xHNKzzDKRNuNs73*1L%x7yQGQcrH~{y@ zjD|D^m+8AES8hgJQKQ6It@U#WHSCEA6y< zuAuY$&d!Mk54`VqZF9pi8hd7in`FG_>*fBNm-efFsj*FyOR6JlHnK32iow+OJSw_% zcKzbnv)gCS*_w0j#s1crol94*>O@|>eBst65xVQ(K7g6vcFb2|NeRuQX zr7LI7oIQ2=p;wD}e%Cv0@$lFSvAq4apxi^pFYiCRPpS5+vC{qg*<)u;ZCSYVWOskv zE34LyU%zzW7eD>@;<)}N5 z29xGz+K6^KX0NtS+5Bv0izRDe+Vs@OO#}vt+{wEZR?($Iy-*+dA(@kDu%%2Jl~iG( z?N5(2&7}6apq4i+_g&kp>+p#-_57$B-Xh>g4Q}*QJA{r^=(HT4aS;Q4DZ47J$f^_| zfhCx!6TEkz!xy|9KZYkc3^`gYy_nap#^_iS)Jc4G4bIXzbhT8SjIa!%yyu)qE*}-d z7m+Xxq*9fT9*zF>KrUmE>_kWbU<8yD)|ll^!Gy} zW@AN#iLC3Z<5L~oN~c{oUuc^XAgbyE6lru7)p;^#yKa-x(wtr3%;l=;?gi9 zo6fKC2g*mZDH*5*kdx`setZqf5*1f@U3_5Y@ENJTJ=!A%o(HW1e)kFcqRR+H43oEm zq$HgS;vyxxM@sah_mR_#yyUNxZ}c>wcd9NruRDKtj<{ej72bYKyRK-YJi4vz6w6S> zFdo-BHzsL(y9mTZ9|uW)alfkcwXnMGigXJI_TywFtVBBW}HCI_RoFLLo*5luQR;!H-W7 zk6hr$tfT2nyIdOK~a&T3GsBFb}cqaHt5+LTA(XvalJ`U^#gwZjE3X*McT zWz~da1F2joSS?)WJbbE1$JBV^MeI@-1a86w)9{wi$2_Av#K8&P5K-v`SV>87q3uT> z6~RgXnGlQ{W1&Q&CW2HUruFTuJ6HvfeuK8)0W&5cnx@K-`eZ@a$gpjkx-@m&Y&p4r zFd!>9DldRC+H(7!l1`SS3I)$Gfpj7W;BHo*8jepUsLsIl8;hL}sw6gK7D6#Twii3T zuEDPt%nY-Av3=mp&=eC<4v#PJV8-hHwK6Uw$s!E3wpEcvXU=Y$UxtGW`vnK z$DJbx^V3)|!g+X`46p=C?GY-cz|W1tap6(4B$cjQ5{A=}4{Z9dr>%K~-eST0Gw;j` zOBLw^CF)Kal#Av>p$r?FqC6L>l^mV$s5P1fp-rc@l!M}%gnS|A3`!tX$e zH5t+1IPEQ&j(MOYiiSsL3vrXUqFR2Qq1q{b(M-Rk)U>3`C2Ap`l86DY#=L;e6WjOZ z`HL5ipY2{cb4AGBogJ@XU2A7XxkZYH2X9`!c=KxC`kw{>!%2raQ0u>$Ycd5|;AlSA zJ7^DHAD+K_b^FYj6I&-O`Zqo7*;5TG3;lP_oD)FHI=N*ji>dtQHub-L?fM`6`j2el z|K%57c})1yr7LnnN-!!ja$YKDXbsGd2u2>qx-gH(O(`)@QYMLq{QC9xKm6dvzxg-+MwZ8<|L}+3fBwZ6cON{Ysk3L!efE=2fBCCl{y+cr-+uAc z7av@?yzd!+Gbi~N8!NddW7Nc4Kl|CwJ=T2Jy94gtbHC-{1*4~Pwxp;}G&Aw`sEtsG z+S3=dT4GQJ`U)m{_L}#8CU%9{SR&(AeDr5e$IUkP3|UU8B!&)i$IwgEdAyZ6u4A zERjTVQA<0JCRIsIs{*x)W!|EeGnJ;fZ`Hw$KI?x1Ep&!o> z*QJ8_<9;jA+2AT~Y2G*~rBacm$WpKPVZc=*SOZ7V;8bo}d08U?sivZZk`_?^{R06F1j7W!=PTzr<1eb$Iq&q$%EIzs2 z3ObI&=CE`>jxr-Ugbs6cwS1Dm8|2 zjI1&z-OashZ90`g&PfyZ4GAqa*(t9}x_d7nh7Ca{Nw8R9M6;sK8@& zis}}55Mvars2L3y*dV3IVP1u31l)}PC3Nn*mj?(SQ{RtuB>JC05gJ`)fC#WiETV=m z1*2{G$W(n$*3~6xwbUW_Q9Rt3nvZKVM9I9ua|Bb3z&SL6A#n8vqLV0b>_pZyL}cP! zdNK_ZW~*n;FbNz}-LtfTr~bBE4V*P4g{aO9Sz;3jT%dFSAoP z7nyhlw3t}-+cgiX$rxb_X&n-x8w4%7sEHwNXfle()3L3EqjcPiPV#q)6j8dx=Bvaw zqifP>EbQykoaHOdzUd{h+*g1~e}h|UvU82$K9YLkzo3b~Tbe(A4jscj;g zh#GW?`;F-&=SV?*7AcPvMF74s2lNsGfx`}FxEGKKFgjK+Cwa1Cxt6`qm~l0FH2YdO z2$Ap12tYWk4Cn7l4@v|n&BoLixQLL^!B1(Un7PxPv{*1|N@jc&Bhlb0HrKL*N;LE0;PCPD z=P%zJ**bNa>^M8mTxQGIZRZxRF5EW&X#Eez%Jsw;pfi8VBHF@?ChIjeH%}ZMIckdi z;e-2{*0#69p4#3QMZSDxLs0>>24OzMdbeFK(t5d3Z_ZO5J5oUW?5Xwcm+fieeWb@=}5%M?ReEpl>{HZM}tyi(m-dr+7o>p}yz`VXa{(t(5Uz+6p&;RK^ zdC!L+Rbc+)$s^nTn3-y6KfST`*+nxa}zmH=~aqzi@8HN_qDg?1*W`?XkhZ%fk!j&fe-x zUJtKaz5bI=K7MfjK9=pO`S};0pWHg>0S0aE(c?#V@7=ax|Fvs3jA}dx@tfcLi7@>? z|JVQeWbYaK)BryI_~U>5um7i8ckcd&zx>Oq*RFc}x7YVR+5O_HudH0UfB&9mc7OHD zUuiU-fBv@yS{E){{OE%l%=pQp2cETtngX4v}beE!xa~- zqq<<@rW5A%|Jz^vw{O1w%7B5b=`%eKO-fcA(Wdq+{x_!brsU&>N=@|jm@yif^~+Uy z>dSkbdkXpabSzf?bE6DP-#o^y`Hr_?+O>@)hH}(30RP;8enL{MtYf8W8jRkr>|9Xi zOS`E()NJQ^DWvpLzRLmHWvy(AtzhM7oc67Zwld9lN>#ebQ*Akqkf<6`!k~+rUcc$8aM+Qtx=Rj09~L zhAdInp(?3atgX|E8?u%FS^e#hyRDaWN!MH@TlbHS5nA15(Gp>&9upz7mI*{CnvkKi zDR4`=L>UR4BSXo^3)dkf2j5y{f#zZm$rin7dj1?Bgn0SZ69QBB-OrE(<4nC7y^Kbx zH^PU5T(p0$g16|@n@4OB&}+ht_*)8OS}naGFI#glCfVt9xCGGTuaCt%bi&`7KgEwN z?uBxFjx3jRdWj9|(L3aNdivl$N9o3ZvNyn%)a{P+ZA0W@34m?d5Eo+Y!Y9ALk%;#Z9#{{F6ap)A^J-~W24I^(N=Q zeW^K3s6Z=ak5kFs*!9wN55tEhYucE(Er(|KlO$z1+2>#{(94t znWM{>E}IMUCMxncd)5~A-6DAD{lUb0`101Ro2W5IZbAB2fB3`YOBes_PyajJ&W+2L zUhM6E^ub3SPQ7clnlFBT`t|8Q`sBk;-oJjT)&ExV*pO}C`})`r;svU2U$1!-;rN># z8pc>AK%6$MJa_|OMn;Ogz4&l%cNaP@9DC1_{%e=FPOP{2&zkqDt-sjYWBXo?yM1!w zz4K=t@4Ye-@q*!7x9@9zKe>GI`Lk!_W7Q1b`SAWjkJVnja@p$zOm>~z+VG44*=}u} zynE+f2LJZRwQE-`8T#hyuRr_gr#A5a_kZyhFAiRKp4baS^rrv(pZxde{J;ON|I-VM zuUxw5jRdE6&V2FvKkSO#FPybR#|B5DMXOe%dg=$C> zQ7}(7PXgV>mI;2V05;rrVjtet>_*SSCy$IerMvn5(yevYy>ctL5p4qB1fK(_UDRfo zvPQxTYb)A<8qq*h;7FtxU zZ@paPpvFagad}Q*MAFJNOj7Pb_Yk<8bDyxTyh+Z!y0vg+<}la8qde`r41RNbi=qPr z`}s7&;g7;uAmCW3fv#F4aU=Z$pNpAeZpNpJdNx9y6RGKlTk)yPbaF%4NFlMu_()`y zLn0l%uyi)Y5UE2T*{t*t{^_Ykj>;{Pn|hj2ssUB0=>xkMzvKsV0ZT8cF`HD7*g3bR zX=GL$dtr^_=}J~7StG(%yvOlqMZt_wrLq@J<8Ll2zv2(@uHasAG^eqf@!19xVFOcu zHf4s+YOsc(cq*ri=u%{{pXLq<(IUT;>eda;9K1kW{e|w=>lT@qZ)g|gCSL+QaL0-w zDqR;emFj>RKiS0;oJ)H@3Yrtw|4Y?IGBNHwD@TftF5GQxasU;O!fFOB}u+Hz8uwtS>af9w(gdLX&Keu8NOk)6V zQ1P6QF+e+lnI15Rsxwy#`VAeWPzKgu3VUjV-qABPW{8F3q*k|l-a%g z0!Y%(`Ss#3jJnRKeN#l7k1C2h;7m~=-^@z_q^U~L_T(+9T#j9IEk(C#t2*1v67X*R zWf^C+$UR`nRkLrJsF4ifj{Y+`lC5=yy97v+P2>!Ugc+PMt0b;?$|$_;vXH57fJcS^ zZLlYDb6~Pe78<;h^`Oc{Wre{VI;LiUldd5evrH$;cA~2ah@8@BNoyf#*zAOqLmi>< z7^&c}P@nkh}58^|LA>TsL8;F*Dq+HsTBK?K=5Y4^KwbMWyeGf||lF(`xvxVwD~ z1+hdH>w$?7MK695XFPS&U@l4+x00*_xmcRs0yw}bS0U_9Ti1~4pk_$i$gZ}SZGxJh zsu1is$tuiEXwvfiFZYSjbeNi2yacdPQ)il60|53sab}v&=`EP+MG8Y?1Cccu;8?)P z3*gxY9et%326nSF-S6BpY)5#WDXd~K_W))U5YmMHM*jH@c4`@AGHf*7a6Dc#-^8$F zxWqAB$jV6lWEtav*aEU$sD#KhwUb1atW#S2C=(Syd}pu$so1-(QgdCCO$~FQ3`4B% z4rWD1)KQ1V{PsxKwUiCPRabJsSW2ByBav}{te0;mn*cM#T1+Ip*na(B>+$yM*N66s z{@uF?h?dN&nbj6S8wogTFc33+QlnyawKJr6v%?g<*Hz{6=)s$-_IQnO52B}lET$)> z!C+Fg;iS0Z&=JoZKk&L-p`TI~`HUsM6^VLJFIufZKXZmQcNqg%((}`w{rvBL{p*{z zZvDkC{(_zT=l}dm55ig;V-{aKI(zP%mnYu4_W(9G0W3Z_u6X#yT7%J{@_89IClR1_l+4&96jod z1U3kIXy4N3FZi;@kDm~y8TzgJ_uFK%w-y{Si{_??+Y@KcoPF?U%U!C!{vZFxpZw&< z|NH;;zgl(k(Z`>@eE$6Osk1k3T*J|ULkDet?PLX#~>vPIa9o7AE<1GT}kGy>cU{o$2N%UR-+nISK0Dr z25yxe6rI+w24WP5z(}6YboeonX9`kaV_~`(MRMg>EoV~&j;aoY{Y(#@Mn_YW*B@@s zqA{u>&RF@pH?HKI@=A$(9sl^TRsE?Wk1}S04 zT7?3Z-u)wn{8qwH9j)a?N9aL?slmqh=wR2GB`iQmJO)O1Ts>&jRb&OUk%4TnOHem) zGat+hT`M+loQ5LEutBNWa`hkz7r1s^ z?~|D9KWuh5ACQw4OV~&G^)3#^Q?@+=@OpRq<_qf`W~~9|5^-vaJaE}eBq`4=q`Z8P zaavanTP|tGZqffNh~>75ChgbOT}yO&j|o3#c~I(x#NA)aT2Q`~ zn|Mou`HHAsG8vp1H*{P<0O1250t)|Y>Cz{13T-uOAPjSyVVxEOvD^g`R$LDjo4J%E zk{1fxgb=BuxSebQIwBjx?UqSv0*B`%2b2(IpDZJDhZl+LI5Ly0y-MU5Is)Z&M3rzB z?5XRbZy(9ZXI?ThxDjgCW%vadXN&NuQpLUqS0iiUDLi)GjSh`xT-$1jgCN0Z+wnf% zY1qJ#V&&%qq<)7gLeV6r^Ypiq86(jchrgyHZ6Z1k?BK^C=J)522VciZ9N!z~Fc!!*CKfhm7$evonCsJj++@6rG4eEs63 z$@mY?oi>&4g}#}_dwcRmTIT9Dva#Kiy+coBi==%A3??9T*S9WaJhT4GD~ru(34G1* zd)XFaJ+OEG(c_0d`skuA&hxt#5}8~Va*1fSo;(qMa~GV)t?#evR&0-YaL*pG{Mj?; z#MWaA=r4Wp>4_8GN_-U8&tE8?GOfU|_U8Qg3kOB6|i+|i*vC{SUv13#}0Nqk}g%30LMl2UDe4w(WN+Nqb z+@1TkpFewX`SPbvpBh6w<>KDlz01ir%s6-$6(0Gh?Y@5P70rMA>WiB(kH;I`gu#}bzW+M6QJ$r3*di2Cew~t7Yp!`_< zDz{dxVQBlAp_=WGO(1G&h_xQAXc|$)rp44Iv>yLVI0snMsSFWRAkSIZO?f2)%wB7I zrLN{O4Nh91R^e%B=IH5?U*$!)&WW+25HBV4<3@8; zM?VE&os{6jCy|3@41s_?u~Z7AP8AMWf?otVbzZq4bb9Lq5(ksqprMNh2x|?f}eZQ=;OiOx+P%i3OniWU^v~@x_d|k?PF3sA7tD_^dpY!xKjg zc_AR*<%`RY;Gm@rAa#yJyF_2-Gk0@Bt*!z%t4gI@fez}(CE_RrF>g4xqYAjQAGUBOF)dqF((t_;@$mNsv(U{V^U$6 zsBp?EDB*zTBZ?GxW?G7!V64$dcJB(tqPKM0VJtH51NHcX$c_+Fi@b^t4aE~RR>ewG zq;{pW+_5J_El-l3>acs7vE`=q)H;@u70#rpK6n~JxmfZVb)*~2B(MlEcLn~S?9h3~ zETNEKzPfJR76U95P0WfcO$}N=&;u!L{ItfPG)iS<*KiO~09?2zl|KEQ;};mGXjDzXHZ zm_*JyI;S7P<|4c9HcRw?z9E<+{Vo-^STb(?Bm|%2%n8s(YaVITF2OIG`;XvQus%zL zm(7qPvL+PGL%5{4?Qc>NK%!?dNhe7qqEM7FCc}#x2&_0+ZiPmM!{U-=0+!s;!60-# z_#$cWhF?k`)ku(=-gLxXjz5)G&XAkDYh&{PZk*=;DJ97UEmF}%O%JxUKOdlULdFEByFzdeW z;GwkZ0UQ#{+mM!|FsyDGEG%^u!a-_G_}{hbDS%``Hs?t#ju-&_2Jr{;i(btp=9r#mQ#t zw2<*}uPyq2@pAj_om=#K>hwvYHv5(ux-rZ{wu0U9o@X%bdC1`A%@;48dape{dC2hV zfXBbv_RqSo?l4VrbDXJR0hEVOJfCsI;-_cZo)`D3h}>yzcPud5&yF?Rv8S$Uqcz%< z%61)#sxwyRP$zW`$Yb$!7*k?8UilM7RsfhnA1KyUgy$BApM|*Y zc^Zji)O#0HG({-d#g!3UmW0Ko(|8;?buOpQvfW7tM@#{Sw4|cH;ehk7XbDXW2cORZ z0naUMS+i|j2W~MJv(v4Vh@CRS2@LiFUl$gHDeoc@2SP_&j*=9}B>Mz=z9j{zDPc*7 zoxVl2pjI)9aLClE7u6|+YYs`2e9nUcaPQ>!v5j0z12z#9BK;B^tk)t&H&G zzA_RY=R`Ljv=7fQ9R&&zhuJR$WVpxF-4ZHB+1zJ|hiQUrmEWjIjn)F**9QyHER0BW zZJU=3$CDGh;7R4|BEfXs4=Bsv69UaSlp53_T$v%M-{_2N71Z@JDP%VCjYW5@vW~_& zC`tykk@{SI%#)=FU;wmRYYVShP?Rc>iXQZK>&bsaX@i-8kE1+cSdCO$G!oc3_8D&~ z!;ebK#u5=!CJ2PH1nP;AU~5FFR=M%X7-D$3gxZjb2~FPEN^DK(B+AIEL_={i!sS^| zIsIC+Lxu{K2dT=#Xh}odV3L*DWx$h-F$H|4TBIt0Ug&bhuA;m4MA9_@Xr^Yx zR?VgM&K0SE)0|N?`>EOw3DmLmZ2SJ!lMC1tC8*j9=?8k1DUq1H4s+VN#Jb(T}S%O_gdi2O~%YN=zcz*BRQ!ll8 zymfK^WzF4c5mPkedFAR=>zB@*JNv==A7IK$Hkdg?_O~%}W6S7ozx~=9!puXTJAKBg zdeb^?9_&86f6vah+fN?t+oikMv18vW%V~IAd!rf&*pc4tOz%lB1)hky8O-YTJb(L? zeEEQaVqguMWsoN<4rW18OkZF|lpLWaZVJdyk%w+N)Q_JtvMG)v0{_ z?bio>{If&G7&~`<_SuiV`Q~ff*(1oGoj?yA6@u^eB(&biQ@`gwyl~>gVViiGD>r$s z8~WtnC)Nmge&82>@#k3nH~;ou>dVfZJ9Fdu)srWV+xFCuNg!@9kJ|`d?tACXy&l+j zYnz=s>9rfT%)8U7E>1vy?dJ7okDnhuags=aInfBN(Y4TOD(fc zsHQ;eZ_3QLsN~5n+y6utgP~@bbn*-P^ng&%wYsEo(GZ1X3^*;x`LNNr)=zu*yuvJ zflaM(Fwk{<0MO@z{nK#`1XiGIWckfSyf;f|l2_>ZkArog_791Z=`x^-ObMBN1|k3( zfFR|QXdyXmLj5iETS`B+cv)eSA7Oq)rz>W*6-TZ4aynEELtmDK3^ z)G&rm8kmy^Nx!3iuT~L6e`uVhy~M-~oQB~lMF&nYHv)SQvY(-1MjPRnJ7{vX5)zyN&REvK z%Rb1N9BL8WB@;N2xern3-la5&0MHCfGVbhKL!<*}$5LvR%M?*H)Rv(F?__*0lBms! z{@t=m1zK*d1I&>s8`M_l8oL6~#E+HQ1|{_DMr)LZ1Y4anw}jh`b{4M#2|pYz=;=X9 z<{~>otnS1ZsIF-yvzd3SAVQ}VCI??7mO{%bG}sJ(R=ZIMX^xZSfJP98I>;7NS;u5$ z5B-f6hSW*lrK#5Dg z{`Omw`WG)>{@{c2Uw-|~uYdd76Q|F8`q^hzxm@}Fd(+z&KmPb#bV>LVk;a+{`rRqq#R+(SPqbu;<{>o!c*n zW^N1Ydhxxu^iR|}joWoiTIV}FMZIM`i>)z8xkGH&&%P@dR7_@0nv#F$Ra;lDU5D$!habK9;LY!T_dC+& zd7i(1E}HkQ0}mBkzjg!iAN}Mf1|EO=H-Gc(*I)naXFnFb-MPK%gY)OzEy4axi>q$j zxOC}~JvV><?cpXq3poE1AO1HqsI)y8#3%aXl$^xwMAvuuU&OwByiMQ01h9_HRP(ta1xz8 zg1lp4fA19Ax0ic(X*()rmkgO<58ooM09hlP^r+lS+)EO7BDwi@DoAL{5 zf!GjG+3`MzfqQ`QX<$rt*f)5_*Ht#B9d#fK1p_^4)T*O;av+cd+msnrBWswi2pfnl zXT>;mx6P`pKA;x;u1BbQuj?6(9b8U7!2yq3jRAp2umaT2pzSb`+F%$;;a=?_8gW4# zLtHc?T*42`4qd}hVCb@vLv0~WTq+M~Q!C>~VJ>(HyW}N`B@VScawM;5q$R(2@riyw z0yUTY)W^+rUeD?3Ii~OcM>FWMiVys}(@q5YE-k!ERl-BSu@>Iy2Q&J{z`|XHt~0L* zk?Vw3siiC;(F)q0O~l|{zpX1wBYL6|$k%RYww=PZS(WB>CSbmj8sAzhZ`9#y^J87 zvM-qR)-!!-GvEmzngMK`#C)t7?9Q{I2}k}+S?c|W9%(eAwTwa`GOH(j0t%wil_d`S z*^0Z>kG|xrst>ESJjG-}5{k0!Mohz^kdV6M-eosYR2Wg?2zLMig~@7SV}cI(EKiC~ zQE;R092ePpaPsjp=^L>*y=1FoM?dp*A`8gU`%z{B3lUb_c0F0u9Xt{4uPNMO8Sw>@ z0$r$6=`!nT#|;E{^gZ%lSimig3+&Leu$EU#3{NS}rYW$A5e=JKrcydNshT_oX<<@i z68*Al8juteqDcwnQ~^}VC(P-Q_^S?TfHTDb8c}1yz)y`(sGBY*qN@p8b3V>TJXj8O z@IvaG)@ZN+fIIf)hAC0X;1KHO_+kKI>29=|jwumzMzu&x8VY59Q4#@*<;V!u!UCRd zY<)azU|vneQh>E*rnJngO}&JwEw}&qi=H+eEua_d01`D?GO%T2@$WN% z-BfZk2r5Y-YXQb(#q1@6PJXoR(_JCwz8yETUfCUN96 z_Yeno?4Q6IZx7qK!r0F={734DVl*}_1t6N#B*r{z#MJ;lQKW;cJDR!*jML=f5UTYX zijiv-rc{DosEKlAP2sEuq;xz*ovPb^00(y`osAMUK$M48+LWwDMrSF;6AR2|u6}R# zOSmEaHS5h$Vt@!?r^WEaV~oumOKpKh`Ed8Z_U_vE(*2L=x=bo3BsY7!N=)U%^4jW> zaPBnq>p|FO+7Fcppe_2h@LRn2```b;>wf?8FaGjx{_bzT`tpyq%=g+UOM@PJMb@+D z;(i^5H77QnJbvPYHT0+U@3mybKATsrTzUWehvHWo1X3zoA zZhCz4-FIHwcKqa-4?q0KporMN_~MJ_PoCbrbJs+K0oB)E|M8Pge{}Bb*+2aL_n&|M zd*nQLaG%OP{`j&7DxN&1=NGr{dLzPvpZ)ys#fumJ)31JQ<&PKo-neo7#;v13a%P?f~o+UtBFf0uEA5cV?_8K;6wdPWmPetuG%L?RdW2dC)C_7iaP^Zkw%*le4iv=m_J9M_3 z-b5kcm=M6pFy*r8))GtD_(Bq?qU%1;B_kY_J~FQM)N`4amZ(c_y% zaUOwD74Csg?0!e2&HO0MBTqtXC5;p^ooZy(Cujjsh4+-=$@Aqj5ZdM+z|Mr#s>#0{atTF6DgywXdx zyLULBc?CeSDa7E6^Rar%XiO%K5vo3Da@!`Tde z6>-d*z>|ZcHpveJDoLdVR-6-n(Gb)%lnS80sc<8AIEk%V_)+YxG@M8I6r7 zXJZ(K6NVvSkU*=X607&y2tG#oO#Nv*L$BIKf}j@85p^*=?$Qf=XCw%o9J%0@z*3`T zvMM!THI=xiNlEG18LtSeWF=hrZsLB$CtgJIkW)pNS1c2bA#w7^iPoJaqk#e1z#L5? zO*;52b=pLBLX6G)3uK(-RFEG4?~AQeTsK9}APwAmO+?}FV5smgt4z9td8};k52<-n zx?swv^qKkMelUz>QZY-$0~FmxP}I%Bqk)KyQYudX<1-V>5^vx-?|%^;r!kimuGya9 zRFFNz0*SJj`c2=4qLJ)Wh9b@E=Tuq=6R0>M%$g4J#QhwaIHJ#LoDJKtR&($(6bN#uC0ND!81d$`z|Q%N+}dyjYvS)x@;{R82bnr+Nx|&$B|< zgNkBX9VCin_Pt$Svii;_^sW=L1B>n@0#Zq9JFi?!< zgqU`NIq=Rf3onSwn)@7%KH z{Kl!1CqC z+Qmzk{_uy-fBW0NKYHZwu3Z2vhN z=8X+JLhXTLcSY9yUV|v_JF_1sHG5})dqhv4^d@ABo!o*rboijUA~5M%8j~-&vn&IWu<8=7TgusWn|#ZdHPE z3&CWrYn2D*z|;()BUDx4gjd7epy;R1Kv}iAm?Q;%Xt*1YfDP;nM5wN-41_ zAco8TaHaI*B6ysnt9O0G;4d;jOz4N?#YqV@EB;tU%+6s_UBsHC4lXt)aT0onI3i^< zwa6?CUbM(j2&9CfTy$5ZXekSt@*~f#a6JWLr?Wm$v7rQzd~d{z?{w%QxD|<=m86&s zY*-UHyaB#D_WFBevr) z*Q$)v!R;e;L_VgHm`ihOedrq@kfW6eeK@F%@mNbk{L? zV1v@~3<;$MMCL^33Q(y15K%7m<;$nrFP=V|?{k|k!~f2#0rHnP zqm<|v&4g{~q-GZr!~V`4o}9Bec}sEaOYUxA0ETKH+QTe(0~O^U$xLNZa*xA9X_4EN z?=UzdDXw)^X+zxbHpLZ0St>`ykTd7mlF?MFXgq}9ShY_jrcm;iI+McP(li7RSZcZr zipH807%`x)frkKcrU)AV%=oIrI2fmrPgX9eO&kw*%{Dzr9JMeZ;;3g}m5+8!IO1pE zIXJ`J2eI1kTQk^?n&5qL&U_SuuCmKT}Ax4pfc;%?o0@XfWGpdt9k-n}25J$3ff ziBm@pGi1B?A3t*F%k5W(jvO^3e&^oZ{f7@<`~K>s%b&PIVubei$+P_j4_Lo{^VUso zPPlXTjyLxjT3o(($y2;~z#skiC!PoX&2Rqx;NgQOPaZ$<=A^qW9s<68{W{rOH}&%6 z!wcs>`t*~_moI(dQE3Z)&VTs6RYo@O@c^+|d=EM6N39O3wHvLq$KA`@_wIP$qF1UN zIAEkpnz#*iMFU{dqYg6EYF85(`}&9{`OX)ze9G3(fwWKAv? zW2T>u49r*|?N3*eul8QH=2iCWd*&{OMzCB!&9HW|8WbC=4KVQ)piOe7H;Ft1b=MY{ zem|=)J1Qxsuu`T%#4K=|MmKqMsoJ!hu>8=nK&?3fJpamge5loxHClNSFFOlyEqLl~ zU=8x1`$l=C26J@Om?K;6kQF07%grtVW$;qdZ*9gkOB~CI1QvuqC^dK)ZzsIv{B+5d ziNP^nKeH_r@kuQ!U5$;7q@s#x7IjNTK+JcN&?0$JoEQoi;T{V7PSRp;Bi+>JZyc<8 zi?d-;0SUuNgD?9Y9rnlO{Wvf)a6b|wKme}%txOVS;=N|vd;2~rRTWS z+{EIVeEcQl;b%?;@~AVmY8u16sBqz2ym+@o7)G%}3PVdyIgZ63cAV)Hqb9NFh@t}2 zCJJMv?o+8*)h$DuMGN&r!Ko9Sjx)X>!da`;^<-=PD{L_f9T^l%3uw6|u|k99i7E*bx>9*^|4ex?T?1`~SL}Rvl5q9r-@7kz*G9x1ylw|EA#i*ZFDgz&f0l1f>Hj z&KNNxmmY8qBC?iSM{40dx(JOJBN^VJ&14X+ITBeDsCkHbUeXiU8o*L-rP*LBT|YQe^}s$4$i{&mjUFLz^E=KVb)dj;b{N zSffM;M+&fl1DL{oGdo7TK8%})n*TOjlp_QNFfwBdMJ#Z`aHQo~8g>mQFxDy+N3lPZ zBHCs8FI+7`8KNUMxMSNGh<=6^H)h^iQ7Z+6BGa(fOb0rG6^%IwQi3u(J5fckaw{;l z6L%O*Y{N2R%N}NJT5_K8101V-Ldh@)!Oq?W0P{ZC75ozbh)KA+HyU6I@>?X(BCcH8 z*u+r59wH3YLnZRae<)j$qpozO0S#Bf!fG^9u_sGSt`<5c6YMiM_q5p9bl!8)k|9W& z8s{leWWmLvb4hCnD|r2v=5ICGu1AiO7*^?7?$GQc7cyVkt*qLvH=?T2TuPX{$RA@0 z&6Mq=B7PHYr>hsLH&kX06;;)FWu`_t4U%dQj}jw4XHZaY`-0bR9$KQl_4H`#F?PQ? z@KSd|Z5*rnR+XT6wgEF%Rw-8!$Rs_gKP4*9(UM&X~N#lrIsSyx^s8NF#D+0lMfz1Hwkd(?n8qO%k&TK z-~ZA158gj>+L9)6^Z4vXA6?LcJ!;sq_sW&;Kl{;-EsU`s#?1V_XWIr*o|O2}kAC9i zXpbL1w&ckSy%kS1XdM(tUI#!p_Tn^uZ;$|iB~_jQ>2_v!}fP*iHC@dy#X1oUP8WohWwBoIUqNulKsj&6%PzoqzJwX$|%3Z@#uR$(Y3Z z0hok2e)~1qebw4tD|U3uHWIyc>-Klwerq=&=Z6p4Ue)c528!;?Fg`u_GG`Ard&0qb zANMZa+r5{Cu_AN-UNx}wWXmXs-J&*IpQfFQt?R3e)Hbv1G4@*7<}@?S+-nN8mM*6; zX7G%-HZWy08BtK}X3Ew=s#mr72V!7MD|I$mwU~i1rFpL!(l}NHDqdv}Lbqbg{u;Aa~=4>RPt1+LZ~M-&(|L@_Mc1d zN?5i&aZpa#n@U|uTq&IvrmSM14%qyOKdgsz9s5ZaKcvM%B>hP*b6L5i0a;Ubk-Ms5 zcuf$_WfV=>2;jfsum43yG=QNM2Dsy7_{mu2Q#HKEowMl!W2MYZg+Ww>6{|5eHNY4u zgXttrbb48$;_PVZPXJO@jE^17E6@e;soyLc&+J+k41*qV# zGHdc=m%koPa8yN`Q4TVpkrOtj=pez-U(c=+{q9Y}}=n>YChm>&XkC+9x#9Yuj#EAK1 zQ~)KB!=BZNpJYD-EJB8{hyuHbdtG{VF@v!*;*+l7brQ7_`U!oZ?(2QO>mL z%=8OLve5n}0(RmlQ7v(-8LDb{47b*s&Bsjo>JLNcPo4dY?MW@99|=YaOj&aMV^k57 z`bn?hYV9osdZa#>8hx55L`gusP^Wq$BI}i!Ay+lD%L*nytcJwuk0BszXeZ>+Z;f*D zX)3ed)mrFvOCoN9^|ttum>g#e(CmLk0x)*dY{LeQLV%_*P5`#?972$Qb36%Zh^fg0sqC!J+Z>vBS#!4@53t3y!E zWy5OxR6RjO#4w$Msl~+SR2$uQfv{S%`l-ND^%r;U-#>P^E%T2s)_LW8WOAr>VrK^}ebJWTkX>)_f+>Ym6_lQT2969%o z{)=Dypa0^Y|Kaza|Kw*s_qH#~cs~E~tJ8M$Jb&I|l9Q*-oH_IU!v}Xohs6KTN|NW# z)^oJz{`%{0-v8i3ugbFI+}=Sr_k^o=+}^)?-=hM$ghPi8Zh4Z{gpiBv?Wg8yUOc<> z@yFj^{rUSo`)6v3u-OR7ZY2bkA>&2UbPD6&3k6JwH7baI(Fpn z`EzGUiK(ADc}nE1{Hc?tZ4zp~o_oeld-s}}Cp$D7K^QXN^s`SsvHPbeoE9-+`Hxp= z-ne#S>&R0tQ{H;)5rX%={qDhw7taqIJg*HRS(&uTs|6{_0o1GywAiwE2Gb zLp+phg%6AU_}FoGELep7mpmbW-*3P94r}fVynOxq(B4B>{pzbPpFewk^7JXA3_`kp z|L)e4#~L*jZa#aWZS#iTeRt)XZ@^t9d|ITpmxM8YYiE`?;Ntq0`g5l zjcJoRElV<0WXKZ(ImsQ?WCp5VdLM{o1T^E&lq)j-HNe$K5gD5Ln>+-+0#yw!j5QUX zMP4NWCMnNTdn%*+kjYYCp{WqkJ~9{@SiJi%nOuTUtxlq%LW8%uE&)+E7qhODQmHw) ze(PiF8fy1`8xfSHOw3IEIW)&IC|2Tvwjj+phGG`SAs!k3kka*}=^g@t+0G`d%GFvw zQ+JZ$U!pb_Hv?H_V{Q;iO4Sk+_CJD%=x8muGD{|*Ok3+Cz5cSFz;z{K&=0H};&6}> z2hvi1g%*U2tW!$z5iF@4J<$-EV`>%1taQJo5;*t5zhaR=RZl&&s)x%`0~*SD7;N&S zQ#2lAh;yCITHz@(v;Y_yfEDR8x}1;-pV%Z<3S=|@tV13KlSQC##pugsM9l(n_dVj%{ZO6x%)qzb9J&A@`}cm zjiGG}21kM~)D&^H(qQfZ)E1&~wFouGp8G905NXZgsiGMhL^ohDkM+{r&tr>!te~}8 zar@O-$LepTW2-rAMdK>2;jS{l53_NR%M5^r+>aZvfk`~!8B~NDLjXjc{MQjj~ zueq#dc{6Fc&9>(2a_m341Bs+?hcJ2~Id-vvZE|lQHYW+Xu~~!L(aDsE2A1Fu{Sy!c zHNen}sDN$Z{ifZxQ&VG4|E#>CQyj)xT zCW{Gk>Y0l0s~97AkGlUB%uqh zcG_o_&kz;Z&WAu`Qo-UbbHQY64di5;@2gci%I@fLP8Y2eEn|cikzs{)3Z}<;)-iT4 zpz8jRG_fDsqK(b7z#vqu<`4is5tF?P4?|Cg0S0Zz;g`E~Y%Ix=L*Qzf-(oz0%q)_o zCDfb|6N?JA0^u9(<_h#t_h3m1v9&nqtm{e5(pR-9;^40wZ1G=H380ZZd2Ce2pr%n- zz3y!Q7MXZ&o;a~J?v+@+jIOgY7h`%r31Vxd2AUw&a{&{6`c)4w!`yBk=#spz_x}BR z;ymk2Z1G+Iz1Mv93Cn)YpFjU+KmRjZ%2)z!lKV$L`tj$Ve{OgGbMJp(2FKj_3EThg z+-0{)YfDVDZ@stU-o5+woU%NE4DZ|sSMI}aW`e)gO{vTxG+r%zw_ z;QbHZ|G?YJh}X>u0kKz=kvo>{_s?!TBfERw-fzD7)|9?J)uQ;vRt5?4L?prTd@S-$|bcaDp^pNz27e4x%zx%suH*a|q*fYUman0lV@2^=bX4ik? z6K&dkfIIhy{ms#1CycMOG7Dk9zw-Unt2apCr$7GbnKNhIrugo=Z@0D{dPjp%*}g;8 zH(4Ka7_P^jcYb2H(yFl6@3jq(J2~#Cd(z;3VBs9U(Z$M( zgq$vToJbCJHM_}1)xc6_wMl5J5G@kcC+chV6<%qp&cH8iF2`Y2YRDgWt9;cu%P+x= z(0aU0A~%pD9cNjO${+Dn(tHM6lSp!>e9kH$`eBV+Vtr%uZR!pzb=IY$-8+XP9c2bLIqV`d^XV+< zErGIwf}?Xb6< z3A(914jlud=7nO6gR6mT-w&~&LJN6K00(c3P) z;JVJdQshAmb*1TqV)~tqhW}Kg5rD-T8}1>V{;3H>U6KAo1XY0Va^rAkhP6>a(N&2m z6kyft;}t#PArxslq!oi`gt())ur+X*Kl;QRGY%N}&E)JT4^=32qd+Ixa+(O}@2wjT zHqLnZY@5e2GiC9Qy`9NB3y0U={EpU><6Mn#Q=BVDA0nvb)jE^Ag2+wlYZ|wl1*Ws) zwJ9}%tGKj*o+!iU*FjP?GoEPavE)?97Rp`9=x4H{C$Xm@5F)^%=;14J)I0^3qqRho z2ty%qh`Smu$yamg7CDqyn*?d_CGicNAY}wlIp&ZYl7C+Y)7_X1yMzC6PRYqvl^k34 zAKQI~lg^=Dq@$&538{Iud=~VAwTcHQXf**2Fr|ao#k+(IwoyP8-qBo(FWBMk#tC%B zgqW7df9FflzxLy)NF&80n3anBb>~i4=6o~Y29ID^Lrp{l-SPfngWgGg4XKscAd$iMKDk?ig+K{=;|i|Vjn3>4wE0~Qs-xtFKP0ounLK=0-X&j zU82zI(O$h4j^DZWV9RXufkTD?d-iUN0+&olBXkye=lWiXHB%cacC*z<4{#gSzT9CS zEZTEF>pffPTLse+c~U*FZ{O)NXP~kUhjB6VN7rwBbn)W;eS1xyT>j)UThY%m!H*6e zIrNEF&px+Z*4vY(PMKA)d|Y+2__uG}a%;%*sm2nX@jZXx!l_edu3x);`IAq*4A-*+ zmo8l-e!~XFO#Gfny>a7#1wYu;5K}_7kAs)ix!=8epA4*d&%f9#jE1-=_5JBZ_wIYW z{S3dQ*P5s@xiKBq&B|VFKlA8~w}w6QTHk=awBDagi08d++lW4QKiWIL$U)F--~H(( zo@%SYdfJz$BcrmUA^iRXFvYwPrv-~^V_#?Yg3l%*zo6f zzyE`K5!hyr9*Ms4-FG*x-~9CQCqMi7Ke~D6?tlD${^R%8zBe`ykpDOTwPcxOMx^qpgR=3MWsT zu$bq;{d?EGzwT+{&p!L~FaPqd4DkQ(`R8AJ@j1qAaAYx(EkQkKfHuByTfa%i?w0I7 z=tjEjO`bmAe!9KQ{O&z`aKlpvj~^MJ83AUvW89e`v5ppM3S-X8vX`w-)5*rW99J`K zrxsh|;piq}tw{`^(t)n6Oo!!!`P4vGNLhZlp&FnSpDQhB>hUPkFjZMthdjl7B7#i? zZD{Qm+=YE8tW;&Olxqt{1Z-#&yunJPNwy{k6FBrhky30g!Z(Nqv&#TVF$wK?s)%0p=d#%(B4=UL)92B|YZYfvNX zAT6i^&iSS^N7*2660}ZALr%~ z`>8-D<8>-gF+a`VbwWSsi3mAe_YS!-4bDuOKNKg7qP>WpdIQyM%+_f{N5lt(adhH3 zHj{1r7{96ADKmr)!m13cq1$;(#7or%L>Y+7vZAeC(BFIlm0%keRX*j539O_M09nc0 zFy&SIXe^@$Qqflb!l@PR7grVjZw3Hm>XBwJP;gOnMGfn>&KvFc|3QX}7c5&E6}xf2 z_|_>!ZJmK8v4W6#?h#9bS5lEauzSYWw`Y!@Z`(c3&OgrGi;%SmI#hsdxa_BlxW|8X zHareBBQ`|?oTr+6cG5g6L8d<(NuB@7Ejt@Uj68vfrtA;cK|68d)LsgTluC_207^u!4p@eyjm_S^$iJ%B%kV;A3V^&cMXN}Q)jA?k?% z4HmZb8O`RjW#h@X<-hvs%d1-tFJJzI zJihwskDpw=WJcT^_S2_N&z?PJJIgP>`0~@wKD~SYK7bssJw2O5fA;j;nKL~a_UeW1 zfEGQ0`f~fZrw2su?n>(<5A1ix+{dm**=x$AuTd91NoE8*$|dMD!eq2-J(UP`+uN`_ z+w0M@TFQ={UJb^1ctejj6`Sg=PXpww-E3Z&2tRgW*S&i-&VS-Ti@OgVJ$0kloq?UE z{CByn!CtIiJ$m4vHf4&y`-87tyDkXs7W&TJc(f>pVH<9FDedbhGTA z{?rS4w;n(H>woq?@ksybU;k_SFM0aHE=hEH!|teO&wX_9BQf`vUw(Dv`t^@K*zw@e zqu>1Qcfb7AZ)}swM7_}P_aM2x9#e``}hCBYMjqL`^@v&Mtv;n z&aFGPSQYN~RKlL!TaO-D3uGzLM;9)5DVu43(fpx<2Y>#vpMCVvMQe(F`O9D3ym^zj zJ!XI;ErCTCx46-#Au^9QFFh>5V0PQf)$WEK@n!~(pFMl%*^y@+r+e!j%~-i6v_4u$ z4oO=r5aPv%KWqO74Ti3=@j3QWQ`&q~9&k;X$TF_eBxp8h0hwa8ZIvl~8mf!Qc!y#k z90Lh>zDFaPQ-tJ*kiBc?(k5{clgpmM4!&}x z)vHu0Tbg1ue&1x5aPm*NqXPaBUdcRDW!`;rUYpe;NwiuWLg0T0E3J;cWNAMHJ0V}} zDlUTzqKx1Nz>r`0N%iX_0vDwf94!p+b*03rv{68SrusUC4VAnA06+jqL_t)-k`w|% ziWP>3w)`nqnN{D#yL#a+Ppv*vXlewcdO=%&{5UQCst=h&Jx745SjvJQxFU>ro`U&p zE7fxzfhvfc?UFmk$_%p&>>#35;zVy=t;scdD62O?gk4!EERNiDUX34YcF4VMMqCehe|M&uaG-)av_pfJAfdhz9K*5`ABoj;jTpp>rpUSi+l>r z;$ef)0~Jku&Z;9}|7Mz3#@GjPvW8elI5FzeX+*9fZKeB)oxeMA7CKS~Rh|FQN!ZkMw zfLP;VsY;ZB!2~`i1cbP2NsraJGq4%l(N+sw$~8mjBx(mqs47K9Q*KfTH`;j2rTz8? zuf^rP3jcws+!P8_B#+i>B8l5E`iH92YA7>ChtOp$8*{+Y@*10yjAkmSyAhCI;?K*Y zpb>%s@EQF~K~s?30~t7@MHPpb3Mqls4#_GDZfO&6bKIN&B^|jbL^*^yZ6i(TUQaCc zXAh!~ykbtx7s9w4HI5jQosu;)AhJ#gB|#b`Ni=npmh7W(iBJx(2wJfYag+`=qSy%GkzU#Og8AC+DC1@sEG>j@vV*Pv5zF+on2% zc>45Nd(;SpckkJ6HVY#nXAkmz{neKizT1)V_1CXFt9tP8A;Np`;L*nyFMG4qmG8el z_x}4Prwss%3QnFl<*j@ky8YnnxdXj{Yo|wADV&DP6cGx7#N4kB9kP~&DGQzJGe@9_ zZ9EU{IVYs9cDP$cHrl$Q_|$rMD^wo6)U&>RW#1x=9PxYh9@qul%N~iIM}9r{+I!;1 zK6iNA802>!*=fm&QMZb{QOmo@_Feek1NSIA8p~jb3;ksE`04X|_a6w9&EFdjSi zj1X?$yKn!_)2C0}f7A<=MgPD0`(L-G=Zojo;&?|d+5YS&KQ{XM)vtf!ISQcM8@YP* z`jum4^6KPt2J0sRd-Fvk4!1kis z+uD>F&6KiQTLwOksBnhQ)L6zO$J%Q!%kmw4rimv1@ozlfIEPwct$VU-yVFD@P8XQd z(!45Ao|6S>yg`{Yv#@P`ho08@g?gH{!#l0>o$o$?l{;<;n1UAgBrcLSSb{OGq-L*J z*yDG87H-FP?YlPa=H2=Ukt%`X%5nn{>Z;@Vn=E(ER5z%A0DdLIsEWAqSKg zUQ{N%2#2Z@5SG&&R&rZajp8ADN>0s^Mn(xZCDcDeQG{}I zR0YU-6Y!)(u**iMqwKN=_K+0FP9l#q_{&6+Vglkqj1mVaMQhp=n+pYsxIlAR+4&Ji z)mnn4qO)iQ0lYYGb@BkQVu8!It3e%Rk`#VLL0Hw8DPM6=n>9yFlb$*OG6C%!%_IC2 zP?P~gRiRQJz9D;rp^A$9rg{)$CaY`XNP{?Ghi19MraD$fOzW80w+c4>FL0`8ZnsIt zYIPzn-W)fwNh!H$1D>E785HptQcO*hZT=9u#Isps5`+GIChXvp8Uyg@)_~4ID99}r zmW*&vp`J0mNuE`Zs?n8 zEDr#WON<4)GyT}EG}%v5RRQu-V=6Al$xRz9NAa3ZRaU(c27Dl#FpG{j7;37#Y6(YX z(U;<>ZPh|+`5QY}&|Rb@sg)cG7V85_`cPLzh7pI^W-_1IMAOmc-C5jZ@MIbJ_ zPF5R2i_a3EKuoFn4w>m`)vF<=v>xsqZH=N*CQ;?mYKodcv9HjCh|R~RLmO_+&bMhg z_KO6Q^1!L_b!VWmwhP2U{#`Gb@> zJSR`Z2Ia^W#)jCVdIqrK=91LOU^dmNHRaWH5|jPZ?2Df>c_kgqSnvVe!^n5VLd;-l zLh2|L+-Ii=7tC}84tL3SkS!!s6TitpMgN`BVW^>-FC9;aJD?ngW9K>c z^$q5ld<#&TOw*l@2M@Phjm0J=;O%uoh=$W`nc3T3_ujmHi`tH#JbC!gkxxGP zOmnk>#>ecAcMRBq$b+t?|Lvb|;@^VSXI_1^>y_IDqF)<#S~zm{+*u1ST2}^yGJ>Z-~IRe+~54|-xw`C-g-nPhj$+F-UUke;QR-~ zXg6-p1V4VUd%kVk*E6cdS;45OcI~+P4Yk|Al%a|| z|EfwYL(_wMp~)&U!ICz}tEub&t~{s$x$qMA)%5(Yv++Uni?`)RRbChZrBWK#bSR*m}^ZC7^|7VUUNFDX$Tu>iEo3A7pXs^Y+iPfc6V^Xf6xh!WWS!aE{a_T8qMJ@6N0!mROSaWmd6jfY95zUQsCZ#41MLw0lQ zu4K_nU0;Ij(=k=`N)=zw~-$Jt=V~@%T>!-IEv*#hJfh)FMV) zm9D8(Ejg0>=r_goPmgm_wFH~#D=9;1$|Qz0VO(Xom8uw7^#fWFVBp!DY?2#0wmvtT zP-^_r6z&ERF@U6Cy03VW9bJInjG!1p_OGBZ;*=b$H5Qekqr?pu1=wvu$gvLh={q67 zzSnJb^)JgY7EeDw(S<1+>@`da3l6{BGq7gj@J%00;SCw|6juMBx)Xa023;zNO< zsA^Lt?dt{-(ioMlm7Wl0HxdMS6;%2lf1Ndz>!s?n8w}P86HB235Q~tOzPvE8FWm6( z?~a{r0ogx{?1 zl430qXSH3$;&z2~H3zaFjF?P)c%(VYdxD(^`{}OOgv{d}QdmQPfujm%g=cFUUDh#Y3Tf+`HTvcCxjN(cjcT;vG&Ady*7DZPLN9nNafjGiqOU)USfQjJ|CKPle zV&tf#;tJiTVgNVVRgHS%g@^{oaFREP7ebqMl%Ihq787_{_H8+ zurAW@Iec%e$Zx}BZrEm&XV07wD_{Bc8xOF008|WWG97AeS5dcH*Z*Z;%0)KB=uIt8>dg6 zdhzTj)He8W((;a(aj+`Jj-uAwAK16&=uvkg4%^~03&#|l^d5sEzueQFeU2Ks0SyVD z&p_Jx9`R@rwsO#Crwv9McNkXe*}bnV_jIyv+WfTbF%KQ`c=5r5=Dd~K=JfI{f?$vT z?%pG^z4+mWCr_NT;1HcO;Xz?O(Nn$$cJBg#i*c*K>x6Bv^WFDX1<;W}U} zaJTCC(W6%C8!bJ5@%*Dpm)xSdbNi;-BR1yWzQ0Xe_a8n$gCT^6n!QTwm%sk42cR#0 zc>d=<|JkRPKKb~g3&#!~`uq=nP~F(^?%g{eoj88nra%`jUb49V^r_SKW#X{C%8MM{ zKX>lG{U`r5m-4Uv?Z5fzt1pipHJ)kJq~V)Ki;o>Y=I(@HLr)odIQX^4o%fn~czWwmg-A9kzB-y>cU1n>t=_C^$ zGuC~y7^y~7@;I-WDwP1XfIbF58E?LwpGJhJ<#**t)@2?-pz3qOFm ze1}%+nVzRUId`n(MVaq9@gt`eUwYN6J!dt@=?qaRktxRRG)K)dgyV2jiC6^&ZuRFA zRCp^Nc9N``iP*c2Z2)KD==uIeg?ed0Y2<*52dWw4LXa3L{N zRqVg>BpD33L(B+t$t_Vt+fWw~oxHnrnxIE-8~FsK64mOLi3C!qxCsGlGW?GKl?=AL zp^QhWWCb>dbWB}~=qyEkfI9>>UY8Vxl}@G3*nA}5PFwp^)0IEf{O1cd2 zRB`6^`B1mfsj9#Xp%J+N08TT7L9E#njN=dH6n8;ZGCq^Q&@V=rQX@_+R)GlDyQo`- zGtjejasHaXlUAYWBZlWaLOFN%{q2t1yX+AF%irh>Y7EM_6|H@;E2JYG| z&g%hzBV%P0Ep}aY77dBMKbtUm+(ycdSM4bx@Cls0rw(CIS7+o{6^kMx8l=%>lL!5A z76jNN!Jr+A$pm6-C$~-Kya>oP)$)Da^qRn}rx(#|-Di!0?FniU{>Z;m`xZM+_n;F< z#Jp8*xNV|+dwGd?P{)P_b0NwC%sah!wa3cMn*RXQ$M5J$*ANRMc)A;wQVwRrpMD5U zxH(gaOT@!MPKe1kv+$X#PX6@CEG3h~w6P=>J$=$$td1t6 zA)Krx+p%-9pSp-Nm?2>15~{K{^mm%p<{)FJTHpuI&^n{r9>^TDS#UycHdI$j=pI;a z#aat)h}hBrg3r3sJ%a?k8$uXh2SC6%m=XgA@2VjY(lT6GXoia5GiDd^vP?AQUZWkv z1#N_)VDz_pk1#8P$(ET_gOg_emn8tU5CnyG0SR{fS1;psA0I>X-_SDj^)Fcvxdyh| zM97ayHBpyRMilvL%M7F`U#a38C2JU>{G#{4k>6c%Wa<%81II&c4K;+Rc{MT63PKE;ZWOjMWR&L^ zJOPma(Tc(vy*WqcA)k3Xc-i z06Z@GI4T$&74KUoSUqHSOTciWO!BM^f}eT7B4)}%M`F~5BySLsAcGhHn8wP$?LuOQ z3T!aBT**m!%BzbO3`EkLBo-n{O0GLYAKSK_Wox_a=<@8@lO1ooqt|Ed)Z-pav0IyN{ERo--B_kCywh=j&Tt<;vd7O3FO}VS;@CG|fAeDd<$;gA z%J@$r+#jlu1(U_CnchYuY+e)O0XFUOA^vDC*pcvONf zIDY(S>&)3RUVkNEH+DaB>NJQNiFKaA@zr!(J2++C6({-H+*c zI}z=+jH)PNC%I`yR`nRtESF|QzM+WM>!wV`YlAf}O3NAg?y(4~wL)|-&qI5I@PV`E zEU>Zs$h(jYIxM5I;OLM6&A~$_PM&lk{^nXu*dIA|?7>#ASk~r^g(v|EQeYm!bfQ#A zg8=&C#gFxoo~X9jDqLiC>eNZP^JV~>FBuFN8{E2ilTjF;UcP+U`z`;)|M4$8)cf(r z7rbKZ!NUheMf-M`Alte1$PQ9Zg#H$K9Y1pHpv7VP58k?a7vm<(3_m;{!Zq$kwE4$Z zmQL;7KOq5`ppd?{pFh9<=<&@5k8VBmj6}qe_wL#an?&=o+tKtuWa1CtCQR4A_?U~DiQKQ?C3X~d8vg$=5)Ms8~Gn}Hlc6WH0;FG`sZ8{NCF7t znA{K^)^}WW0(lH1Bw|p*Rdj_`>he-K!c{F^{S8f1uu}e=S7M{@$({;w%a>U#r&DZc zYA{Z^0x>s`QVbzZjAIR|jmFcoB~Ikh%0aF{Iu5bHiOXoYA|9-U?S_Z=xu zFTf8KDsV^zKBKk(A-Rg^mRQBCUbXsQdO}zK3!2P|8!r{9)x~Dx303bM=}t%QMwnSh zvJaWEb`2VNt}YSLVw%WH28|zblpyNN-|pB`mtfyBujf`wF9`jBS}IuNC^skpnJ#u7 zwYSCS>TU}qGLJYKx@;292@M}lGc z$ZaowEKpqX1am^$)>0(1Xc&Q}qDb$6hgx2fmgJ0pWKqUMI;u?+lJ6+3iiK6h9pJG> z2|1$bfimJu=}}o`q()1v0+0b5rJ%KzqJX6m;V&=n z^S^33J7iNYm=#*J!5pfVXolurfb?lDcF=}ylyz*6Gyr^K=Z)J)5! z6x=e^E=MShFU)MdvelvJM@Bv)QVst1>}tJ)s(?+rm~{J~IlAg~WR1bLU}f_M?l6(q z5Q*?nK)E^7>cmB#1e&IzcJk7Nb13FlHeR49lvl2-RlKUbrb+qa*rmw4LV-x8wG0GM zgIji;HeDyC){X{gaB`8xz;lC5U@#}jhrL)f}I?OzlLM_-yJ^#&Ek|lX;Nfc#n-Ry4c`%+j7%lzIG1$wgT-kTX2arQWIA~GT}MK)>=724J5kKa$FLflw|4`{`$Ou+dhV3_EZzd@gLxTVrLK98v9I&i&E~flkj0BS- zx)Fnb@K!RRFA0~BG$pS(nhDk0^Cw=`-g^n_3$g!;S35i2pJl#}5pwy(-xkrDq(X`e z+!U8NKj7`N$4|aB>iq1$Q&FaMts*(Ig-QDepr?myD$7OI+^!r%V>58qhcdhy}~=f-00-MQ@vn+VlTGBVph1_6e*3E=Mi z`{wTLTJG)Px=>zCp*rcdNhmr8H*+L$J>M&xPqx8RF zjOq3Bm(M*#@qmf;wKT=drAr)E$+-W7gdym-kLK$>YIS5KZi z6v0ypi?q_foFt0%O0RpjZr%Oh_KVNH{L$B6>l`6c;Le^dT)Y73JeE0Ni_d2-p5Ffc zmNBp=Po8fb*|O>Cum1H{ckbN&vp@gEC!buo`os73E3$jhNppnWyw=}fVV;)v&V@H` zaAB8H?*_BppoDh^EMPF6ra$)Fj3DaX+SD0(`SKBoZf>BT*D4A93|8sM zQ(=ivAph!ezY^q>ea5|1-B>&M(cv=f_h&BCXXdY}_ATdBg{kRPM{*B}8|5gIi+sG) z^$WDrspENyP~mXKOPmFG+CAda+Sjj zWDOg4QIVSAj{W@3xI%Cyx0C7Q?SoQ0ifX+4OrDC)`IK`Wp^3~N#v)=eu3y;J?YS>@ z9cEKZ=%;Wc=4whvUU)DL@@nV`GzTeqGzYb3C1npO!-czDCm|eHuwG(TIPjRqh{5+9 zK`47eHW)O#nJag=zsREVq*#0d9z8S&+DLQHUer97Zj3U~jY(O88Q6rm1er}%n6oG` z$~3i#<+3=fK*Qu#3K}+$HY84_ko&KFku_{V=Mrld{bIo`3O!vr(34*^A5@w0Ca*G{ z*f9cR^sl2bGo@qXQ0X{MJRRxBm7tmU6k~gJ(cbNC@ZcJ!ms$cau@M+bo5u^L)IyaK#|G(l*v8{f zxQ6}^M1Ewl4yBbzs?5YUv6Y1P@kI%~Pa+3p7{!Gw5nV9_6Db(5Wyn?0c#FFIGO|zZ z@jyyI9n4NjAKj61N!;u78Qq*2NI@z)Tv1Ztby~4SKt?K;3!m<;(~zQq9i=#%ZqEaj zz8m-#{Wp@f>cSK@fD5A+LBqzhZ1Vr-4qoUwO|EP^Dfz3GBOUU5_&_*C$YHOIvNGY;=DMXM1x5;eq zi{S|9PqC@DbfH@Na*Lw=7#r5(hB4JSKpR?N@)vI1S4vWe`PBkAui7p&Sg%Q| zdz#`3uIpP4Q4UuyF1Ab&cHlix>D(eFPNytziVZr!#!kS7BIeebLK)xZ48Tq7IU8&`h#_>sEy;~)L#^5x6_ z?|=ITna`X#WjNn_7z{zI*+06ZU6Mbk<)!-!0;{RDkE0?osIoMI!T!Cfc$<-X%kRdsC$Dy%c+;?ygPYr==edNaW@BsH zH2o$60kf?^+BQ&@3ccre=a>_LzPT{l-jbV$4G#J3BOF&B_<&>EM?6q*($uP^4QHwp zZx$2r(vwIwUo=(@>!-v1y-R=47kqvK7@U8&z*H1ff{Dl3i zYxNHww#4GU{@1^?()YrJ^LKCG@u2U??qh`g^rwG%{^EJM``x#HFh=KP!@2=ao;)T- z+j}aBk4KLl*wM!V2Ja4|>pY;&?3*_a9zT5c;sxG3V7Ju7X0HsMyY%Xuy}cLS1o6oJ z{|rdT(V$aONFCWQ-8dP8WysuDt!03hkr#`rM0vULuUnPP-X-MJeKq+5psz3#6U=$k3=y4RjEv`q*?35-PP&NB zp(o}O$fOkshhB_>{j#ibzMUd2B9!j=p~8W@owk@~oR&bQwo2@{nw;|M++^bGNdC z#&<1Zrm>H9ah?Lz=q`Ya&(YSBGI|;sWd4PieT;hcy88n1EL@?H4fixA79}L#cgXtf zG`1#|809wHCE0jQ~7MMh>lTaWQiFBpckV~lbwsg;H)?NgJmC82rDLoTbX)b2T&>1H&ifT zK=K4~##|-3BLhK^USInaCuIeFDSX5?);VbqW$p}E^EZO%!AiDjq)bEE|5Suhmi$nY zBga5b0nFyL^Ny$Z{u4*Mk()n5{j(|ne&UVOkN`3Qz}N7W8f4Vdl##F zn)NXn$;NeqPAUQcj+&GhdBx^Pta=TmUa*PUE%D8Dq#~h%yj{7p?yu?=(W!0yCkEzX z6u`WEj3QD>BoR(_5hN$M%m_}oF$doKPOdTpZfRvMAgO%E0?cCi_T2Pgc zE}t8L$rwgl0p6#_II#U*24}bbsQ0LA+@GK5x%`a%H&g-vL3Vezu>e1lZ`h94$XYWU zzXm#>=8LLHReG3}8SzA6B9^k`;5#Y_a42g~0IUREvP(5<87qBs)Qj>x_MkBYCwt8z zhJwf=hyaXokr0Zpl)-?S_mWL`V6C#ZF0+pWP^QIu6a>d5ae(3u=N1bqPo*n@g36X}ebP0!E*2e=e;7m@Q6$T1 zbU6lozk{Pv?hjZRc~dNA=Ss>|Q{}SUS6J_aL^ro~9z4AN_}NpQ_vov3{XgFFog?}M z_VY2cf6fbE4;=Q2F4IwN-n_v_nxJAT3!i%W^qH@}`q9;ES8v?B1(;=~rqp|9?S~(p zy>RhkqXA}#*nQvg1k-rF{`zY@1XI%O7isJI2lq@idGPa}|I7rHU;pOU{6+)Yuit!h z{^Er3n~Y=gxY;FU%)c<7h-=#4`p zd-KLfYA=#vV-tIaJiFL?@oZOweQ5WA2fv3lx4j&r!EIaF2y@#BI;mOwtFF|u9{E!> zO?$7fUnZj)Wd=;@EOcY}Eh^CPY_&Rh%R+?N>!@3RRVCD!ak@hsz2xuof1W;jJ`?V3 z7X`rUqZ`Ko)<`L7n;qQ>Q?O{%7DIqvzI+8jyYZ+#H*Q|l6QE)H_v=g@H;d@_vE7G{ zjA3X!|M2|}_wU{nqvo7M?F{}#F4la+NP>{As=+>ZZy^V|P|{?rQM# z9VO`RI!W%|nsZ5ID7*;Gd(ljRZY7T#ZIrn0hk>tr_)k6EnJhyqRQdTJ`hstcQp2@4K`94fc*ZbRNPb+)k_Sfe2{N>FAU zYoSkT4GSu{jp9rSs>~24t6(IN(M-unJ%m9{>}n!r3m?|4YKH0o9cNZF!8RH-#c!G0 zHfMMc+!$C5zq|(8!WXUSrVk$JFa9S{K7+E`n;ToR0~f?LKsoSsGjMz@q|@4#ceTC) z0)*Atk+Se0$ceOMA2V@qgTCRbJ7N)tOQ@bpRzil9BgU{Xs2j!e=q0--S3isv8#YdW z7E=|3=f_7%;{ak;Xr}fcT8+%bH_{IgWGMnM0+dx9aO)%zx{;z1Xd-QbI`GJ!9zv)laohtgDeJPl3r8|O#5wfo~08vK}}44 zknD2!TJjrhB-asZx?mUsSeheG(@*@5*L5vxC18rVQERI2laW~` zNLk||I;*1DZ&i1hRoo;gow6DMs?0MumF$MBOf1)L0&x&wR&5Q+f>GO*y^i6Q9U`K@ zTcwjl5YgOaDM`p*`iL6e%|D2wN=0i_1r#GAd_KhK&{Wqwo%kp-O$s;J)du>K!o-K} zfe=84R4go}K`CL7Po|-%jN|Alg@h%av4kkDUgf=p(U|pF0Hu{!reP|y?~l&9S4oC3 z2k?^^khpDWhv56A$tOFTMB2jveFC?dW^?4cJuc1B5DMEfbZh(F@e^BT&Yo)xK#K;f zyL~6VHd^l;y<*c%^XzSHv-jGzLXdk<;MG?~olWll{EIJz?>BGV-n{=nKf=a7JI9WB z2d#CY=7$*LIeq4|_iWh})8n?^{qDCge0t^6oh`%g2QOZ{{KK`YMv<>v`NR{0_Q!m@ zE4sD(01?^Ef~eeg>@9xH=lZb@(@+f#pj=Y{_S^vxOeZqfZXgXvi*x+{MnT&S1d{X z&2N7F?YG}Oe!9z!btFjc#*N!nG3c0WZf|;TfP*#Lp~$P%av8P&+TFcL-}igPTq`R~ z^`X)_DtN`aSGBW9a^q0n_4;VTJ=Ll#HuJ3B`C0ARc=@S9WvI_oU6OP8MER=NUC&)r{7oiz$B>ur7`g-D?tDiT zTs0G*A_pwqE5dR>(VWz}69NeJ#-V<7P?l?FOgqqu#&g<%RslQXi3$u#KNS!Oh> zCK5Tzj+7|_#z<%79?3@-cB8d>O3Nxv9Xv~xC=)F9EnBVAOrc0O);P`1V{Ln4rf~6k z+6AXWT&(&L4Lq&QlWX` z^?&w0#8oUQU}>=fQ;p7~f{26pUmI;^7xh?vWETNN2&!FJ>bvS-XF12 zB?ThQ0hbc!WI7#5V{9x83!M{*+M1AMY5Rhi`%4d#<}L>_G=WRdrS?srL*PIj!x;+P z4$`_;3qv1aT4^JThzm@|`=)xrm)4yVo-Vi z4sdCuATMDS3R}-L9jW5}=>r&?dmpn5z2QhGjs|`NhZbA zM0n|7g#KQt$#;C7?5R0*CpCD8Wl@ufalKjR!SR5$XU~Uxuz2i*jQulocND~U> z;Xo#yY+SFjLT=VWS3Qwk)GHO}eRrMu8uu?(l|@0`qHQg!AnMe+_FGC2s30!DoyKaF zxVbXTO&PRV5~%SJd>C{oLn!sqs}kATej$LdULF#n%j|jVZ&~WHKgJ-tJGD3K zKb51CsIW}m41Q~EjTV1+!A$YPqDu2S&Ye3i&@{@s_tG?*XFysu@X)Ki z&Ybcl0Xw>Lx;3p@>6-?)DL{{06RJ~}6M{N3+=kD4#P{MtsA_a59c@NaE_JvGs@ zySpp;{N@keUb%e9PMYKbDVbiqdex}*XP96fts3~J<(E%iH>lkbx6usIb>6*6)QO_p9+z}E93rx(uwOFUIhcNYO?dGr0yYb&~+OrQ>g}~+yKiL z{5dJ-)ci%pkOTW*n@p=t!#U)PCz<=2h4AuLl?+RzI;!h`JR54eVkIE(x8tB|4p$Kb z<|1;{rIo5Q^8f!AZib#Xi&iFRnA-!LY$kw^QoIwRi8?K$rTCn*Adhc;&x~I;64w6x z{fudE?({|ejSh7-fIu(5!!Oi22P2ue9WK~uGX0cueB*>+sTZeptLH8Nlj5Q>08TOI zgCK6)Pw@j|U4}NmkyYCuOB&vXFn2=c1|yJdsDc%wj;vL=oT+SyM&70Xc@!NQV(!o` z6;7JfS2b5fdP7J-B|*4bFp&TnDgzO|4%Z{>xRA3J%q^5Y!jM(Uj83s`Rv$AK8X)jq+-~0iRs33J`oy`5Nx*M2ya1Y!r?TljZ1-Fsk*PjewoB1R|XW9{27Uvn)5|6arWc3F!gaH{; z=zHMW90g&v-r=11j+V%0dg|pGi={-l4aFWZK3#_LU^ZbuS5WUEyX`(+A2Gf6r7?P7 zaG=^4#h(s0(_W9>%hn9(Husg~0Y>eZwrxWC z6Pw5v$4O!RCC$W|4MxL~vg>G0e17(U&Haq5T#0-PBVh5xpmK~=vC_xw)ghR88Powc zo@QU01DS=^^2*-{pkPhXP~!%1CDg=Uw`oq8(!PgE5com+!h{S;Lxj*W4v4LOb>z4- zRlMFyEtR_kY>Bb#QH{@h%HT?ub+z6NL#{O4B~TR}8PSR>oR^f~Lgt>Y70kZR zj%JE)M+UM%XU=&jFPkBv`f-r;msAI2ip%@IeDUnIzeq`gICj#}8r%sxka`MDEJrHyEa|Lm__wMzpgKu6LZV-7hEJ_jH z<}tj+82d4v>ii_T{xGw4J$hDrDvo~h^5B-`4bN?6x3PItLry0Q+lafzAjh|NHnxs# zyfU+Gv(;%WDqXPeXmRwJC)**>=8C1u&F^{^>M<~EvR<#rL> z_5`G-EN$Jh-o#)@KrdT(w5uQX;?kwd?NDtcg4Y|rd9(ZY@$;w86!EadLl`}Q>(_rk zyXPn7{X_A|l`B?Xn96_RgiVsPNyOS5;BAV%E|HP2EI z0Ikf%^4)v)Z1J<@)zXI#iT0_c9TL#2xA4-cXb#`xf}XP@$;GmN)>u_#fl^gvHJ0Yy zTX(WbY4fSVW%9gKXZy8p6zOvLMD;7>0+|;oJ8Kz=LX}vfzcc;j6NRe&I==12cLmORqJrl^$15O8ZKg0m?c*gQ#MIB6-NE zm7gXIH_=gw$qybP*0pXvxpsdIe@3rXW?C8*UpK(tfKX}EKMyrrEKIP7!?;sRhYZp5 zh?CN5Urjg|0TV}(Q4l4~pT;S$u4-9pbgln6yz%;gt%qxX8Oe}Z12Cb`=%6GpR=q1+ zOHMiiy}QI?f!7;7QXI)wJst7`Z-f!;GDq_WE_p7o$^u3?<&i5z|%F*mj>q&@o)pyt3ik~qql;-RUNE+P@Q8}Vm zxG)BY{NX*>)@JlYdeJ`psO~0A07gYI8w~|Sz@aDc^&M?dsFp`*zot2tQDivlq+;Se2 zZYHTUDalQs6f2u2UQ-{8;zWqQX`x2%$X8I37^30p1l2c9b*tDqdNAS406LkD!V zQW&H^Y#YMYrt!A_c!r)&8%pKEtDxVkBK_>mde)uL5!X0MnX!$jp@|?Z=n@TGGC&ln zSd2;LM{z2d7t(8RP5sTdn4DysGQvE7qX*NTuN%!X>aQKvPSUsNpNfoyxBe8_R`}Vw zLnzQXNJvalgRFMMl`5~rwjEY$2gf3ChOUVbdsH!42VE%12n3>Td^_2rJ0-CFv{6E% zH=4d)?kZf*-AA@O1<&P(jSLGDORpe!>L9-a%9bHrIZqHWS86%2l%jRXZ@`mG3*fVt z0U9N)a$c)ujY$W|AL@kmW5}4Uz6mlTDw+!i?WJh4pnQTdV-Vy*)b-_>oJfy7q#Od2 zl^a{cP=r#y%SHv!fqoqT&llV+ygNkv2fdti;Lh^Q`UlD{&Vv}sTuzbRNEl!FF|b32(_T(G|!Hk!7CzO2-58Dszx8PAEpxMI#8y#YSH6i@!yT z`nMc3OmyH8)xlEL^L1NO;HM^@avrMFZntV9U*b(h=Ar4bSd3JpdH6GDg#WJO=sT@DV#)*u~Q`fX1?qSeU!>;^1>$z(59I z#A4jcBzL0>&z?NCV0dSH^VqhScUqQ)v1?e)8G>@xT4=HWXrb#`R3fvBH=NJVP)@iKNfJ z_}r8#%Q@`4WHAQOcpor@Uij#|X9M5;@ST?-!_Oz0OvdJ;DEF*S&wJruAhs{H|;dyn+ z|0=EeLEto$nUBr-F5PEp%~TranZJrK>UBcS6`$&y@lFMirW}k#5rL`dDOc?+zn^MQ znWk*0%fzkWpgj4!ftk&iDj$gOa1Qw<`d0tsLW@#ML(l^?4M%KDA< z;8NK>2A2>9J4)g&wkpKiKtVWV8L{SQY7Ym5Gj}-GJZu!X4hqKNjpwc?$hBaA2Czf= zw5MpmayJ%*%fq{hwtb~kN<`d%$AT}N+9wAuUP;YQ^hEN^;tajKNkj5Ttg`*pm@tvj z`-$nfB?jPrc!+|4*m6e%R4*6dE+R8-Vz?zfjv3ry{V0P@8_AwV9s5wZ0f=HILD2U^ zIHc5;WFqOArW2N-!8a)iNoE@rO|0T2LC)*&Nha}#MJ6;G+d&9DJo%M`?E2N~rWHrbeCPLU{n8WX6G#}DGYGgwB*Dv1~ZS%zb zMIm$3+uv?@e`ldQ-JnVYQxxFGzSdF5&g2@|s3Z~3@Z{7+S0&MlrKyn!aISrS&00LZ zFAPUj0ex1IbPwCgCPK5lMXB$2$j+lYKwU)j`rTjmK-0tYh)HdZBr+f6PHrW}8sa>x zBPx;~dzpkWny4tzmmoCcYQF551JsGI?-AxA6tT#cT8$oRqXd#MY-5zR^24}Cw67w?{KDuSGzviFl+r+Flh5{mH*xOGcI0vH>?;xe0hD z!`(_klrm6600S)EzPK%^SC^nAPJ#pu?=M`q@bQHU-~8d5-+%MX&wu`N68M82!#CgB@kad0TWsxYtD`oieCElT*?j!- zf!^8v?)8gi*LX7DY}nQvHI390fQD>&knFA|A+549Hpc_Cx<8vplGdZqrTux1zB|0d zH5@YiC7W*%t1iJivF*;$!}hiJ%1Gk|+sBT)cuw@SL0&|2_=rc0dH~OQAK23iXV08C zd2GvYT3!Dv$6sb+XiwWx+va)WYx{DZK6UE)&D&4gQ}wmZfSFOC9NpOxVN|;kmPh^I6iTfBMOlk1s)L zkAHkHlyAQI#_QE=F0^^>yxsqfA3gfz=U-&hW>CG*E7(4M`IApBT|Rg2EG^%@L*CEO zZ5iH)6UTHsR1_Z3OU?B9%*DEK>$>+w5TJ1!Q-0L`qsLEZHZ9$L+!UauQsEpWDm|k& zB`AkK&wkgg3L4oAyLvS>f(xjnOIztBFO`71Kdwp5h}x-}%5~DmQPy~ztP9$H7X$y3 z=v%hQ18@Ztt;Y445)l&Ge1Aq{Zmxr(i+H^Jb4&@<7UcR0X1}-(u7oT znd5UWORKI<0Tx+WR~7c@SKj0n%)jO=H2d%%?DoCnu9)W-PJcl?u?`&u$1)eX2d#ID#;ERNRl-) z>r=xycGBHGqq5Q{%{|9FLTXN_T_p_GvP5@TR8#JHd4+PUWdqDW0Q zorY&%CdtIv?{prjWsbf(2~mh)&NNbva1uRaXr1Gd3+dX1|IK*K-iMYJq7M2L*_c?RyfdA3nfjmI-KM$xVlvJzOG(^z)m zf9cx2sV_l{@3>{LdKDZMzzX-9K;n1uq{#3I0n^9$ibYhp2KcZ(Wu{2lo~nlUD)8h6 zEvz#&s>+waNC^No=59p7?QYbT2wWoK^DC0|kh=?}A0e%3f;;UN{TqKEG}i&v`KXUs zwi&s0%Vc8CGtp-n^&;yt?j5$+fLinq{hNywzW{I?+T-Ym$Dw{)O~Oh0{uk{^j8V8A zzN3(19SM$(FgKUkFKN3o@WzJ8ySflj>?BKz_?RMyv;w4oP2#Lr%c6&z@+nv7*1*C|J{(Z}-vy1C7 zD;qzYJ$=Rwo990|XW{JKo7ZpLY)^V_&Y=6w+wBj|o;_)|$3utg>}lJ6lRauDUaeKZ zueW!$Edw=#Ebr|*caEJpbK&Ae@3Z>u`|nNrv0B%Y0wc=S5^Qa4J$d~2?%lg5PM)w^ zjIm@D%T_aoe|F^efB3^c|MHh+mHhdi|KhLy>aQ+-e32_Nt?2rV>$)GGe)idSfB62l zzy0l3Uwv&7&JREQU}3Qh@jXc}^@I#~ZEJ3ggg4Sv<GM>cof9NKJI!Co%(*h%x?y)9td4xw9z{p}cr zCoAvu-SIT>#o;%5FQ|IM3(;Oa=g%H{Jo5JSPygg=XsryUu-50k+S@oHc%GM9ywmog zF~@hZ4K$C+U}C*S?TEMkKHhz^vvu-#uU}vs-je{=t!T zmhf&IJGNus-;*4Br&1H*Q>q+As+F;iG#{aOIQZCr+^WCr_WSC(At4xz9fR zjF7#h;ntnI+qy?vn@?Z7ynW-=GY*vWss-7nD#2;CZl}GFY9_RYD$p3W`x$DLPi>l{ zYJnw~Sp@9Yld13! zp+LD+hg7MCYQvHDfMSBF;i4X8K4+M~Z5P#1MPq`qAD*0tmPH1n$~mRv94ZD^LXc&j zw(gJQqAvuf8EN`1@!=yKT0M=GFKqH}gb)>|a02)a@nFbn` z$qu*RB^TR}$YrQ1S4U$QO8v2rv|_p%;|~uCRxCO8(9%z19|admlw}bKtbOinv$ntA)oj3f+TT=F(Q)vWQZ=79ezCk+Gn)zQQ{8THe0!F`u#W;g`@TX;=zP2m zM_UPr)(8+ADKcMgA~Ld@3=^qhUcAe;PgK%Q(beFNN?~*K>3ljoC5mlECAc<>PG?+A z0E;*w8+;1^Rgbb~8i_KoG!P0;orWYD2a(%drB?^x)XE{(WmJvhQF`4^I?Y^`KmxVn z13V+>);^@<)3ARgkl2G72o9tWKjM%kGH!BLeS!V9EU64`QdGoqmDdLkb3-iH1J-0J zUNWAd4)U~i4oLEr1=J?IH^M3tkkJ>>5jc;LqgSF-8gb#)d{vHA(pS!Fs!8-`h|G-r zD<(aar~D#?VZZ#LWwf$6V{)sm301i0h>mr4=J{7BI4XuD!ZV<*>ligT-0456ToF4R zq^cE~uoW8!MRJVzWESuJc%EGyrmeYIT^qhrP_z|1MLK5tR9*H+JbT(?IMCG?`#P#AUOR=( zdw1`C^4aHh>$jEU-~IjH{rJa!^3nMVkm1%;d2gk?cjwOUzxln%HfPVC)fm}0=-h`N z{nfwy%g;Xh{2%}6pZ@#*_{Wbv`q0||%r5%ygLA_6d-v{~+y3y%Czl^T+I{fo(dVCi z_TZs@#FL$)$300nu=~^_1^gJ7fBDAjfBR#4rXkv(lD)S_Ug&9!qbE1N*r|iy80#ji znRM9j!%LpMnJ}jCncBa(ZBD!CD+hYq(d4zBGc_@FtNNxVvAM1J-Z)~Z@D>Ss)h(DW zHa1>8e_{~)?cV0gXHN}u6Y-f-r>|VT^w4|0-oE5`Iy5n!QOpc-0eY8)-C8`r3Ue;n zAZODQNsn%YPM<8!E)pKIIrT{(M zeRki=8g_RxPW`TCAKU*Y)6CR+@S!d=XQFzMb!X&SYV|sYR>d5X9se^kKBf+p1&noM z^EJKMa!2RxR-0EAaQvx)#90lU^tqL+%aj>(GPkN3AXK*{^%%6fC9Eo^h_rZEMWTAf zJ*t~i5GQr{*<c~ zb;d*TNs%-mq`YahWB+@8Vq%iy8~-a+v|OnO8DbUf=*m~2=$v*L5esc4WE^;Nqnnc@ z-OHv-+z!_LTZ?>|CX2)+lqnY5Lp+fWh3>5z<7)_FCP(yoDiBVH{rHC~dfd;RcDB4HrUP8&z z6m34U#1ZF2gatrDsMfPLkjwyEh=#IwYQ_SzULE*KAHY^)EWtd8p3?VBfLp|I3NN9$ z#1f`usw{vB&R!o-=wAKl3n69?^KZ3wF-a<5!8^%=&!w``pEJT14 zi*X&4RLMsa?DG^k&cZU++!T*G7BMn~CZ+c^_5#3E!N%4`NCef=3}VxI*5S+H&b-f; zoZ}k{CVFjKwU~H_W9!|fJyBx|%mEoE$!NT~08%Bx?R=Ff@kFu{jKfhS9GWD^R{C(3 z*yPNlmNez6Vt2SDlQ?l+eZVqM+cTUD1}qd1Y;j3ldwND2o0&n0?6kl$walMvN@RGE zKbawz49g*OCt8v;Xmgqt8c-c_hoo^7GDKSC{rA;yZ!#XHd__y-#TyxiFxrHFs|cwN z&4Qt5dShn@AzcNAV!%_&*d~|=aJV&J$u%-4E%b4XvC)9eGk`(0MD zXkGoM;(a#9Xyi(X$+#{6h$CW%7fmr$EyEc~r{Q3}8D6S)a1mN4+%ag8NjF8o7*TML zvABH@OBo~gIA5RYVP+XOAPNjQ?Y_x42&Xbt z^TLpWOPm8%1|V=5qljrWOTURI$<+aPAN7d?VT%as$n_yREt1S9`9)z>YSClnU#ikd zltaEmpVyC=^tVn1sUrLmRKxz77J~(-DdrN5h)?^Y^Z<5dH zGiMn+Ohjx9*RGK^hw++QsBAuiK)`!0W)8=@x5llt6t;g+O`g4cYFPTxl~2F=@=LQu ze*3%M!lZLSa5nc;l?1O}ojhR*kjDZhu@J~_e)rpJH?Eo0^Pm3HfAV&!-~9U5fB5eE zbEi+4_j2X(W$>?GyXvv4E$5-cyVU~sA3Wyty@iT0CYa>uijr+(#54%qBSjqm-GSFW zY8Q&LeO5sm>6hw88RGAkZ{NJK{74>H2;AfR&7-F_wvVZN`6v#T`mJ3#V9T)a1oC(Y zV40tn2b)$zxxFFyo$;fj>K%rMTQb3Q7*;=Hu>RO_QuMAs1MrcTYvS*!>TBr%qD-bY zw}(U7X|O--)b!S{wGKhw%rqyP`g_)R_M?wce*eKe`7d9(bnE6#O9FoK)1Q6x;aLTL z{p)}I{qKHr|Ka`ibbaZCVsCbOBip;vr%&71hY{$ZwGC0PC)>M!{~k+1$juwqp*(x; z?3K$`^aZJA_vtR@=aIrRlN-GSd3V=SpBJxQKDqgX#{CI%bO=2=Vrmrq;Nc_hFtfG4 z8C~}uKYpSAWrET64kq3_x18F;y3AEmrnO%cA;FH3bvgdF5k0=vOVX;q87<)BgEIF_ znGx5Twp=qs&i#?4v#H+nJI!Kurak0yDqH52{r#!xG47~1raX-mR;sZB^^Pj12%@3w z@Dn-6KogVeR4&0%n-N!4BhQiB%Og$^e9T&pZcMFKvK#wftK!bh8kBHF#(?iUNC6Hr zA2;Xa%b5)n=@?0q>ir_wd7eTG>HQTH8`V}fX@+z z5X6cGLRx6cJhF&`k;DjMuujn-sWOpN(Kvs9zG$qG+b2%8rR}g{mC~hzm70bgtDMX@ z%xcy1oXMfO@qB#(7>sP+UO=!IzUObOcN||3b0m8N?)?7v{((!qT4W<}BkEYh5b0|( zECJY4jSPEr9%*P_ErS7!H}?B6Zl(Z{rng~JL|1zskcLE}Kk1J+Yuga?a_``gm+E6# ztzJ-f)X5pD(87U4r9qp@6=Bi>?Hk;IRgGnIt}?waCR$lE6CH`o;A>%hTsunVjv(?( zunD})3K+?1{c%tOh&v(=mF!{H@HcGI(O3#0qie={<9N(0s=}^RUw#1akM$AMY-s&>(GoQ^e|hS+FCsx>AM)s3NQF7H*i}l% z!iOPNfQM{Z{#9|(OPV4(xxh|CLvu1glop8N6x!6yo6u%tS1A%nE1l2~6hWA_PNMVh zvt_Lc{JnlMtk#4~9ZNhxLJE4O8bnd#_K^@Naz$?hmlep8N}o~@JGT>MY?fIrr!%yk ziwOEEqSC}0mB~kF&a#w<4_$!EJGsiplr1O|E7b zb;cWOTNvcbw&}o6c3Ll6-#6zX2QX&U&^=V>Vf3KXAhkF5^iEKM!F@r*`XE<(jQgvY zs?^N+PGfS0CWc}w_FWa&;ma@?qsrtOlSEQ^WdHz;>&SIXsvmj$xtVCetkYl-qOdds zDbgr6cl{2)Iuc4ku4xfFOV+kFc1?M08U{V%0k0=Y(>4}Q z$+oDfmo~Hyi&H+p`^+f36k2h%G2?xDfvGRl0BPtgq8@xwhU~N)2*fb?&4X53+Jj75 zL%SdXNx7m3OE@gX_Q(L7VIJfO|D#l`nGtOAi6Dgn(J4xX`c8+7p*WVpvVT}?j7$?P z^#SUKBSnU@dM_;iV1N>Xt@dUM>7)9c zliKeKz?Jz76Fp4AmmI|?GzRFBK2{_$<475>yuuKStJobqAl$z)$+p?-CyySz_~D12 zT)cSd*bazKcc0uh%>L*hN$eaw`tgN}moHraj*By9e|V#JPNCk~PW};V2C#+uR>@qx zeEHbMhQ$D;=sayt&wIKMUgdlJ`VHt%qDmQHesJ&p4>xXnVBXIMXKHa?y$~OZCT&9h zfBonGeC3l*fBy4d{Gb2fKm6;z{_6X0e{T;;FA%Uh5IF;gb2HklthEiLQZHY>5uWy% zf|>nq;E?WL+6zcM#ARK>Jwxt0FQ3hv(*%%p5pV?MEDhdR+OLx9EeLjv{vX-4Z_mNa zqbIxpz%zYA{CRLsAG4k2`R=`V&Lq6~S9{Mi^%PoZkmDfDJ-cSu+S9mECcv-sa)T{o zD6G?EnM4MNOq&ODC_=XJ#vVR+@^I+LhB0^z+m`ng8<_ICkv*LRRi<$`A0zr+*7nhb z%N&%Y)SgWm>^DErdv^cLU;JCIUtS4yBx9#JwE(;|o<@(=o_-bp=}DJr=^8v=S-VPC zee71reaR5e?&W<~#+m^BVYLpEy*R|bANi+v4O&2o^<>Uj1}8|ZHy0~7O~)u&$NXxt z^Oz*j*PN{%sY<+OE;OuZ*ob3xH?XBB#D!G%BE?b3m30y(XUI#N?;Pc7Ss7eOviJ#m zvT2s9i$Puk8a>gsl7bnT@=lhNlekxF<`>JgIyo8@1F;i%l)EyZBvuNUrfE=zK0Pe- zld1Tr%f65h$U)pmqDs_|B=mtA`TIP)zh3f*;SM)b{T*yoPAem&TVX_M_uJJxMkiir*)cVtI9JT_`s_@m_Y2&Oi9D$SBe^aZ$|QvBDP zO-RxlFc9;Bq3R)iEtTo@y89jy&Wq98jm2PKK2?pXq01=d7fA55p0%Hq1I0HWvAZaP ziSXVtRvS6;#q}tMUlkJY#r^fQ8nW6{<$y7wb@<>5gSj61YFAVTbXzq>Kcg&mOJ*v=XcqmR;lYGPFwMu;Egid2l7w|O z01i(52S*0yRuyH88#bg4kw}`(HoIg*8A`+?shsIPjr8nb0NO7muEy0LV~y3dNKeh> zp90t-nGAc1vG*zsik6)7Kl}NjK4p&t(d3Sji7UU9(2`F|sYy~$+Wfl&Vn5H&FzKDS zq0%IPDr!a(;_AT&KcIE&knLpsrt^@5B|1?Jz!P<*nEL2&#HHvOrmgdT@p9N44N~5bPKGq9INHIDT zFN`I~re6#a&`ha*6_v}TdWCydBkT*6btLJ=-NwUb(8`UHXmY{MMq+Vagd>H)K3rtH znPcKjFbP6Arof-8>8d=c>-xhnTEj)lB_>ni%w({lL|=4iGBb}O zo}1W&SvcfZj5k}LdiM0Goy{Yn=Our0(CyQ&+UN?LK6&zsFTW7h@^w&%+XU{l37*>? zK6L!VF*AVv`IrBE>-Md`{L8=m@~bbu`Q}@S5z{BnLmMBSzu;}X9vb}c-M8|eIDX6` zs;k$2Fk|G@$>X^ncq{0A>n#?qTN7mieALCr%yQKEAPi)I$Jke2v@nG(}mn)dc<<+irPlCdJeZ z$l}SmM|&C-ug+|jtG7yNMmA2I+(rs&8%EfBZ88-H_{`!M6r_Y~1kL)LpAKkrs8&CFl zwZG4Uhxfdw*AisU9`x~U@7}Ud+p0Iyu0H+jqf3{r2oor)U|s7sFF|YU|HaFvy&2#Y z^@v4s!3VrS;m9#-z_;vAYB@loo)VYmSZyFPAs-C)lb205Jp`!grQmT%D+431lBvJd zf?NtMW@v$2F0YKit5!Lirl&%kcc-J;wsLh=y@R+S$-OEPjo}$2rIo1~j+U~`b@f_u zbjX?qoQ!>sV?k_3wurhJc$B2R`kJIETj@@K^ydZyp`L>+44p*MD9Lab^r&6PfEv#u zHKL`WdW!9Xk;I7wcuI5<_0vj~AGC^8(kH>Loc$z$uExU+2$Xfvq>$Bw=W?PU=?!dG z!((OhqHaO(NB1aspLp?IFbG2$1LRa~Gc4;E=tz+E6^_3@YHo!5d_#J$*! zh-rJuvyNc88=ax>S8G0!?`o*n&nh|J>S~$XSVm>EBGC+ryUZghE7Jh6#Db^j)rJZP zj8L^G*l^APZzDX>5+7ad4SoNutryC@e*H`g3DU^ukCF+`tQ5_uje4s~4-Jyn(5l9OGO{S8^Tat)5h{5$X#$R6k%Eq- zkC6oIY#6P9NqzxI2-v87)!T4UjSXiBnQl1%4zI#X{Amo>iJG8BR)J%C87FC4It zf*{VzENYLFLnIb4>V{=6kyj;Od80s~L{a&)!+$jDk6>@rYT!f*A|2HU)gQ>(^JY`} zC)DE6f|U`$`U;REED5g8o^sV?40$Fasd*{GpA1cQ9Ul{w14v$^1~kfsD`cWMZAr`o z`;O#r6L}cA_1<)bD~Z+mDro9)UgOs-2LxJ<04f=}k<04|tJzRtOrdGmm9J##cn_^a z?6cNDl8U<5caC8YDmNwxge!r37&wGUZhlH46mO#TbQP5OUq=k4?oq8RU}&?KzJK>E2fB+ygW6s zd#@>sm$4s3)Kvy0$ZAhvj5moxs0L_5AELirN}<3wu%UhQi2RON41EJ489)mbvgqo+ z+nLT7HUf=KW}w-NqhfC2w6NWOM?NF~ZV#iwk?dj0y#$CbfaZ zNQg~_>)AuZcRun~G;29eOc=(G-4p@~!NY}0i1V(2%<^|wF&{PVy6 zhky9{fB0{|__JTw6Xxo*YnwaAzWnh|Z1wVd@7ckf9czaXcjxx4-QB0^iXr3Mw{KZC z`tipXr3|+lmo9uP?)TQPAAb13&Nkkhb>)*!3NUTB@WP_vm(OoJ`|WRky>;y9#~)uY z2cJ##5M}RG9z5Tk&(*vd_Qx^aJsi(DRyTVH=v&VxFR}uYX8$|P>haLxC1&{bjm@3I z*8XlCGr8x;*3lz7$E;>;3D|)*FDy4V;4gqT@Bgrc!l6x=IpQ|NX_ui{4{+ekD?0K< zuo+hOqOc<}h=C0^QKvB4L?GJ;A>QcPYUwQG#VM`z6k;B}qm9{UdfGp9XR9faZ{M6d z`+*)8u>)ZMo-UBEwX-kTifzp2TvPL=m&oE(S?g2v$m^O zul;cKdxmdTm|Zcv3(eCZlf>>n)E{|b$*&iCfl!C5O@THK97`)$e7xla5S|IDj&$m~!V= zYNqDD`kzB%>k%NI9OG#tqe|(R8tPl+?~)pocO9X!cOLEQv@1|0P)hDFlk7x}TJVw; zRVN+eN(X0ngXR;tXdGt)SIp-b$I*3ThTr-ve@A(T9%l+j$kF1o^TpZwpKcDm`HB-C zi8+E$MfpAG2#P=@B!6Xh52C*xfNtrq--NQp}$D?Y~3gq^Z-K|5%c5GXGWN&=X ztG(?Pc0Ow=OM!#k*gR@(Qg29c!Z{etD3nQP{Ebe1f*4A;{GtkCD{H(I)V3Q2_rVFa zoAAqxR~v8k>^6{Z;cc3#3Q#sb?4WTBGV5H9$sb)J1e< z4gI`7WMuD2KfSDjTJYCMT%**ZCB}z{2-nMB4?<1+QKsk07^@aBhayBAPv)>px5^cY zI1i-xI6~tl0{1a4rIfgIm_FtzXqwGhRj9nhvDdTl<6RHY1InY@*Ut_-A5Zjp)Am36 zorI|)8hCq=*c%O#)hr^M4=WIpjL)fN%uv@yU>MD@xju<` z6pLqUWPbUXaUm5PtsJs{oein2H7htUh?MeUj~LA}yXu%hfI>%Ae1JvSNOIU=>-!Fl9c7xlyT1g<6D!FO(lI!r zLFb@Iz%M#|y*Gk=c@*?zn^}+`t|)peK}g4XJaQV#8t1=CMmE9~hjXwbz3wId5JQKD zs(J)vv^ZS6WO^}2DAh6*xG5g1BU@JHCrjlMVDi?Edsw}1`uWJrW+?OxMP22rxq1wE z^K;w}RF9SEO(ybbc2Ho}Zm#OAS07AoWmR#$0aV`%G*GHe~b zxlsS*U@RywcV+o!RDqU_j9*1bbPqR7NPHA;D2&L=Y0l!k#JQ}-{u6ikH=~wnK zkaAOQ0czr$^6yL%S(-@CJ=be81rdx$TvWj+hRo{4kU`LV7E39rOY)$e0U7xE!Zj$U z^F^0N5Jnzl!Zg2NV3xu3))dbe`71-|sLhs)ox_g%Ap7kr)-X}OV35Qb@vvC>M5^8u zWJXdVah8GB-v$TdB#(5DE?1e{TYbr~+j<~D3`fi->f|JE4c1hQVJwlWJxqoJ?-AvU zJO~rS`7WxqxdLCjI~fEOi3uvd25Kh~L6g9g<4*-^)>0w`f~@p_*fM^`fwb#0Q?cIn z!?UMNqY=fiGbVOGoO;kRx(sTY4I(ms z^x%M{rT^y7{_NX7eEaP;zx(*|rLCtMzy9^Vp!kzdKR>>+{klN)@b=MTm#=(kq}hmZ zb46Y}zj^z%N$0|1XxjrH^y4Q^pE+menQPas-`ah2YU!r}9$PM)wS z(ADq1G5_cE2WMw+o0z@z>$!^4e*FYR+5*p})_GL1!MX@djg+WimUsSZr1c$$#8O+$ zY}H(xZnpZtBZoG3%vRZ0+gFym{_1wf%n%=@J-I^MbpBM?l~s#{AX( z8aa*m_>jXpM^C)idHnA6b7T7&Ebac$jh!b?ckkc7>-@~Qb1h-)Xur_Z$Q@nasgWdxqY*dw2KQ3ubxt?1!*f zlW_C?0~;b~@1E%^zSD+!BC!Qs-J1i4UU-ee?voeB8fW!33fXKX4a78zAyLtDPgLb{#Z4`QCh0*Lc753`(5rXhbhC9q`9h-qZ4(M7v~w= z3ooFV*WBxeC(1wcU-tAd9!m8M21s+1W3s(ZTG=@r#40mDM&u&mOmYCG6Q|2Voh z$no`yvS>~)b2tOFXh_(Z6Ab#a>O(l%BfGCvVn`mA;|3Bq-5DJuntj@&sa=Yt8v`pM zv}hpJ!aM~E=OC>K9zhSZDeblOPab{UVp@r&Mt5?P!o<84`M?TYcvR3Ga zsS()-Z49F(2mWMG4p8+RjG6ND#Yvp}L=jw}9N?Rs#~m4~es#$1X79{%E{s=XksPns zh`WI}g#nt&stX2-(n5p*6eb2`eF$b!+zLb%(3jNmHb^@y>PRiYl`WDLWd_x%bQM&I z>^8s}JY&6O!KXj1UC72NCiB3C!BM)~#WA=}BvpX@3;`P|WsKPfD*sAdNL^J4RN@|x z<{Uv8-WN@84^YjU*UtY{^!-!m%nGkiX(4ds0qAEmIRR)}deO-)j1E1K#5qTrs;*Aa{CS@5> zYA7F@>M2Eu{)v&B(mr>&tRV~;0G;|`!SrU@f}6#(sd;>>0t0diD4ML()h% z`Ady~^2AJ(ItEB4xv!S)%Q#0O8R=C#cdN!vC`Abrk%D4Axn_c0O?{L^70RY_g0@j_ z`6YK0xH(n|Mr%GOMc+GxiaQ+Tlnc>B^wMiZ?5<*l*`7g!WMS4E7Kk@W_Z2Xr#+Eem zinx9=;;sJyBf6GA)V)e90VybK^_MG=sgkmeWESqEfReq!p zh|c}yJcm`^dXhqXb+pz%MgXI=PY=wJt+1yseaxKHWXUi`5v--iXD&uJ>COM}l}g9{ ze1n?ELDRZ#n7HlR$iD#x8#V$;FF6!A_~7OpkwtYrqy+=+EctXI85fb7+vU`<6#}W{ z(ggNA0YKCPyt)<%1_i|r844k(o}*7-l3`*CMeL0pM)q05;32m@>;h+LJ5GvL^)u|1wkd*_tVYj$iZIkL?Q zQc18kEigs&%5On+-2&z=k|fXltNl zFSKa@ilu!0)z|tYzy9^F?%lmTPU+y$?fP{l=o5mW{pU}gT0?9df?b1@Vfoe?c%2vd zd*PZH3O8@v^u92={xdyK2E6a>lR?%cW0 zmb`fQ_|fBbiF)$%>F>Y$UQMI2ah7V^8?&GFMc8sVnbyXTRk%#I= zgmQ_By3RRh|sD~s2o-KNa~=GuKTM3B0ZCcxOJ4Hv$^jA;DI@%rm=5I4HldlITZU6d?D1Ir_><1Tn($GC@z05{N|2Pf8VQb_Jo8C)`8kFfo*r z#~s*Y|6{l~X$$PMxydwtn}qX>{s-?su3!j9HnPnjR}nM~cW1G_2$`^N8pve1q?y{f z@{6ke{RVKvtQ@ZPS>12+LjrOvcXx0UdeKGM104V4ALP!;o4B2HijRNc92%1DwOHSqHARiJ0bjSkprYL2cRD-te|;Z`<(fO!c<} z39#XS^FY^VF5(8-Fai0{HhPAZEVYi>C3ESy~P;Y7T7^|E%>{(Ij8-0}M>x>QEa*&h+ri4%dl4 zcELpFgscd`R03GStn#~1ydF;kHhZViZj6+(w`D@{?!Q_)dZE*(_gRXT>Q>z4W_ku^ z{o;h6YUtdEx&D?-r#w^u*-(MmX@x#Ig^?z#=s6FWW*rTg%@Y8|m9weenF0tW5gKUD z6!LBZy{k@>zl8n&eT9uCu%8M+^V{Pa%*G~jgg%mdm8rzaQ>pcDi8DF~6Jzc+HEI59 z208Zc%SH|AbbXAy-Urhv~$0|a3h8pD3*fDLASgGm91 zjrbXa&MiY8P5vk?wIqYGIvqh~`kgOh81=ZBM6#J=5x1F0iN2?F{8w@+N)j45GM9*m zmMTESsw!gwGc2L(x|vDHHC7j{YJwO#F9>vMQgchM^=Nf1qZ1RP*%751tfaPIs`!6; z+C@fvM(vzlic2Crk8zF_Ux9BY4odWF#dz~sGEfePh8lJlQCHuZuGeKrNzG5E!w7d?*CjVOt@zwYTA2~-rV|Rt zPTS61$Z58Xw^i5K*?9HxojA`#4inAY5wHqq#ecPgjN$O%ogI^f)LapyRjoUmB!*1Z z{~u2tU;OxEe)7ik8=}DT7cUBb@7=v820eM^jAzidZrmUmO9FH$jQ1aZb)0eq!MjiG z7jH_;Cd#}O7NX6q4U_)2zxeXQv*&ib3!v%g_lfq*>62bki;!D4u6gEg{^LvPBblf> z&tJSewggO=U~C-mz{R5tT^lQU+ohZCbhu(R4^4nL((O%o3&tly0Wu=>1 zzb(w(`rzD|y%*2^<-h->k9Mq)_OfM5uGJ+-k!NBPy3pXM3~K@b4Ia6G z`}pBQdbxY&o@W%j_Tk{6+qZ8%dHDF!vRMASRS(Ih!6*AXIZ<(kpy@180YP9ngO<(AV_AakX~M64MV?nBPcMA7(5H8ElnMg&M#Skh28HKK+*Z!0~?tjunc#xxIZNb7dQ zEjW=8xW1f5pCdTMG|5KkNhHBq-&{^k+DV$JM4k{-7p_`S&Cgb@lJgC42qJdK9Xt|L z>InezYy%?QjQ}MCc~SgFd5sSC>5jquBMuyk+Da-1RZK2Ei~c)BK|GD|j3C2+2vpTz z)wmT8PqWICcXts^18_u;|BxlYDAh%eUN980x`4(oTT>v9BN_AiaBMnGm1doWJv=cw^D}3YE*!Fj2nN#49j&la0o$fqosm^k3^` zwSd{Iw)88CiCPB(XrPukL)izZ*+lHXgpkN+-F0*;Wy*pMqI_9qstSse$v-Jkh^)|( z0H|dv2>^d+oj^285I54~)We`;G4o4qMHxA^@DQn+Rb1NlLnl#Y4g(4^cZACq?GPi9 z5Tk5~SYmQUe^ur^l7%iIlyOK*6X3|2%_?1&l}gYj7LzA4NGD|qe;*|X#0l#Y;Q_Be z!C$l_JS7&`WR@}Y7l?iaO+@R0`|_ET?#6v3_k}X2lRkD31WC%FgyYH%h8VyIT%#)J zBf1gIFr5m51wo`mcs#+-A2vyqVKJ!&9##6>oO~=>?VR|bg`Qp@o>gUhGA}4qJbb6< zAu91Tft&bpsp>UKJLp`d!t}{1#+73TPvFWXHN=fvq(|w9eF%2GFSL!RG9qjfk>u8C4^pXDR5!267IS<(M^8Qy3e43BIQr?M4KsUvC-9N>~HEEw3wo*D<8 zY+1lMUleB8T-ouhV;`S8d;Z*6!@!AqBUK2xxCy~G|6pjDd1zL|WRD&{eevAf{dZ^1 z*m(1Vm-^bx622p+ka|!*K?maY&6|dYKl|)+Vds-4yLWHj`QYq1uco?w{hHZ6_Mb5c z1S?grgNKBz_R$B>4ikdu-UR&U@$qBF4XC_%{PxtT3=mN}$BtPajQxjpv%i1Wz`q$O zSFU_=*YuTZ-)kf+cU31;oUM(+mY8N@9tiC1J=uLCPG$piGZo9J{~kn%QH{M(Ni%>VMsV)*iqmnvake(BjW zh;|4Sc)zq3Vk)o*_{r|BhbM5qdiCPSn<8lgN!ICVC8{qy2k@rJEvBVY^6JI&O%>iy z2sGYrY*nYXEU;g?3W9rU->`Jl&zmu8FBr}uAwMbS~S9eKj zZ6m>h!2p9X?0?t5=vNzt{{RE_SjJcy&0y3$t=?3suC8KnCv#7d{(MioEa~RU_wJ1w zarQWIV!3fG{r}0AKRG-+^jhCnuU{GTKfH6BYq8^!p{$4Z?;HMSo<~RTUcY+v^*{ZS zw*+Wvkt_r;5A>rSePPcakLrS8xsK-IVE^{Rdk=JDzJ2=b$@TOvq$Q>P>FfGp{cyht(CMb)7h{DW<&JHd> z^XB7sC#P?YPTuKK=rGXAgqW%-))OT`v-*BH{Ham-o*WYu*BDK`%S~3{rUI^~t0|p} zu;5Jy(K2g8UZku*j;7xFxXVCR&BKt_c0HzA{mLVRmp_IO=qx29ox2E`;$tN;DqH|H z=AdR5T(Ln3CHk-u_Ebye3-G=j1Fo;`p>UdO)F2?&I@#tbD{#g2RhI}wA$0d6xexAQ z51d+&#Z(;RamG(qCJN!H;C#jSI>wy^DX94x<=|9A^%-|QgLpYx4tdB4+f=6V4%}j{ znyXN`SqA@sp(E6V3X@X96mI{eF6sGQ1SX9RGKl!e6s)pm34;~nMdviSpr^;Ou8_)! zi;dLPQpHkNIh>75Y6W1P8`F~yK1~fdLm`nyU$P238UX00RDQ-Fk$Gp(g2@SR6`Hp4 zNAAau0{xTI3vcjNIVFx;+PY<^T6Q8mQtER={=NbIn1T40?E6i`FBU2xi^$@Ohc%w5hSf8?ER)kDAPn%Hrk2W z&A&|J>M<9uqs4aB%8qivIu2u4I_YO%8ps4d>I21VFxlKbEQbFN!96=BjXV zPRt2tOk~M5Ifpi%-T9_ucpOZ`_grjJ5>s|Q2*Z=^0^H<_-wfKC^r=TClx{Y}+mnGk ziIT;qBOsHI+M21Vw=7j%7NE6xZ&p|sfxaPTG*Cw&BS!_;8F@-~J+U7^F`V4|EP6W` z>JnkXi_5;9uj({vlQ^~k%Z5Y0HI%uQjaVY6&U!h@)Qz&wU!X@C13F{sq&D1%`qGLu z8=Ds$Hoe3**UfM%mF%(9AVTbreBV|Kzt*ubL%Zx5(Xt z(LD^XZ8>!lII`7|;0Z7x^iW-pPCWIf{3b{KWC;fkjWFcrn(p8snvELeW+XNSRn8b? zMQe)aME*58u#p9r_z7N;Q6|-dXzV`)Br8gv_&@(69BZbLF;F>|lw*>i(pLviVAP)g zYkV6D0FI8h2sd)a1k`v9+jK)mDcQGs9PE=vpZw%UpFe(Z_x9dF?v_$~S~;`z*t4FG z9@)1?bdEN30DFFMTR;hqw@bA;<$7D%AH98RBYES#-t7A)fAYtM;ag>X@7{y^53ubK ztGxqp;LGoO)z>3Wk#dptim^M*oqP9mE^HffdHUhrU8DK;J%}r?MxCK?4H7LOxUGDf zthn*%Fw;?RL3-{`KA#KU0r^(3Mg~&ha=Lpl3h(889M-N?e6>eaKoA?9e^DbwrW)9zdL)? zwG~4ip@ygaWBSMjsS1EeWME4rLVueGF>SKYhG{Z9D=eV4)#eQUdwMM8d)P;}ps9%+ zVmC)%T80kEQzuw8P_B6=o2{On?(W;F)V@mlURGwh06Q;VzWCkOU+I|r_Yd#gyKh<7AOF!GfBBQ2Sh)Y&-~9U3>lceK)@I7U z)}q!f>7TrN`)2R)6LS%)%_{(KK##x1bWxr@fBO9S3sc_+GSL6(%_|c}Kk=N$AN&z< zK0kT(?Ac4R1fIY+pCF#!wp(dENs_8v^=s7@`n?o_Thl*hT0L>L$D+2pYpmGp;=@V( z|9YU-O3d!hG8MKHHMwHTp-vF;)GaPAvnOUc0H8Xho~k+1Bu$OGw$pI5F8+i*^v+E- zu2of(p*k~KTpqAWK9I5vM-^+MV;O6&D>i3(8JlVx;!sbO1+cOHs*ebuGQ`}pQ&}gPa{>Bl_h4%T9Hi6hii^ZhdYzBQYKe7 zq}Cn|StV9zB+_Z_rJwQ2iAa5!*1Je2*ONC+jBIdjgn#&gINuz!)haE>>hhJe7d5^~ zX3+^gE>dVeD4MgH43e1gM5jP7Bdw%RM1yaasFizem8n~VCMI5tDiz?44T@>{!a1st zj_y&BgdtTDGee64eFpnr4z?r2(kDq+9}%cc9h9eS8*s8(KY9A$@@U7q&`-H_dxW*B zyqbUAGyZp#GUnik;F`9?LAMPK0~Zg2c~L#m*>KsNf@lL7h&Z$CAl|BMy1Gp@Wq9n6 zrEoddY?@F&G0Q(e#bPXAa3-302R@WPE&f-mHn=yn0?<=-L3YGwTXYUz(Tdjuow{Wj znIsK_PrT-&8tWWG;}vy2*vlrp5>Oa>hyB*PGsNa(IsrCIo%mlI&`e;vAt%%;4F`YF zpd&6;{ZiAG3Pchls(LCe8wm|^^}S&f)2MEdv6#I#BzilEkw7GkL$A%P>qH54z)ZFh zCArylO@k;7u&%TD@;U)<*A+Gako)E!Ev!{Fuo~0u4sqiv=y0@{J!5))s(V;UrX)&e z(VCUU(wvUKX`ovUy3@>O5^l44-9_oCCD}B@+)ugvgq%E5U>B27u|P?NRce`{oRZVI{n-EKK)G`Je@8<_66?naCy^Z~UbfD+^t#AU83 zax51Q4-6d6?%rsw(kSo7+#2j@Jg`0GAYpS*aXP%wsM8GU&$F-%<~O$5$fIb_@Sb@9 zb|(t^0U2WzZx;Va@>#%VaA2*2ipr6F66tUSe`=iCnh{FG-{RBHa9iY<>Kd^iV6YR1 zYZ;af(Z9H6ItgjS1sJCDj*~mk8=^`Na+hgwPTW^88cU$9#N#wP0o43YRIQEg;cjcP zH7DBUB|4Q#wGya_c?{Y5h$)&7XzQ~Yq3YO%AuXd?i7;uTXjs5>q5WSjCbnm-f_^c& z8B(wJZZ};)HUs30G^0EtdCe}FYLmgu&u38!HT8Rb_T6{Ko<=o@qd)LZ|MZpjh1tO8(W58Ecs;txsiVcSr`S6^ zHKKiD?o)A{Kd9d*nx!^6l%Auw!7Q92B3lJ*RNijpI#WHhWD)p zkUxI#2!pw6EywY-kH`7|H*Vg4@c8A2??k=l?>`)$F`PU;L8(t8x)we%sIj9g8n3w+ z8N}=AIUCV)usCOHcZQgVmxd6$4)kuSWmRdJzA!yqj|tV*3Bo)_;3WY2A1^KMnLu6( zasB$?T`%OaC@6zBY}6Y4j}55Tm`rPc1RbcDQ@z z@#7~y{^ASIRolM@F+_KA{5wXXKWHAnS}A?Gtvh!GD~R;Dd&e6a4&EHS`|kViU%!3* z`t3Um3)HUOd3W?o7wqj@ts~roVeDrx7hR zx@e*n6d48+_#l@xLXmr+m?uofSQos`8v!j;ZB1_=rc z{2&U|P`@(dw33kmreu+Ttw>3V9d1@p)C@|4G{(Fc_f8FZdGz`N`Tm`e- zKt({mdH2o+eyDl5`I$y!B{Na*5=K#>+yFXSMivcSHq-7LRF~zbJuO-s;G=km6sBGo z*cnkBof!VFO&2B=<=U?^9(l$V>#GMk%XZM!@^Qm~#F9TejeH8MC0t_{E& zkaDLCU`HyabK#MX)sTDj>w{1NTCf=3ERP}F?ilT*?ds_CQvhP-k@vB z6{c=bC&ndA?y>%6vu#t`uB?DKlT9M>JqFTmT64jUF*hw_Pb@SdLLLSNCjPe)h?q#p z8s2#}qOG~P)GRHd11RAq`3cYBnN$DeN@#pI)`!iG8WgN$=$+xVlW*|vKS zE068SJjNcza<;Es?s|G@j|2QbdnfG|VL@hHHw|M8xdWWj=CAxj9I_%UwwL0%PJeSG z@CZk`oTJuJW^&EFMMGt)&;&fNYWcIlp*j>9Jk;eLgC4q@Qt;38mHWnj|F=i5kY`7)Xf14$-W zlN1RNv@VOa)8;Td5*d(#IFOTbBmrXB2TXa&QboWjh|VM=v4AY`bviPPi_k4LB4x7U zw)$F`AF2`?NN-Bk8DPe;P66T;L@2j};JjPzsV)@6u!EAD>}1RvLgSdVJ#`G+jSLWc z;B8baqpi!Sb^=?Eu1a4Pzw^TdHZF!qm&0FpTM)HWTrPhhT5vVO%HeXWz!Srgn=+${C9-L0Wqj!-Z_}5xk{qE z$Lxq9F6uFEu*u%@OXp9 z{(&*Bp{;AphE1^QqcnnRsKajW$M4=jfA{dvi~+EygXr_mf7EgyL&49V8z~m0dg|As zzzP2ShrRtHg9KN*%$y%dp?b_K;ty&10 zgB%UP`xR$T{j@Tyg=Y5qoDr!W0NA!$$Bu5YHT@`RNYD6nZ3Xd(>Zbk^tfS7@RB5)R zr6p3TlBc->t*`zWIA>m%)dRt1ODIrtLZ0}3d(<|=XpvZpDrg5bOHrW@PV zP(X+TJ1}3!gsw?6W`kgCE2E?YF;IZDujg?(H7B z*riJ_y+iHQ>(@5^q(|@nvY`Jj{_-yc{)8`Bx3$0RMBltV#=z%~A0OP_w>baRQF>{z z;>Hau*vRPl%a_ldzf?xEZ5P*DyU$O(m+a!g1H@(+Y6`90dGg3u*|V3gkIqhY^Q4`6 zpiVH)7c#XA=UEg(&y5rgDnud4lf1Q2?Sfb>)t{F%uB=5B)y6!~eCNW}G3BPZWIt+u zwX>K_hU%IErv6$GRKv$z*WB_1dI20&F^+mO4{hZF6k;k@pphC?Q>XZ2)B=#B1bC?B z77Rp42xRwtDltZ~&ZJRP|9!8UzH7x@S7efTO)e#8(!EYlp8F7C4Mk4m)&RIW)x^@K zJe2csGQr6Lk+R6o6^sxR+sICqIbeb3%E?qM#sN1-guz5W6q>UT_eDpDAiWhjXm;xo z$W-o5_~bZO5<#0dUy>TK+|lqgEMp1Y5KhgkJ*+K^=Q($}js-9Ff(=ba7e0~Gof2z+ z<#m>A)cRCDvi$Jfqm>d$9^k&JZs|*EzZK}7lZF&*-rBMDUZ+GC zoG9qHt%Dds9MYf603_Yo)e)Bs)e^i|lwWE3p#UIFOO1%3zq4$?Pr5>h|1p_F$trjN zn{mP|_P^!fv7Q6qn?`ORA+fn&Sz%MIPrObLUt-lHV!%@5mceFJS@SyPB!2@M zxD&(2aN@73rujL^dNQt@g?fxH=ZP*crT2tZ0ZFlk-1jrmFybRcmR`6|DN7||86(di zkJ*g11b!4DbJ9y7M4-~%B{*@yw8daYOjAEOV`=w}#7gykAtImw<7 zzlK#%7Z-iT`GNy-CJ?Y_Q4G70ubOTS^j3ieOOPuu80Xs+EMavg>W4izFcn#Wg(r?c zD0Zz~<>^t88JDFzzLf;cr}QvPbFzF6v4PUi1{eV+2B;=h%=C@NVfvNeo8`7mcYYGt8gaES~F|yNo3{}LuWe%I$oV6@q zr$W`ZRY1b3fLDi{_-+YMkGvw&qgydFH6C(9Qtg7ARJaL3p!+badK7g_GFPdPkPgA( zw&Ahlm+Ut-LwuQSnx5t;&KOm-ES*dmpF9CyzN?R8tb&Gol#GN#r!eo;9GkLL*S^O} zMx_Khu4raAQe&l}XIxuxu7xhh@H)3^_I|(S)lw!}&1M31d@P(7|6N!Ca`%q?X|xMB2y*`@LZ*TnAvUNr2Y>7&!J2iK!mZe=5@0u(qrs>H6|BhV80GVOyUxrB^HH zs|Ot@0rZo)F&7jg3U_HY9^>jKr#9?wh9osI2@fOo-TOs$-_^d zeCoZzZhrju?zYWCZ{6~!@!PjYul%1_)_-D0pi`rnX3l4UG2=1PXkAb3?6{Cd`Y(GU zfIh#8KDNTNa=2=V0V-lrVQUAu@FOs=aoTv6^i#q_U&=GOeS zh>~RoQg!0)g2Mf6eo4+y`QRO#7HZhj>LJ$ z@rQ@G;cKdQ`Ihkve6d9YFDB#xXtEI&>H;SP(i!vM4#3W$u|2DS@3;gA8{Wx)108#8kT6)Xf9lzH)f`kI%Mo`c8c~xLgPp~$?265}$hhZJ z#0dt{f!Ax|x~?KO{X}E>L2Yt1K4P$G82Q>0Fgr$$NgwtvQ@d0a%o%T&ieJF0| zn)pOA(Hz+z(|-%Khd_fd`Xn1K9<# zz$|_0o`moz)lEuS0^%B{UMs4lq3ATGMkbFbo0gYQOPJ!J?g+{Hj;uk6-xQ)SIZhPC z1`{G3H-+yYmg;u!=Ul-3w3Y0nB610m3qw9}op#WE~}_LW9i1a5cCC#EJV|xd$?2tKnGpBSAL* zN;{bxoai)4Y{jj_ZW+j6U8+qCqsTPxkgDv!UkUS#VM!Z_%a8l1t?9gOxJ)Rj?7&(U zk_2W{Cqfl39H|9NA#9hPa(FHna5{kkxhCV8C2R`{J^OJ2^SXv@er^Q7BzS(d1+KW^2Uz1 zoX0IZb(0@qmTk{2pLKdg{}yuiGxk4KxV_vlO&B9cR+$WG%i10}i5Nkgl$8ZWUpf8p zu-A=$w&ZW?<8IFZdrPCYdixCyh9aG491YpVY;y3wp|=VFOR{=UKI5~W6gKRo zHZn*ZDY}}F_WjGl_)~9EQp5RZuEdY8nN_S+Du0?=LXR?d=_064`zNjMk&d2ICBN3R+%NpPWMv^S_mY3(1M>$>m| zT`vTC^Y+MU6CIVF7Ms>W%=+ro(*@d(TTedwF}pI&Ac{Br;ALJO?X?v@jG8Pu^G7UV zaq`TOOS%-}TAyR;T&1`EXV=;kgb~>?0doij>uN0PqP2`F_2&MFYE$#N&SOh-x4jiK z^!|Ey<>NI0v^6#+TWb6*aFXljgJ@ZYhpy6pQL=Gwi-OLL`_qB&z_`}qM~^@I{0lDQ zci(>NH3<(M+&?<9{^lJ+uy)MjvliR5bjqHdNAD=>5C8b5J#KN~nZw5QO_=C~={T`6 z^CyS<`&LXbe-k)f_seRdXMGCi&Ys59}+&BCgrK9*?E<)N>gZ(;k!Kook~>IOiOCX z)P^aFXDY(F(SWI&s!>5gEtB$A3ORLDd#m4J7&N@a;43{>DjR9$DS(=a+Ssof3wqc|@h8WqrxjTSV@%2sCy z6>%^am0g&lDopEcQmHnqFg8b_^m5z)I{^!1?M_V((bOn8aO|rg7;t3kB8XvgB$|95 zq_JMuO;mHEB+~Z44Nyy)`bcBT>=l-R!EUiJ6TCJOjM()P2feFXqIm3%& z(SPAba*wE?EnQAAkan{T5p4?Mjc{ad#s371OJ|jKJj_s&D4Z<1d}4Do>ek75h@h`^HDlpDpW&QC-EE0 zo00CMj*gTz=}v->WD+33IU!5D&;6`&tYBjjRVqaQ>CwlGGg(iuLDj9Oo+DF8i;TayTf(J>}>)qDWWaF^8cM{lm=^Pv}9X z%S6|S%b_v^&Rlj4#T0-m(nkmz_?yf4R$`+2ra_b6$|!YC=a#P_FdX>;(5^6*Ey-1I zYFWk66(jAmGpxBgMX9xIJOBt&s0Ta*}ifeQQlAZk`)-^9-o@78PZEgy;q6NQa69S? z4aKo5*VsE5CscebdmGl1e=KSn-JTKUIwrL*I2d~dFJy{28_U6a%cRog*h6(~mH;g} zi~fgal+|{`&LVn1MEs}$S%LnzbFjC6uXnf>{Zqa1j@BPpuCupqQ4V(}ZWRJHz;C_K zEI?mDn}(`E@sDpV&OOw2Y2}c40Jqj&K$etz@uSZf>`QM4c==N3`{>h81)NPZ2&r!w z^~XZb0l$BDgkZkkn2%ORqTzqm$^Fxq0Op|P*POWs_Qd;T%_UHNYMn3-pYxb$TeRW* zmPg3E=Jvcdl+nBQc|CaisWnKAZ|z!ibYM7NxTlDSo?tZdsnNQ!iwm#G(R4UBhoC8B zQgG<8cs9qfUgy+Jn0D?UJU5Di>9n+dq@M+541*1!Mop^nq(i`RPxhfkh-su%I{l^Kt7 zkd4RFpvc=h)Rek?{ji|z`qnq!eCuHh%k;Hbbh>|VOIJb1qu2DFpX=eZ^Q#$#d8NaQ zLiU89_W^q?0M3o~pB^7wUYySE#1?&d=hvNw4|nvbD0lzxFhMX9!~QJ;yLRC9aF>@y z&tG8N`F`&Ywu9=9U50c)ME@oPyr`_2YhcmN8d-Qtu=Fmqyoqz_v08vCk6>?E`UTTC zjK0aadYX6>c{ZNEswP&pD&{XJIrOOlGMUeZC|v-+%oQ39ku06sq(#NInYly5)of(2 zE6?%Def0;Y79A0*vQ5RD%u~^7MW%(Ra-qn>D#FPeqe25(m>zXm%I32Oja3O zBu3W-tgMVUk8+ zH9#9ORC4#Q-EAHqN;a~Z8a_Of0X@Fw%*5flqf>S11FvegfLRnnQXn**12RPt95wkG zyVv#Q?{1yH-?0;zH-DI=H7K=sTyzDRb;Vf7Hkg*kCq3~~NT3-os%hp@Cm`P4+l8vO zPkL=9 zg~cHumK3L`5-2}%)=h?71l)eb;)0qTK_bJ4j-{)}_vhVsbQsEi5DEWIv_NMhFh z)O;EJsRO{aa6jWZy(t^B{-=|0iHA^Djfj$C;BZ{U@uiBEOAhfixitdduW@;E@L7N+iseXS1 zM{TWTRF*tK9g?b5ax8~2x^x*gj_IzVV`W%$WjDFp$pL!k!5hG(4Fe@4oAAU}6kc5n z=T{d4ssOXw@<2UW2!Mo2F8-q_4=fFXlP|o04Q?s$(bQy*U`10K%;RJeYlf2W<>4e2 z>QM&e=su5!OM0QMbbcstuUa|mJEh!2np6W{W=FZSH2|e)fiUSMNTY=;uGKG$z0jdH z()=M+ZDw5+s}&fg#HdgYQ@Ktv#NP&&sW145|$gKQWXb#|5_eUF?~rPLH{vEaygQi3xGhWWq* zDH$74E~1?Ffq+NyWZF$xa#=vTN?Lg$5?~d1q(l)5^cc{DI^6rlHC;MK{Dn{mR#rXL)gYCC82bJ^w7;7_c2^_aXi-<%6TDWVuUD_4&;^YXe zM_$8wcJlt+dE3T_FE1`U_4~yaKfZJ4uCeZ8W72l>)Zf~<)I@4+G>05DKj}eX2od%2 z`7%@XNh@%w#OO@7%rp$%6+^9zFQ% z$>S%VJP`A-q!G9SaN4XCQb>?)11sT=}8MPSWT4p?7f+;q>5tBLJ1k(%{yKfK38{goy$&a8$y z_;AqJ6w|(blX;KZ@j_oy1fDUo?g)J!1kEWIG&podD^35A>Vr~u_ zyA-&6{L~}4Z(qK)cK=N=dV}V56%j(YcI322=aWZ|9@rYASr3F2Qjm>H?d}RPraYx>LI1IZ|Ae-Eu*e7$= z0nm~k+rsG5%De2iJw0yTOpUXz6XK<6_V>N4&1R?BvuzU1iqx@~nvj2}(WojmUBf#P zZdF@_QynLl^M5|bxe#b7F~I~YvxfN{VkNCnyWvvX1#490BpaJ_9-+QLp?Ui93Ab zdCHSTb$RS?`YV~t$YYTzk=osxTd3m1gTI5O7P^vAkt(M{qv5Aqx2B&k>UTS__ES~@ z+LXw`61;n3AFP8NsxVrCWR`x0|=T8|J)Gz zTSlY{kqrqbW};9&-b$^EB3|4lb6FL`5I$?s*VjPPgcntsmNEU>YgWuH@9wmQD4b>x zAWj*z)>^={TJ6RO?uZOAb*MA670;2}hR={KIgC^iK$iqFRy7xym!b;EgLR$=&sCri zG{Z4BN)NHBlx&;tWRSW>7K4VK-FBM&I}clxVlw5L8Y|#TqjZOv(O=M5Qno%>J$0c( zNTM?G;|!<}U<>QS3-Mr>d*h{Q@NiY$3et?9{7+gU^;l2xeKLw9gg%!ca1CDosH=73!0Fb|sfio+kUQ9#!LQ!JkvbeGwjPDS!?=t|XY)gAH? zB2_H=X-Yyx-rhLF+sK$-H^8@+do{i0svDa4HTYsPfFwfLXvC4&=p5@Bv(QM95 zQu66C%}Gkp5f(GfV5f^NoFHvZsi+Z($vHv}d?BL3$x5#NNDdam|9n$MwWJiHVL!>% zkdM^MUJF;~KTAq@r2L_n1vq|Td&1EH(HCZbjRQ1S?UKpJG%`G1b?4ylfsH!v-s5ed z#6tt##oukV(l!(#)7pt^eDlz2m8NYPy-Fbq;sFgf3*wKG4x_^&dFnvashC9_=fZad z_k^3G=#iX~)!zxtD(e*Wm8rA_8(4I%OQIKBJu(N?T}kAOtGn6uO{lG6xEY4RPj7-Ptc6`mst zksHV5p_O2&%+%=+`dgx7cbzxy-kR7b(oOXysxLjaXZ~9>P`hvW4RL(1?%z22O&)+2 z?mc)&AHV#~uV1`=>#b?N>uBA%2hF$NJ$?TAwVs6({vmbJ{j?KRBT28{{Ql{)kJo=D zw(!%PAD{PQn9eN2TSjzau{w3#t(c{kCW!-GA$3F!%V_(Id{p@QenpKBG#NG;Td zd2@X9>WvqColb}Grew>qZts7z1lN!|;$J*}c6@S%HO9RJ3%KV;2#M*M)yw*cJ=U7K zCrHnkxTQ!Bomc`bFlywk#WWz`B0pxei>!67dVWX^C+epAYmur$PN%}U?I2ziph^dC z&2pOb_>YMTcdMk;F@S+Ea&9Vs;HIMFL3kJ18Tr*-q9H)@104FN74<{oQRXbI=eE>c zX9d}-OkQNsP(c}$C}*e2MIlPMn*Wuw7=gDVzZ_9;4P;dU%bDSHKl3dEF;9Z#!~qzg z4F`b-V1ZgKTqawDAgAPIngo?&a4*KKMIdIvBuEDP`?Wh9fBN(86_X5_T;4gh9Tsq)fd zaEPXyQVKMkN(p>wR!JLwDtI^IU>a8h-z@XwAYX`-P*+oOr`esuFGX=39LTI(60qOg z+qIkUJUv`B#J$xyAI`UJv|J+_tZ2?;zyqAgP#`;YeZKwv{M2kV&tq2L{psGh4v9t) zPOSs)TDM?)1{02F)Ppk;riEz+s}{?~;+GZx&WsbDBl?#-mJ~|>D0uVBV|5vE4Y>lH z75cRvN>rB*t)I5)O1j1r7bnwjQ&9lrrz4069p-MUTar3!SXTdKyBExUu)eCM$t7zc z&n4kR6Sb);!LY%RRzl0Ri|ZjC4WvdKHGT_WE@0s;6~gcrlPxt)R@_aH?bc?DA^`mj z^}b`daTC#0?br+uYSsPNCr#ELmib zSe{5{XzZ;9RgrOD1VuqgQPL7t%=h0nn@EMBR;K%tM?%%o;j+wtwjBlOHg8+mv<`wB zn^2EiDCTSQTf1o`%+k^TO2^R(TJU_0Urp>OtrntgS#r+cnE)(?jTHf9aUV9AD?2 zp$h+1Y%{p02eoE&V1a=@TLKA6NtxP)C*UH+Z^Y57Xt+2pbjHQ0LJRvOJb1MDwt!P8Dh zZGfZE5`;7Ib=UYI6QUmEw&}#)e(QWJs@XfR>&|b!{{4}umdlS`nWh(@yY=q)%%iqe zTSXfS@}npPfX)os(V_uq`XWNl^1gcg_UZGN`g#BDAO7L9Pd@pBFM11v7rWiRbBGed z|D)NS2K1T%t>5{{yO%GXefRV`uUs$@W7{GPpnl%|!QPWkKGAIN>BVhrTgXEuVi!i) z!m)a%9}W)oyr1Ft?948uG87CGa&@{^g2r3e&%`eZQpn1!Q7oCxdVg`iN0QM^Q%M`U z#1)B;CFh;AgIQV?CuR%8(rBmoMt5>@J(^XIc^q9Wo&QDKc{s8+)8=aXjbon~rcY zH7d?Gbf-Vsn2aN#jTqfVMbs6sRQom!L|l%f*B~7=YWxs3@`=7g2rDJFGiF?8kJJ{* z@I;lW?i^a@S87lC6-X_Uac-7-o#<3=IO1SE%g(4(d2UdJswPiEicshSUCdZNZc!`M zDt8aNO~v`JbN+GrX0QBVcrm7;RhT|#1|{5(`0AM0zTY2sJ5Ys5udc#NoY&>;3Y&~jy%_4y_PiCL%30&B&PIKsiXiWg;LrJ!v zFq{K%yp6BP1n{&t@e_PM*?X`B6zy|J>?4^_k10k;I<0iyV;Liwwf0K(prTNbv6?(t z?H5(mN$^*{$*-J!oFwb;3^`7#t&tOsIdOAhSW;}S#C)`W-(H875E5x*mp?@(9#Zq2 z8^A@Z>n$LxQ+Nta6p2*8vDu%LxvGLOogr|HH|i391O{}Sj2Q%CKGFc19EY4t*Xvw{Dn9U4we~PcWkvs~w`OcXz~ItcjI8MAfjM$Ha3B&a5E{^K z=O|5Vaw`chxh&KqvvNrte;S$~`(FsgI?5fe%7!_lk&@6{b-0vaENU|N8j^=ExfJQM zO0P13fy94I0cAsv_&}VGpn=RipGBKlf0G*32=aN)-KeYvD?NPJXaNRW{0Rt=|x)9Wl7b-)2y)iJj&Zd#0DLCb&r#ee_i+uwcu_|c#L z^r!#&FaMQ&+g`nXr_W)VQ3GAH*z?ud7g{y1nP-EJLO)fSv20&!2Kv^%B6DZPS~fz( zyId?_M%Ig0W+gx-O-O5mXXKqtS9?~wNhU^(D3i23V9)+W<^s`3~9^Jot-;~hpeZ3}+ za9DJtPr^d*gKBHKu>SDj{r%H}kKaFqx(8~mU%zdx=k{%e=f%vrV)oVdxcJ%08IgE7 z@uxrel&!qhZ+NTcaBTSyfv_Bok2VEJ?G4p~(bZ{65dxTi(DG|;{Z<=$9oRM5e3~@K zlxua876I!RCU!<{;WH_YrMoSmgLbo6+R2HnTC`kb9<90zU=Ny9r>CM@WMuWOI@Rj0 zV%<864R93p6o(ZQ)4DrVZPLj#y2C$=Q(&WO&;?{}z1aq5zBh-Y&{nUZ1b9(f{a!Q= z-QDs}_Wn0_s>bVx(HuoEy zK6t|k146nBRi!wz(}KWI=ZUKHIi$JcUgZ%L5X29Er7TpWE(hcd$fI%H7?)W7KAciK zIDGY){{n6x(=O~zT%tHShBQf8#n2Hox#4sa8=#Efr-<}Pj>ixVkw~JUu@Qh=jUh~P z7S^sJb(kT>dW8xWMWQ!aMnPBQ${pBVwdo!!E|?z9N+gVdpuZQU@KNH7*@)vo6vtCn|X-hL63;n);XiE0|Tt z)u6JWfZ5czxnC$p2XYpk^nhJk7t9I-Cuq~_3(}$|oM9adF-`uF0C3pA&Eqi=&Z5C!?}h-+-0UCr)bpcq@Acd8c@<71tnOX0+c5NlwIL zf7@p!uVXOLjWB}P$vV_cvCC8!&bE?Vt?>hksQpV({yQ7&1iHke7d#%E`|UA zKmbWZK~xR(TBVNLz?*EijDqwOE141e}~P!V)tXRdMKP9o@oW{|PCcN7{&~s?Sj)kWZR>C6=GYa~;91Ui&9*;4_r; zU+CZPshXIw09U6Ca5R3BixWMncxK9}VX6V|8Cub$v3L&y^g_v2%(pnc_gfhiIXgKw2K~+sp!Nj4I6L=9=+QeH z#-Aa@3Kqd@`zMck7SS3D7yjBT$O~gdhSrE*lKsx^&wlxvU;XwUA3S*QqtCtV_o2~c zPa+r=e}8e3KaG8hX6;paY`Vbm_t&2EjlGn+x8qH~H$Pq;nz}tbI{xj~-@I&${q_I+ z@Bf|cDXr%;Fl}fEGgyzO7WmuXhZQi}G)$(t8Sd;I9Nav(ZR=0>sZ2vu&|0WPL5Z6( z8Y&~ual0<5Z;X*(s~suVcO*{)wrB@kA7vi5O-s39vZRW@(xu@->P3Yw39Jq>;X`OQ+0XkL z)>B57fU<42M}|Z__Rx|%NfCrkg>xZ(hd%kIR0)bfnIltNegFoj9Fyp>(}k#$Fci4ffl+4EqK?;{ z(5GF2M333BE4C2mJXnX5jM%sHJ|XKmvNp*H1Z2uMJ6Px_lI6<_)L&O!SwZCLOQk0sD+ zs&2g&W-h=UwJ;_3!m?-?VW92`uXnA~^b@|V>>Ad|8r;WvMW%ljcd*ral(*DTVNe2E zE;LCbLtbDi)6kX#lhu&t_9&-XH=$%JOOktZ#!#}xxaEZ6LsOT{Ue+ko{v#n7rODQg zr-$Jr-`dQcg!u?4H;|&ZY(SS(zUKTGEuy5MGD0b2{*z<|5Cv)bs(5{~if=G9^rds9 zaoQF4+a=n^(^4KGFpUt#mj2h>-nr}%rKAEgd{M626evONi!b}HY%xJ|TlqU$!a4T1)Ib07jv0urJW|N$Rfb`?4qJitj zMB>Vj88qJ1T+NJ0HC;l#wf6i#W7E}?sH=!VtHzN)CEcnGV-h{LH%&w6)RwW&I(G8s zkVm`KnxQ%5bU5*-JQ=1UM*HdoY;B#@FhFigl}p>fg{+4JISYjN3rAc{RdQnFOQvKV zovWUvgWQ;YJw;P3V#Uq;s#~+Cfov$#h*_r^|p=xuK*2 zpbh4}D~aH`jRe=txx0R1DH0aY$x^{d=uWsY33aHdiKHQX@i!$2OcI%rcVqCi5i{7| zW8?F<{bEDI*NE3N9fkhVVzf0&`DqP+fUMIvWaqp|QAmt5^(D_gkN7Pm~@ z(jBO>784|#w#nEAFMWC@|FAeYeF+eMBI zoXl5Vp#>xw2?$pk0$2Sc643@3koMLyZa|-`l=*`>Wr7Ct$yS@9rP|^rxRb{^am*|Ngyu;&WmX3Sx(N2*N<{(`Vmr zzj$%_-FIgO!}*%&>*%;xIqM|?kQ3mGXD`lA&VT;PUlGCo^gsUl2lwwMG6E+SwyCCQ z3R6k%dRes`pR+^XH4i0wKXUFq1@W#8Mmbv5QG-gaRi&jeMA3s7HCqcscj=MV3!zk8 zrWs!E0Ji$yqAEM(O7YqVm1OL}7>q+sNOBY$=v>M{;8^?C( zZ?%qBae4N4XUk|kWxcy?hfuQt2KSBbZylc=Yw<++nZm~aZ$Oa0)+Rw=4_2J1#`<-K z_5rJgnCzlr-nbv3!Y_+d?p{A6DFDZDhy3lJ0n?^91DlrZcqN%qPtrEH*zHxqP8g(`b zQ^%m^72q#ZsLBNaII~uh>ZvLe0BLH2nU_JHq)xh2lAF!M>Vk5H{l$_aAb81OE(vkQ#)1e)G9{7?;Y0mMF5%QR z$kF`>fuekikHo@OhpsC@{=lL9YRax~vO~ER_R>WJ00suMc@&AvS52B2a^eoTI~XB_ zlkv1TLOd1-zJFiD&_Ha1ItYxE`Nt|~_(gkGl~0CeL~1X~7Q}ZR&m%D+igkapCp4*7 z<%pih*`%8pt6WgE9HyC8V=Ik}w>TA%sfjb4CLj?x{LH<~Agc0!8O(_Ahp))eDUF)p zH(pcAGNrT}XZ(Y8GhnoOQbnpIj93&p_}|qskd4KxwlllEcl*x%ZQq=Z1}(3u(0FRl z&fcHpY4Nz^FarntOTuWDPF>>Oam}zh)!~gG-5pC~PcOE2&NPpFYFsB+AzLl0Onu=S zj>yJt!k57UyzWk1;v~6Gj>$}Xo`uVvq6hQAB0fbEV~#LQ=oYPOpmOCJbU(C|ckk0= zsS^n>%mGZaAY=NGLl#hSX!;bZ(O^)uZIKHt8NJ_WFwx&AYyebn=#6sov@VM~{0Tv< zjj%37CCF4SulIswdu521mNPKM%Apmw)V@YojJY={B|wa%*dNxj>*{P4Au~fzhT&9l zzA+WOG1rj@0y&o}v`T^l6spjr>qZ}xQYU(;tAS)iJbqL&#iQ5w zb$bL`xfB+3zabuXnfpmC0hU!VnM8n6btYXpqr56!`2vGGz~veRD5F%AbFo>eY{@QN z$J@Y&;E={>2x2!nhGq8v2zz(KXhG*pKx_5BrvrivCeE2{gK-L1>X-&Xv(70{avmX7F zBkzmdsbTqe0=L176(iU8g^#AaZ*Ey@a&Ld{rU7pvz%xDxp~qWIt)9KN9%yfOYyalq zt=o65-PjY?{`xmx+1cjHFaOZm{ZF4fxpU{X_mGK75BBdE+wo9Xp3ZxGw~Sk#S{3uZ z{~!O`cR%~eqr~)W>mSi)`vTp(v468|;2z!EZ-vd#>31)_|NCG3;@|$8e{*MdH+x3* zm~12yO0sO~u(9QZiY2GEWtN zu~3k4X`}i2l^lw3eoNuy(o^Z>1-rXe$pdWF7rkMjc@jHdTL61~%jzIKrPK3EYkb7EN>yWRXY!@`!N&K4A|wnqqin-g#JjM{u`652DTjb#x}xWtZL6b zT1O|p`)2D~T9!xCfN$#GKRB$pU^5rq4rU@^>+s;9nRQ(h+w^LpWvzO!NM9nFT4z@| zf|_Q6>}}t=e0*R}KoxgZ@v1}WLp6Imi+Y)-K>$4c*{K+f@R;5?kc z`N54)lA2aFyAsV185ALE@ehhBjkBvVIln5jZ;<+y1v1q)H1*z#5E}ePFj{44Ldow~ z#E_Qjmqp9&P%I}L#@xa93T4LKM0QcaAnk%btS1#r6e%3aXzeG9%kf!8qGOc2Jnt-D zEgFZKKgQAKH+K9-@42{>f;Y)`aM$mz2xeVX&1Z zL>Uo3(pJ8hDJjTSB`Xt5=u7yU21${;RZtd5g%&d~+`3C+5$gT4C5Uc)nMO7N)zQo# zrc_+5BTg1Sp{1PRJ{$=q>Liy1v`lv7fvAMVjqoqM!B-GI6AuhN{U^xLdJt+-vfaeLuo?eD!?g$K!rhj&${EJfiFM;4jw7q3j9hzu?Mph6)PFf=a>G8qX* zErni0z{v?gPU$}7YA0iAsB@RjMGi~m@i@1XfGWmEHcL*~R`m-399V;>8Ez5r6dGME zNe$ApD{CglDAI4F_o?QL5kQh9E~TVUw}$p+yjwb?92KxqcoV55(c`LDfKi0tZdCUp zxHJ4Fi0H{k9iM!Zs7TN3nz!ZHhZ{xwZG#)`zzy~&vzwc9aQt=0U*-^aCpb!IncON8 zMv2K)7I6H!N@j8ohVqvhgX1u-C=Kc)mpj_YWCLo@I1gXKqgYqvPCV5FLQIM2wst*K ziMN|mD0v4j#EX#m?kdJA$D-i%q04FnX-TNHsojc%aF!*^<;%20u0^EoSEHiAIi=JI zr5TpQ<$*TNmtx$IKBcP>PW0SYfQ-OAZT!^XLyvQ$B^G(>`OS7@&de_$8D7<6MZ+}^ zRSz0OHn@E$n0byYA(mT~0fi3dWK+n5dG;Q6gAg5sv2ZUzCON&F(dYo_p4@A5f*e65 z0G9^?Sen~@)P|uZe&t{UL(^~+qe%*y3^M4lRDV>-rRS_g@R&lahk}i@%ieySTf@U zdQ{39r+cQCX&%>7PH*|L73AH+yM}4Sh3t%zRK;a)A&e#Wt?%veDK4Wu6?ShS(yXVD zPMy=hHnK>7R<)cQdka7xlLm&X?Qz*_Zas5*;uN%|5A5r2iw3Le&DQ!AHBJ=W)UB6@ zUEh86%p<(Nvt{Qej~?AVy!-HzM|bbtH=(HrQ>NCy;LKFVwe8)z*S4P9kml|G zdU<_M3uO1vE&YjtYytEQ>su@wx^Zyxx<0_kg3`P$jnO-KI~>W*KQR3H?XYMCB;?gX`H}`Q-zvg0J0*AOkn9%fl+TZ*$OS+ zC1f;(P93Q9m89o+*_1X9ylLRI@&uy~Cntevbt(Vk^!)hj;`QhDQ2YE8B0>aNOj#VSZkzQ3deOPb$ zinovY20ec_oB13a9l!Kk2fJbb?r}4i)|h{UE8`#pvcv;1uHx13g0oFUV&}~QWa~9f zMFLE(8Zn*gERb1MqhfW{dD&a>s{@^5qZY7pAXUf|;YbHlIp;j^HIrS852_g$Bw`wT zjxTAQ(+um#LFc$}FqqM9P{#za0uoUwr-G!$fV<>Ztt83;&a=yUfwM>B5JLirCA{3;d3sDysJSXHUNZZazB86GCPuV{9A z@;FP}3#+6?JhBZXH?5$qS+phN5eh+pr}z(w8J9~wLu(izNWhMyCi8&g^rEKLR(+ql zBN?V4zG|F!cqxr?$J%nzLuLdHk4TAPr1VrOAT>>eKW-N?g$#rrXsg_UF*LhWHgOFI zH#@D7Qg?x3)2J^89O`gLQ}UIiyW>WsrAI_Yfm!h_t5W;f$I1a_W)pr52*SFR+*3|O z4Fo3>I8+@7Ybv}(LF14)$Bu4Jrf&aF;lyY%wC;o^QoF1|k1(Qj1HMuV$q+ppLB``S zf3(p_&4xT4`Rd){Iv@Txkktq1d#DeSh?d{+SX1wVZfr1W>!8s-?Pf*%F`5P-=6vx2z!*v%7^SkKELBVmDb$dmcs9)# zCaVIawBn1xTb7#A%8+1&k*p_HyB?n!5sum~rAN&HmKc-rmImwm1t`bA0YNF{V>nZ= zn?YM}Dv$o$S%u2On!zHpVPcm1Thh>?%dK6n0`vs1mtM65OS`R~RR38~7{*afY!Nvy z))<1;r)o4`;R}u5PNFMf6d!z3aLDV{Q0;D+T?5d?IOF9#YB4HX)^0|J(x3?Fgmgyz zJ!ISJf8qapw@L@an}M`K0nauR|Myu;N^!9}(muzBW`a)x^Z(%J|uK~th+ve#VseB(lQY|kuRyf+j zDFxBmR1Y<_>w54NbyG9uNO>Zc9;d5m7R^Z3XR&puv2`dEkquh&QLm3r$mOfwfB)U{ z7b>Oh!OdGY4)$)|Ik39MNY|||zx>m`_{)F$lP`YC1ONP&zx$6hl2DQ=FeKip9(rbD@aM?#^(=Y+og1 zY*}VE693iV+y}gLRGdcb{wGVhm!?I5dIEKbM*q{8m5XrGcI9~jc`;EHhL6!#s1cTN z7iNCSR;MoaI?b1F-OQ=%$jfI#2v@d3b}q*=Mg;iz1lgPpb>L8F{$OH3JeEiqF!OnH;FBILS#5gwJY>IxTW4WuQ&|MTJvX#ulO}P)xP0OrYGrvn);!r#N)t{wY%)_iFli1EnQ&8Idr=i_Bw-AcfAuMG*{lij1xF zO|8mEfFsW4XTcbzhG}O>sSK5iI){sJX62h)Sw}s~W-Ln14$JGitaA=^7L4Rdjle-F z%3*eyx?AOhaG6#_MPH(gh$tNBOgUOufQs64qSE!ko9bcbG8zv~c-adOWvFJvz>stW zSg{~hJ_HL3>x?`QBN&AIpA9(A#=JBXCJ@igxr})T!Gt9OZc=m`YJEAx`;TqZ0_j5`jj~NTX6+SH@}bOXblzvh61Y zY;KjSk)uRA?pv@v>Q{{PmsG{Y&S_ zlUwN{q${+j0nz%3%u9xv!nEIN?Z9evm!9QU@uQUD;U%%F;_n5UEDGZ)x&e5uAxc#;C>p0o7$t zCs*pKQVf7d=_bU|0gn`SK^a3fYEz>n-coAhlm#f`Cy5ukRi$QBvCcrnLept^V6C_x zix@bfAiLBBj<1ZoRRCja3N%fjAvJK9HnuH<=a^bAkOAcTk~1+;HO9xPVkUpL7AYqh7QhT3!M4>PkITQYmzn7r<_#9)&UNEcp%S(QvJwYZS{SrCUq(lT!i` zLIUd;SMqcsn1?X}%ePusmbejUGzKLz8>WU!c}X?tg+re$ti$TF`=H{_==&;r_)GTzs{UT&zOZPByC^~B>7Yv!Y3=|nM3fxM5P^p zSjQMvrdrI;+5L;7l&ZN^MQS|%0N8Mz)G|qV4TtRS*?-^TQU~obW60L(1pUL7VTbk% z45Yv?_&G+k@^|ws~#K9-v20U%U{o-GBH%^q-HKQ5G82H}20fsg3Gdy3UK; z_=jKp;y?V=Us(|UlOKO-p5gl02_`K3G-J~kdYe~5Ju7xTT)Vw*Y16G={PG`v{@;K0 zl5%U*t?3iDYCy>Y8(Mtle6=#%on)bX4t} z1`6f)Yyc{~jUdIjB72vJWUo_m@(D0l&PhZG=c4x+GYn$0*XPT($6tT@?04Tjx8736 zMyF2Wc57EBXY1hB&clZf|LjkG`sI&*bob7kt(&*LIP<>ycCi6~h)D~sWjWOuzFc{a zZ8uf4y-TK^58l}lty^DYUdV>~BJ&$}4~o5KzaU_GX!+b5i!VGXpb-Wlxd3;)NrRK7 zjf_CGyoda4*y=rHCnh&ePHr)$9^1Hnc<1i^?L#J`JU>GsJI5#92W&L~5vk4Oz$6-} z>p-VTC-N1`+hnRhpQyPl-Q3zn9SHtUBiQm5-IW}=YMDc=j>jjY6A==kgz7gvk_z!^ z!{karSL_fcshAs~W zcv_ibNFH7Eg@ug=bQMdWlTw+kpcOxqXEe6pyG6aA_8Ey(dek#NGv`rPi@MOhoGnLK zuKeYa3se9#b4?C&Zvr?(REQ)sn#w}eNI>d|JXJR2l>~?g!s1`D^Jy3mIYjlA@r1Nk ziLb;pWfd5)rNMzFAt{w5ERhge-yJ9cDv4~wHOO60r>9c=bvIgu#_~!64#vPlaX?0n zp-a@&g}EtJ8VAO>Sw^*M1sGBTir3sm<4HA4wC)%T%u2tuOxhI`!@p_*gi+`(7Ev%q zEXATqN#?`gkQW*f?OQ(Ked8ZzHjADycOIr!CVC;cea>9e_T=0WCp1HxX=Fiv^N;PE z&=^on9Aa?Dzk>}laxghOF5Hr(L0j#MHsI1H>Xw>OGsMrAtYO6CWO5d{qiux>SCUR$ zCvb+iQNU=Mp~)JGZU$~lA`2M>O8lV;RjC3FpWzNCm25nh71|eB>V$*5d+#!>V$xQ$ zEdRFv4Y7-(==4#V95#oPTufS!zc>%+(LabYkiH#?x>Peg}!6$v3kYzQbYBHuD$y2;A zGw665EF9?kmZSoMl9>D|mkS@L6`pL0rp~Moc|b@_*X5SvFSYkwxhu6xp0qfLt@3Zk zKxT{LT}}}{i93CzBHx{l^m4bw8d1hl9FJPX}0G;C1(?k@F{ed>~)M*0f zVY>U_jI1n$7KcLs-lf30bIAsp1i3t z$+%dgcH3oWOPf3ER>V?^hH!Y_JPGhYklJ#f9a8}f#UhMwab;ytwaI@xCZ-~dwABAn z@91vxRVy%(jh=*{WCDmvAYyrB9bY4{1UG%A4fUJcL9TrNSu+w$0B)WG`WMl^pqHa| ztIA@OBfsK!#x%>-Pov`?rn$x1@*nFr&-KTU%tS&7IwY(~A$M z=kM**DdfH-#bB>vp8W+&#OS%%kMEBu$&&V+oxO`|H~zz4{V#v_cYnKcYx}FOT24X( zYZipQ3~#q>v#l<@OlGy6w$v6J|LDorUqAcRuYdjc(WmgHb#Bi@G3(S{+SS*y3%3r& z`{p!>Awh<}it8FybdsrF_RnkLo5> z%@-O<)n%I>`{wI! zzxxhYxZrbw+_J}sdeu09NIhLD^HG$KvhDYGZoGYWD#UL<-%9;I=c*uFFbqJRN74ESaM$Blha3 zwue-Wf~=^v2ct~qO~_#_5i2!KvC$zRrL#@hs>r5#ELDy^GXXkXqAY(Z3V+I^{iVcl z3;_dSfJ{ZjPfjX=I-;nj&w5%;F^tOV)j|MyDMi=0iSe%(jx!1>pZ~~Mr)BIf^bR-3 zWI-gg8W^6k=xpU`n77y$;RRilYX~F2K15qLQF2Z8!1IfP5%#n!2+?K3NztalL$xny z^PgUZ2%w#8#1nWF44I_8A$F74jqda}T&(XHT!Tju(%l>)rn)`yNo>%%wVDNra4+Qs z343GcSxFiD&-^D#CfaHJtj#BTBJ!SJ-C&D5kM<~~G*?u6-CRs{XX=qo|mFAR;^Rg$6M?=|eL1 zkt%->A*qcFSHS&OX;dZ(_gfx{_E#GmzG2sCcQaEBoW{>MwBWm`w=^7F2Y+lo{+V39 z@ZJ@$I5&{A5e6KL9m$e$0k5`%Y$^g8B1J8TsHEyF33VhF;z~M$MS1F1j=L=4KV!$9 zp>ZkR*8vdsH%7rl`EMQ4s%pekx7o5+%NBbY)~BbQcsbC(YswA+_-Ru7U+^l7jld+7 z8WI%olq(j>-mN(=E{@2#W;_Xzb_hM)`%k1(XX2@`L_g9T5O6A8S2zknH@60M32D$B zIg=6q5_szT;CZN`VWA*9?3kuze|(=Odqjj2p=;-Jq`l8nMGojl9r%v4Z5XD*Efj}z zVnC-XcB16GV25u$x=>#WUi)7JKw_;HwJ$@Wo2CuA!hP=vqO3$(0qq?+P>`qV(2eM9 z2pYY^T@67Vgd>O~+x1vy3cDh-v&7X=o`K^Wg#)Ex<%pZyopX?DDbh^$rc@=nJ2nqq zWr92SB4z#-n8CCNuU2o!1_8_sl%Oxzp_X@L3R!@p4Lo!VdkY&WC_jdfIh2W)p=QOC zIGMqrDvaasB?B;NL>d+AXVLET0cz+f-{0vFjv2TDzL}{z4OEs-mYR{ zfmNo~7WW#sh911Bs)vBz+f!W-dtsQdXMpvH^X`ngKb;gNQUr+fB-pqKiYADNmO0$) zSa=A2G9)a!p*7IjKZgIF^6lvn#LL#|5+MT4%? zMgG?N?>|0%b@9n>e*2>z|MXwoe}G6b?J0Yu!sK~Jo~t+0B3(=$kzwUTZ@SSbC-R@Y zJR4ztVAT?9}_NapgPYaNd}Z>7ug(EaSGl~p&U z|4%Njzxw9c@%csFO)93?;#(PTcL=;WJ3ap7*T3RtAAETE-OIE8gu|{$f3*`MaXwZMRu`^TH^@BDj`&9qeuO9)P=dZ{I?gM+|v( zDoS@RE^fSf?AYsz=gtHVU^=&A#|Xp~_w8-Z_&+&4H*hnefqj>!Z10>~J7oQjVe~e) zdE;(_dhki=;i{)=D5!e#-x4fx)J+p9F|c20hcgLz%CP1re{twUMmd?Of^^xP3EJ*- z8$f-HN4gqT^bQ_t;HDKBGNgKLHPWXBX|SJ;9QdTQe$#5WRURd)Q!K0j)OqjxR4VkU zm|ufc9o9o%1JSXwZ@QUtCs>@)Gs&Bjx8z_vG*d#&u!Qi&*?=}V{e5ro&i;x7VnB$8 z3qQWAH-onM>Zk&x2bdPH%Jo;oA>Xpg4*HN)E4Pr=#sAf5Ou@DgqDvWch<17fwwm2LUojF2j- zNrjARGClwhmnzy4ZF+@KLE5WhnRlE-Mf{FfomdKs)7TYzshA6S0zgB6h7Q0qV)7}i z77B>DQA!}kd}N_Ci=(^{1tNE!WjxT08V%K%V{Q(AX^M)IVhz_ZlN}`Zq};0zFT1mt zO%^U)9JCTqr`4UnGd`(T!d7C{w$`Ss3Xc=?1Uj-AD;vFx^&~32yB@vUiA!Wdx6yxa z2AHxX$N`JQ5~ z55H4e04&KETTZ-YZc>xVJW>oJfzlB2+48(*Mq{j+;eml*>?Y6-qOz&f;_G0kG-Mhj z#bOiT$m7Xp68?r+WG-gQ>f0x@vi#92%{JMusUY6{89KMoi+xL0CwP}tuCS3%U|SMl zP9v45KvmW{5>OhW*+&AHO*Ip3&6m)kx}!PM*-m#ZHU|rsqav-)r*I-+@l~BL6c-kZ z@s6siqA2n~|Ijxn1K4{U~ znc-~(-s1lBhdbRH0P!2SkPuqRl*u3_l7@bVOU_wTr6WA&6PR8m6ZfAmn1?nA6sg3M zBsv1GE+l(T3c;x65L6gl-Uo|_0&_O2R#>M}EYyljkQY;0R!|_!5<<~;*;eJ5RF$2# zhpkjQq^yde4${o{V!sg-Q=i$*9^Bkuz?-WyjoND4QdhJsz4}2;nragvsiprwtG=*B zL_9}a4M7}n24}Z_i9D21u(-ar;dGG`f+k{8-oWo z4EV_1Jd60~@M!x9D#`!Yok%#S_&?(5@MFOTm3`d6QP@taTnDf7h$ zi)b`6mw|e!&z4IJvUC?lU@}$4TAa-#OdQ?5?J-2jtA2j_?*6VtQwRH9M^WF3tm}sl z4Gj(9SldS*{=j|7-sxox&$HuK$7k-ccH!Xl>E(?ZK6I}QRNP!$T2Htzc4giLf|1kB z+-b41SCFaOCbwBWR-Z#?G)Z{lf$*})C5#Z6_xNNRCJA({p(I6705}B(%rU*r*fvTA5^!#pM#6$)Y z3;Fb>4@qZA7;D7y;eIvLr69Zq5K9S!`9bT2it+niKgoq;G2lNsXDYKl`On)J$+s1mSUD`4d65yEBJ#l7m7GL ztbiR($jo;tN&Iz%!Vbl4{yG`&o5(jV2S{OXI2w_T^(R6*18Q)=257aqTX%h8v(~{0 z8)glI{e@qZIE>OVgN|dotbyDYzE~CgyCKXj1|~oQv@%{$S|wS_W?DusKrMZG4*-}e zMljFON%)l3V^K=hf9kCuRuElL|&SCqEb9??tswLyiJqP60f?fF*@jym;J{7f{Vp2cxS8B25WvzMD)ML6sqPw48mFC$cCTI|ntcH)d-{ zHe-!p3S$w?g)pNLxI@`c11cFW^IBwafBvvE5PR_s1<=YPBe4zUt^y1{!`7^LTq!)4 zElHW_RHB&YwOPi!A#X`wL@>aYY_nWLyY0CeG=1i>9+LDi>}z_z>nlCsVXg1AHCLms z@lxV50Y^Z2$qa;TI&8|6x{;(tV*(jpb~Gx( z^%x-48joSt+);-MT%PQIs;L;R+)AzS^XjIzHqstyZ zYrdRVQ!jSGo0>HUlT&1p;!&FF#+;lfG0YS|E2K? za($w00N}A9vorPq-m>3-Lz|3g1Tm3Ceex(0_$>8;jNe#BSzOP|R~Q4J;mCl{oTwYAj~-!1 zI++gqbm-U#G-;bES4R>VVH>Ff<#$6{TzBap?7>PG_Nt(o!^)_9*qQcI))^<-P!vHX@O%>;>g0_vh3yisJ_IY-XvI!dO*&al7e4sX^P zT@swyB0e2OOjk=rb$~||-dle@va9acaYD<)r(e*inw1fm>m>TFUMSNK);L1w5he)| z@eJjCJkD8aE8?0_i9LDqWS39~!9u(`eBKm2N?S^^XBxdVDsp*(n54G(Y?+mnJf6iV zO!Tn$JxV$2VA-fBpLN zSzywdAJ-RTpd+vrLXv7OLilgDPLH48T%YT`@*%?t&F^em%H-M4rmEqe)W0!^zqM$I z#0kN@so%Z0{PeTWzxeX2M~|NXIzC#8(j}E{oZfolnWD)(rz8%&9k85~dbmW3ftW0mn7DoJCk!ArzM>EDpMkw?6x+9e83Zi3o_w}jSf9rqF zzWBqp=NC7U8U%8>L*R4>EGzw++RXA^a|6FmxaamWU*t#K)DJ0DyTK|0*X-7I zcYhB=Pn)tHNds9wbJC@wR;#?R$zBIUJ>6#wiZ)n};!WgGhNcX4c*`+}NH-4-yV~?H zS*D;e%qNd#l>y!EVqTY>*MFjd3xFbFFu_JUGvDyN!4{y!bn^3$-6}T|Fxr?zGd1+* zOefI*YMTjMYSU)JpNiQAk~hgL_eLFpd4Wf?##C^4p>P!6^I5*7)+m53 z5<%A{3$DWgabP^}sP$L{F{TG5lAclssA{yUN=A?*`b%aVJC$Kqf04iO|9}t?$b}9T zbyJsWJSsfqGOhxc+#1x2UhHEilM6nfQfH8d-ewuPdM6D0x+elp%%I35gZj1Jd>2=<= zx@wO-j06?s*O0_hvTUYefpcz3vu||<%UJfXJHTt+R6zLDu71@T!G|YExm|n^YMO|? zst$_GJ8B8Li+>r_fEk|CuTQ!knj;Uu5e<+~l+D*FSRu&Oz#Bpbelz_W8(?Rx(94q` zVI@VCKEqhPnfR%xJ~be7H2_#=Qo;aLH;Dov7yzYkZq=Y?b}DX>#`JxYZiX3-u=y*Z zaOsI{?uBF%usDtPfwUx&WPI6B!FuK*;1I5-8erKvGia4YF-0Ijj3OQy<1Hfv4-kjL zA1WRCR2U=7v4k)@sR$UMnXh{sdIo2e-+Fp z;6Bi)goG7DNPC`CZZ*r>9cvuAIwRDHQ-s}hmE;h)VD`mr7LffX!K49ZUwh8`8*Udl z@fF%QTqj4DHbd+PCb?vG(=adACfO?zgHlAx9J>ZEk+e`WC*yeoHCaP1K5}qz)L+k1 zIKOmJ>CT>cUN-@E2XGJe(OWfWGk4pAz>G%V!jwn;3!9Qbc&zNlV=vkDaG%rD(^s!v z9ltnx_0n@|9(qv!gCh@LlQK5#j20Uyr!^)${DeJaDh&XZ=BV2(sLjr%XUwSSMN&vT zIrC=U_jCjEG69p~u6B$k>k4uKX1{fJ_xAqwmNx^JYgi?6`}Wb@-G@h>9JK3AwR^kP zxR+c?b6XD7RlBedoFwh&=-%(Ydiv{6KK;cnej&$AE6zoNdjRsO+dCm2A!lmyAxfx^7DKoGmv=Vzw|HiRMXrxxd|oPYTE{kumG zw3)~7(2&=c(bCm7UZ4Bw#KT}+fk~vBPz7ZKhi3*K=hQ^V9=&&b?#*l_k#0OM%2@iP{cIjnOE1aH53{jF z1Dc}z?OxM!-bj$3>T$+4yz$v& z^NJjtOq|Hw%O5mPKeoWEBis5qO~#RhTqiQUrR28N%+GyjwzH|&pW^iiYGp-?EI`b# z`UjbH;+*Zut@wjAW*XqtQ4ZxZ9zuc2#j5(w;5Ka8KrC9t4ervrpE);GJJ8Xw)mRcj zYRsFJ#o=Z>QPioV4thZ5`ovUM1YCVvex@UvCNVOYYCpKiadZ&9`av`Vd>jeEIYv-w zovpTt1xS|Ixezb`Q7iQnXMCI=i>+KUB{o5a82k)8cB6TIr;o_@RYkz+%xZ_goHN!Y z%Cfc;UB$>s_9D{&jY{7%HntIXyYbi#1B?Pk&HmRSvK=&&!2?>0=UpC!-+E1OswPDS zf|+_VVthsjXqjI$4DdKm8u~cMI-Ko0;_6K2{KK@)!EO4#>)W#1a@z(JfdDW0k%3DAFwZ1fYh?xR81rsf28_^mk^aQP6w1)T9c4Or68ry z4ws_Ckg~yZqO*Y?DsgHkAS8C=fAnInAQ=gu)E(c^1NR(2B)dG^0ud~j1t?^d=nzhK zW$Adx$12MRG&cqjfhKT4V9+cN0l~*DD zmcK6BSu#WN2&J0<36MIJO#JBaP(K31q6Ob4Ca~ zBupn3r24Ncx+cg>r_CZq3DG(L06+jqL_t)dAfx1?sW^Em8L*>z%ngdTbE*lMcg;CN z*=momp@RYgGmcZ{*C=qs3W)-{bJCEqYXZ49J`-^NNtEyP({*Z{_Lx3}7VD!DoiIUd zxRMm4{5iAO8=UW5Ru32b!gUn;$L*-5T2Lf^XBk(+H^^}mkog%f?^+eU8Zp$5XG*a? zl#CvgB`D=fE_o=taHWEvPy0bo#GJ-y3*~%=Ymm1Iy6YRs zZ*CcDy#MIltMk*YB$%<@+j?~W;L$xV4Qm>`?B5iY6uV0REq38zvP^lO1zk$aHhuMn zZ%$6mta49YH5zbJp4O*JszSybJmG_?6W8`OA|FNpuE_Ms`6= z_362y|49z=wHC^<*Gud8>A)z0c|xoiZ>2i&qeG)S?-om$jRER>T?#0#xV8KGwwLvu z8Y8Ok@W|x9H<9hTY1_ch8X%Vdpm%Aw9$>oI;$qQ>L@f{kV}@FL2M=_yZ*Dz%`O4ee zE?>X(#2^DHFD`J0_ZjQEb{8=+>iJD!dEh7_T&>yn03jDze5U6TR+a~03M1|mF_*a0 z%Q!N~LSF*rcT_UzP6C+W7*RM=KLbL}*z>zRyt~L}79^i?a@5sc-EXB6Uy1?)HbYY} zr8V1<7dY2%4%H_(L&Uhqro8^NMh!S@Y7;;;1q_51H%zlkb|@PmK^odAIS!TuWHWVA{O08sn4uYyuk70Vo! zVohPF=9}$@CE&^4uHdye7ZzQq4JncpRx@F=*YA8vE|bh0K)PgD%mq$yX3}0*w9aW) z^d5Yj(B!5hA%}BVoh-yo1Gi}yILxWq&`1a0^+Xd2CDmD3!|?2Ck&#D7u*$fEKB6>G zc}!-+OX6yc^XK-_5o?R5PDJk<&2MgjZd4>ybQyqrOI=B1p30iCE56Qw91xLPiK7!q>hf?+_biteEv z1N02KZqH_(Lpg>tIooI-G@fEyWlW)c;bx&T&C_g!hD&ZqstjqAHFMQcR*9=Tzlr{A zD*>UTfq~@qqB#|pl~s}O0!qN_C+bF+qjX_38BC0Mr{iRtSp8?eS=U(U^uS{U&ZJ~b zQ-RF@tW{!8cnd-b%ozGOGbcM*TB~Rp*eLN`*+CYh2vwOD`%GmDl?{sd?5|Ob z-6is@!NkSP+B3u&-UDQ33a=&>rYzH~l(xHu6M|o|wXK z859wgR)GplL~^ID_~|t!ssyBNH_Mpl5jHYR(&XTX(z1sLUg6x-f!PFAO16=$0kk9O zXsPNPj8tn{1p;6@@7PunXdCCXHMCQ!YW)p!>v+%t;V+J})2K4!V4TR5+=yOAzVm9l zt!qYwZ?{!&B%J|Lg_jk@2Wkp$jHY=i0?pn49)dy__lX1Jv9^$Bc_k-RCCtd~oTkyn z0x*}gc7CWWv`;@d0o_5ud)M=Y1sp*&-G6`G_#)QK#9PnwQ-Od znQ+O=q>Qp;xtdwySuSD@$faR7+xBv=3$y>Hr*0Lvxx2UPIijBH?^V5?tFyhgFBb%H zr5O<&S7nWWx87uHtHAES0ujs?%B!dseFE({&4ZeQ)pVC z2EAd7r;MIlT;AN>-Tt#*{M`EU3&gwgcYo*ohn{F;MAMXi>wYX(>d7-@n`D};HhaB$ z$Hkm_+U<*r%j1)iAH4tmC?y4E(tft@-0Zw|KXBcHcj(^PBaI|77Ug_uwBaw4H)L3S z>xW1#W|zt5uz77I9K_PfCZ1eYy*hId;Nn8Uj>d-%-+TYVADIVm*TU^xAZ~6QKR@^C z1b5S(T^IxKeOhLIS%20Mek{>gL_x3A_O;sv&UDxJj}Ff~Q>g$@O)|IU$9s|YJr{M? zfTIC`K3FEurGOpRXIwrxxVg1^etO0}-hX)CBmD0k9MI~$2M>+tfG5x_Q2yEwzzD>0 zMAzrH)wt{V|Cd)gXD7xlu3@+_z{?}t(rh5`>gh8_E-NR3;-Rnm&GW2oK^930tE{z0 zfp(Rqxxe0xaMf+($&OTuCIuIkiDZn9ISH02G%KT)aiC(+9F0S!qp&7>l>?X07CjyD zbQQ9CQp{~ft|QsW)+1P8qdO2fNV9&y0BPWko7<~2r}K}rY`{HQgS>;{&uA^2tVwj~ zb-aB_VlAJh`MEx8q13G%R~^vB+L~gH{RuIlPZU{MOh)Yhh`qH7OUia!&Caj5*-pmq zH|kl8RNL=hm+P0@6P&UBX5k8A)$!OiBow2CP@e@oQ;YY&kJN`Qb>JGTaXJZn4Jro! zUO=J0BJE_U*_^~e@>r^HVpp4&MoE&I~i)s#EMGZ*0ubz;GC!{u=zKl=ZNHd6RPu=Ap^ZCX1|UD4iB6 z!Q3oaV?RM$<%kQ@I|-)FY&INC$=Sl4{9_>Ko%_Cp(;X=$)WkhC23}MLc9kYUQC)A% zuib(rjPvr|4rvCYOKw5MtIIo4kWxceh#CVQ3%@jd*ZhQyJLf%CEz-EDJbO~YvOj%f zIwAHFid?z3!;T!{GH9ivk)3SNm4Yb`$Q6?dME_jp)A`34YKYwULcbxa9aaq&s=O>D zbY?}JnA!#jx~1+#ySO|wi+!aRU|q5le6Fc4h36Oj^#n36rYu#~I0gcPU@Q*VMnQuT zd->Y%HuggBD5M`m=qc)kv6XTYWG8X}Wg<@Iy3yu<0Nbfl4BjYJo!Lu1HS*&GhNfyt z8Gwtp#HX6C6J2M5KQKs1;C&syS?{e2pT$>(YjH&0UB7{y*gaFjh}l9!5HxBARVX)N z5~g)ellj~0P6=bAQvVs-g(*SgYpEkap=i9peo9UV5#rYwb$rSn5n6p;q9T;38|`XM zi^PK)nCi$c?bqe5$a^rEku(j)KHRou-LgZ=^EH0I`WHd|I5nvzPdX>5(>X6fu z+p3=87I}eO^bK$7yu&C&2O$=tE#y(qaYCMYPUQD-6?v7;W=SD#oT+kOAHSNkBXdI_ zx>i*_q&u11sp!Vh>J`<`Nmc6$@jw|2;Zs2GF6G7-iBq89t^=)K(U1zkfM(IP=p(4S zOH4@47QG+@70Gx?M2+SG>bBQ}V!;8?Dgb8AKacYE+^aaDWrZ_XUC}|Eu-V4?ket6o zKyssygor%2`b%)m#~pr(FICbsbymZPfs^}c*|!4UB~%5gc47+!Zc}onH|_0KwsOt zjNY@7P43b;SHIrvxr8V+L@-M$G}3Nqp@2~a?Qt%%00#9fs7CB4iqYuA*_RSx@eP-Nt_wMd{kG;=$7W{Z6ycBV~BBR;XTzhqgK{PgoN44 zxh}IA`YqAV!r4H<3C|kURS= z=cR{F-k>~&dx`;tw z1&TBUZzI-PIquhyPtHXT(WuPcgy9rDl^y1P`Es=b6X8&Yt|RTsBBt3)kU+0b>5Qop zG+X^3rtqriCni*fXh@3W0?C+!pTI4!%2zDGPI%t^NA4WpyQ(8%>GlA0zF+8Of10%0xj9p895~4j6On*@TB0pE zD}GDu)zYSZhaSNjnO15iBbnKlU~+q1DySkh2=IVV=KwH)EuzT~T_XZ!mgKSme`wE~ zholWo?Wd~RLk2=r1XiW7C&R|5h-r_Z7?+75Z%li5 zj8^}#S0*rfGyq(!$TbQTr|N9XZf=ocgO`3s0fA1Nm8G4m5#^g~h`N{MtVR(q{{*62 zZj1?LZUZjV8TI-$3`mG*8EzPYskC`qPg4%fUI_#GsaZ88L$EWer^+|SQt4}Ofv}`e zU_wmmKcl8B7s-h(J3>Bzq;Rao<*7{E{|)++e@u6n>Lh^Wv-uj7yp?y8MYdKjYKUX^i` z$Q;rcnF?y>~X!YkZA`q2-6{oCJu_PZ}X{?WhK-??{wc4i*J zvLBhKJP9-hUCq6|@sN!lzyH`OEurnf{kxCu-|hV`Iz1~^_6-4~qcCJAHgamWzrKEJ zq080r@mW#E>u*ob$jIdYDi|7z8kxfc0>8uA+{Vz8i-3yPcKCzYJ?)luM`Wp4l@I7&J(Kk01pn4mSt5js=PzICS1k|% z>H6&KMF7!@11Gsz!pmD? zc-U39%4CwF2*q}NI-G+-))GjPtJ*XOIjJRKS@`l;f)oqJMRT3vHGSEGvb?-A6@=4K zb?t4cqyte`Fj6W$WZAjYv|AfZtBx9bS|f*7#-_}_itd0$Rp$+U1k8!z8IAvO?XrZ{ zu=E!&%tU%99WKh^al&A{I^)EuK@Mnzvyl?vl-`C}S)HDWP8VuX#3_y4S`}Efm^Ozss9Q)R zA)U@Fi$`g#wXNF$o)Ya>F!Cc7XKN5^(-xEjG5aH`(N`?j{f;QcE}QQnzbuocM#{L$ zTp(GWWH{pPd^AUgVH1eeQ*A_|a(!|g)|ST|bflU}37W|2isyIG;4R`7v&j7j*OoL(Tj9l@K1+Kf4fv65`7daH2&7@U?HOO1+ zj-Ko{Jj{qtzSLU~OC~Yq6#5D&CZvj1jTf|^c#23Uxl;X?DQv0#ndgxmoOvOnWrbk8 z))RTLW2bsbodLk9OOl>VP?wmzG$s33UxFkV9YSM+VFL#l!-xU6#2eWNNGwC7g9xrc zBxi&D&nL|*98Sl_qkT$j$|J~TQ?b&GnTC0cV?k_Ia54=ofgk1Sc~fo|d1hEwj+^81 zw+TJuGov{yrq{6@bN{cmU9#WpRtaLfk-@plm(i1jI=x>#8fXgqJ3`KMF)3km!63Vn zLYH*gLq|RDI}Q8#q4B8uXG|xx$XzY%Pu7#+M`^0BizZLc(DqLZUKhcPynE!6baW*`|Dv2Vk@+A5@x7yr0r(r8z3Jspgoa+9JpZ1lFvx za1h>wJ*-Ot9cvURv)3j#eGJgE1Dbvp_!7uscL5p#Nj@dbB}qTodG=60U6uTxO#PXM$|Oo_Nt=~u@d$KD4zro$Mmb}PvA<#=LbZWCfl!mvkuu897Lrb;P&MjM zP?#XFaax5Nu_t^WS!n-DSQRChL>V77Uu4AR@a$vEhb9_Qtzn z-khHuyIVle5?nmhe(&(Wo51eu9;((~5rj;#0+$9p@sKmO5&W}v_M=DSZn z|KjNI=jaf2OyTImkmUk^RX~)`1oNA>?>)R{L5+U)@smg1Shcsa)!KFMqv}oYJ_*rm zFDu&FadW`ot+)FxPL7|wc;yNP-f&{h!*vi`k8!A9Crt-FUH90Q{kI5;d}QURfezNJ zLS{(t1*t4?bRaa~Iw@}~j-JQ1_&X&u%czkBX2w+Bh2XZ@19|q{d-up$`OeMd;o;GX zojW=x*}trLAV*t?37~#caMqo9rZ6ZYDixwhjKZTIdyzPNt-#TQ?g-uuZU-pp~9OR9WH!yZd7K?lF^GR$7jbEI&5=9`_>R0x~=TZl>wo-?&i6dobrsK z+*;DbnhVBxbpom@!cZDkXL)V1%^%!<^3&7%HU*YsB%jf5ea%IA3o!C!#=c0*0&$N(M})Ra=Uhu z0UhWyqO$*YB+mH`vdK~EkDx{;(Fx~UdheYPo) z`imLv8%T~IvVmC*v1FISWnka~*7y|kD;!>>6eQ)qt&6?BbW_&#n@cf41HkY_;qA=9 z{3dA>P67B1{EtcKxyE}PDZ_PCFID@=%wjE#rQ&2|)}Lf?(Yyr~2#S-q*MKoK*sgt; zgbsa!SJ<4Ldl%uEdz%1}h2X}r7TN!)Q)j0x5a{;gD?ae0SU~#3HTf|N+=7_XZ77)v zLszt-F|9@Dd?2?^NNFQd-T_Rhaj=Ng1r>uP;h1*89|^uQwo1vPI?g0@kb?kVNSm~% zo9?13b zd#ekpvIak)n!id?1<=txUaVy#IAv${0z1r%29u4}tU3vVtjhgsj)7G<_n*&{IKApS zYgfGy-VhuB&=8r$nw3niflvb~~zpKM{a_5?Gr zThi~v+SU(nL{kY30Lmd62>~dWbe|`1_21UUbeG9?%fRafQo&*Zmc$TS*#VIci)qED z(1w(;nSxxiLy;Fj-~T7_CE=P!Bsl?4JX5Lv8L@S0RN`#q8zIGNRs3l*Vu)p&Y$#M& zvF&8{OV*($sO0275aC=RYbQE@EqI-Z?RI>!GZ+&LA6zAKWsp^%o8N5BMbXn&J!9wX z!M+E50$BVg*;SV+rFwgNY|0#pds5w-`|`!hSEttJpII&FzETq{yOLy&hH+B}DVm|` z0x*MsH3euZ|sN2-PKDc+Vb^ATJ^1XX^CH@c}n){!R%++-p zMyZFIhu?Ss+2PIA_Gh1er3ZWOy@wWC=nJfRf_lXX{HKoI)oL+DJh_rCHE#tBT`>8~ zru>M_5&P0+`X|{wVl4X{O)o30xPJ|4%&4T-6L^&VdN|UYX;k3Kq?JiyTUJ~V``oV9 z8+mIty-m)N?G56o2`^LycIEROjc`9+`FiJId)tl0yE_l>-Mx3TYlkAN?C+k4)V#B2 zcK*(;?B5vVi%)+0<(FSSJi7NMKmGW_4}S38d+$AZ{GJy!@9!N77dH}pvO9E>#~rK4 z%%GjTdPNPCM;Jb?&-iOmuwK=DytlWzgTdvC+(y@I^eIe~WK9y3r6e;ESB62Y%}6n6 zVy!m{N_RbV5Z45tZoDhmgx(sLGSV#bX=mUi9Ak9S?*0s};F%QsquSAM2?I!Ru^4QS%!Lgb=Nkts)MTe4@@2^CfeF*!)AfpCx~ zIoDa`<7SzBb#RpWaM3h-+4e|wHd6uFmKytppH?dV7XhKMO#R zbT(pV>D6>R$rTgC^#Q-2VxdOnmP5(34$(S*oO(%;rAOgQb1yhJRCO3rf zOfnc-AQSLv494Sb_S|k}>{a6Yu)QB+xn7vpK91ej3@d%z{|bTqhk@br{(BjECIuaP3*s zB^#TEId^R)p|R079>39K?6wCNw!*c6oH#mi9JI)TxDWN`*`sB3JES;+?be{VWF60$O;SiQ{)S$GOq=%OkZNNkFY!NB7Gvq$iE6ID zWUMHJtVHXI`|;~=%q)n34QjhZ^ub%LX<{1^h5SiNTR&yZYS(xzavxpIBu*gK+16ba z=VB#TK10h;640a=!%;ox!xedG8yHTiQ|Va891f_FLB-R0_?nt3D_U|ua}JF%5W*P= z3viMM^&kU&mD*0Fj;Dc2c$t=t7pjasw0+rx-KCF?t25N5>zNx0Apm9#HhE_b*e5rX zh7zfmTf-SwLddDuiU3A;V=hDM#$^Id&FLzmz}_g52_`0R#u$f)IIN`YG>eXQB3jdl z=R~md9>4ykDk;!fqEiA^jl>dxsahkNswM#&K&#+Cz!Qcdd#n-cKsuO8Ro7wxdLf-*JN2zw_;1- zl68nxz)qbE+rw(y)@@8$$U);SW9~%8*c>bO*Q6Pfu9_a3n{@+KPtZTGa=sB>qrPtU zmi@yg?muFWOD9A2iwSS)NVO-`X-Oe0Bcjrcpza#5-fths-$5Jl}vkZNY8N59?eSH7u7eD)>ufF=)Glo9-^!NAg9e(iS zQOk>Z$W(0QsOCzsOuE}!k00D~S8*?zTCba0YusI^;sXMda7eo)@fP*p`QqzuKmFZT zp10%v2VBj?;Y*rhh@qJNS2t7FebvLkOt*BMx!%g@%`SlHD$ij6ZBsky`-s27N%lAA zr&g8HqbN(HUaos`oK#ziBPdZ$cW4mQ+iriYe@aPWkd1&m;cHZ#nWS>(%8v(4GLd0t z5KLpsn-JKDmn`e0&E7wd{kuW?&^jNFd+M$Ta++0qyX0;ta&UO>^z7;%|MAz5{q$#l z^urH7xPNqn!kJ#OFlo<2mtb^Du+i1ar{DhW_rGH;j~_kql3CK$8f3e%K3i(jUloe!nRB^c&VP~?* zyo|Z_HhB-(q#B?TP~ve|#c_^&WX=FSex!yfb_3O}k7m{B5XM0`F-Db5e1fPjKHe4i z>eZ~ww0N@tXjkL1672>aqnb_>q<~kkNEhm!r5aOBv^L#a1SrITKF&Gpfu9ZJyCtGP z43SGJU==$zpXslLrzwER2ce5e2;*;wcc`wr#E`3)DWD9S6u4EkHn#u>8B1R&CzfVw z6I$&1C)82FXki4I0p`(8X)H>m*8V!2n5ATi3pwd)RFIKRm5$gk%HSceFvZkVjQk>b z?P?trL9VGLbxHYa$AdUZJ5jq~-w8}3AJ5p~HxTG&)66Wbnbh4r{G)^f?(9NUkD5q> zxGV^L18~SPSD<{8hPR*e#nN6d4r$XchKyb4}({FVtRZ)7&#I2T|*DZ>%{*v;lyZF1j>e zaNr7tM|(8ai^maM^ad?MNXKOEj~^FkhPQaE25zbsqwRD~nKNn3!9=(5`D)apQ!@z+<5>|s%%N<|Y>_a!XcH7&vmVqV|ux&Cu` z=jzVRg`)Ggo@X8~5rGQqk>tiptmkfw7`EY9<&fuc74;#5jU-3wk?)iLk^?I#G}vxL zIjthpaTHSEI8=rLSkryPWzWBoaS8yFB{-wdTqpBO$1y)*jf(6jqmK3H3v{}p_Drvs z7FAW6geG57JL?-wmC71N@prT>xjLhg1}Kx8WKRRk+1sA+yV5 zcwe%dWt=mt(-75hUd`aQkD<}<+KvUqY4OIoPd-aeT${;Ylp7G$FjbW(C%%z?z@Uz* zNNrk|7YnoVUXi3yutGwubYNXw>QMn+DpkOBV!lq0b@IdpjCtIsGaWWLQssz9P3dWU zRgSJCH;P`2PCa(>HIfDb`i)e$24O#dATVY3 z6QC>&Hk)QKWl5SF5h}O9g5D}JkK(&EL9NAbzPju$GXX-5*E3qol|w7oNC!ofSq!%8>7Qno4-d%jjR*GM@`#zk zyV&SiYv=9(ozMlY%Y`#>@cODC{N3}XfZTue$kG>2{dfOs>r;_uvFgEnkAFG1d-KyD z|L}{S{q*mC`S)La_w?cCUwI#`RYAKHc+2?W+)@&fFd!ur!jf>CR+mdtji_j$i;;;W z*S!IrJxSo^OM1Hf_TjVBtN-}d|Ic?XUjFchA9)!99?#D%Em8W>PkuzlAm{|eY04m< zUV_f1Nu#M0GHnG*7MImqwVjEkwZzr>u#N=7dC;fAr7)E;oD4$$m+jcP+~Ep<%he7? zk@W6+Rw0c5Acty&75W-=3|JnUNiOY7+5t-!ze+&Js` zxly8M!2q^z@6$%p$!40zR+-kuWx8cx*=RlRL<7c9XGG!!)rPY^nX?Zmn5o?aIkZnc zp_AevBtDL#smAMod4Qg7=xu(A{BzhaBi|TIx#aw(j{%@nPC1n#R3Y4zF!-T`|mnK<+p0lq}iTD-}5_m`bggh+ehQe&I8j zID*oNsEPQ%4o>~`v>)NlxsgfyLo~_^7+8o&ssbIiW|G2BF_WvOKQ%jB&B=NrFoGk{ z51|a!sD&kRoyHSclTp6dSNb4OhPf^a(nhs$z@f{Q>`h;Y7U0Nz=GB1qsXKkWvvoBy z-I{{kXRvF}2U1ALV~dK?!cr#NOfreF!YBzo1xyB{vhx;}ccEzn6{UDB>4*<<9{0qH zoq5JXE50Wc>mdbwOsqgVu`k`_8TDH*j3h=-qu1~*)QM_LUqNb$+b z($5@#iB7fzKn0Pw#4#};{F%$>NEr1jpv%+qbFolPa@loNubjU=Q{)Lj9sp$7tG+SO z2V_}c5~sK^lXw?+62mAvmD-6a{LTzfwii9(qzG|~l~SXo3yQWcVAkmIpUO(WXg zP#&g$1h*&MY2w~AV(gQoiDWS7z%@+Jl&O5^ZV}Py(jQFy~ zb~Y5nXxOll;%$d=wDntQV56hoO6u%KYQyt}59h`+qPd!6Kz6tr_7sk&Y{#T3?j(Sd zAZZbsqvSz|S~RT_+8Xf;4r+JM8U6~II4JH$`U9;PUd$shxi zt+B~@Zf~q->o&jkodhdMIe;mEA=?(V&#gZo~)B@d@fkBIQZ8Uu;#EsqnvIzJ;ZOIfH)kdPw^bUF?z zMa1RPSI5tvJ$vxr;jNE8e)OdI|77yUt);tnkM7DipFDo}=YRIEp1*qa>)(F*>F1wA zA>@4cE|B0?^=O7!{~?my0YQ%a0drRS$XC#@Ao7E7vaf5pp#zqtV}yepLnvg(NzJ3Q>FD} z-*xfUvw&;~#Ux^#RcYL%DYUG(>j1lZo{x_VoxZ`*-kaCRhH>d8NzL^cFe}ssTpk(tr?^4k|z}i=A6JbVBi{N4RCcMB`HQXD^Tc z{#U>9;JiQnqd)rKgAcul+1r?HcMlGcy#|XHXUEUKeR_6!>}?M}c<-?>Jq*HJ65&mDhj$;psE42HSPV@!LJ7$gCNj9nqFQv+1TXY15pfdLrF< z+kMvP-SIl^vNJ;1Sa{N{Nj}BG4QL&*m~wFas?m7PNk3|&6#ps6K@jIR<&hmnA-xdh zkQ+5BwO55XSJT|CL_|Sp8a^y02;}scg1`C#sg^B`4r}_Z9c@KfpBnhgF9g&{8Q8A_ zsa}LsOVKpopDnE3 zW7>a_XXM}%Y8X!H$fE}eOBzxG6tN+5YGq^WDTgU8C0<$x$ja7=JIroXV0J@Nny`a( zu`@_fHN+2{MO45Qw=<`Q0a#H{60`}V6;x|VYcqB})tnY|uBDVn8BCF<`$w6Y7KcwGz zShhIp1chc{Xr55B5v5fehTaI2s(TY__AvV61UQO}rUF4FlR9H(7q?i5^G4Me94*!$ zf#PeZPD{?3BCdCz0C_D5mF#B=Julp_Bi?O??U^;ghG=D&~qc zuA!P|A{o7)00_7hqOJcqv##>0AYCL0Wk#Ugs3ZwRi%%@1CwsO4`H85cQ^6z_L_3ep zm2V@vC?*mC9esbsn0xawd>On`^ICvDK{O(JEqe@I&c(; zGlFlR(WoU!Lb3DKldJ1sr#@k>R3p3YC6Z|XbhJlN8gU}I*#oib0qyzAUg04&`;Cf6 zyJXXGc47;ZZ`AnPaIpaAWoxf`umoI#kpmnH$lz3A&J3vfN3cn9G+YJo&Ky9<2&%_1 z5Cf`{qD2s0vB7TBz4REpSsX9um833kct1@Ym8VvZvE-4(iQ#+QPBRfb>Y%8W^CKRD znRty@yH2O`ni4@ISKQrEr5Ej};KIFRaP3c+-3SE4(K^Z|Sy+6nZKOm|cu{0`_@0t5GhNz>(Al7A zLnL!c1<=+n*&m?}6g3mMEyjn_0Qr^`l{@M&Mk#qt;0U(UnLcBQk-Qm!1q!e5^7<`{ zLqMG8glaMHr98k`_Gwy0Cp2MK^znEdiL1x;^jOKBEZM4Z>&X##ad!Ul)ytFP=cch+ z46?sZ%Ek|F%|e?sC~{Gm?xP3yUmPEw9KZO(mrZA21Xx*MIW!fAQp@4Bs@rK}G5BYfpynU1nKp9hsNiHgS#ULcJ;5 z=9l-j_wHW5z4KRp`(OUUU;eE(TfP6@`)-B3#Ozfa!1?*5TN>`($6v3^Nda?nDux?j zbiH-YBmg7m3hXE$P0lAhBzopg_@(~tDDb0RQYOZwm9atvg0k5gw<&u{(c2qKYYO1x zW=zww`s-*tkF{K0+b$-gvN921=u|iDK@MUO&Rp7-9_!8AqU(vJLeeb>v>sx+uH<_A z2E`4IG7}tm+#YZ7oSa?$*Z=pI|L_mL{`e<9{*#~o$&)8P=#J#0 z2m1#{hP=Intr6eps~69XUp~d#*;!ssVlR(hFs1k2e}C_=FcQEC4`KHIIT0xY|^uUN^!E;rcyWpCab!ii&Bd7_<%RP%NY8Dw;{&)%h$Q>rv z$R~=-qV$ST%zG&ZLV{ZWnY<>oqOE#!=d$s76nns9>z}a^W*+`KA%&x(lm7}Y^GXM$Bd+cc9fa@M)XQCJ*tj3`)NRyh>7!ej(rIK^iA zc{jTj9$_H-197txsu?BO314tV+2V(u(q#n4u~w*i;}y5Bw|gv>OLtAlK$fKDR;n4I zM|;>xSlbbtIGw6CO9lf zY!)v&a~eAU3uJJ6E~C!UrbZ^-ILL}5LYukgLP)d%k&XDouORGAUt71%+%zVfz(LBut2P!KZpF&d<_I+1N)u~^D1k)#}vPdGpttI;nE0UC*cWsAk0^}jW zvP`;sUDsQ$hN zQitaiKRvb8x!14nvh85G_Qv#ydRP}!f!w%7wi30Gz4;`F5Ju&cW46AJ4c_gR>e1kus50*&a>)CUHYTmf% zfDBdXd{?WioCPFCq~>lpBO{Jy-#Cc&QEM~>a?w?3L|6K~P!=~63X~4!euq6&OqaSi zt6|O3wvyp$f$liwQ$YH69ATr{d}3jZt&A=jlTwEP=`3D`adEnSYs^7DYktd8i95Gk z0xG$bcr^&mlnSxqb|Y~h#y2)TK5+d{VPuqvVWTijSETqaQ049S<=ITw95;iY8X^!O#I?z*@5 z_H8RLU%WgPxJgc4Xfe7$gu(UctCwGY@w{C?Co@~rR?=W@x<{aHdcX{(3 zK0IC2qdd%5SloO4|NZhG{_-#X`pM(RfAK&4&p-X>qx+U*-Fx8SJ+cDnzJAndu5$ym zXHQRFy?p-s`R~8{^5po`gc|iddGh%7(a!1fx4X`iF#JXydpma&-N4U*|9i;li=SeAuTp*}8Z`C!PCSK}@>N$|G zp$Rzc15iQR*&UQxCx9h9v5c;&Rh40ZR(E6?+n%x6!`Qt)XBv|fD;8(T5!K}obI^|(A+HAc| z7qhFVLl_IqUzRm4vQug_j%b_(h6FRb`K*S}y6o12t*W{aUTuhK_!UMvi0EW9!2HJ5 zs*wC@TZcMw{9c;Mf;m$tjWS`ZS5&rSP7IhBI=aMM4-=JB*sgQI&EBJ;b8B6h3nmC| z3{b{7-uRAWX8_lkPk|qb_P^L|608BS&^?UL<{m<%U4Z)k5iNO8_MYRg= z6{s|SQ>J#s)g0+We|rl{`;mh%6fKKnda_2d}KQO?vZCk#*MLl z6X@grnz3`)n~|+xD-|?h435wfy0ae+oCt-Pu19+_tvvu5kIMfBbXv zIDhpwfBo%u-+l4L*B;^UFaP+$V`>hIe{Nc6%z^X-lv3)jyIgTi3yyAIzJB}V@4x)} zU;XCq|LL=f%WErJj-Nkwz2eq^*B8uO%+@Wny;NY?W6Gq$CN{du*$oBTyZgcbBd8l+ z-=z`)8dDpMC;X&L&UH&>%Y@+xH0l&_8iUn&3MWOM?LBus-0=uh`Sq#ZQr4`MNvh-H z=blw})}u~u8rSUhz=bg&j*+$K!cI*(U;gqR{==t`gi~CfBToe_|v1iefB&isOOX7qZ_2% z)O~j9rtMeWxa=8FO|BWBSjcsC`QpXPxn+=IWrMd+jLk=npBUUATUdB;EdM`UqEVye zrM-~&PLKUzFwNs-oaCH47KvwYuXMs!;wl`(ToYpgSe4T>_hdh8%>5p{qhYS16z_?s zzww+R&m8t^kG?7=j&@hYaHmz+^Ry7Gh`Np@qBIp8iKFoR)&eBQv$SJ$I%{?6jS7n9 z-e<{9dKV>egaN7ez%NtFj|Xiu6fAU{8C|DCUb7LOjV{Z z9p$7s1r8dR4A#B_xKm1b!w}rrX^Kd{{z3pcBA^{7#l2V=s-r2T z=3&Z)Fvrlo!!)UT4YIS4CF+4nl*;C!jRq+;Jxza?AQ-G@7S?9LH`pR_D0X`N;R`7< z0&1svEyJjZip4a!2FFQBz&YG(M= zeh7#kpE{>O!$#0amoO`s{vQR{@R*7+J$9JX`^{-5thj9Z8%Z zV@QtGgjx<-xQ!d%8DSRx*lZ>^NkFo*DmuD`%q&2Qq|v{wD#@kse47c5XVnQ#1$7o- zl(Z~jO4=QM{OXV<(3X6|mEjJ^8aANQ|9Rk?Q(e=*s64f&14-7JiU@w9i6b9v!?gYv zO3KPKA~K_}je~TW39eSl<02p*Dyf>^$XrcBqZNo*?l_8wU8pR0jdxU)pYx#}q=T}V zd~S%t_mw0z_aN3O@-!KV*&88Gm$Ya4->0QGowSRqhz1Ndr8;rFWUN)Fc z6LR{`irOW`emgOrs4bMEZqAObJ6X6(V$(F1rf|Us+D88QX9kGTb~FgyR3nN^6hEVZ z1XM}fF$TJ@vlJR{V*tufFE=u9Vh;n6WC^?kLq}@1iI9VjVoQ#V-s#Qzj}}9ynJevt zEYGryoI+?qhzBs$+;G~j3d;MexKkNbWMEDujn2h=?vhQZNiZ8@2d=3>aGU@s=2>lo z1F@txD~E$Jc(YQTA7SZx<9Y?d#!3hz7Ola_(V`0ENLgg##h!E7xk^?ht36X(HE9gM z?+OJ8Fp-CLBsnBid0+%PHk|HXvDz^i{5XkvhoPRrf~jKr-?v#TRXussX4*z;&;cGn zPJ>t~bE@LduSUXJaxLsQ=}dIELd#O*_k3+@Yq$4Swsk|L!GOq@4NQk8q}i#V&(NE7 zBcwPCrlSBxk55E}GOT^`(Eg#0rc8RBh(7;|pjxHT?`3J9*!rIysYkdijntQGUO!QX-|%fXMOa(c_Ol{`ip_I;~V~VNJ6N1sX&AW^_%LNR3O=WNe-S zBw9??C3&qTn;-b$DsqrRxe0~@5li`Z4i64bPxiKsCHfZV@1i2ow%%GM^v$>5e)sI@ z&C8c>x)QMe^2IaF0C_B=Ns-;CdU&@>1jV@RJOBF6TyyxVzy9m*o@(~|^z{6rr!PMK z$qyggzkm4V?f&a(^+KhD=v`vTnkXchCVf8r?e9MM^t0#BUrE}qx&8KL|Mr_l_wLD# zo$VT`D}PtsDsT%X3MqwS%a<1lw|1-@Iy3-q-L@$Z2q<9kX`A zTx*A;ozW5;WwkP*uq1BR^{Sgim(%*FGskPUqw{RxLLt5e9EW1F>X{Uan}hca{AyZ4JP`bzPJQ)o=(%=NB7Q13IHB zlY>d!s_8HNk1RtwF=3oC(<>2((|#kzkAny#5g-yvVD-03Ruq(*bhCK@v($zP8jR2E zi3-ZgC2bHqe@k8Q0Y+ew0&nq~SVv4@RC#l9^fKtHUghDVN_J`=QA=ZysIcO(fb@_l zYc)z+$XH#CwW6dP$2@_euBp_FEvk&7G0)ltpp}r}&kV{KBDhxJUp>{ZG$ytRI}ust zl7dje-Cw1!*H5^Z6B*VJyA~VWiK*-)GP1m~L05k?5qlJ%K==`LMhJ>Z4eN)(r4WrZ ztQtIFA`cDR&Q+%Y2S9ZSb9Ew`Mif;mEK#1c&+-vg*ZlFI(gqwG_>B&VUR+u{%tjN( zAZIUZ#9;;9q=qiFp$i#+k02VP(gq1W9$nlAdhzCm>EF=mIRjOt#IQw>I(hC5X zKxe;HOqJlz*2$q!#exS+Y!?Z}Ai_O}`O-W>dL^l8Pbsh{9Qq+0{f19@Ue->C6KJMS zoH3CnMIZ9Q-lhGj+Hmwjx1)IhR zJ={Vs)w2Id&0Xuv5@z zKzJfrjAg)6URqjv+WMYvV;Zz(n2`W#thIh6zRY38JsI1hEk4hWf7ugxu5{MHW z_y>D#4Pq7$jzH%(*_Z4S&1I-OTn zW#Jh+3asPVU#%xwoH`>2Bf{xpBN9jQ{xn`1Wt$teQ-(EGp-$J?M%)2TP*e>e3W6Wm zBZozUq9RQP>+DE`FdCT*tu(v|D4XV^g_I6btp-F4q=Dp23>=LvLnq=%J$2o_@C4Ie z=W+}`9Fu|vcGEN|;c)-({rBEGIyf|Nkh$WpT)PlIy&Vv^e!F9+U^3i_N{jM~THD?f zW&w|t`j(&Uc>0VPZ^l+kA}O@C)ZC)L+g6Xb4Zz!@tXi?)T~pMXf&qz*c!mt0^vD~J zm%M-f9_u(cJ^t$Vzs+ENaBE-kY#!RnY3~bqW_kAx4ju0N=l}e7$0zPDJ^SR>pMCko z*H4~!Ta|}VuJuH_h|?ih?dk92)!B0|EO`F%`1qI=?(Q7iyX$-J(S3_tj!bof*0LX= z(7g55t$px!_sKFDBJ1Kzf3|z*#ksQoF3O|O$x%jJj;1L=iKMLvn|qFa^49tmQvw#- zCy@+AlwKKw75F&XKQN7c_wK#((_g?i1rdXJDdj<7b z)dcF;vPbcm+e-D2GSMC_YEgNCUj!?6Y~e^)^#EopbQgQOEa&9>>iF9ipZ)GD(mT9+ z?=Sx1fBDyc`rkcy^mu>&@WI1Jp87xbVk!U0i>F^dfBFY^YI^}294FrVaB@P>2M60W z*nQU3fUA2)o)@%hpvWd3dUf&bTh`gWdUZOf!e;v&J+F%v4NjZ0%xZ^Ys6ClPP!uIZCKffpk zT3{!3EEN}U@}A=NLRAQlpdi|4sLj9h^nm<5QVFLEF01P7HB)Ff{MCS z+hSMX*qk)9B}AC_rD!ZDb>{>vHnxRYI}Tca3yn*N+Nu{SgfhPg>RDSNQJ~rc;X34u zq59#=j-yhYzC{g|Wxt*A07dF58UDkaR)PwQUsgG?meJdOxdQNf%jO4qdiPzEvU33%cN4OUl4ZBwv+C_-Z$#nhh6TQ8IT z;WI-GOjp3%3otf>ZeL`U3ejNlPESYEf8oKaC%P{rXsuVZT)S!oaPMjifBd3iXhN5QB-vr&*Ej+Ru5vVmdZ?*|?})xebzAW^W(wsP zuOg?`Ult1}VuHwTx`dK2Tc04tj=YyAk2V80`cr9sLqLRR3v;-0omAcKQ~{qL9IsK- zG=Q(}J2M%rB@~uWvWyWf+vvD#lEieXf}FzANwsi4LJHrVHE2ru$v&s*b?ZH~S@!R8 zuttFR$-^?Hkw22MIzQ3P6wouG%Di#W#R)-IH}4@HF6&;&{w)CTMrQ%sG0|k>_yV-S zMlX+hotAgEFI`~Wn`HnuyKg+G!sGVf7irN~8y!>wLOH?3?#=Grh?s<3aTt(jN+}~$ zR3@W%9%;v7;vU%|G?7PdCFotTkp=SBYGr25RjKoel6TQ)nx#RNAYpBN$sDejDjF@L zYQJ_XcqvV4fYg+&f*;k%GQJXpEYMg>7w5$bbeUZtgR?bGGacGZz&)_Sb+cZrKH)aE zTOlKgW124&pd#Og-hF8|+RzZjkqVQpj@J2b^%v)XQQn_|5p29ptI|8W8Ni4n44ada z(axd9V`gy}q&_GFFlKYk!K;$-e~|)|Rszk=cvUkaLp}W26#A4W>CtEm+f#zNA!LYC z8owUXn19!h=mov7IRQlYhYVs`!E>02FgvK|#VIk=NkzjP7B;P>pwr2w^9TVtfod?eY+zdi<%L*k`VE&PbrACAt)luJ|m~&G}3U`#}tMv3_Wd+PLnh z|LB4)IrgNzE2t-EEGBuh2em3u5H>00n#0HD-EOU12=K1Yg78SZvlczQYSaGHa>O0=_Wj05sU?dL7Nbu?HuGVsQ z%dfb)OQCD1j9tKu7kHfq-5WS(Nw-h=a4XlXB)PB_$l8`(_bZxtkUX-C41^|*c64~? zR^i`&_K86C-Un}Y-dxL*t?q|td*_~Y{{Q`-|M|mv_dfZ}Cr_V#=ea%4U%dF@%Wr=F zFZcf;~b$NCvNZ$m^mAOGn;|Chh}&2JdC*8@E1onQM!M2~TQJUe>()A0xIU^u$4wd@e2F%1H=6jO>@)HJn@Yo!w~0ja)f?$;QYz_@-+!=xnD zTR9ymgOo=&OA9eGKl`2^Bl(e=tN0n+#CFIhp=-B(Ippal9M0qazGAK$Bl0q&6{q*pWL+aX~h51*jN*eYwctx;Xqui|1Rof z5jBEGYx!#u%LL~vwG_qp6xdM^i0J?rd(-;Y^? z&SD?A%-z!6#CLn>dC+D&lXD_0aPu>(D&hJ|jDYiueqpP;&eO?uZO;LspCsw&f$ zA4^uthY>e~%`7OqgJ%3)&X8^h5bBFP8!vB>jA8yY@XzB5wsj1O;65>~++Ymgq9sU> zk4zWNK_E(K8poxdL-!D@FfB9FDTT0+fLksxsFx{!lA;O4liY;EI2};#vPSoCBWh7-99mE z+^YJ^R3xUH`4NuKuXgB1I5&8H`swWI;_~F={NbTJRQIjS6v@+|fgUmb(sL7@;E?^x z?!o>){PuVM+wcD4@BY_+)a_BF9PHn>l1ddbE_8iX(D54>umVaE*G^qnsFgqJdCqu$1V#^34DxS*vZ~WW6p#;Lp>Si#xplz zWVEJurJUDQKS!y6q+7~eJI=dN0&FX{Y%``hsutu$h-kqm#6m?CcWv$BY5omqkze8E z#>+EFCcH|{txgbda`jitsUH|4{tT69Ynlug>|0t;w>&7Sqf~6tsdBw8Q9a1o11VnOfvcfshT9T{@}(_QXWhhL;jTR`{zzky3S#!U!nrNv+5E4}M0qi(z<&kj2NZ@J$aD9W(rxrSw)5r_WB#q9_<&?AK;QEzrM37yn(rP}!( z&SaJ}1>llLs)R10tP6x>=}8}*sEp|NYtNmv0@N;<(5$I&&et2eG?%H#rakG4yf$Gf z=@T2{Yf{hn19o@}=Wg8`*hbdv)9f=7?oF`6&BYLe`4|Hrk{3s0mjL|Eu7p~HT->>3 zybbxAdH-~7!Ow*{sX^i<4E5Io4Or@(!ILcbxE-^W*w4^JVO`dX(G9Hr>M4cfQMGH5N;Q; zk?};j zf~GgdI5bZ|eC1q~T9z?X)j6cgMIo4|jbT8e3|y|qaXhm|d>IDJX|15^;gn2#IV&1R z1OutqGlTQ&?70q1D9$`DQPrcs{dPGv?MW^h+xQw>3}C z&(4^&VB|8H*{1W-I`falmbGaS?!j04+ZaIj`NzMVo}Qe1Jbn87tG$E6o!tXVQ#2y$ ziFt?a%a<>nKK1glqj&G#ynT1{{=Ho)bC?{QZt!5i%sr1O+iCRS;e#iSpLk<|S7KSn zZik&eet73uR@)P)M0@Ml;eC%lAiyFgeFIBW&Q3pwmU{;ecke$kks;z2S%`2ccSN1A z`=r*5_-iEzq;*4z;|=3hMyXQLLHeN}qN=B%_13chmQwA4;OS&NnGeTD9?boC@#*yP zGuc!39qn%QGFxKXw{P!k-9Fgfx_@|Z@A(shO0a*tIREhR#QvzB)Ry)5aroRz z7)G8Nz}w$Dc=7eqmoHy_`|8_Y{p!~*o<8?xV2Ak?O4s#bI;w;>Z+`me$G^P!`TN&D z|M2njSbq->Rx3Swv7;Vg_Fe|eRJ`O3HZK&j{lChIIa&vH{r%tQ>S3?%6!q!@#K$1UqpoRysx!h~9{$P(So0qbVZ)$R=UtI!ke~PjTF4 z99f8qM7^#XvCq#C%ZmEu#3i|KB&;OWIgCpp^2S_Q+$pat>(&UZH+ZGmj3jK~3-Nq{ zH^9n3tA)g%xX6$}fB|ThH1ZoG8ZcQ? zeE>Ywe)FG-GHoS$O_l=n0vbrrN$!AFuelU4JK4BV)eVe)14nG4nEE;m-QM&qcdmQf3?>LAT4Y}!h4X4e}& zk1d!WMO%u69hnkqCAQ`TB1=UD(-zoR|D*Lk9e|Npv5%itK5s@b#oVwRF&h77T|ceLAdNXB-w{rFDrKGp|9sV0_1#P3Y1>y8a$fAHDlWY_ z$gbb2%=(>lxcUGM{#94@2*EZ(PZH!|+`(m)qyzAby-TH%^Q8HeM>wa2q!2SWER!!u zBT)AWla(Grs90qkX~-!*pC2R3Vo+Y`pp0otD>D7@bW$Yw6V>Km4yuUsp_f=s8r@M0wACl5WnK~CKMnJ3F%DMD5G>wO5 z1MB&b33B=VEPEWQz!Jiu2V^jZn!0(G95;7$;WUq;AWS!+pOq>b3`w)M=767aU^dyjB+DC`?`Lnj5 zh23>bgFj>8T*pK1pEUg~={|k?{=KI>_iOU)Dty%zSUbEM`K|7-q{cXNgAJXCP>EpNGLqCGMpoUWxwO!TZhG@^a|_`oy*%OMosoZP5x`p>15q^# zdAgw8Y6nO^e>&XRwh^ioO-7Xj1%g*~*L9lAmv!7)l;eMI|In+}wDq5Lym<2D>4S$4 zHR4q!j~?8obe#r`Os%Ag(+@xX^-q8J{r`CX;jQO3Ji9?U_a8j)urIcNBIF-__|Zgy zb_b#Ih-fj<+rpmPFTei!$@5#^|NPTWKfU>Q_R*U*wMGcX>VJF1?y5WtC~a-u-ti8A z>pN$yI=$4hwViaWrHotz%tAdl(Sr^IcKbSGEQXF4!c}PDJHky%h0rc*+p3Q~5)hFQ zy1$7xT@YO-S8gU!R|`*fWSi36D@WPEs{<=1VyJH+rMYP-`sMDw^RDuRZ+hjwe2?;O zsEuY&X`2ZwI|}v$p$e~?)w&jwHePI?zZwpwx9(}x0G4AIT4YS^?y>3(CN+qI*}r)M zDHWX1)XEY$HH$Les^lyXkND_EJ=Xqu-hj69;kx^z%%NvYO3hto0QDk#gAT*#)TM=( zrrK%_wL3zAu_3c20i=f}i_P}sOkyZ>M1spTlr>bBnhG&FQYtZT%R5GGn~Q9uvIs8c z6dO&()7(jr0Uv1cK{`iz%XCoeuDqM_uE>fEQ^BAL5SXK?5R?dHLamw{rKJ`r3)axl zNtVd(qFYfh0F)NmYN2{5D`Q@5rUJqsA@BNIgzongpD3e=nn6m{7@(MOh@^>zTK_)N z`ZrcnPe5xWh_mKvIy^Pu!H`0+`r$w<@1aZ;`IQ-fc9v~dmx}sJB)aA~6xU@lgpa22 zf|vr#-RuQ1NkEFi0jPZo$^bEF9nM~961yHWFwL5sC70~65jH;(^w`I=^w-l7$e3!j z9uBBkmbSA#W_^@siBMA23&6z)=dLHzW>&y$bg>XT5ZqF*s|Nh=y*3RcL z{x?P!TDq-AvY}wz$YVHDTAkV_(@RrPG_udSGeSRx1LSs$ozQlBMXL_&wrsS`ZgLmi z<nem}@1?nWap&q%HF@u1`|5J{ zCZPa;j0%i}p_cQTqW+?7Kq{o5Qc8hPWaEp01U)#FTpW*n;O^HLqsvr=C9-fsX*VT~ znsQPC85_(5<;ayCg}9|E-^ndm^PG`hCjb)K44+5%kPoiRIamb2Siw4b8DJ?%J(pt2 zI4xEZOU~Pt7STD7237-g(+5D&fSchXt%18X6xq4AyMJ%5*CttD6+%{9)8(QLt!Hm6 zpU%_^x*!fEsR(1SCA$DH6iv!Vhys&P?Zj)#ESg6eda#x-B7OoAs^~U5OB3Xk7!tX< z6kPF9m@L_@6W$Os2wk$lqW>DL4_eL=NExa_s+$lq))=_?OI%ATOP)kxLH0-^az@5e zB045L;x;;R7v*6fpFtQZSN#~Mi8y}aGm0l2{ODSiRPq>4z~*)&EgKpVrzAMZU{N;k zMq+uqCL`_7RA(Js2pcw8{r@~zLWxTl%Ln^CVB;B40gSdK{RA>9Shf_U-CrV$k&Y=} zhS*Frd1Lh#I^!>e*G?Fq^9urysIE}KTx&A1kQ9Bzxw;X|Lu#fzp*wE2Z$)HdVlf$zTN1j@1ZAcb3|S1R;swGi|qpc zA|0g_&8<;w3?T8w{$u>75bUR&I^#A0=%l$`Bb`BqDUW?jeKQ*2bT$*u8ZBDd7 zp1dF(K~u%mko&bwd8STGrmQK2=n~vgGg$UR+LiQJK{NZ89>0dvE;T=*;yU3u(@# z1>8t-22xWD==2(D>$SEY9_~MT>}d#FZSD{SnYu8a;X0dX#evOszL*y9GKEKvpI8=T z<o_<525D%Psgu+`uQiD ze&$@a@7%wC`1H{e9RssBn*PMQWB=H*)4zTH{aa7|dMY3*v{tD7tra0*hc?cpbu#1J zs=~WhOw|iiG79Rb#dxRHF2q@ME;hBTv>~wUIS&oHd()K2q63`JQ*R1Q%LBjXRvzG2s-nId$Bc%t=E=5h*TNW2>1Lkj%gC7aOJSI9+VRWWJOyAEgc(S zwV8{|(GN(@tz^_FR^$`d`{Ta(J4@sGRHfFfI&Vy~hI6!mTlggNrniiL&eUkF(N_K- zD|HAzbqYE3V%r2p7}1z8ay`xgtBUyS!F~wcN z)y1Du>PAbV{BW?GB>9g+D6O_oM1C#&8tyaje|CB1A-=Y5?#VX;Vd8GJ3t@sdh|)Bc z12q@Y!Ia`Ic))VPWgm z?*8`Pd@MxXHT|ZINocnBgT0MezTCbL_kmP}r+;z~g}(Gq-`HWqL?kRxI50=3vA!s~wjk=Ox0iwU4yeln?%djNtmG|D{J^5a9!P()7P zv!KXNZv7_FA_4stvs@)4`1&2X`AV-qPFeTTIB**Mg`x&YxGzJq&ZYHkT$s)#L}l}1 zfK#}>WCk8PNR5w>L^X1TM(1HyjxJ(hcpF|8T0{*8l2(r7TG&^%X7jMiqh_K_P7=ba z^I#&m2A=DhBt&Vr$gbx)u?C$avSACP^hUjs$@;>3+uC+-kKVc8vNS98M|76rynp}U z2Rm*aeV7&Gp6j}O|IvNJ#yyfd!ysCaz2VD)WwjjceNiV_9dgfup?8knULBvjSETKN zj5pIuqAGA{mpm(@a?z1e)ja4)#X;vP4hbwKvd7jE*ETajU@V4mdhpup!3eV_W$wx zvWHps>WFK*da-S=>6~547&p%QhD>M+?1Gna#;YLhp@185-CL zB_F-{ske7sc;5HN-~W&Q_QyZ`UO;;K7n7$Yzux7E3aMwD~b$!7K?HpqSII4A|_|_$jm?KIe=L;4X`1;>F3&Qz%-zhMhS*~s+n2WpfQzM_~8Tx5jv=!pcsiy1Lo z$)bUfpv6Cv092_J(xDl-2C%1i*@h zzAw2_;{0M5NnDSZ$!y3`yfHSan`FOoKEsnW1ikafTWhtqhxC#48YgE|vE`HJyVy9u zFf)SZM%AiX#yw1|!V)idcqWU%4$j@sPVsxpX=<^WFmo74X?-ulM7phvrfMTIw-Fve zsd8vf{QbK@%*#4_VH&T28<3+ar;_>_on?bvN9YKFh>Zjlb<)B-7+O57p7J-G0gxUI z@(fOboev^68qeG~@yMcif{lx{MoR}kyRV)%i*YfuIU8PU!KIO5o?I`1qah0nlFcY1 zvZVQ2BGQx`y%UDWZF1SiqhJR9&Fj?xpnYgGDQI)UJ<{WMBoOOK0^|Yj)>x8Ot!`v> zI5V&`=+m|*eszFmERNnPRYXB&HZ|h}dR>0i50Jzz)g+TIbTN@*RgTg};MsBb=bz0* zf3c>mMATAijT%Zlr(};d}W^ba`MhxDh+>LfR;EIX^t@6 znI&JL&5Nc84e_H||26&-FNOluj1+jfh}G5rbuML0Lz+c6#Z(NOUbtBVAK1>7#c zu1m>RM)jNap<**xHd2q6#8XNArBDH!aMDW-sU~s(K+IPI2Qys@i5-4LMCV-_^GLNHkX4X%>i=rRXM$)uM?qHMA0-l9VC<|?XJ zp2`i@VXGnBET^7(({~zgW?S=clUiLgqD*fn%n-vIT3^tnIndB6eNIbU$tw2C${Kwu z7~w8cjl@j3gt<{gC1X-s=qO3`z$i_IT?|IdH--FJ^3n!0%K?8VoX z`ApO0i#|3cXvZ-2jFsN8VwzEQ_ygss8T7#>BiUPJH}&s^qXch9eHZ9gzCa&X_` zpL@Fp8qD*y^9;pbpVZKq@iLmFc#`eH-K0J-Kgk#Uv#e^fJhplS7TIj%v%8P?v(syj z_ zX~{4(vUyITQW<%VBf2X}H_HGV$BpEUDBpagLch~v{|;zltgE@_dVY>syB@Z4M!N14 ztOeGsfPha!Q-3ul0C|Sdo$`)Vbh%7K!r*g~GDxR$QN22tLgI&Z*=#8at_>Nz@2g^> zK`&CCLrn4QwlfkZ1kr9S!2q@%)$)n%ob-%N^60lX9j?QG*H|} zRj1i76<=gyN^$%Zuo^I%2&pO5lX4uY4(2rMkOu@1=nqsFY7ge8LMbH+WbPM%E7i@euTh>< zuoj&lL=@^YA<0lSUo{HW43BoT)Qq50pyaCnG9XXC_Tck~o^6|$&ZI4#M_M95`UyP{ zn_}2^RQV}TyqKy4O05UZkeK4Re~CEgcTko@=AO)8BYo`OeFV1I9x~}lr5|wafuN${Xz{Ja3{lv8TDIXMkCC%TMjpP z_pmYfa3Os_EgmCj5DZZ$%;GbYo0UoFz0`;x7-MWhjY4^BEcwIsgvk{su~EXPi-a=N zYW9e9;qA+e#eE-+)zqh0VG{*Ym@uO!?9zQtTf8v>0i{_0x z%`-_5=a3^&7@$GVo}|0?j1z}ih4(=4H!{7bK|J_@({^ei)~{;0;YU$)dfugc(|?^M5f6PS%Do=9o{QkTc056 z%%SMwgIDk(tmiZEan;KjuReS@`nP}k!^e{ks3Z$3MDHK$KRDPUmXqTXQ1tB@-e~eq zkL;TEpFVlYz&rtB-sa@wR;+{^)^hZ;vs>_(MlzvkXhRZz(XhnjLs$l(=*fJ78@ zKeINNNW%I_*L^Y=7oe2~;=m3H*SREXo^aq!lr2&U{5iYCTz-~OC!>a63t1B;3)lo< z5U(SSi6$Fc5sQ(H2wH&7Hv-=!pY877y4D2>0V(1y<>fdF7(?`8t_-v`@75_$gp-F)J&0%dVi zHW`vpAOBr+J3+5f-5(f(tz-nHNh}0oih=?YA_UZwsqAE$vcZjxGKHk@J|#m{DW^qf z;dpbn8U)cO*?DVLx@nwNIwjF#BCLlJ85>eSs7Ao&*nHrs?zO*r5_H=AwU;&i8++!b zE&maSMEPVVQNaIQX?S%lrte~D5_K}5BMoL%BxgV zplalwea~kW0rdFw?8@J!H&YQtY`9uJuxLi)Mtb8Beh9-U<*u2RTPD7HI0Cagj04Vz zRBK~K3n8RQKf0A8dJ^c~Cy&q#87ZUL*f`4%p%~*%y6}YA$4I3;MJ?K~C4+%~&u0ti z6@hovO>z}s*u^+qg22T{V+amZKQo9f61rflK8cx6`i@&x1$ns6$aki?9BX-M=9fkS zIzs0y0b+L&GQN~DN6@oPRxOFx zi3E~65|Qbc0a8>CF2><8H}o&%4!^Rqc`%YomKRuMA-ypGvUQ1Q+*T$2~fKNZOn&7)d!dk9@?ir5Jkn6$nHA2o#x|f03_i7BN&1u=Y$bP z5red(Qt2`_oY+WTMR-zp+cJ%_}|0VpdVnVx2>nB45BK>Q$bV(1CcaFB9krlR=0;xfJH+F=073Z z!1KYwN1At@l$FtF`1$8^o9g`K`@jD9^UpS#1jsP_<3~2qdFb;9gnihxe~8K3*N{Bip0_d8zoCApde`6H4pJKV8}T%#xN1F~r`5rdK@~b#`IXGYu?H zL_B!*^{a=^UYf0=^~5+L%M}u1O2O#pJgKg7Tqq_w>(yQjZ(17o~G^bl|~Wv_;>kama995hK-I zK#C}B-IPT@dv#?M7FB#Y{_su~jds%3*Yb=t@FJN$mzIMei7#LFR57O?-V%hddS$^d z<)VF0cxG?}JxvZe^}w-O@6*+u6a)E?dZw73Kb|s}6I(I;@Y7E!MRXv;J0i^HP*Lrc zYMh=%z>9gLLXiqQf9Y-aF{n3GN2U(6?is#W1uMZmaVJNzE?gC4CJuGcgMgX2e?^xU zWE6QpLX${1lYu_UHu_3~d^QTGgY<&mtzyQC8w0Y;DPNKt(NHwuG}RFtO34|=0BxZQ z;hPZLqtr)P%(*=x z!?58slW_pgxM#tl5F~HDAfli(o+ikP=1>&5TFNoi8u_c@Bo!ov?V(GNAuKHo;_=Sn z^d3_UIyOdEx{6(!|yN>t8>SdHnzd4g`dQ|n(OEZEisX!x)h z*#5`qm1J>m!v5NLEy zgBMLGWJl>O!T6c#mL8??C{rCQe#|hc!#Uw2HH#qi43t+4(uCuT;Y@xV8zUd0gCA69 zBeC{pU+%nF+Gdd3QXj+rdMxfyfym}Ds>WbCYdWN?$^e1waP>EzI3a+EI86ocO+Ef5fS3pNxR>dm zKHeW*ETE>G0D$xY&%n(BM@2crN}=kx*&x1~3^tLofc_AQ3aHp!DuJ^#2}E9T|4|W* zHL=ox_R(0XbbjH~PzS7vDZ1e)fUiyCM;0BlC<-&TOtiP;s&IQvdItS3_u zYOx2%STiKmEk@&W;W+_2$`a5Z7{MVT!%)1C5prY%AQn0`@ttXgrJSL_4~h62l_$4E z<}P*19Q&$)mR7UyJ&OyBPPS^=TNFZ_b7Jd?vRYnN2ShimhN-Ht2s6awKA1|^Ffkmi zA`>}aJClw$LaZNLt=J3;MzIxv2^GXt!X0P?hxys)e9Df3lmSfYsDT)A;gnkbyw^=R zNLj%VyTITEE~3G@Jf}5;GJ>gaReVu}Jj5ZyRNWw`F>K@NMO4}9WM>!CyPQJi61kW;pNR%1U2eE zPKikJK0SG_`F?H0{p#YZO_{xa*FGd7zXy(+#NP7&QtuEuKeH2@7Pkh);qE?a5oO)~ z(b0!@o(Z|Tb@1Tuo3FpI@~fD&-A1lfjd{1S#v;Cua^CvTc~!jm3R?k1WiD`S|2eN>)2}E}@hl>1MUyus-Tq>GIxSK}6C0-X~Lm~(C}2w7ZEfF~8E`r%(@=K!UPuIGrZRiGj(eRE3`NWt{m zOlU?SW~EP%q2y6jZp)Lik0GM1dN5I>HPex=PJJ*q^UdPtM1~ATAj)UqF94Q+8aZ&L z>d(N2pedLA6fCo}K`SBb6X1&aSq;n#CfUIY$gDQ48X!-(@#00NRfc@+) zkm(>5gea22ndK(XV&e??qn6AiB5Fc+fia0E*<8pG)ARA=ujWjS)Qy1(-^#Lr(<a2R0S+llC4(;K(3S{XmvaQ1aJ5#(iXp2y zB<0z3pA4FZZsnhdP?umjy;B?EJXHnCQ3&oc3%+Nx64yTXiqrI0Yc&(n7Sm>quTezh z{l`K8eL#Z0l`So1IIVeA!C*L^qCD2BfP1jm``#~2Ln#yjs<#pe1xuss7D5=^&A5kz zoTbY3reo8p(MCiCfgXMkv`tLb0zr#PDj`;v<}2`y$dbMgP%Z3(+%%N^*6Ly)r6lS& z!IRq&-p`GcQg%ij57`8M=YBktEq8P^$MnI-p{bV6dI5skSb1VZNm&tCzX(;))6a!# z;Y1xmN|HHQ)V@(EMPv3WJqfyt4p#`$Hb6EiIk07_3_GX7VJ#*{Wnyj)-a9V8q|vR~ zEuNA~CPwD8h;D7yVH4!dgV$ZmB*PwnAP@A#z9%-2Ji?w8UV@*2ce&UL!DkW*}ATBhE?GI)~xP)jBUSl71E^o6?1okylb23AU(o;)vf#(`P&+ z0&|B&%GgTYLl%JoC>x6&G^T_hh96G>s1u~J6Hj8TLa2bGzln-N0G|3xbefR69DhP) zA-5RNR}?H64a!7Xm?B==cw{tvf-MPZ*^%EV7NRFDNU5$V+#+q-dr1-58_0^PuDivZ=L;#jWLh-lSUb6wwD3v;{4L4 zpUiYsNp-R~Hqu{m;+v`QX7r zbb3Ve`29PtyNVSYSRKCIN|lzt-MVX&nqU9sJA+z3{P@HB4@VY^pjhbh9I#gqP?)~+ zvllN+p@DYt{^;%7ce-j;yVyFBz`U&LQ~?V$%IGqW>DVBKI_ zYMS~-p>}qwna+9$7-L?zYmbIIchApG&d-nAc48txf|wYYM)vO*-!yyJ)FEAWRS4w zT#lr(`^=uKh_A0C8TcfWXea*p((y$6FOD!GURp@Xku-9bbLfk}JPW8Qz){d-2c=)Y zuPUU*A>r|#pd+PZlH~3M4~6RnkGEIplxLkfi&6#8sGfinGxKv$Ew&~tq_7OH41{n} zo}se1Lg>IR;7^@}9bmI7lqQr=c_DqR$Wj3lS1l-u-WbjAiJ_atp~e8lB#bP&uGdvU zqOADXwo|cr1e&ZN#`v7Nh~;h~<>s>ZRoqh2AIT21#qOMk!RUziK@+;UbbmvoD@=o3 z#yMOLw*hA}6&Svj|WD-`1P5AYr!lRggB{4}d`7DFNnH)pMVHBU4B$qF8GvDO@+>xG1Ry!1ylf;pfO!CX8s(;N&_7iH^BNIX z7lN0dCB^HssF4^fg{na`AS^4svJ0oCd2I@#7nB;P><1R5l^iI+5ExK^NV0LDck{p0 zDq}IW#&(U4?k0H{GaPPi0Z5|J;97Sz{!D_$RCu4`Lw_(@+G~y1-hxinqA;TfWG5 z?lL0%?ir0$EPQFE26Yv&gD7fhis>VWiS2or^uo>PWr?e9aV2uR$d75ivx(Ol}UlwKbs$=%{2hqy!G+88J6%v~_I? zSG8`Nj;rl@h9+g}17vebj#;=;dP2}2z4$wfa zr0mWc3yjR5lCN`g4{%w^){!pQnRLQsc(sfg>S2P0l5TsE!rdmpECkZ}N(2&sY0 z#%JJ2zvB^$jV5ag6=M;#2^A#2Is+As**W@}27Z=OY6h!h!a;JK3Y=mgm5cZ_2BU4$ zB;+3m&M4d(4HY_N(N}NH-q^XWyOiMNB8JRbS)|>h!sfaBHH{~S5jlW}NKD9&2|wUq z?;vD-x&0h{$yaIyk7hqpq>?O&0mhHyOe%nwNCU-Cg%U(857!kQXj23^X?se87d2($ z_im@iW; zhzAG87ae1F6NWMg;Gn6L@cIvfLyB=R@%PPRcbdbCmz%!nfe&?!r=(Xorbq(zMY0!5@g|M#iiXo4bd!c3=|VvUL9sP)XRHbNHdms>!PkwgBK!+%|!?{ zD`!5IPEgyj+@zfv#&-Bb;-n?I3TS>Ka|~BDRQagfBZFn^ftb_6$PGu*cI~HWYy((u^-sz3CH%r=cy! z^a}hej1)p;j|(8k4QC4W>A%Q1qLA7xXPg(k7fExw;zP*>WF$J0GmDPhJ|UYMC5<}f zccBrt3q%;-f}rM7RUd2ONrPG=ApJ;FDJG#I!7aYDNEks5 z5sN;8$`WL#-bfPF&k=H`O#eCqD?J&=avv! zeK)}AhwCWW!Y&}Fh(0wOY7CY#`)DX!e5fVSi=Tvx_SoWw?bd>=tQxW?H_Vcf`pX1- z;;j@wvT$FeCPUX%%2QdJsVvk>%SROI`JkReuaX-{MxCLPV96u&G?~OV5|71;t$0s> z6v$hlM@j2eYf^WbS`}vc#-s;bwg2&?AJJ?-*q)xMh?*-I4&g&%bVn8FA@FPFZg4@L zu`x={*3x=bKUxTE$fU0Wq2TOpdT3ekI!KdI#QMmO~c?KtwITRFh`B6 z=&8Np>EV{I*sHIqQ1{AvAw0UBU{g>M1dqM~S+RKO*Mb(}Fd`%z5>@#MMpgfQ-iWy? zNjaxTK8FN3#`5w0Oa6EX*ZP;UqEIH80SM7ZY1aME@PDSVT_3>2x!z+lMpH>tcsOZ{ z3s7QEn4+h3JXFUgjn|C}cq*!9%3c?{*-o)9B{)s3p3`e*83>e2)zgSHB~z7N;Se=_)+?C}m)Qu2&&~!-<2X&sZLpF`S9K^{pt;k? zDvd;*Ng`?KL*pgTYszy!pPOM`^%o;xp{^nT? z1~=^Ksy`ni73p_q9A+c18gCG(WizCSud>hxVjAL)s6;aY@{>aPp%^e`HKQbaI zHk+}@F%poYToRF79Ln5~(#NgR!AeCNvXl%<)kR||SyacMQ-B5X7fa-|Q?+?_c5U!) zp2`hNM>HaR8cn>tXNJ#S_1*+hO>9vhY!b(o_zKA>2sCp_N>t>G9=HK$WYQT3BwJr6CHOm z!zM#wDf8wa;@bWYZxz+EyEG>+fmcV1Fq+Cl5SL^F7%%OUOh8@hi6fPJgh~8uHCyS{5^Zx35zi7iguirACFXv-9&0AKsJOvuDq(0YeyeOaiq3 z)6Vwiy+f~W)(QOi{cAgcW@)#sbuPxbQ1xe1h9seOtX-OD$FFBY;zd~D=p&vJ7C zUTo#n=(azybyKrUGx|5Oo}4M7rSOgs0m%bSMWSp$!@jXLf67_W4zJVAk$-2TDKPV& zf7eTsD|yZ(F?Ya3E`tF; za=;hB7n|PDSaaD72WYo=& zL}4&T24lMYj@rhd1PL*Mf#8TWPu`nyNt8g)l~{=aHzQ|uPMy_D>rR9UUa6p{w=jh| zrqK~8g(F|%I=S+&9D&_y5U-0(?$LF*hy03Ea@{X&pn#C*65BWhGodRR z3`W+f$ar=6N*lOQ+7D@S#6oG63jd&RF0~|?P@$JmcMPQDwuBL1#biq+E8Y`76O79< zrJXi$B`Bl(E?iARdCn9Uo|FhZ*O09{opLWY&>I&JwL24ZZHp58l#RqQwYcER?1wci z%c}pP2#bK)jSNIxo8ONAPKiurd^dvB!GPgbty8!xgqD2w+BVbnIVPdhy*V<-EI;@V zxa(8Z>6j#F*Jo5V1Zx+7iD!KP$G8R3WQ54AgkS0iYYNK*hfn67eV0*Uw!JR>8aWNb zxksns{rHv27qn6daGE|A=ai-w)kX$*^j}wsgyqCC^={-I0r3=s{>MhkS)cQ~?gOQ0YP-YPy~oW%8;m@wCJ_!y|U-+~{b{*K_m^gDz z_#<0XX!iFY&Zc?jDveA_GAc&?T0?1JerBZ9UA=D@)hm$BFIB@DJ-xE6y|J?k(ochn zKAxhQ-*`(3-DrWyU_x66?fG0>;Aspbc1EW`DJ@P%0|_L^`Qc1IxtriYjlNR->L`C1 zorj{$FO0LB&({1;6_f!BPqkZXVJ=&Wq5=R`utPOz00n5LrW8Ba@v?SvrN=~6U7m$R zh0JJZx$)%ItrvdicUv<{qxZn$_#lups>Ek_?K$mqn4?s5T+rXlR@o zqji6OR^;u@id0G2m|;9|qmh#=djL}irL?q|y45VHydnC|elv{l?8 zKma^Kx_5AZQE8S5dL^!9Bo_DV@9lW%_t3_*yEbx6S3PXHYZI&8*41R-!0Bk&!wO+{ zw(V}o^lR|hlH!xSMB&=qzkB=q&iVPNIk1DhLtWwQzu6JHsO>1D{yG16Vm-PxrFZlm zA0KOX+Q}YC`Xz`x`>Tgoj}Px3XiUC-{l;5=zkT&B6?wtHp{@Gw+~Gm@4-fbE4m~Qc z(|dottNeU-*HU>~WNs%)a_QZalI>&Tjpp7Lux0T1_}vF>?_M$Xt6zWp&9~3L{q_C( zk8kTg#MXLg%WwaW|MGvq`jcPYNdp6|X-SJ2P zD#7#kho^hZobX-7V5_pM?)0PO|E2{f9V?lnrhKn7*(VT|0-Yy}jR@)Q3(mW@+8zUs zg~H;XX|$THRdNDCj?4BmVQSQ3##9EF*{F3cTUXnkZ(k~<;WWz^Fk9WjUCx zhe@UgLMRQvj|U@H986IioG5=tkvZ0QSjlB7wsdwBbILOqDv`3su;rLsV*)`MaI`-y z1U>|-`nwRs4>yyS1TjC@gct#|AeOmgIRjzP4n9BUTT1kWH`@Ci0`hqAL?n?j?={D1 zC=^iCN8$We0)m#m4i2&paDf(Dk1NDo6cm_>^olQO%9uK`(R4Ga=?C=1GPwZitW3?u zTF~T=J)pDmIU?21o8TMi!}weTg&qdUfK82l4M|<}OZf;YTaAKA4eI7j@JBS2B2>ps z@%c_-t07j{jP?;>LcvuglSDzC3M!yw7&yTyA(W23gFfZwTRz}M7^9j5J=RoWRW1Ro zy&q(k25Jx#TXc4L-Olwx@np=Y)wA!Ivj$T3wtJRwbRQZh`*X{=Tk5*j73 z8!|FfI7DW>sMs?K7`{fzX>N!bdP`@^k(T#bRN#7Se|(^TJxZJ$<$^mWSTc^Cv7*H6 zA8vFkXh|iqrp%yEN|78BKEqrRsVIpw0wz4R&!^u-sOs*_PN!$gcEz^l!S4pABWZ$&z789oD0#nA>Icj`Hc0+fFX=<8E z-g?|a?Ov1IrsSkDIUCnmPv(*#f?_GMtBMG*yEBDVJ29#DPtv6#XMOyL*8)a;47-%~ zg0_p(^Y+Z{#h>T7t?E37kDHC)bC#OMs$77Bg}cO*mP`$2FsL_Cjj-fqcQ^f{&5`sArTxNvMYWi+XsS%JXNb{kQe1WO->Ox_buu@upv-Oc_emf); z(}pdIprG0@K18W6*TABeeY)fpxksjtn1$TA$&l$oplH$;T_iDgWS1~$t`bW_JG^B& zD47I_L>yKsKm5z#chqR7B~_ZW$VaoIVSjYh0dQ=t$7O!eu3^m7ItB+N76TR?@B+xu zNu}06^{|8+;UW7U+kP@zwyU~#_mSaoTh=~#?u1SWS%`3%<%J9-8^7ocI5gake-h&+(`>EZ$|KbgD{=-pfPhtWj3W21T! zo+UZ&Z4hMe-S$-jy}9Eo@>Y+Y9m{_^6>usmtTGDDOKvZf9UPS_dHts z@X-Ukg71I)0a~x8z5nGtUcJ$4+OJ!{Y0>t;u`NgI0a#O_i?Zj!X7yU9Uw!lKo$dVx z9x(p;Rqq(~yi^9W4uZ-YQ^x2aH*-}}B8%R{#z{%bbCVG#W{JLa6tj9@5e&(^` zB}D=?HL9iqAh4$r3fcYbZ~qZ%|MkEA-+%hk?+FLA8on(8D!?F#aC9-fr1{?EC(i)1 z+E*ATc71;Ou9ynpuNrH=9a|6elT-Q^ulTw!;iD_3 z?kjLHksEG{1FWTl5nvtu+^={cAQ_WtV%EO29u3dDJ;YdViK$jXv9att%E1M)VX>m| z%&sQEz4_!H{ihQ}SDAFOuAB>s?oUFMvrIYE%~^U1%&!MZfIt)g-3c^0l#053Xrc5F z0J0cAN+)%>*|<4UeDcE_Ff=gV*p&;o7Lf@9(lNAHb}>g*321?%HP=L$Sln6#Bfm1b zVF5@q3<+|SZ3z=iBcb6+ z{sdN`4j1tgo#hE`*F&ot-MP74_VNtBZuC1O5=S;z<8+j^p{!f5q>X`I5yhRSa3>s5 zP~GqR2By3=1!!b-)79cH9L2^&2ExMaJR($jfMxBAuOTWhluvqb3-Mxkh#5ix6&4lk zwNCotdwnT25(Fa%iJZ$Q zu>Q}n>7vNKGC%<%Nypr>zb@vApTt@iLNXML5Zq^?9gQ7`4m}Ut^ld|jOY=2bdeatT z9pZv8|GrXd1|liE4B2ljlrd~R)2$hKi2ck8MK|SFnP5SrT>7AK5S6PT9-eh51H5xd zon%JxrgJQ92^F9MeAVyB0n;%?2l5sShOYYONxgtla0%Slu3};+C2C#Kbxk|XLwI|w zU)wblQwUp$E|V?hMy`I~VZ@;1;F~t8P+l~*jA6?(5#1z_)C3{L1gn4soy)Y=30}7X zyuwbu=b`)rFuKiml1LgVtoj4GvPfQ#0u^KZR36VOWs;qS{`oNy=q5;-gvjYGrZ+^4 z#>#|l9qYJc67smH`2%;Kk;n~_VQw;{3ivCAF`2as)Jc>~(ppOfGcCyR2%D=E6jD!3 zn?j+v<#*QJl^&LKz*4KVvsm99&MLP4QuHD`;xx?pDI~e7k-GCst&@)*&pvuCQ3Iy| zL4+im6flLg{+s>SJo5l9a49IBtKGq@ou?s^mGvih;LdkCN!%%J{2tV#8XYG8!ZjZ{^k}OiEiN7-y~Hnk^0H z+$*0<31SUVd>C+`>4lD~qUvnqyLSGz3BG@Gj z{BGboualh0nq#53Is^4g>IKvfaBDQX`Aur0RZJuqv%6TXZ@x5>xx_+MXDQI;9c3LU zN~~v6BPchEwmweflDPqt08W7+b{t@GC=6tuF^;$)qsAbI*JL+>V;Fg!Hg}`{#X8^z zVrpYG#w_b_Io$Bpr(;{t9DLbsOgFRG_pJKEL))EM`(fKEtLTlj8xXe60tJX7JHctf z6Pts=>J&rQW*K*SN4;)(VdT@jR>0?LJ$|&aV{-cF{fF(XQ=9vPX~2rdnQgE>i2aLi z4<0>!^7NT??DhzqCGH-LvV(!w=N-uU(d)o2uWTiRMBQ@?S#$F9>(?e0bWXe!jAT%w z>1mP5)2D|dNDjM}^cZ7iwBs+vuJ7*bc}HGH|zWCZ!IWo|aethPjryA^; z091k4vTW(Q?|#ej*wXL+`M(+Z)gPVejP>8v1V6vL+}Se^moYOoFzL0$S9kAfrhhp- zYd*jm&aSV>%`@8vwnjQU6!`@^jrFD;&d-%P!S9{hx*+;pbob56m(99-`C=O;)G-sw zbG$IlGvY=?1V%jsyDeE{yzh~1b3^OV-kx6YHgksC8wdEmIiUD6_C-e7$ci0oL59pS zc@Rv-n`8>Mq;*F>9ILl>XG0C|WbPt8fw z&FHhs(bLTZ0&?QqD(&TU<0asX#><`s1U1$|BP+4Eu82wi7L5b+P)?y?DRJD?ua<^P zyr{OV_~vVjXv7tILK#BKFo!h!jL1q`rXJ`>2 zt6PX6hD{R!8n`i=s9QJ*K&~o?)^9D= z^4r>35nC^6ktCnGW`HDweA}@Y8hV!fmgpuqERq2Ox7@{tKzB9&sbOd}*`>iMISRq7 zf-4UOH|K8s&tvjf=4Z$q`5c@B(SWrZZ7=Q}ao)fscN(B(tLrJ6qt)&gMr%o2^khwO zzox)ohI--5%{a;;A_@0l(CH(wPm|Z(awt(V(hX;!0u}x4PTE4ZwE9$}&Nh6=c=r0|PZA{Dx(?(D?jRH(f ztm#fdQ>M?D#Wc9Yt14Ir>bfy-DavO$3UNetYyYO2iuQ?@q!(m3q*o`_^vP4981nkk zTtR;@G2!1YDn%?QMDkGTnn*PqlI)O)3euHHNT^NMk>r#o!Erd;iV|QaL5fS&-73c@ zbtANqV8R0olyO;TM~F*IX}}fjt-C->)(vr!X(*_8v|utUv%nnAC3!|S>Lo6un=`MI z5XaFSDBr^H1gz$1Yc&Qim&^Flp~~0R)fo#R;> zQ|&}8oC0abTeu;LSjc@%XiUT#@umprj-@V0xTLxdiVK6H7nhzp)bzKD_LYU(m0YrMIgB(m7+}TjzH@fT8I>mU27w-rm_iZ0rNY_1P3Vg-ZWn z8P?cw%i&p=2?R4mP2l5p+Qu;_uHu#bZO`+IRd#v;!!mKR@@+!_202;)OrE;?*Oy6U zEO!%TW=~c9^Ud14851cGk9g-a{vUv0G?Tya*3nje>1G-of67iLci)-Re7c?TOUNht z5gnCUpu0Pv({Qb$U|RbnP28R5=_Ve)Rmd$~hE4TRJ8JAD4+cN1*%06%Bc5^9K?9`> zk>WPI{bEjlf>PN5>oB${R!*WyuPy_0N`HHNpy8f4dlcBqtH zI4+h|ea1=Rkh-E^#Ui013Z|;USK+CWb90#N&fJVuSE%g!mmC!?_ezYLDIGC+ofQvL z^;cklAJ9{N>>`~6b7TVwLx?B_6n(g)!`h}0M||{wJXk9^t{q*Aia3h2xnGhCpOxfS z0!ks1!7V&F11Dt9Wtpcy{R6ys1^^*$N;D`j_sRirUs6XP6X~Uh3A9rQgbNYbl;%T^ zX@6q({BKB7{Q90INe{|uScnH#v*&X2ry@WN%IcSzKoQeXn;5ym1G14_aBwA z$0y$6@t5CmU$r^~q@u7+e5Yr1r;`I(5-3AED7(%kFMy9rLZ3EJx~+oo&* zY(e}ikiOe)C|Glff%GuUPOcK88kghAOOy#_WZ^sK7*UPtqrM{Sp|?yA3?4KMio^!z zKwnX7Sw)Epje_ZUpgNW63`&>XskFaxbw6ZLi?&}bgHLHbt;(sLMck1#QCBx)2N;n6g}=N%(|OXQDt{DsJ+$BNM>6FQb-jvYG&N38Z6Q>MBCMg z?;}*D6szbc-H&AIC$7dfGLgC)7!&o`0yV3(HT?~td00*r>cyG{KUJbsS=)U?rInQn zY-!x6(ANX0{Goncbl`zb&Ev|g1;%-F9D3f7u{GaG1o6zJWQIGPC6$yOfaJzhdaj^; z0OqFhawoHngJ3fA;U);F3C8gr;K;NjS4zxBUu#_?_)0he6%{FS*s5d?a!iZAmuYO< zho;9TED`opw{8v~J|HJE=sw$JK!i z3m-XRM%hU8kF2y*U{VIaLTyfloJ=&bvtq=S=yoRFuXN;Vuma(p^Fr>i9M?9^C zj5OdR^;fZZfnIByssx9Vb~M)h-xhutq=$gDATKigMqYkTMUi1n%tD%s(c(u+4^A}3pt->r#zpUK zo2T@UfJVOthcOGo7nU1w22U!RBHA+-z>;PHi`JUAwD$?#I%ZdrY?2k{XDAc^zTB%N z!i34CXBB5CkQ45TOV;Au_I8L*=Xbm@KwBn<&kxdy`OKcu5jtikbR-s9@@WUU*Ox2a zBzb;M{X$7;N2W68dcK@|XKE&#VuwTz)PVp^-ucL;t#i2viSoL?|c~VTJ`+3*a)h z(R-fmA*-mbzuzC|^AO+^mL7LWq>G%`bxCp-an=Z>MXaN7 z2~24zLSo|hF&{byeqthnO2$ zQ3siM5;L4i~o+Oh=fi4gebqGammkjw9Q$On;F3fvc_1J8o4z>U>QmHl?-It zq(rWSZfE#DQ=KJZsH1fbqYs6$N>Q6fh>U<3u<@Zpt%26E5|7~~2=ok+*@RQPh!FTK zLcmc5_#+|?on(kehXh>7HW^EiJ8Juiz;1UgMfSoJ?j8sEr5mT`(?Qih2LuFA0t zBL^yT8OjmZWVWmq7We#-&nN=4V961*Ky*9#Dm@J#zwlV5?nw)lBlg1@i9RP4V%RG@)f#GkHsA;R}FfL`FPZ6;&Wj65#0hP_bKSLu|8XW6H zaI!350LcIwi|CVzN6p=8FC1^BBYfYiLq>S3pyOkVp^?9O=vMxC5YJ5Axf;|q>lDt5 zW!E;-R?@9?3Mnj>4ZF==s(4Tq49WNac{PH3r=&DRNdv8xUgSxO+u*n=*TlyF zyXmerp_ZbW-z$ME#Ly1CcW0+s96fbLWC>+`m!w*pfnU>rghK~y77&BEbn;K$Q}5A6 zR#SwC*n-O#Lxjgf-QDs4Pn96R#i^49=gpS1iirgc(Lp4>Xi}bjEq=(6fkB9_v9J&Y z6g63d;5|Q`NPU@5*heZu7R=R;@Cz72J_akW8;$*dgA(Ql!EPk*b}mh`g=s7 z-9oQDNO0C}p{oMfu|QqJl!9XR)2HpdgBN>;AJ4qd>de~&UcCCvlcz84?Hy>#YSVr^ zyL$G`L+hB9^mAW&p}>hNISKt#MT1pkniMkHe9Y)TQYmG`J0hkCn(*n@|IgH&HQBLb zXL`nzn1M6Y$Yyu5Mbb!!^l)3vNAvYXASQ*{Om2_O@Lp7-;ODn{$C5FT^Yyh{Xrc<&DGE9yW34wf0sqVt{%e)rkt`8r`1Fuk_vCNUHRd#K&;(7G2n$$wv(N}OUNAkR+Pkvp>FRhcEXYmSGe7LgzIZ&5_ zoaD<(LFOD(Yj|0xuX%!S0_7AlNpNBYqLo2D-P!Jm1uuh6S%z2l;XbBb++AD>C%SeJ zSoP5tR^5;seqmkFaGFAnXr2I1Gh0AZCtumrlLS)k;I0www32?yJ$~YRPiiF-V3g=n zY!PiMU#+?Bg5GL@A&5FW4KN%Ex0;l(X@1v2S zo;x$J@ZfH*BySSpCUBeXCTMY+V6Ffq4+8%I7)wWH;2<9|;a8ulYUL{PzdJ%|cXkus zxh9MimH6~{;MVRm^pbt=en<)*BrN{Pvmr_iUx^aUBdkH{=4jndHH)mvU@&;wKk%eh zPYMmuKSmfNZc|9hEI-;5`*Z-BzoWwW`rf*1V5CtbalbiolcXOXe>4{M>7i9(cKfLr zm`=oOWUGzRaK}zy#}BCfXa|ZhD3Lf4|KUlcb|a3&7u-2~agCbH1Mn4&Z%22g2bTh} zC*UpIT)edoML+{o!PNzoqZ$N%DjDFKlyNeRq1&#)+^U0=gO2G5QvqvoYF0G#6vxBC zZF8q<5T;cVS$<-7$y22Xu9Olna!oaP zIq1oz1@^MgFSml>rs8w_<_U0trUB7I`JRc?0Vwou20#ELC8F30y(&60N(1+lu2nIO z{TtWI=JVHxLCEwxITC3tXOa~uF?qFQ0f`GQ$Ae16=h!KoH`m3OvKk*;Al{|c3Gj)f zJ>KQp8@7)GM|Pz_ucehQP;236nFtVu;&=mcM}t~~Nhd^sH9xh0%AEF0q$bFI7-<%S zSdC%xK1094?wb3?i^Eo7hE=`#031n_fEieI7ys{}7$+y}j$-hAqcj2SbiYZS3`&jw zCyv9B@g%`A+D(M8WZuhFR(;yLau4bXUMw39jjaFxKmbWZK~z`c891{?HOqqwG`10h zKcbnf!Turxi7hEx+La8L=^qzh5)G3 ziP$SR-ufg%EaYP?3FtHQ&6cs%97~ef8t0QsmO+EXoq12kN>~Fp1=o1 zMez1yR~fbAWa~A(jRS3N#)_nS|M-vpySE7bU;p+0{Gb2*pG>wtdi0dl;klJK$@A|0 zHdnfv1Z|4+v!|qg`Pr(do{O-{s$N_VetEj~@o6ewz5b~sKn)&}u6H=R`|-ygX#p|X z?&(**`nBhJO$TWu-slvVFu8x%-a@u1^ZC$o#>SvD$0;<+;~a19e0k@MU{;Bl&NH_d z>wHK)CGEV$T3%P2mrd(23eVAtU*>#FE_qeWvZm;4+zzpNku}1fJQd){?Y1=M7?TQR zlxsl1S*${n>x>fl3^=n&IMu$!#QBV*q|*CcmSY^t+!tY1|0$d_6MFj4tTQV}w34S1 zoyzQQgmNB<5e70DZ;_7+!;Ko5DTH#{J1yNvrTj9NGYZ{O51ClnN*)YWRb1(~{P?ZT zWtc{$sIDR;UHofFA)BtoQgWo~(@>_hT8;2t&P#r30LWnpjxF{(9%Ceyz0YddR~W_0 zVV9RxenD0)VCg_6-XXBxOrfKq$3QNnXDs7{lIJfFRq3e2i6B?DwAv#1%9^q)^S%ar z>|HTmjkBYT-rw`=d#_VNmSV2qJLlVl`=cf1=hl7u-Ct6HOP81Lk+fK3Wz;fs$^g<= zE{0^jGb3qP{!;}SEiV#L=G(&RT9!<%V`=%i-0!OR-Q zhV0bk6KDWPG2$oLRCoe+ceLQl=m0qKhFyx~4`-CklSS}3Nk%i7q_0RsRcK1DhS=JD zb(F)Wd5es(2qWTBE9Gkc3j0o#>eie;Sm^5+2L;^F5{E5of54~MmBNV3195+15+_^# z2`{T7WFAMX-lzj({1qrQgQ1$C7p<{)lfrWRkR1=(0&li?Pqda%tz z@%^U3*Bz0fHE(LOr&kp&|vWU8}D9ucP*P3 zUZHftub6&|e~bg!xQ;Kh1?(tDJ*@p(1-JJ;o540m`|;D2=fm~;Zref4du%@SwAdas zf>qRxH*t7T?p@48mU1b$;!0Gm z!m;K>dOc!b6<57dPjna%_dG$KB7jkiIQk+Qrg{yHYqn!SuG1hNOzw3gS_aU?Z(dwKcYLr<#ZC}36n4uo#vOF=FTO9190%7OLZvBITiS; zT@oXXE@!K|&|7xWIUs{t9l6e`a16S{g}?Qyos)?^WFOja^s1bB?-fB?tPDW>JYQ8B zj{QkWjP;-5TRIWa5R)5BH}QIGg34MnWrH_1MK3Tblf48EUDJWYtml01fjG}|p&VS} zyDcJo*e04!tn=a7Y_Iv~krnxmpI8kPUwD*;mp%Dnx@3wHYop%Ry?SIi=X8A2HDi(+ z=GoHhHm^7_+CA^^NY3pq`S{t3m;dSQ-%aRshr9}-A<~{Dc=YREIle=;2?(CnxUz=? zypD?L{L?@E)4%-7fB*gO|K|69^SAmc&=T^th1TsS+v@@C=ku>&xhJ zt@2_+yOG}9etz(`|K%Tl{p;V~^N@1XWr5ePf59S?AJD^*RDpc5>nY!1C!ENDSHgAL z8GDwzkOY=0Jisd4+_)s7xm}zG{#|S65~RO%9&7>I}dYZ)+QP4YPpf# zphoR$hpiSAGyu|1f~fP{BLSWJd}@-A0B=0X3_|N;KYytGA^|E>SGK;DZy*Mts@1PW z_6L(mB$U`m(dm*(OsLc?|B@6P``YhHP4=>PYX=em=(_tERkqhD$W5;n?iLK(HZdmk z%bnW>oK0g0!fG3YX>#kA$~yCJe?pE|rff-T!Ry4fi|NLfDw_ch*;N__P8|EnIn5!( zg@1=5zUCBZ@uAjT<2s@*<_|&)88Km6E4v6TccDIpaGKf4r4(lGXWXq^ZV?DIgu_85 zFnK4oo@0PxV-xN!i|_pSl9xYRFWKfxy6>c1epCr;p^41Z7rnd0AZZhMiB67W0=?Yl z>yj%iPXZTvp+swX*2gp=WY3mv7Jz+WIr>hSlV&ghZZorw7RDoQC zY8>{XmjtBpIMRGYPr)o3+73gL-#9T^Ys9mUxE2urI*+K$FX~-rbBtyff-U;zW7h|; zid8$P{cJ6N-GIjbJ7N?<=q@@8J2i*x->X~h-hUJVc?YhYMj%1-uL%(SzxB#?Q@I?Q zHuS;@eSpSv<;$G3p~t~RbBqdIWO`iHEjw8()Ue|d?|7?=BD%z?cF?GQiNR^J?Ao-( zC;6GQp(A|Lt2opl9T6#P;_ch*UC9^)r3I}2sq?w2d>~||;{_N4DPJIfRE@upCn;BR zpsslLWWvVGy1`r4;g}?r@i4+x=isvr3JM+RGMnb%Dl*EnbdutBq3CZ9HnlaLJ~oQ7!eKg)0S@mQPzY);L_@ zXF&Xizh8jIw-curb&WGvIMo&Uo2VC#>tKj3#sL{Evd5eCi;k$ptcNKhI6NU66N>1( zFGzp*@gx0n(SILR0D!kfTB_Vv!D;>n-&AghQq#y}+DMXfXc8u0=KCB~5l>K>WV?*x zsqCKR!Gz2L$0eotS=`}ZS&t-K1L7-B-VvO0Ue@qmZPiPr)vgaPmxV>p4OakCm5oX% zRDZ8`MoFq}Io2zv`i1_(2TG6<{nrOzZFN-6hu|;SDxLnv*I-Rb)7v3h#U(GG=;lR~ zksB6c5Bfkt?O#O%;r9U~4A7jJcx^ST$A7x1vieyxD4$*+ye-eV?sfrb{z7SHGnH{4D_xMv{u26GR8r_SVIiTTx| z@4mZw`OUX`+#b_DerZtrT|FmE%TnT*mk0<$;qcuz-x`zF2eHe|D!cp3k3atN>(@37 z`t`4W{o>_|mv>(()L-JO6=~lesIin!H{=Y8gceUed;Hz+zI*xIZyr8+Dx-;$+xOQ$ z{^AFYpi@b~}lk3awPlX3kY{`?~be)G*YW7S3EOt#o3)Iux$D31qNl=}MB>o!FAWUbMAo0Yx3zAkVFi$!u`cfWW=v>u~} zMwouufP}b__`pR)Tw+eW7!F1@`AjDS0#~1UKiZg8OIpiM?wle$d*IP+I$%fp-iDg1 z@l~Y2Jz=8e&r3pq^^(jiRkgj=rag^rZzc$Os>b;3D;Dpsgv7ZU8N(6jvMg(uCdx8& z?z@u9=41k`R5zrr;IS{~u`%~2?y578{khOS^y(su;=FUSI(r7-wuYwxsGQgqc=UV?o^=#bd@ftMGWCP(V{bKco3bpnOY}?>_gb0f%Eo=}UAR>1pmvSZ0ak zFM~5Muw@lXi0R=Ksu{paEorPAb*V=gmN8^x9A3D|d$J5n>84z2h?Wak_4^zravG(l zx-y@d&u{A-P|eBZ=jK*VbN01q=*nVNV|zf7MGJ-yl%b&cDzPdZ78yU}9I8T}WJkvK zRq}}(u*Gn|pL8!eQmoaD zN@E`Uf_*;1rs;U~2~Nyao4uHLFMQfcce)t^eBwpl_)_KF zsqFN0O7c~=yUCO=3m*QKjWlHCAi67i>NFMyX;olx5KfsYBKuV}%4J&-T2YCPlH?mN z0L7|2V zN#vV`K1CHqvJ2yTG^}SIXorFSg~G=E(*VKdXk6zoj6qUCgdO4zPnoLW|31u9ySp*? z9yLyaTBn%ZqjnA~|@IYrFb?X6~Cxoes6GUdwX>Sl} zU;9gh7p03QBduR25LpkT6M`BlLumNKfGJLIX1Dqn{_!F-Tj7`^aV05h?VlJR$i94f zw6k#feAgRrKcC|RpNdX4e({MhVa5gV6eIbkBEz<7IMd`%Ij%mMGY34zH(U(i7^h^Q zT;eVWovl=+7?n|ttSvF;oLVR@f8PSbcTcZ=Qw(Ax-{^&ix#x;LJv4X~2OHR1HTzOc z_8a>AQ@oF|L%Lf%Cu~ux;5{$jZ2WCy|C+ikF1J-;Y_(O zpANi#|4~G%h@E^GkzRrI{kZ)0_F5%sG*FEy`;FS%=9!`KN47G$FE-?gd+dh=KYH@w z*~`aIU)s*btIr7L=It-9fBD&>dCw-Bn`nPpvs$@L2AN=@)De4~)$$(h_Ipt_fKML3 z@-3r;^gHREPy+~D;-(D?nFtWDx-C&7f=f;NrX}L-G{}^$bwYPPyt}!5>b+(jdd38D z@v^UPzW?ptKe&4Q=-I#h>pyFT9z3#h>w`xoZ=O8|js03<`0mxK@4o%+x4-)vL4B`Q z-tdpjk__0tdiB#U?cMVMODTZIF!a2fcv>1ioAi0|UjzKQo@N&WK#hlzFH^3o#HTXN4(G4y7JRM;ev3#RQ&+0 znBP|kyWho0>^d28S^GwZbkeGvJ#4&>5l#^cO+sj9h-Nv@*)xO}JGs(@znpZGNl!>(-`p z?6sCljgZT2L{FtlZ``n^Nq!X0V*ANr{pVCds;ZoH7AmN4fvN&R?g??j0k!A;n4FB4 zJ2WfhZ!Rq&54y`2X&`2>Dhib20w_XCD~bJ`lfXrzD+N|fKQ&+#E))KawYEUh4U5tb!vZMTq_VeJ2rGXUXCSE5ij&WI3 zLQVR4`?ZP*;N&Z=A1V-27yhX`JqTV?DQRT}%qp8~m1TPu+Gatt&=YnkTnIz8qZR{% zrU>1McUgkN`)N7*)e63m5WBd2v?rZ?5U(C+j2n%sN#;7KV`MQPs~6S#(CGh|mp@Us zu%_D7#St|j%}F&rca~w&Y(@Wez<6o<3vV&k))d^ZY+;l88wSqjMCPG( zY=8yKto-RIPoqiK@87)t05QvGAKJEcfg3ys!{Q(lz~X+3f8M?Ac~Jhe9$CKZpdCN_ zUwW)+fqhX-W;=dE992|N4clT$^+4lX+lCqsX~WPKFd2e2UOKbKV^z81`0$N zjNBZRNKWp6&p-%XKJtqdI+SvxbQBKv6l5L6h^sP0WDe~E#8^zKFmHgJ3J(xQhq=-j zLnk=j@5CIE8tLNn64carNC}m&lmL2c*0@x|W14QQ?25?Ou1T~8IGR)Ud-P@^LwU$Y zw3DwW5@bjw&C{P03W$%?{ZXw=BK4lK%@Nhp*dhxpA^A1)alie@$l|LP4zuvdnN&jEdrD!Hiw@+ z3z8$!~b3b|I z3&J_jT&!I2EzUUL(_exvo`=WOC(%2b%*%_1DyzYSde3!Rd~9U6G2|4_|1g(lh=osh z4L6y^EoS)&<%+!A(_Kjek&vJwxbKjiWyFa7bDI7yLJPTYKKJB3LNcsVQ4|Ek-Ow4) z2RqrN=V?I$%SDoLlpm@)!N-k3g48?dbffP*`Tp^D-+p5&m$5=Fl-z#u#=ypX&r%h9 zQ{QSG0}gh=n&Nj9Te)Zt3f@P573WgoX|p+jY?)Fup^zdTa~i`j$cXscH^0~q0f%ij zx$Pf6*^lNAfB3_zS3g>aV&@+Z_yX4US#RHYzwdwk&wuy*_rJ0vhS9a1kj}xEFP>fa z?z`ttU%b$DF}d;ck3VzZcEc3;vI9f@kDk4-wT=gCuO2^vys#UKykOW1kG*y5(erPf zzIdsklw%EMY;(luNh_+mn3*k=1WWzrc1;twG_6rlr(b zJH4IzdQ_mx77W%`xNRZ+BctO6qf8*^I|%+CJ?U-7&!7MKPk*pJ_VLr_w)CPAurrhR z{N*>8YzI&Kgg)2LdHlHX1JeR|t}nJ%%DMFl1fPiGj@xgpR9l+cDT7nmz7b zwYYgG4CN6X7Ab4D!lIu5jHdEbN>PVk3Wom9Va<}$(!;w}7aozyb_XQ1oB|F9n9K&o zg`#<3SWn|A7N-y9F$e-aL#WhUNj)^|M(7<&TJ!Hpfss|=)f5m%kiTYjQKGNQugby` ze?xNFeZ}%Q&X}$NDA%}h%7s_tt%5;v0XHd#t^7g#z%E}t)^`N{rTPI_`l>67h&-bL z3IZp{TV%SDAcI7ZIqhIpA*M%vkh_eJ_&j+eiHXKl*hY}sovVuVu&}0Oh2`OTY1DO!vwiH5ceLM(e&Q?=g(fgupfq( z+90C-%j}~XLTzMX--wO{4MgU)?b_K>p7u$wsY1_AwcgbO4ux`BQMhuBE&j3k|Juy6 zZ&UD^n#RbIDSjlz{*D@;6Z9qSiAtvvchsSC{edLwKi`5z%f$6Da2`)|1Moi5%H6Ge zDEHY|<&=SRGj1HUyG@T+tGfTA@T=qW&R47Ja8Y^8U!KH9jtb&V=LbR}>g*6wdO}$q zKdWR?y3kVxveFfZ2j%xoOLm(0Fnu6Mt2PM+)KpVsa7o)V9jb>$XOI@lVxe!L$+S&c zG-u9P!R0LFlakUN?9%#)HgQnqw6G2UbbN=T)Qk=^;Nlo?^vo?y?57R>3voZuoFd*5 zE4SviS%Q?EMmY=Ks2C2<6eaT4fxSt%0s_&Uppbe9`$dCjI6D17#$EeXvCy3 z3X-Giur=aRu!Zl2ITp; zA26s2u;M z=s~#gp4cFps>nIngmNe6)`KZ2M0Ir?1S}<&-0Z!I8c>+XstM!? z4K_u(ofrLwukTE4o$XTO01as}9RfoWcGZ{k&S+dX!^{n`azFkh!ISFT2Y}d?$!2i= zT&0uP+gYUscDW<8?12kzUI5z5gnT_gi)w_4zw&79x8GV-zf#4ici-cy;H+`u1CbnS zKGT;Fi63!5wJ2=Cx`_j7tJ_fbrw0JAqRi;0x)t>ho}RK0Fw8v)A~dgOdPi6C?#r9^ z_O7|LhU}(EmzuKgzWY{J)4rYF_H|xI;AK{S`yc+(uYdDfSbcc+!K1Fmtc_dWd-~J} zIny$LO+9}2;~)S0<4=%${LsS&7RbDO^7O^yr!O8petPxj#e;`WYR&eBfh!$qEi>%I z)ZhO4@$=`O+e!_C5Ew_Dc12!G$$y;V!scahp5Wr_e#dqB$z(G6TBadv(U;$o+x_xt<8=eHq(%^`Gk6n6La8RKKJyRrmO{qdJ&lO z&=c6tw2QqG0&ZFKrZPOq;7^o}Du!g?mw3bU0%9mXd1qH!nRLY)VtP)(!ycD00Or8O zp*sFFQ2?;+dQ5bd^S7ZsTd%P(+VbVyM{HtZIdz26DZq7aW_w@m*j$P$n<;-~^3i?b zfOTSAy2K-a#5D8xtCVI!Qe|EN=qpw|dBJn28gkAr7SWi#n7z=hTDE$DSww3t*1-m{ z8^kqM;1PWB82EXH>WWY_stZGmn4|{l14Q?HM)hUhzQ9p+|3j>}J@C;UH4(RaU%$ef z6=LINB0?*!41%0)L;kBXTmuzZYb#*x+yQpPmhVEBq{s?ZpET58YAPvAsw3ODoSr(% zxEvMy>rv^Pi4vE5iYXYyU#pSZO`IA}s-^#msH3w}^-Nc$@Y#*vuuFM#HL+bqh7J)j zCJam24=D$Tde>ZBY8XyaoIIYKsrsqBYIjUQMZ@cqieqgF&>jsPj?atO*>D3WnmbM&`X<#NbaG3!H`-apt@qT|Ig6#Mr-WsJS}riVDB> zO|siH@vPQvtSMfhj#10pcHTj7X4uKtzb)S1ys`f07Q~NNi(15O#&PZ2+T#HqKfKpZ zXzJAxS(wFEtsvUrd=3o6#BF(Cs6<+RW!<6lXTlwoqQaChW(wbpLhI^X-TQodPb2Z^ z%7UmnzB#7^>Zv4SCk@XPk${tKjZam5bP#CgYEPt{Pctyv{raR>UOHHycF}pcN<9GG ziw>XgP9x-4Dc>O*{!Vr;S@)N1;ii)yrltej&&dwqDwxPQCmqZOhV*&@wdYX#r1TD? z(IWj_k*K<$T=0R|hX2!z08@^y$7;;?o<0~x8AYoX?PQ6nW|ZAE@B9 z`qQJXkW&pWlM&Pfh8lZivkW(d>gbWVqg(KC5K4gP7A0+*@b0}Eh|-R{jV8`BaKft= z1v6?7krevZST;n|(i!rCt~6+0_#duP>lZmxuFfU^+AgGLaDBrO-!r*Ns>)J}3QdMU z0X+>saWZpDZ6=ql*rN3mo+WMHkB0S{bu6#-c(!KJI_F~p9vf4);sJ-0c@0udYwD2Z z8ah4hBm))_a99$)c6E4+o-9T&n{F@Gdb^0<_go3UYempdC8A3ez`n23!||(fN$ZXw z9WUS=N9w8Nk%&#)YL5YLm=K6H2;(XdBu{#gCd89eKDi$R>WuL#J&m3Ch>@Ko&)R!* zNKY5l5sNSF4|c7_xE*(6b{v{Qa??f0JD{B;{&JB5r^O}2;)$)+pyuMFxR(UakMG^+ z*{A@y!S#J?(_j(*x8HvE^2G}&KmPdVcW)ov{r+1H8-;DOd(d~T?tHJb+!F$g3;d@)l zUthBhFPVMz?D>~FzZ0<96v#IVvZ}{R2@L5AhQyc{%ZbtxIKh#Mi(7wbqxyGttJJ>z z_V>Rx9biKv>p1O6lqJ~X-=oiFA-D*OMRDPV3_sGAr_Wz_*;#HDdo*Ir5Mi)k8@h*c zTGU}0ljqN9b(9tLQ5_;b(nthZrObULc|ZHEU7bf@Fke1Cq`g$2hkcFr$E)5y!aX+t z&QKw3I9>-xCZwhtpOj{zc0ZGg0sroA{_e%gZ#`1bSh~qy)!g2CkfInlLD^R)D^r~p zA`FRC)?;UX{__u>MVNnR)t~7C2V7LBi)H5f$<>qlH+NsX_NIFq(Cd)an8h`30V|s9~;&u!tET82bvqC6)lKGb(11C z4wH@ECj#Q6X3%XI1cS_eXt+P>12q3h2Mt5W%(aoyp;oEmva6?Ks!A%yP%iFAuKkre ziOwKW!5tG@_e|M&VZyA+s!#nL8i9j^hDZeUj$`swoy#p3=i7PQ{-*zSDjV`-WS5 z3u`|i>tt+oENjuE zX*2Rzvs!XM53(yLtDLjsb+b@FU?c^dVxB`z;6@oV$*}k`oUkAg!q_FPaG5?eHij%iY7c%k2#NfB3l3W715m{a=qdZikM+ZX-%Z!5G#JIa;gwdHdH-@1Rs?4A z5DTlLzG6kHF(bsSR0UGu49a!ILyPlH4tH`QUVL%Sv$KK=Br!`82^swBR4Cc|F3qeu zo>~jJJ+RR+Vl`bP!=10Tk`^MQfN4>!gds4Cf8M@*|3(}1+HMJVY{pP8zxiGEh=#U< z)MjbDEO7l`Vsq~}!7OqPIPOoakk_G1Y>AvhT?!n_BF!<0g-TiTM{?&*ZZu_zGk)Og_H{uCi|Kh|2h3=_-LJS5%Q5x<99xsu5+zTg`m zr^*hoiX8Z5y^twU5tXlsWCun_tl(-HdSNLSLP?#hgw!5FwXK{m^zoTjHWi?vb_^m9 z83`7*qP5@y-+ZZs5LZ`K9{rK4zeq{Tj*ZRyz@x=8IS+dr?Tm;r*ZoVe7=S=9b_J2{ zXW#*bL3RKuOW_!#Mo+`;ImXlc%^~nJ>cAx-qwNsaNYoD2prrTK{SY@ z)`i8jF)?Z?qpn408crk>uQW_CUhWo4@2WhIHW-)aU>wII2ABNr7#%UB@)b%Z6NS*D zxD*MkjC)w0yk)&TySLZ+6vptMH;3})kS zE#OLwryIF??e8~rAmVWCh!AB`vk*%r>i~3~V$|36ON2u*oCzRwgAt-h0+J&m?zq3n znvm~y=$9V0e&`r|Ny+MOP60;J>AuuflUg!ye-Re#XtPv-;nX* z*|XpO_P3wEyrh*4Ap7CT!%Ppx+j51%{6q#3G*$bZ1eijP2iNZAB2(DBlu;6Wr=rKE zpvN6CyWWvTjE&A#sDID(dXKNsWlF__&GFh6f#c@27>w0ER<|20F{5g*|KZaoxM1|e z8gxxd!~c&Ti4lML`Da)aJwABw!p=9n-0G2nqJN8kP;Hdx;p68`0z7K49Gj3m z)}UAoRm7oH*8CEv_afhZc>P+O?+5UCvqI~F?ArR~`YlBwUu=PrW`DkhO0Wn^0zclry{m&l zxAg_avP?*zl`}UsS`#|OLSKHOA37uWH(s_U)eYG>Vzk@7bRP7k~se;U$~pOHw#IQI@7i1uDD0V!a7ATw>ylA2C<5aRuJ5ban&hd{2lgxkjcM!!-sT zFA^Hn=Zhw!J0TE1uqnD+ONDS%ht<@ms4Z<;Y>2O@ys&jeT3UNKBF`lph%*NH|I4Qu z#-^wWBbh>>#IFDj?_dtEu&u=I?Lvv<)Bhkibv|wf#wBi?62bgyafzo~^>wRaJ~1x+ zfNwan9G0|pT0-HS3IZE0F*x^{T(YX5I*eb{J>g5r{P-_pg4g{HL4iay#L8V8(y{&Ii%4 zVXc@VPfHpWndi)R1gi!GwvAhcLF(c&9hq9NTbQIz5f!1+UvyYeqPN8VV=Aq^fP@uD z3KfxL)rAk;lcWLkpQ)@WU+zUm2o#M@PJ$36}I&-vzC(qtYChjgJd_{AfF zwUSwBCXR0f-X2`D!}Oci@7{V5&GiSZ(U(W@XkNa6MNtu)=Y!w9y}7aY=bbHk3M1n} zVFb&fk*?AosZ+_W;!7f)ez;GirCn&AVxC5XlX}&r$*H9yvJy*K7VA%mi;Ak12V646*)0&xTgwZX-0fuO@X7Ds;Y`& zksrEi#5&8UWe=g?1kM3Os*>hsxU@IQ_lzPONx7&%N?#+#9zDm`F0u~D z)@b<;74T|WWD_yT4~{T02h}p!96#MN4{xg|FNbc!_C!d06AgrR68|#nzTxWf!b$Ww zB*Hc;D&$!{{;RAZyPKcBtOF(sE6H_IFPal=H1-YVo6pwt(0fkt;#14gLmk)=NEe{YJ#US}QH z_WI3R`_ohXqUI;hT6Ib_@3j{fF8XGF&#do%ZqbV;ePapi&AQbpnj-D-G&1fbnx7HL zgzkCtERBlwPEfG_UF0QVZi(}IJcJv|8fQ9=Y%s|!LRnax`NY!`vO;4jLUlAQMy00d zRIf=wYjgZrVQ09#4nSfJ=fs>Rm#~=$o<$8Yb}Z7huuzH)LSXX}*_+Ont^994T&Je? z4*iHxhR@^Wci;cY_m4mQY#8e5@#EGC+`0Sy`u66h*PlQCkq(%J`Tjc^@m%+bWIbiA zIwI3s_~C~ie){oeT{U&!FhqI}Fh>la$7FRXGv#*l}F1{CDz?9#@q6cxCH(*a;go5)Ve z>YnAq+EjAcpg&o_-eDTFxasiQ(ldYQBex3tq8?Z%z$J2) z+XX9bhq{AOR645f)Ec!Vt*V_?G-_qIfJ_m(lqor!tHA%I(vi;YJ>pW0WroM}!%0=p zc!iG~)mNQyZRi{nhES+n?$kv|@e+!k2=db)B~r4Anq=QJ5bt}b?&h`W&-^f5Y$?FR zj3J6u!KO5^3mtWTqAW8jmpsc1&kKL#NxymV(~LIA6&A*JVuXGwpp8>$L%%XcQE+n( zR3sMfZ%C&j(~a5%b%TRIb152RHYMP#W5pNR*GSQMfX&H;Hh?h9*6>Q zPo5YzR1Js&=`Xw(3RL7ENL9$gJIcs1_)bqi$uqObRq z#3Zh+r8PJ2Z|&;y_Km4fe(TE}@2?~wO=L6VhIsDkDH;2}u`<*C7NUP1m1RR`Gl3a+ zL!lFG1gLV~sM&#!@hJXcQ7Samcd-mo6XOy!XKVqEe%hXr{A#A;Bd!qww}44HOpC;{ z@l+fZGyfThlCaFPRq@m%NY3I+2(92w{|fHl6{eK~x6`jcg3@0~GP;J$Y`4tQX<%VF z$6OWaBWCkfdnZHNvs!&_@v!ganXIr&4gt+9W@Ktcwj{;FAVdH!2c>r69_>o!A*L=) zS!B#+YQ1`GtrA`Mgnyzwl$W}j)dcq}=15}^cG}k=mns=J{c1D6%>yvajgT==^_PWW z4Nkd>8Uvngchm?5E2gEaTI+F0Yq<- z_G*gu+rpUR%qoI-On`=bQ5O1>Ruq8Whs&^hsLtl?v4nQq`%L(v78*pW@ zHv_b!m{=z05vrCb#$5XhFz(`QsK=rz4Q;pAW6yrY(S0-FTF1kg-91vMrt)73#_tXs zSQ1Q>+A$#=UPny|l1ijo<$)^71sFGMAW?`^Du)v$&QptfMlkISAJ1d_tU*aE!ZrV4 zyKCP9J6BDxo!(2X-NA)b8R8E|oWb=VSotKH7nEED@Qki9sdnGHmd8CD3&GSKAyQ5r zDJQYQ6ff|qA>!!E`t2nOTzkA{lPdgsy8Y1Mj5lr*Ptz;y;`yV|5 zqE{O}K=m@hcOT!qxq0`??Yq}TgH2YcLm$X1uf=U!d7Ag5#x3QrynLM^YcB=mM*h`o z9seuu{(AM=bEq=WX5YX!(UdQK@Z!Zc|M5Tm?XQ3Py;*`SB=I6dk84@`4dH7g3BW=o zW7nhk9$#yndx4@pz@58~?Cx*7%BRl-iHVf!&I#&FzrAyFedp%wqAn}ftg4Ai#R#(~ z9(MW_)6$u-GmcJY#(^$^N#~BADpeu8Nz&aJ?*p7ErsO*Vf^uMq$EF+mk7daI_~{bL zw((JOFf)!If`X}ry!!d}_Mz#QBJ~fqZ?50}`16liD0wIcn0M~{z}S0x+LOoM+6n083+F;c&6d~V zJb3y{xNH4RZ6T)UyhJ|2VJh_PT?(87_sWdw!);!Pn!63j8BSIb)?NhWQR4PhYw=G+ z!8_+2E-f$7Z9Q64gOFdQpmCT9Wsqs}m=a?$q#*f8(I-WG&(;E1p}!`7wUo(sRboZbd zVoGGj&N{lPWa<0qtV;d@u1Z=@=_nuuw2XtvG&J~ST~Z+y)ph|!f^rOK@&mMwUq2PV zXh^1mTy%7}c667{<@_3PAA%eoGm~M2%^bN`GETWNiz%r(V*^(fwc+br4vON+N`_Op z3yiN4@QTe{CqC`u6r>nTy(+dCWqd?oY9Bg<5o@_3L#+z0R_G}JKb~3K&JpZY#*!#h z?Q=*1tk7|OZ3dJ&#KwSFu|xEm-IzfuoKp15N3A&&R27LEqr+1(zC8fYYUZ~6 z7qCd8@exM!*478!1 z6t&{i`k!lyf8N_Dq3sNsG07`5aA24*w6fzkEFPH}$J-QUN6d}z_?tRR3a%qKk;xmM zTq!2J0>luUhWo;{sBR>s`;{4>d5kWm*i{iCm+ED4PSmTfno}c;ncNTkD4XwU-a1n0NxXKeAYODCqy;g zXAuF2djSWjS!_Uu!ZdFTaE)uZQ+_uM+=_VR=t@JtedqTMTf-(21`*%L&rOv#C4f|l zoWhcO9{VsjEu~HjscyL=tIL|k8q~ouV^*LriIRv5+A6BgdutsUnrpLe@l#q+eGRG6 z&=Yw#0n$SY5QuFeH8EzxNKVqzx;N->((*UU!1S`;%Y%7Qn1r76fMRDA(*L&Fj4((a7<&$F_PdLK2x z$oMg#vpFbL5+0RvUSTIWj?NCO&g1+nK<0}RJ@?_0w91g=&gIyh1ImbvDv+XO%UyJ$ z<>gK)qcvul9?N4x%A&@R=+{q+L)hvo^jsC!EV#)ZNGfG4J*YdahK_#5s0K)=2fSjj zL}G=Vs=7UDhjQ63B&2((Yxb4y$rX)JhXvsn#beQ06fhV1EML+-ki)uRXMoa8ir{f1 zb4``Td() z8-0HJ{UiHaUN!vxOtTh86QH-lwrRb|)*pZR$%6Uc{`Pl<{|&S?&DJ^}D%$?dt!jDH zyI8zl5yB0Mcp%Hpp*jG!@9%tmpQ&Kc)wM}tt%%;bw|Hb4)#3|>(wwd)3*y1c2{8uu zC&_Kh?L2WfDmNFF7EA+mh1K7I9RkG;YJeH2zgC zY^xnpd9p80AH}u~7JU6HnB;AYnq6JJd~UPUn?L>O57%$61^qTs67<8|Yrkv*szuXR z!+$zwJ0z2Q`XbP$>N1obSa_??^YA?hlumZ50GWV#EGDEN>paIK&fG?ZnkC zb1{}FU(7BJr%_R^{%vdxpPhj(SJfU{P$(m;qV#0`ytslqq$wIs2SW0y@jJB+!fL~%Dn3<1o<=~jlKz9t z2`GdyAyM@!=G3hV?qk$&-2I@n86(4-=BFklsVV3Ht8yfy4QbKP0@s|l30;lhrp!&bO~4B@7E2KuhRz9!w?+M)V}&j5tz$`9^9I?G(BO*MCfh_EG0RH4(f zI?FsWGk=DOD45ctxOhayt4#IhTd}!k9SKp55gJnluJwc!5d)O zC{&qxAO_x2{h8+9&U~?Cj^tmm^R8MwkX_YwF*RQ;F@}^ zu;Kyt4|0C~Kn=8GdQhPW&{ucUSF|i^07?u6#5+N12uPe1{)(EUi32!UR_21KLa`r^ zQ>v5FW>kl5`L(b}YKqU(Ch0qZ&9n2*R?`=u43v2@=ZI`1O_=ZFPvd|WS&zfibzHT&<}hWM_hJ5C%oN~ zt_IR(AJ8g zt5syy!##QGhc5>Xes37PANagIO()I|*LLsn_R$&k+>t9W``SFSZ5 zr11%l2H(9CP5;VTsSnx!A03=FPSwp431kIqtT{sX=T8Q>I-@EoW;6_$z4n+;W|Lh| z(Dsn_?7cO0aMINRyz)grfZo%~_~nZ-LPdz#uD;K|se;<4zuzHkdWCOB}Sw&4Kq%yOv zb3wJm>%xLhZp$}Axt!yp!b#LOYe=4HjhZu#leJs1-`B*Fyd_Qa%acXrf%;XDO&i6K zW!|7cqhRLd_bQ`q$T?5iBr_agk@8f-sHvuyH=m(Z^Q%W?P6~fb?KlK*ItGS;s!Z6Y zzIm6`8a1OTEISLWRqAY)-OYvdB+}Q3_u-r+3Q3YqWQzLZy?6#1wER z=twyuAn!$_T{k@f2Yx7b7nOn8MWvU&GM(2N?)2O_U?qVT+b*P)6Rh&fSBXfn{$AzYDyON;RmJht!M*}lgp{9{FMi`I$~oNs!39(b*R?VCsCEr|2Vy(C56;V zcQcfyZPaTkSB~aMr6;sDIC6{tdDFknk0_nNY^kw2|KMJURk1&6G-sn)z)~F(v^=oG zOFZ%?3I@`@enJWpaEE(+l5}lbH4c9nWAuc#euDNSWsqFm3|^pc;Cf~6~CX*K?#H4$eCso-H= zjxr%l=T7*G zus)j1|9HQ*Vpg+s@vXAHtf&zom;ktM@`-i>hXCD^7^5t5S#y~aK{g9!%)=i2$AyJ^ z0KhW<=3zX9-MfSX<5T;ml1&7In~HXfNnC`+yB?8~$Ok{k$5IU$z7k_&ft@boN@n4~ zwu0(#X9#2AO2JV|!ws=tmSG+aQ7|3rERCvoX=<$I+9+0sekCKbN|wVa2~L6#iM?2< z!MA**EDd19BZiza!$7@{8qGbcd)>;#Z)hs7Q}Jd- zMDg#b6u3lu_3-V(C(k`vYx!MUA|X+j|KW`}&AT5h{z>t57q~VZ)-8$jfatZr(R*ay zTKAX|2!|0S#JzfY=Z;nWVGN@TwjQ6QCXXLI8AFB54I7I=h4?84%TKWspe?OjGPZ9< z0b_sltKV4rKh2_+?N|>MXk_P{tEj!P>l+uV8_*+AH}^ihziU?5NK*?r8UIHQo87s& z^YN|Dk8Mwl0@;clXU41f^bktp#yqfMWG*0%Gn{&~A(k@nBQtFlxcNA%J#Rcm@ln72 z-J5rBUcGaZ= z_-kUdfi|NvDT)}M1Gbsimc-z74y?Ck#bXGHmOK{ zAZBNTc`n6CBnx5nA`Np@?gd6FY*`Le6&)G@X~T?v>U z&=m4wwlzs*&yIoY0jnOD4a=M(3*R!3<|u;gzn?PpX5qSg`~Ob1mK-?9gi+4 zD;^*qoXa}#Uka`yZwu@04R=|Oc=7__up&;1?qED=Rqe`D?$lr2b42RuU!L%b&qE@~ zg+Li)`OB}Hqp$Q}H(>27atV3*#x6`{{op*x**tfbyvBETy48 z;tOOjE1RT=E>6fruk?e2 zF=}&rE<}dl$%?`-n1SNk@QK$6u3D;U8LQ8g42daLuci`%;6K4Sp--nk!kD?w=*m_% zwKTk~(qL2&xcyW%1Z0>PyHmE5vnrgV4rvI1rwSF1#h8!*X*KY=_r)5%5`Q#PIy;W6U>JhFF z-l3|W_~60qRlD5jl6ya!HJuTl3Q}*L0MI(@GQO*OdG_+?MRMNN+0J@2uFhf_U}R9F zZ}fD#?>`XwdpVC_p@$@EPhzToNn4z$=+;6cMlBX{E9Mh)uPYYN-DE{md@buZJ~}d< zASI{6BERGPoR5uC8oup0;AUcU!}`=7ACaYBSUbTgmi$xX`X%`x?*Ly^mu`Wpl1qD& zNK=fQPnJG#UqKoKv*_EHj;%R^?Yuf@wh_-vr+TDF&ZGi7)5zOWkLpU?flGU52)UKi zixBm)D-rodRzHiSOxc^P&1ZR1LeP>$gwX*I6r@$};Z@fzFZ_zyZiW`3<*sfz)jVnD zoqi7@k>yQd{Q9$FQlpxKHd*w7HkU!FaC*8=xXtHZvYlrZXVHstlIw?F>)`o|xxuirohn=*_UU<(wD zsrNv}qpWXCoF0)7u^OA2`)xXM!~e7=1SCVsPfegKD^y-(N?d)T|g@{u@b~}l%ddXpU10Uq%oTbR`=Je%tpCvuYCr-_f^f^W9^X8;ePZ9$ z^d(AZ&F7DAU%l!LWm>1kB|PV?tzAnBp(%J6#RbvGMzOd*OTX@V!lB@j>S)26LTrbk zX5Sf|KR2&z(?<-@?QLS8-rpF``0QP1`bqfHR)TVED}pMBVl_^uA%eEtO(Td7S99Aopz~K5uSEG# zn&wnuK-VG!(=o+?qelP=nQY-xUN6~MV`@o;TRC(GJk{#PDTN;NW7&$}taLfo5f8AewHHyYriP0W zK%@QyFc4~^xT++gtiP)K?Hi3{ldqSnw#(pZzF`un9Ycfl z6bBNr`l~8dPe!07s-zG!-)^>4e`-Le0K%eT10Q^(G(OQ@+4pRZHcc?ry~l#H*eRA) zvLLXAF|5?sLcH;lpo6jeMF6q-tJ%#oK-g{h<@0rr zJn(ZlieF;nqb_6LV7soS=}5&?jGZcgK-dTO5Kx(cS0s#TKTfroNX)5hd5oUYg|Sr{ z6;OZ@#WSMBAM#oxLnt@51ZcSJ@?>2dGvalOj7WxCP&*D6b`OV%5pE5KRyokLg(2}A z+Vza#&i#jh5*DV2iqCN281rh(VW**@-TGU%ys3`j&DE9QUaNk4b@SkpdGIRdfk-R- zEk_UlZJ#^Azr{cA-`iT-5};<(m8NG@3#~=@V(wnN<#YAK4oGdz-clSxxU2{UG>BqC zLh#_t*|oGsu&6;~9e~*%TJqUSWWiXS^~VcE#~}CF4wB?S6sC}*;i5JGO{;rIW-~C( z|6)@Cj}ud~Q?d$HM+ns!LrkkJ(&)*gnvLh_gNXuq>IcrP39bP}mD|nJ;HDN%7P9GE zJt=!x^p5qcr48938f0!RLh@HAnPpz2o?W1lfWQj1tfm$40Tb{2h{1UZMnA1a5j$QH135>QD?o)%5teER78qAle<@06-$Ebml*R zRqrg_^#?5nHtu8)%2sxVx@A9C<=a_!9(aKZZa9T$a`9gPedp~@KRtNy^vSm`pL23v zF>5?pE=w;sz)$9b@*f`B8pJu9Vj1NfiB!>wpP72c#9m6pF_xAJqU+#Egz=Ow> z0RQs!jVMfdIGikIZ)rT8JiIL4NJh0$gg(r0R`@%meKVC53HdCZy0(JYdP0nlNza%n-7P2@-H17POANmAUv*LU## zbi0{h-IC`?r7ds(Y8;e)~|c%AU-Vjawb$pHq=b2nBZDtV;65 z`L&9KYCnlIn^hmBmrDj1>jDc-sCgcuTa>b2Z4%0dgFATvx~Yzmc}Yb>&h-x2JLlf! zF;i{@`lABeg7c4_)f*p5*Wwrw>&M57J4#eD`Qmz^HMY zquPDc9xDJ0+o?qjDun8grMJ2B>JII6+*iU2OO=2Tq0R~*&w5%tPGf*^`^y6wRQ`tM zYmY)WDq@(1wPHih+2Lsz262v&H_Ml!FA@t_ixfjfwkG`ivG#0!YxYr49pKAxG`zYo zK6wkaVe3oKO$7-7`i1K8CD0&8W58DhJi>3+A?0qYSca+yl@V9cN{l7LaY%wa)8;AM zucTV$FR(ExeXCXV1$a2gznG}qP`x;w>&GUJM|>s_D;g_Sb0ucV2NB&Em;M~cYRK5& z#xt?Bx?dA93iDaUKse+Tl&-LHqr@q35vKyRPmvo=%e4z<ymZlB;hhyb3j7-S+AVvY_ibB*YSiDvF;TH4* z>r@yXr=|=3K_+PVPR0p6>XAP*OHNd->LP|I#)ktjJotF`X@d?Mrz3J3Yok=g>@~uBNU~A?^X#t(tPw;s z^3fLDwA{ZH0#6@3cEo}bkeBjcIi0>Mb#63hcFR~(qZCb7<>uqZHh zT2*RV2;~$r_yBi^r7>8Q8Y~&5>W&k$=s0ru4{HT-eJSp&!mX+qS38ARFdZ3>uL-2= zkp-~ZT5va~V3R{-3x->NZV8I9;C7#M37-Z%Za;g$lkJTmqD%1NMLW53ZmO`3#dMXZ(4Yt?rcZSV z=(^v-0MYwH3saBZEz0mG=4eM41HUPNsPc5g^nn-~;Mq4BpzB)%p zQgeV>CNkF zPv5@!*$Vv!g4YHnZt0MoN*i~x=K48C2{({ueN1?$2~&&Wt@WTC^S|0Kk$GM!{Snkm zPp92ZZK`D5VXFh5zxY-kz|z%ZWOU*Bz3~3&GtXVT;3&4b(7Jk36MPr*z4g9jc@{h1 zm-#*EsEMT9eFU2-u(Y_IeEq>%M!e!0y`=|Iw%JI@`bNGl-jwPVsllFid=D&-Vbp&Q z4xv_d185-5rDShdnBr#(7Aj)AL5);|azP~S=01ST+*?W107%?o%9MV$>Kn}V%L#vX z{RVTUd%j�`gXCVz2J5WlW&125mP6!PjmMUO6uKoRjXAmIvu1gUZMb??zSaBYx(w zz;G+A$;s{7SxGe_)*b4s8Z&jHaDNU?ZQ+;7N=XMz3ZWR~bRdrqW9Av6mIy0QT~!d7 z9{Zd%Q6SN%9&kaXuP(DUXB#AiV@%jMIRKQsm=#mhwhCisxRM-}hC&>y@{sRTVn6bf z(+!7v%SlH7oc8*PkZ$c}=jSihIZ8Op3OI(zTBt&Alz>55#*Tifb7++d9CTBP#8lxU zAH#VQOd7gc4;(m+M}g=sU~w+If|H{HM;=$c%HU+GKJ)w*@fSE^bU660y9K>AsVKBVk{9E{CfpB`9nJJ+U%DmerSb z_PA}iS0FOF;X4J~IEFtSHo_07>kC;pWsP6o+ocsVDA z@Kd5Jk&`W1%xeL4dGo(FpMg#4Tir5X$9KMkRE5a?&9Skuq~F~t%mR3PT> zaD~g}Sxt^ZfUeho(;XkvKb~Dpzs9R+mmzx(x3#LQrDYBIw{yze7W0tLC=H*m?e-0B zKD;XWccN|Y`K?@3%u>}y?+sR|Kp5xhU8vZwTud1G$pK>-;}k%lH3yW`Aqn07BNUDe z8SDsBGE5A2Ct!-)inLsOCNioxk{>2uhm(tG{g32@A<2!PO+D0_5U< zROJKt>Ke^a>ASr|SmNy&b7g@M>!&ArrXVyUyo2rwgAZ!eM9rp~lJm5x0hlRv8p5et zWamZYPdi3#BzFfS|KaK+An9!?b7@&hTEfLy#+C?;lJ)7i6nMof-+`xl^-FSC)M`kD zVU@X|P-u@$JC}UA!NE-)G9I)7Sb^ACDdox%W1}LztB@InB23b+u9Fr%>svl z#c?5l?a;gpcKSkptyUNA(NmQ+2k`ElOJI3N5$m??(NuU`y^YSoie3`Qy@vcBKYi9i z!`|GX)6hD89JM&3z}n&~&h_*A0=@emKU<1^@4=%=&)5=-SRAh+$Kt!xJQ0iv8~Z^2 z`i-#ljkj4ndH2rhd7A*)X+{@D2Y|Y9fKu@o;YHD*({5)y^h5BLx62TLwMIqx6{IrL{kMc%-Zu(39#7bh?UhjGy| zms%l2q&5kf8i<|mn`n7vz5e^_*FPE0zw_nAH!m8+YN@`CP)5by=o#=Kh6{~r=>YIO z5U)S5hD*>71DuIEhjXr3Qp-?PY!F1n#yT*Cen6vuAP{8bJFLn05lsZ(|r()>?J0-ihET|maO`2HLWWuV!EsX=Ui+5w- zzWF^E#j`FmldRYueg$I5*|MZ?AHtpN?ylS|XY2*}Yqo@RE-JLLpfOsBIDAIL)LnI1 zVY(AcC-D<2Q0Bys(_lxTKng$j!$vmnYx-Kvj>x*(8`Y9}no9=Ne0;3DQ3!H1i7b?W zjtV?@nLDlfy$=m>QWtOE#WE*-c=?L3IlFYK8i70CKddi9+;(A_L z4aJ$1NI~6QxUYu39E)BK*J9a#ex)?xJ3_0U=|NBv;N|8aC`e^<(^qH|5Jy9)P4SKc z-3?>ZjF+y}m=Yw6`2)ivxyk~7U{=6?0MuIZ=%U(iP&I__5@E$6h!lW$qe-^-sV%qG zj*^u*_N?=CyK9tofQmg~2&%-aykH$t`T#v4#mUF`)t^LD?3~ST$|)CV%+>)^E^Ww# z`!2ji*U(N!#u7OC<3m>PfNM&CG7oqJHLVbLiX@!54^~&m7%#wo0n98y7$-B^A{< zq#l`;yZgQ)tfPwet?Dk$)e53wm?5Qlu(K}!lJ!ny-gQU^t~xI;L)j+|m6TKX3U5^) zmUvEP-&uUX2C?gu%KfEPt7ANK0 zIF)tGLcaC^?$yJrIdagcaBOm5djSfav}G0KsTmMOV??!+M*2nk-e}rOO?wE>D|0lD z+TVlpsJGxD57<&hkO8VDV|6Yjzu1a?K(~&zwG|t7v&brbVCyp)tv}jasXjy6M_6LL zi*wygzEwPRKS9kt1b8Z7_t$TM12xm`VMnmeYW#1Trl!5KZz!)7*}BP(W`C0 z{OoPl_Z}vbE$65);8Xh8)(sTVYJ0UOxsBy<1wDPAGUR|7{{Qs;ofi|dq~~;d^nCcN z7D3sXf{T(_)%8J)G#LdjCqORgZ>|GZteQF3Jre5C&-R4D0#sOW(uRlw^8)V>=*e%yRb5x03EqyC(XCbNMKb2!K{K_H`U4KejeQ>XJ`OOF6Y z3Lev8OZ?d$#_JLiKr@+XQ&Zu%!EVD(cmWDaB&NKdmY`MHTS zqdWVlTqn!xEd7xy?ht5==BlVYkrW4l(W@KQU#(aO;2UOmJ_X}NAqTIJ$PZ8P*DwSc zGTcj_6vc1qGPCu)app#w&-$OtGq*ML$s4TsxeEhGB)!hT-AirjnxuB4J%mb#?L&I|*nENu61A0nE{D z7-Cq873+s;?aj1&aUV#^o!hacRi1+GC~;@RsX1h28^^TIi8Q`%;JtclT=V2*ybz|A zTvx+4`KhRg&LH}cE1}Hw2Odxd$CNRqhq1zKIaOxBh0o@M)HG_8_ioH;Qn;2S)vwr6 zmZ~aQm7s|s8=E^YBrv&(fSnT69w5a#2`B^XWJiD;!6;YhNfq>QB%sRy%D50ek+Gbx zs8li>aH(14gr1jC;c6XMR>>e&btMO5z)S7~T15q5o`REINXMu$g<{2ap+OC2OxUfG zq2vC{ur~hxr|Q0*Em^YrzPCT7PrK`JXE2xnKnR3H*(XVSl0pnV>t9yT=v9nhFa!bw zVaHtC-nTyAU+uo2>g=<3nat(a%9UAJStN1P>%8mz&jUk{z#wFi%{=49-Rx@6E!7l- zd^HHv9RMGLQ8JL$zu&!SU|=h*}@(yT%F1dxmxbD-;ic&}^*VM{;J_ zA1!n*K3?iInI*t@APG4jlI@3QwQa0*shb=tycO2bep+ zVBeSEaWq=x`b%&UWwPYVw?n2qlQx^Tp(8FvmBZUPgj_%jgHL>i=)mu6+VY=G`p>T& zAX(%fsKO;ux{zlCrsMVop)fycnpIOZ3?S%zwDMN^5SKe{{bv5&aONO&@_E*K~E9C zx4LJc-u%g1-Ygq^C+)I@y1R=2jS4ZZ=F;&h#F00b?#4p*!`_-Ocjx?##YT@ZnPUdP z@_Xl8%2|7kBwY?n=nJUZIPm*-ckbvNYj`Nf#1X#L(x1})S|jnfHlxQ_mbN6@44>s7 zqF*-57i37@7JNrX-zeI<=TB}ocG!&(qO&ptM72;VWZ3DC1I^HrX zL!Pb(pY&n`FF|n03m1S(m&hhkS^o9w&tJVT41am=KAq&(IhLYHad*YE7ewZMOO&9= zuRA8K<5cFBu2XwiUNfvN?_LqE(BZ}QPoMtanc#<3|9SAxivRe%>+-!T5X>0zG=Lyv zeT$r6t;*rn-H&7z%~ldf#I6KZfZ~6i#LwI794Pe4 zLS}YIY)4Lx>x{ls;n?fxJ9gq31ePiV&`&Tf$9OnXJqVlkSO`f*`GLS#!2wDK@<@8U{C* z0PCJ@9dMo1<@!ze*%Yt(jRU>yXwpJTtm4y!QXm4ilCC!Hpc;ZM`sS>X_6Y}GiaDI}|N0OI<{ zlY+!ygAXU4Kb1km?xa8vs5VF;JmfItI;FHL`-fhUkrg4Ub}+HpE_AVgqZ;-$U>s5N z?Qwo0J+kQtFgD=MrH>I1w*-4Q3XrNFFyY?^jLn?PLK>?Ok>V#nCnB8_xrM2mqWehk zU6hC_=UP(0hP^{&RYfY6eQWi%Ul6K&B{f0n)ZmRBb{LfS@~v5(l>itM85~qTL|AWg zMqbvm-AaLm)am|UNE}IA9S{+0SmOPH0oX^JjYQQZ^vc%TW!yVmLsw2xd(yRF_mjK} zUBJ}P1uO+jp4IQK5n*!CoS@5miEOeAktk|!H3N2QI>g>aWBF7}Mo0Z@c=7<~osF#7 zGYN($IW9OwOVkW@P6;|W0d)xHBhl5Vl>~J{I#?c0od9nlIMAL>_mlP##bLA6!h;5R z`qY23M;pl^f)XLxesW7nw)rk{f_@D`3nWtdi{YW`PyV2*s}uwdN6 z7i%MPMOQt@`;Q15MAK+cZ@W%3rW2thlqpin7*-065~Mh%c{TRmV*q4$iUVo{GmbxM zDGCOeITH1hmZY-%D6e2(QIZ-X%GLmuZazEs~=VZt6oL zp9g;AB0L30UCpX(<6E>I?v-siNS58G*aiXsIr;z|`PH&-oUEK<-T~#>-A2ncS4|ex zf-2^1H~)tmPK1?oseCNoX{E%yMA=%R4Ks{B!f0ypI;%H0-|j+S>d^Q}IH`bTmwd$s zW_nhpQkfX9U8;bztvj$%v6KAr8(cQ~fUn&=#o|0EFDN$%#=KGR(Ae*RV_~omqJ=^~ z0%N*`4uDK#$lgR zKYt>2x%W1wb9cH~IvjDlBt(f9ebvY5$%k#n2!!{J2}uOf(h;8Sx1iTEv+sK<;I$b* zP4(lmme(Z`eX!qt`^~NMGaUfNp6&^eqisCnT8|1?>c97Q^aMr{Je)=R=!uHb@@xuw zvSpd!EpJ$JpCtFrOq=MhJ0#xRetn1f$mECbzLC2VCVfftd+A`W93)B|A3!>rlT#Ud zd~2h~p^7$J;^&WxIInqrBm8vb2KolCDW(|&xhaZr5;4QrP0jF#FDnH~tKIrf{n7*-;Mj%)D1~;VMo50cG@X~uYxoF<|ahzuhY}M z?_HQ}Vk~D+)=rm&uK7wu0#ShjQWtX7!Tbq0s2NoaRW)o0Yn3{yv@=S>0k$37%^chB zQ&#UpZIaFx{nZ{0XxTqu}0zLB!hss15k3YC4h{f zO11!Z@+j5`2a5>DPo<~bN*&0L@{=zVr^=N(sWegW4c_nscu zxL9e3&wW)fkfuV}X9P+10{{@uK$rKW@KuF4!3fp3iqV3A#ai zvZzr@K2Bvl94=-Eq2?5`LtMyVAAe%&s{FSqh1!h)bMDeelB`lH{-=vz4d_rh@*v z(D#0mioN5y?_I|@ilZzqgbfr4#+b}@q98Nj6m;D*E?Ih798J6mqJfQe|G_V3O^)Hq zz)A~7Pw?$0LM2x$dOHU{t#*?K!I2zy0My!$G$UN!&;z?SEn0YS_x$d~`Q?Qf&|aG( zf!8pG6M7J4CUmNOM`#_~WEZURMn)klmjmla_-e@=AR4xjf)ugSbqxBH5P$G&c^`TI z7S6L&ho@Fbfs(jVlY)_cODUe8-MqN)7_l95igW<91Sk1)>%D;{o^|WptyT~5e4msf zZeusyFreh0TQ$llPY{SYKeJ^2PS(#U9j`zxcht}DG8Z;tXObsh9Q${^cAu__V0JAK z?iOfwK}E#c{{2Np!-KgEI{kUdTVB`-gWV}8^P$1U8nANcZ*n4nc<7txD06B~ew{kJ zV)~BOwh*QA?Qo=&9wSOKHs^P4wj%jU0G*^eTd}C|f4Oa}AwvtTz{jAnb3r?Hq6jK5 z4ivf2$Nyn+d_q1YM0Gq=NzG7f8~vxwMOBhdR4y*;L{+L2C9~)1F(&Ko-?}Upu(7oJ zS61n{O?T4$&Xq$Nj=#8f?}3Dr6?u=9X9jFckvKAf31EhM=gviQBjY;3q9J!OjD1j( zp@gVP=Tdn{_Mj#u%@Zqv9P9zD=%KWU>^7G-A9sz zL6g{Kt#0(cyMev-t5a@^z4buC-K(qgN>JRj1|oGP%2H2n7;)Fhorm*lEp~rTt8JNq zm(Slk{r-m+kAJj4=F6wgtdu0K-@SM5-lw0PU)|$jh%fS!1IH`%&$U}iT2|eH{CZU; z+dsEnz8OkB&!0cjW4Up0$J75$oP7V@O8ehGethTh^7sGfkF8<4sNsDdcQ8vD?u+nl zk)RQ%B0+LZXyj$P>Cp*Y;!&=y!A{Qxm!`^-H$*1@c8j}-kbK?Tic*E@b3wV zx-8zOCw1?cV0BqWZvg8_fRaaDJ2~A#5|8V4oScM7(1~8cix1CaAEfA!BV6d7;mwswOQ&o0ag_=&kqQqEr3DumAnq{={>nSM%_dY`-=MUM{`vg9 zHa53&0@3BBjL-aI)Wt59GP~ewdg23O+}b<|Lkpl$QwEQsnk|?LEcJX1W>^mc_7dRQ z<4r{oPF>(F<-|3tB?`Grs%$?%ZOhU7aqKT4_9^0mgduI%(Rdx5&0G*~bphB~k}T%2 zg`&{OGG-7i00zktnA&m-2_T`M zzk`|{Rk#1a#l(?}Y1k~~*Ulv{SO*vEZ#HkIE`b$i_ zIDoVomm-xYaEPxd7tzzY%stVGqx6KnX*-@2i0gTyvUyy0?P}7p1=ow+R9WmbnUb5& z)DSc0Ry_KUJ;e-ZJfdSvgdiv>IZWZB@fc2MyIeuwlJ^L5HW{dw7^-!oRnD%wVkBJQ z_TqHqD;xq9>qsA!L%xvVO0^Q@>Bpz1;w+|M%N?!~l zr?(ZOp^|*)`=?0BeMtnNVaF zP;-G?&Sq@A(}N^+02<0%d*P*Fb!&4lv?rv5ECSWRD1G2MDYU4PwfEGut8$Rc`G{A% zcs55!l7~Xb+*d>-4MNM=2)SU9W6ayuBoP8hZ&YM)q;j%|?r6?7-F|p?8>yVq>W7jw zK`BM7wU6XkRsko{qX8xUFdMmW(wWnT5zbyZsYq7GDqM0R8|XJ98QuyxDxkF$PcoBz zoeZK!^E&0qv0Ime9#$UKNXkWKgAf6?)n!lGt($b2l$AC(8{H8~ge&#vy?)9Iu?c69bZu5CiLNX($hLCqp@ z?}j}6Z{0X2VjX2~W?9Qn??T@__^9P;o<+WMVa}{OAbK=h z^2u+%`Qf{7fBRQ|_1FL5-~aB{zqazxSAX!U|K)%C-#`EQXA2(Uz9jdRSHa7_bA{6X z*Kb}tE8&0Pp309;pFVzO{ro47Ey=?J3Gv6rPk#5^x6j|a(&~Tm#phpr{q^TxeBrHU zEtYB_CU@&hh+kgbBGj*6@E`M?uDK+wpLdb&&E7Najm4<|N>5&JQMGy;f*ZPD;pKrk zd2cS!r}^rs>r6RoE@UzC>pfH9(GI}6?UTVw21X$dWeP$b2oz+z$gDTvu#OG4a7TPI zE#|AqVp5HfF{JBfZ?AbxfTsdBSnc5puokGr>hR`)z0i=vH-m|BvjRaw7vc?H(OA$a z${Ji*Uex>FtLb8)9@8Z&LR&_k8wh0GEb^B}%nABE9BDk&DmW46fi4!o+o_*6cIw(u z`hXHGfjI#@LCkSFQ$>5W6O`4?RULPsf`0qV+bUP6J^BCTnlbLFc26X(6VP+=fgelcr;(~(xCW$DgCO;A3#%nE%l=D2LYmZyNC&wf z1m?gT(qU$C_9nd;#$+Bi&h1D2AjQ*1aU3kieZeN+S<9gHjf26&MOS~4(#juUvf_Y` zf^Z>Wnp1J7y8>tcR`dj>c;I@`u=9a0wNrQ1vekng7ZpT(|Na(P+lhwL#Ukd+E&h-4 zL2`LWGMV;9fZf9``&lK|Dyoa@&`zo(amumpce(lWFF)Y&jknyCMjXBEr4Gq8Q58SA zh^5)a5~nlrWdv$TGBu}aAz`n-)o`cU@|3)M_fb=Mxdq+NFkB{O-7&HC1|Q3uh_SeS$WQWe2-A1NNo|lXJhR!<1mbKUjwUugsdUyrieJ71_c79;R~TkDo8}q??> zm>L%(()l{hGQMVwYD#Y1YTd1RX#_RoxeU5s>z^n>TkmPM21czwH$DAD735Tm`h?Tm zhQkLZ7HN~^Bk4BHZgmOF&E9G~lU1t9U@{r*9`dXx0xIf!W_Du3bR|kb%|slVNhk6U zn9eHQ=|dNhP>*B-+###0ysA~1gH#iO87d-H&sT-}0Pv5{3QY~I%v0N`3{x11qvI6S zQq^=Hj*Pr0sPM@Nx}fL zma*~U;|w$MK;;^TOnc2t#fu$#XsTKWm=gl2nn4=Z57EuS4%%+KMjk%6PZ6`idWU&ML2TBLbE;+;$hXBt=DB7DhHUkYp5?h&j>lyG5 z7w4@S^g-wM)w7q+or#KmWzm{fD>CF6*YRx5EH}TfhAE zvuBoJz476Acd}7`0ffU}yLLwyvBHw($+%N%iFw|4GU6V1@ZLbHx}IHlzToW6nZ(^h zAa7n1cxDE)(?b;Z@891$$m((9#VQJ9+~T1xy|~znuO=$iomdwx{qqWg6TKxd%cB@x zQ?u>AA^(b=$Kah;-I?43MfdJttx@6`93<~!AGv9o?$1-ijf;shtW|PD4Y$0L&ipGe z+jW?PY9oZ=N^e{dw3@Z3J9N8VLDVIN!{H&ya+Lho!1v$GBNH~ zC1?L}<1)ETZEpVRauxjsCT~2B+Y5N%H&GzU&c3j}-__`{h^m=l^!iaHW@{*LA9OR; z%49Tk*xo(uRr`pvlkgaSNi1UX&ARMGtJ+>yoELn27kqN1#9YQf!TM7RvsBpvC*)v9OHGE!d;_;C!`^W z8_x6qc8yR?8yIc%TsEm9J=vVj2D+G`X^h&L8->oMFG8$2by*Ea^4*{-mCYVPs&W$c zTiTyba3)GWimk$`W%(n#K2yr6R_dOFXvLCB`;-oVw9e%uQ$2S3LR6Nkt(3 z99?lkMc~Cr(j=A*PY@2BazqjS$PMFKuxoXTWXi2~2P9r&q{bYEP`M0p;)`*6JDF7{ zA9Bvux`x7uvpW(wcdRR6XKb$OI8F|=@Eyd<>Lr5}Jz?0zszwi;`JD*VId)bz=pQ#6 zRCs4-F%*HWdXyfNY_5h~=vZ7VhHpOy1^Jkbl&6_UOvjQt<9fqsReo3qB~&KfYY=1z%ZqAh{#O3L);Bl6?Kim|=|>|3OlcOJ&nmZzDC^`Mif7WhWcBD!a?N zYq6MlW18= zx_vz=iJ#r{Qc)URXM7=ydq_5n{cES+Sc)gs^~P>*;lL!Xp_0~l(9uUaC*mC*jIjb; z3`PuG$XfeEbok2)U79dOt?-Lt{Txa>7%hz(; zqkhrxpk%vFcGSg5e$q!3jC|MGc}oPZ+=!i_2gVUH8iugtW2iatK?Pk6*Jucm>*=<# zUGW7$zy7M-nSC^e^ra{H%Ak06*A|GlQ%Qo!A07kHxwO)!68z7sjNVSyjF-9!dTfZw zs1144aPzA*T`8u~gmThCw;x!tD4w=hm{q%>jnHt--pSnrHs+M|B8EsEGq+}jE6v8d zW~`~|FRwpW2QhA$-+%Pr-o2}L?>-NQc<-NV>PR<|{ouhvL-Eg_JmoNNo;{J*{`&9! z=CA+iFF*h4t6%+-KRvU6=*>H=F+dn4RM}HK=o({fV#+bq=AuC1z0|kw-hXiSKmMgm zoww-s7$RfuBuPR#0E-AWtpD@UnB}3 zc<{;RUwr=cSD$|M)%{PtxO@MhTZa~2GWiD3bf3!Fl3TA~doTD1Lg-C^dil>3c4b-)vJMubJmmqWd)}s2?$0kUEeQ9Ez_BgtwZ?Xk{FuqT+oY zx(cr&2Nf?_#btyKS?Wjx34%54XhZ^52W!lPKvi4fR!`Zg0p+h-(vdgpfkIc?WRVtC zIHmyc+QeK{g*9>%WuG=|zEZC*8KV1W`q-t3sY>D6QZs)tgTkl5TB{1{{<`tKhrdBB zP=Nt(V2-iob^_d|wqlqgTuC)zI`Za4xk!eYctDB9QF}-wNgt#A<-qCyr3l(sYDHsRCLc7=Z!!omKq{uN)6ijK5QsrI$571 zhN5Lg!4*bM?`x9HpD;SO$b2|g(d-z;DcKs-}lVUFr_J@~bz}khWZRQ#! zlWAoDn!|mBc|cSJ?p;3t2>i~Zk&pG~w!`aE7Q(x_*fP;;uQRKP7auaX<6(`3y7*dV z!}zjk0ANtxE!6cP^S{@#TO*(is0cQc;dEJZx8V%=n=XHTe)pa)1CFi5*%F9+bPAk} zLn@_qqLDjoE9FZ6vK>r_NpsbV3{&H5Qb_=s zwT9ue%f=4-f9;{mHy*w8_Apt^OW9eEK`W9$9UK1Yq%4)+dmlBZrOS-~$aA7F(IuTJ z*MS}9O24y~DL0|BYZ-E6%lcm5G=(AOa12*iw8Zh*bB%Qp_B;j#Gm~Oy4tp4Ez|~DO zg^A-1jp(9{hf^}yJ5y5SsZ0T|5sj*Wb+b8O)%96&+6;k#0wc@lE2Ik$PKg;XV+wi* z<}+H)P*<;Wc3FmyNi~r>OOwOOV~rlfn5uXOrWiZXX1-Z#2vu>qwlKF~mGufC>Pp10 z!b-xWj1xaq7_loB!HNS($W^M9&$DAg4POC3+V+BQIBo>T^1z}3iczKb2w+pEvQ=Ib zwiBHdEi?hR#$0}|$v@wJ|Gm7B8T&r2N}*+?9}FSCdj8y0n`b|oJj1}vaBIR&ZPodW zv-3MhPCpy3UGKSJ4;mdFNsD!Ydg1ylbH^C4m-fAS@#f|8r)Kk8Vl+bf*q zEAQ5vIYEE%AOHM6{JZ~$ApXgp{j*1(eF>FDH95P-n~|^&2^PyRaL~+&_h(vK47|Vn z_U-LgZ_eMm`SMr4x_b24&GYl(eKn=Q-2||fC&~5dOep{%?iYe zK=R=JqtCzoy-&aT`oX84=}z3axNGd+u)#6)Ux-TAHI5k^0=)OY+t|1_C+oGj!|8xG z-f|~l_g7uA83qs{RK{$tIBVyIXM*WXp3Wcc-M??55nV_0+c!V{@WacOFF}6w;>FYF z&xDdXdc9Gkj8jazZt($2BQ>JYyNTMC!IjL{sETQCy^;|$u7XW+m|ekNtM6?R=S}n{ zJT+Z8-0kp;#&b}O5qBA0%P%BT^A?Y7Px{?=7fH>HVC7Kw%OePeXh7=@9p9A`6?{Aq zoDqP@Su0pm$AxC>lbIUFNs!hYq&>T*pAc*d&pHswiyU2^hjpa3u zBRQ&yhV6xMP=ep9-nk|NP1^^(=_yQ<+jlxiH{G2{W!rk_M!T49jD$2s?13tq7ZDk~ zG>GR(*;{Wpau3YbL?OVX#x{lZIog1ZEx?`XY(&Q<3jCE}A({y-Ngj)_8rdDJbj)=Z zU_^s}RJJJs2)7L3Fel$#>Y(e{A6(jU^p}1m{5%Cjl22KaDX=BDKON1PV5Y&YA3NE~ zju%w08gZTEG6xvntzA9(2hbcQccB+Xv3lESjeMB`R@|Rg+P8P}}z* zyc+t`iBy9SsJhrEoLGq_At;#K6X5hnr=^Lqf1|LT^(H=mIPJ21b+tcLj%l~4(nqU^ z#Cd{S=`o9zqgZ8!6NdG~8pK?|QHzBL-1u7U09sY7XiasljKj<^^0Uj6f)DoMB?1r3 zjygVwsLlkkN)=z0l|@HjEyL7YbtVH-mBSsPXbWR!ot~9-94SK+qArIdknNuUfWX*& z_YZKP^pHt~8?5TtS>s;(XqE?6O3xY+wFBji9Ffd!LNuzp<{q3BCJkch{I0?NTj!Po zF{l6H?EbyG=V#67H*`mdKJ@_Rt@-N?>J^kl`L6d~5YD;ae*5O81;SrwDQ}v(;AES8 zfne6{j>Zuh+b8amjgNveDJ%#f9vrmE`%U_h|C^8N)xmAX@0{zToSZ2n&m*`;QZw54 zWOuVjbHT=M20^mE5vlxW*f+_{Rw@u-P7)ddgfUYP&Hle`95}s{qUgJpjS;aH!-JRI z0q`jk$jagel3J#8f5e59vl!8pjXn1Y|`XFaDPS+ynxL7ks;4JWC`IE0G9&F`KsX@$CDS(xkWTJkUMox$Z zHiAy@BiHF_GN;8lhp5C@#aFPgb)&!1LGm;&+@>c>v9C8}ry=4d>{v0<%dMW#p!bKJ zUtW5;_Rfdv9(HHYjNnchC#I;|cN&uY&lDI%qxqt=xg4HMx zUtC>WUU_|(i87+5@PO(ZmwBZbzx=5=DrOQbI7-x8Hc#v7?u0LzYBm}Fci-ar^+#V_ zGquJoFRFfHVSewxs*n2i?t_P4efiZ_Kl_F0{>70Djy+hD#~7=7e{GBZQC|`!l7IN% zMF|&1*l*tY{PO!00!^y$Ub0|WoPD2ip=;# zkHNG<=)At3j3nPWOItV0%sif`F4E}u{T(yXy93aaB+BVFpgR#gPRw1UBlLeuDA0N7 z@quQ=(qkseC?&3rYPBg^*1n{HuFUvHc^yp5AI_8MGOD7CuRbaB4+Rj#HU@EI(uzVLLMwI-(;x&&h7jFL@Kgk*^Ji{`kpKgbHTPRgthd0)T97n-z_rDr15 zQ6bk(8-1v@xFvbnp&9CHUP4QJC*l^>rpUEuAL ze-B^Z+e2KYp0ze(6(zoi&&kwZVf%I~?+9mrt+iS2!a3&+a^w^B@;kK2Ibv4v608H; zibh5-`);SNBK5?m_D|@$6xQJ!+yu{6=pZ(8po-Wab~d+WYO9;$y9QOlU*_l z0W@8<)5U+Rg%TuypA@6zpyy-+AH_KY0!*cxV^EcY>0Ez~eLPv-kr&@}Z<2L_0 z-`TjwFfokYAs}KJIMEahTndxm&0N*B!DLXJBYGh4wmClrATO<$tq*W{@!;~(5SY0= z4Mj?ejvl)2QvYMDzob2J9%R*0=ub%B*M$~4KgVhvz1TIQav z7BJ?9_BXQrPP(sUmF*j7l#XrfKFg=uUTLF^+#N%N6KqO|IsXWv?4`a=)@mOcWuy_Z z%$CrFvKi#|x~HDA^$FO^lqF(fY0q$Ek(pdABCgjaR3!ZZ z8;Tmk?TEBpaCYWK5@}~~!=j5?KS>P+yX!bTpn|`puB!i|^gbS|omrCjD2uic=+LFy zVV9z_&PtbmI#o$DJ9pAvRb0(soSZSC?J)I@73&9BNO2VsyHk1is_O8-Ls-Y9iic`j zp)RSHN;Wcqaw*=!hY$BCaF3TZ+01Bvx0cw4rb@Einn3lLjM&Np%}b%H^?BVg6RY`} z+d`%~ln?LcddQrl{N(`lOjQOp^`S}k&BxX&TQ+t6%uN*t~ z96S3c?$w7*Js#wJ$04mxR1Bb5!=NmxRtbvhx6iJK=C$GHH}wr(zO5r-t)gH2@{j-U zkN&HB5AMsq35I#sb62b9j9c|TGa|vsXBwre-ZK3D?bW;c?=J}K>h+zAs~hL%BK5Uv zf2gv4q01TfVgFDl3RzD?mpbTd-wkRdyhW3|M}MsKl}3P;U|E2t_XI@ z%AiwFH~RE)M3KZ)22f^jcITXg>(?XIl3tA@ogv&1g+tYEy*1mXOrLxV3AoX*n}d4m zA3PKM{r69dvR_=?ziNCWg)#b4^_C@l_3YYn&(x>TW)QL~`kE~fA4!;N{NB5adwAf3 zTS0}vJF_qz&M@8~l4ho{4`+HwyAkn9XC)6ZBCto5tYdofx><_Sh6G#G@k#iYHeBEw zUVBl=t=nEhy%(M^je-cl%+eVJ31!T=XXh-jt|yq0K=tn39d_#8W;T%d9}d?pzsPej z6M(!dpKi|~3K-1CEpj?L$D4~ftbVR|l+HB=r1Oyzh5_rkh}S9qE+w++`w8rXaOP80bop)&E2~zeFBM%#m6Tb5 zN`g0l6U1mkM1qaxI%;%he~05J(FGg|+yD@`pSA*oDBbDB;3;=ZD>CpMB#mlg`=Hv1 z!R-`)*za)S4wZ}GRkOb!{j?V;Df*~b9@F&p-e@$?b>O#s|N!s zFdj*-CM+ToS{1ukL!kI!HeIOQ7<7ee8kHE~ffRPuZB-n*POl(Oo;#O(l+1I;<4-t8C$z3ZDj7r zG#^%(k~3MuhT_)t!#RjKL>ShKW8g-SkxzfTyInJWE^ z)(j!w97w7=V<|^d45x?Im9J*m&B!^pXKLD^{4o#8MMvc7y7Ag$#*O?C7uKn9i=Ant zno^=CAb_XY%+SMTG7Rh>;Wl?3+isK_&E|Y%tEP`7_&97jL*BY?z+o6$WE5WkA+P)} zyg*Gh$ayqya)`f|A~S z4O~bAzF*=}bzB`pcOl$`vcbjhshS!=@5C&uat;EK|D`5ay5prXUU3tv!u8%c64NDqDGt61)zZqP50(NVL^IM1` zuARu}rozyoRKrGpse6uiB14R&UQT;->W}5D$(lp0TLMgMw`J2t;d)b&5kMpUl7CD8 zG#a>3|2N)$TvN+hmaMA~Ur7^56ME3v&3-ZhO{&L|2xG+vMB|yU8O7@^K8&6(cGh0M zgtaB9O~3KHY<>dJDJ_0H2tq-gWky+n`>1lH7v4<@yubQK{>o;zre7mI7>Fjk^+1(LdR^p_b z+^iy)NgT=O0EL*4_)ANHm>dQ(r=1V+Kl~j9ln(HLJ~$^p$Jq(g<&o)15`g+hFlE18 zu-Jh?2GXd2NJ^e&@lfpXhAdQA57+GX3qwgN zHxSDgh$E@!2uw9e+%_pP6nz`r(`PcC?8pFO3kqe(Qj~k6!~E*z%bR9@)lZkP5KZote--$GWaE*=8erw5eA`%ufP&K+3Tlf;|{IuE$sm!?D`nCeD)u5^8?tbsIJ36v;)8}krE;^h`wCtN@Y6w2;(TpEK@(3$52gr1(hDm z!1Rp9>>w(lr-rJM+&WJx=O`s?P`E~F?O&QhGo4RIs}&z9GisW{TQ}g^?gqd}Gi*JQ z$L~?9qGYU&%PQr1p3iPCY_TGuoelS!%WLGIwli>hQBY=Bme|)HUJ~27twM;4Q2BKGOFMq$;pCL zr1>r}T$^q`6*oIYTb8KAx}=fqxL;hD(`bYyNg6C}GEgk|M|&A-aT76yiEBck9a%y^ z0T=ebjirT`WS!7b-?o@-6d0Mx=YMT@PCX$`7LMxePCfz41OjBVj6d& zzEG)fYEygI2Z3s>Hy6?6{^^~LD2y|@SAKqJJ;aNvtGf@bbN~$gn+LSFo^Ir=E`a2} zewM?O<>tN$v9npptL>3YBlki!jZ#UxJ;A#TS`dmFNxRX zsB!I5_n?Xg*fcCz?0Q3P_yW??o{3J*#Zit-%nMrAa-`biuAIuY(rz%y&YP@tgt^A4 z?B+J}oWnad0sE+x!E!SUmz3}bIY#S}9PT}N27DG2Sp?d14jimtRUgB5VK$5&D@4Ht z$NcuEhLRKA#Mm?^jC;=tbJ7tX2aQlu5zV-S*28YPG#Qg&(wVC5?stb+MUHHTA4TFC zheUTsW(-z0FG@Oxjih=YlT-$8uM5!|Hpe?^FHfQamr^-!8JkKGq133Ng8^|s{R&hr z*-zZFhxgJ%@&rR#e)hZD9;bC5Kt1-(%CEgYqw05e&bFG=nFDK~q46{kvHo=_=4UV6%^CSvxI`DR4*vW|!6YXuE3 z_lk-d&G?9?p4!jZbing+a`eEmW&WE&f_HYD+3r=2`k!ifR7bO#li^EtD``Hvpl$tj z%rXc>cube5z_^w9sS$4{TKQ}Lq(Rv&$O|MM>&eg5UcPrtbP=~pH@8L!v50X~g$pji-hbSrw< zg?NNw&sumVTdx7^?qi99UaTP{l$I`NOBgpeiKV{+MiNTtws-F>-}&PC%Q=Kmos@n zn$S+3&$R1!yR+RYVN#WmWmhlb!(JLdk!i82wX->wz4V3-&1dQ;ZgC;T;@cR7>N5)3 zS<&NWkoypxh#--4Zmf|K83r#(k?9eI zQ9)3hTMMS{3`25OnQj+Fx$lF}6%;980LaOG4tj}g!$4WH&3@0RX?@UF^%RVp!pnpa zh&~782_Q0(OEOJ(t5(%2wuNr8NxHdhTRL1Oau8G*5ESk!G8GK~a6{asz;%@GgvO;x zB=#<^#Lj=D`D0ZDS>DKyUz<$Okx~kI_T=fd&w^gQ_qtgNO)P3+JJs8r1_^ z6;O%zJp`NtPa7XQunr(1~ccwHmp%EaD$_SHyH8gRvlvl~k z6ALRE*IG>PF3s0{vqk5rXRs#JsMY7rl`^DRULNU&1+&EfKl9n+ z0XeJjtJA!>zj1TuXRTZ8RKw5OwG%!Bve*@)ftPzq1k|;T6dG zo<1VMbf6>0idz^FOEC+J$3eWDbX^)WM#EJQJcQ&Ifrqukar*931`(iim_}UKRg5W= zNhdbinih_Xx@sD!gM~`?RJFP*U%jp#Ck8YfR+fY=m)7*Z zynB9O@{h&&diu8^|9b4FP8?$5qZF=fbY(ou0_mZvHKNJ-atHQEDQKghFP#^I%lW*U z^?1w@XKpMM(4z3-a`}@QAt^P`Z|qiB4}Jm6Y$l3?G#c7b^!SDl5gv0k7uWN8`BO+5 zVN83_8NBcHJyreFz-l#EKhflGecq*|u`^0QOeH(y93K}(D9xxWcu71XS9+RUybW}c==wkrJ_nzvT{Z+`c?Pab~aZC4pu55-bM~v?MRjzfv3PI5ArRS2qB~$V@D`*5I)FVK2>5s@J)~gR9Xcw>7*< z5Mz(xKX3hYfArucVv8Yr( zi*cE`gThO33lVN(UKbkO$FbqKU~6qMyR&kgjz;QKLhs=o5sQwT@W|GR;Aw(=#Y(09 z?Ayl^BJO(FE-jbJx0Nte&SahreQH{o2HRTXhw;L7l5HKe+(+YQ~N)5+PTQ%bi3>(+zsX>Dq^`{Y9L0}2K z3>$5w0UA|_g;u#HLiq^|0wQ8y5sd7}{ z&@(*(D(7dLnpLfC{l!xDNM)$3v_1d~>MzxAl=Q*XkLL5zY+f<1i` z$kC9bX<%qds?iv4y_78+-^8a*QejN8AKs~w;s2&Wrz-i)=H0b(^aP|zkQQWf5dHS_ zRin`hAu^1HGk6)<`QU=s(IOkn?lZ7Wdqii<*)|<9OHV?+Q{kk)o35#sc9few>>oRj zwmVCimVSe(*^l!IgLLDD_ep>I-S>b0o8R=l*bld?R(aO|_W4~aWhORKV3Iu;`uw$o_u12b z_=mrNB2Sv1~?WoqayjXZAr4P$yhEB_wx|{4CVG z^JGDH74NfYMaSu*Lb(KQY5A%{ZN9 zo=hL=AqO)FpI8p{$@k`8sY5-zz~tic=MNv9-8S(nDLXHAxk-?kpFDo*=E2j)q-}l? z0oF07WlBnXO0dlX>S3p&7^smyh60iw>!B7MnOhzJZ#oiNZfp^Qji>NdqAj6Sn;kGt zcL~~JHhwZuT!{zCq#m6_WZsYih&-Y}qa^O#EiNYBZf5A0hDY7QX_ZBC;;{o*;!M_% zCcXXob~qW-g`Uh$Lak0_kbL}SpVf1;I?Nl%1;F<9-&Jzz=u`wLU;zV+%ef+oYZ>MU7qcs7}n*tRnKKWOot0P-Sd3nFU zpdVnr071ves?vVv{=fh>a!_`(@xw5fN9n4-kpLio&gsN7zI;y>ovq=l&5Bd!D|Mht zONUnCwu*yTnNAN_vZ8B)qtyEX#4(iwkI7`5>n-BnY&0#0@ipFvzHLg?~3 zs)%oNarL7I)jWKo&RRte0IvQjkisl9!j=Ub$h5K_^_9|DVCJ4Pq?7uA<5_F|>`0LpU8I zZwS#unF|KM#eYcrx8F9bi85KCZ)KJ@q|y1$WLH5Vw8aJx(bEa%14dG{R+SyFNjp@P zI7+&Z?pMqH$T>8KUEoR(Q+w3}n^aNYQHqQLPN3(RAZY3CFty!Oo_ zXzs-|*FT4p#mIXm&a91^SlcmYC_J5$(_j#`ZDW~i6=ewYwkD}3cehecGiyq~8e;M5 zdi*f1@e+ARD$CTJIY376$PT=Y{uxP?dVjD4zsCY@8PG>WTxEWZgMpgpW)}!i(g8cM zihtdV48~#a5Zs}vi1Eat7kA2JOsvpJc>PL7XB=0O9nVP2chZV5JBmBZuhpbM+|L!2lCL8KCBcFW9u;If>PLLT(VF18f*$Cg-B-y`3NUgSl zLQg&+&ul34C2AGlFVAc=q?gk6u+2>`gl*~_EjC{(q>Yyz&)Pr+I!)=bmmqc{B`G$j z82QFh9b_DhAh#Hh>T94e&S~g=nu%H1nOP>%mIrkz<}pG<9cQ6kn{4#Lr7s^{!Szb0 zHs4t7dC_rURq}%f%MgNds+>E7NOi;}L*Yk$gm2c%h{dSVv1{3&!Fd{TP(x>+oW1M% zY;fU7WnamvJxOm-{OgyE^M7r9eGg?O2O2O;{L+hcAHR6={qx7)J%94U6W>oCOTDNNS3>mSibgNi{2&C<87uA z-MMJ~oqmO9TQ}IPIbp7Etr7F?^-U|1%3A0TFi*%w-HYd-5C`3x8}B{9zy0+;=m2nP zxX-OG^k-)b?RUTZ?XUm#Z@>HIcf9e-uYdNZ|Mb7H7=DA(BJ?KT=P>*%yCAZ+4TRe|`Mfey_ryB2Vn!M2ar!wq1 z&EysJpcpzpKfvwZy1Kge>g6ZTy`%W?_b>E^zW?sGzwX5X+?lUf+o^k>I&^2Z?_S>f zieps*Iu~!EWf#;yjPUOxFWm0{a0)H~5S3hnxX6nCi$P~D2 z9-%N(K=^H~Xg-oNvNNlTFv+7^gz_%m@sLqzGK(kT zEUEF1#%&?UV*yL12}IRH%xU8AA$y^Tj*xYbU&qMda$Q`}R8x_F>_6fMFqj3+uC^bo zG*`?H%Aum8)=}O!fu#13R>KgY{VE<{hR3x@0*$U_g$n}kBs=o-;ndx8cyBs39W-8kAFQRCAFqL^f!+^#~o)K~T4629*yz_=fcwF^I*sN%_Dy*yCj4OFjWP$^di14Ni9d z6OoPm=hmm~Fo=Ms?$odg$tMa1Bv7XlNZazd$HlZc;!k=l?Wo4S#$R;kV5hC*@242; zsnFIt?)fN@&8!1HBVZFWctl=rHvHq5C>?EDZnILC&d3n>R$QfpDZlJrMhWN zhC0NlsuSS=Icqp=Rk3?~?I+W#A&CMV7#D$QRJAiawSP)eQK$(ed?5mKz*Jr)?^@^* z?GWH9qng6VI>%_A*Jmq=If<0%aL|G{ob)mk4S$`I3aS$kQts@vK*M(Jn-38M^T~sD zr6MmXItyE@1jDZ(kP4nP{*KB*}% zT#JGlwq{I}z|A0|DGqXIY##YUV<=f@)c3Bc1w6?KBuhGxdC*nL(mfL;Vk@Ppfg0mo z8vO{XpFuk)f@jS7sx2fS#|<;HA_29wuD|FAl+BNW{K3APAU5 zgEGY@fZQE}yL5m_-h&g9DKsEXM56r_X1<+Yy{;TUgBmRbBR#@8g;YWEy$^*-OQ>lQ zF}X2~4Kf=Jg<|pLimNC93g0oDjY%qG|DMve2E~^9lu5>ilGsvV?P=kW+$VDV;fAi{ zSq*S1tMqD2=2HJ->b~j0E!fk?2}6dpg6(kfQ#YcO%%Nd5`uM$fZnyQt??DCIFW(FN zNhndXv91$|nY*{FTE5<#ZMV}nKx_d!8E7V>E@d>BAT?oTq|k8gPO>uDe|>IIBNplcb!I058)tc3L%U7Pfk@u?$E$(V$>w zq>C7=eBw(nl<$W}`|rEp*Yb_mUbO&{)mBRVTWQ_PkC$SAUv^%nLRx#{T++IoExB7C zKu-=5E|K%1B^0waSsyy<43&bxopM`gnL2QOt#WZ~g?-%(m{@N^76>T@FS)O$2pXZba{$jV4v2u}t4z{D z`o0e`==#murudU3Iz|Z@WVTR}XOEOOc3M~N(WjsO$shmmfBmQb^q0T?^Q+5CD*g61 zzr}MFo{N$YX?cC@t!tNO*WaCecy;^3>l?3LvYwakUze&jVNjxtfJ-WPm&%}Ap>QbjEL*>2bVBf4hkzl@@ zrIQ~%xD{buyXowv_@J3Y-g=>xIW1407~w|njT`z8H?6`mgZ-4Xb#j6|c zTHL*VB?duK+QAdw3&-MW`&l^z>U&@uM!1zoppdQu^4^n$Z9H%#0XFi+@C zk*zAPp1pazD3|rsPQ1lV3^QL6RPwDy=?jM59Si- zBN+84VlI_L;p?{_Uc7w!yeGZcMXhaZR7Q$jA}>xI!MIR>_1^QSnQV9^fpa66gfpd= z4b{+BusJ|)tcw1oW!6Z`BXu*ndI6=YYZ3vPM-|HQaw6hU3rc*LOBJqM$#$mUX`SBD zXIF4oMPem?C+sM8=qugqr0M{G&05-@?C7~L*kM)_ZgI$$6O^y3R`G3i3K|h{2;Eud ze5wgW_zu*OkyJ+QM2Aquf(I%Vt&T)Z2^S;k#5ZO@FBSmnLfZzV))-|gn-i!ba15+e zaAS{T8JdIcBj-ynsBxw94*VudyGdcJcK#y+e&vM0+vlKRq0cehSN344okSF<;vr^H z>*ydMtV&GKIb`@(^!OnIrFSk8N8PGBSFN<|JEKdP?FVf?C*sxN{$ngLt)6p7>F6Sj zDjmS>4@sh3sj%3M{^_h|=%pF~h#T$SnSp>UH14S?FZVMyZ?k{W8jUnnF?tHg@N*%t z#Ts0)e|^J{9V8^eXeqYB37w{cjHeleSW`$9$&M?qs&TaQqY{Wrf3&Z7;g6+lKbWOc z{s-eDNJlQY-(KIKCLwXC$IWUi=TKQ0oYR55iR`d?#j&{vI6n|*2?|cdid{}toG=c; z^HYZHAV$L*yJ@3;kdBN@3e~5>12#K2*iHmWj^sE>snM1IB0cmmx`Tl}E=1&d7v&*# zyNwfj23+3i14zp21T@V*@cET?c2OM_!SM|?Di{;xMTge(ozOR*TyKI*kBkIP93Rq; z!NyfHn>q_6iUG`-ViGRcI_@XwCinnpdwaC3@lM++?ba4Iu5c+~oJgU2k~j|aVMhK4 z^#$z&3pxOUo)I1Yj3GUQ@^q#W?IRDPWI85y!dG4dTG4kF!uH506Tw}>x?$rmN5k7$7_*YhWuYKESJD!(=JoDbj~e&swX~@n|1p*M)Db12&^1)^ z_NGzT8^+)u*-dXLZDx*L!`;;XGlO8Rk>rD!I?>1-2rGc+8|55pkpBJ z9#|wDX)3G8`kBUT&Z+Tf2-rzEIElaarbRwTL5)`K7UD7VP+Ugd{9`Iad<16!nYS1b z4t5ss?RI4cDLk2EEJvs-pzT^;>f(L3fO6D=uVU5c!Y;3NI0kC$C{V*<;8xFvpUXkf z(-7V$joApycn0&D{%8}^+>X3{>(M8l|NQ5_|A)W&qtCzij9P#A_S=8`Z~m=L&}R=H ze(~j(j~+fWKh*-0#FrQvYiEJ7u-9)d@5;V(Ce3}f)aLUi-=o7s{PW9ux9(gd_-=EX z=auccRQ~Q!R0GCdVeOk1g&%WbRCw_4>i#1y0kGbu%&^dH!FfRzj|DlB%);FIF~+as zV!#JYvd(6wGc#KlsW?)V~nEaF2a;95%E-$anuI>}WCtsbNd$q^?Pp;j# z{rkWA!+W28apTU#b^VRA%j;+7nrF_PxF>Y-YA1V7wGC01t)>a;L4?j}w;rUtd>pB7 z-#&Zu>Y4dPSC?NC>F@vY>d_aTmXNEPg#$?Z)HCu&9%cVtAT05AHlv4XVIJf843TFY zg4u!^sSrf%V^pCDV=eN3cE-F3h*AwT@CwSZQq$9?Pu*29pmF!!m6t5^bcFghZuVBg zi`(zs+ z{Q~?v9Nhk%KYHh;pA$`hU}V<}y6P6>eiekn>EJ z**=d+yz1M@gHz}=zabq}(>vt59wfC*}phR zcS$>CWS&AEcqKrRn})g4m4+W1+m5T{4p8!=Ca6ZX955o1S|t&nSql>U2MG}xE%{!; z>OhFzjl%ln#WiDJD|LuiDZm>uPz9Yysj699Ew&9h=erv4C`{HEsmMwURgA@*;vJ9A zGSQ(d%I#9;{dE9M0pj4g>8zPg6bxwE$-5fz7XsLGDESfF`iLP2jwXU65Iy}@FG>a~ zF(_6Y@a>!&x1;u>+E)e&2Qs&OoWk$io-L4}$c5AexxI$86vjdsEMgDf4d%!#s^dCD zn>HCh=O*)=jgH9Hf>SSWM=*)6b#j7lY$ZoZ%%M+*1?@JCT_MnT2pFntunZ-P1(HE= zHYuxzFf_^fH^zDNG&_W%Gu07*naR8pFN z^B(DQzxvds&Og`38p1gd^JkRBNUoy_74yHhJg;*=I@88oXb?-cKHRn@d2im-%64~#MCW;B1X0_pFiAJFplC&J-13sKH&kB3-*9&H z6dU}?V`r8XHaSH16U9aVqZ#vLs3gW3^=as#w4~k$s{vuYf_au)gj#Z}+A+#tM;iuS z-jSgufHc)PxTBn5t2sMw;Gb$5vh2aqbpz@aM6wN-fK?@EnyIq?kF%0Y`;qkN_|+;$y_Y;$c#l9OgoBPArwnR%ic3wscjn!+zxhQ-z~) z@;hWQKM$vm|BSsW%3%$cXs*Lq>c8N2TuZVC72U!hF1`(vX(Wlm}yytlsN}`SEn}y5EIm+2(oidb| z^dY%q%FXCf$b_*SqiW{=>!H8+RXiD)$%M{_5Vt8y6QJJblWY?_6BF*-d~%KU*Q-!g{(uHBC(P z>DaPn>z?*{wc?itjqI7efy0;eEks=(xkNy z#4%mJ?9>fXeQV;-;c>g7caF$Z-njGe{MnDc?FQ0ZpB8W5g6Y8n-9{geJ%7y%NJ@wA z@#81I`{ujf{_c0*e)r9fPoL^{=q>Th%^E703m`14VDiC=2w8R6ME|F|j|7t@#(OR> zMI;V8J(1V#20e^ds^sE{iZI!~$ar!nuoK`M3_JFCRJxp-#h_X>*>(@T=h_QQ<9;>( zUzfR|`pINYeC$p+H3jV~)vodanhK}l6kC0pLK&En^{G0@U5Vki#-JA$DyquR*{!5< zpcTuw_p#+S?d_M?;wg#6NZ+!75{Ef)6fXPE5I2}IdlXbhI@`?1XJ{0GpzcYmPqmWdPGI8%FRUx>LR|E9h^M0jr-aj# z5ul*1d8#P!^+zgI?xMZG`L5)Njy!DyxJGy@S?S7F(AOdJ_4*x-w2@VkhwlSm7gSCM zz=djr87X9c5waA`tJ7~cyY7M)-sJY+29I~?1iR8pi4E9$nViZ$z z*bKODk?wg(-b39DP3Ny zEDnj7DiAxU0=D*}6{ULoaid3BkRp1KZdZue*2PDQ1wW9ZM;T7GM0Z#s6L3yu26RGG zJjumQB(}NW>?Bg$bv{{w8Gu#Vxs^KP;AG}BLkr}14Ou5u3&%-cS)_3gYfZkpI=lV9 z{ty5CfA`P+O!p4qG$0$cz@K~2pFDiX*4_Dk^6aVjg;&WQ(VFG__{mdEjs<+=H1dLb zS6BS&>5G@Y`OU9owx53bnbwk3k&k9q!$nwim2SuvjS(8Qydg&+bxor_0YQ^K1;O>Z zZ-4jWlgDm=YuUUO08vfpGHiC^wsk@`!p!_jf(S+PW~z;jiseK>W}nYqJp1kU-~PwH z`Kv$w`@bv8K{soS|5*b{(BY*$SatQ3*Pp~xN5oft8PT3>ZW z5*bkKT9aVdVhhQ8W$blhe`V^s3E=D+=!gp9=*~;;vpBonp2Y!5BHSzxZ&8 z4*BZsXGZo(fi3EIdsUcKa|!j;%V&?9nJ*u1y&deB10yxOyt;qy{(U*CK_z)16fa(U z!{^A^3ZSI*N{brlLda!%I-!i=&gBP_e{P>&x5iORm-a;R+%37Ln_QHcud%3s@o7*eOrXwY z^hS;Uztq=w^XA8A&z`=1@ujr-XTSgOi{HC_X}M3&zOt3-NR@Tc^h$KYx(^_mcHh7a zk)-g<4!bctFFl8~g#)a)*&7J!ljP;doBwA~SVUPZ>O%KgPd==-_2S869SiZ@oeA9w zw*UOOq4}p|&RdQyNV2j*QLo(UeQck9_SvToA2gHaS<3;L%JcpA->D9RxRf3aYg)*0 z{~!~W0rbS3w*p5BXHwmx!ry-TJkLUaE z%rEG*g#BzZU_ZVl5-4rI>0QDe1 z$&K&2j?7jEqI(L3+YE!{6l~Hgx_2fHkdR9qfE^!uzGq!sRw@t`cA0Zlwou!F?etSn zU10rq(t%5=7Oz5m5CF6bBv-uydIYqR(=fRf5y$sJ`P%17A0+JCxnl!6wH)LXAgTym zcn^B$gaJ3gY*bmJ%3L&c#J15bQg+fsEmJrma$CbbB~&6i z5Q(4^9b;+mj-nBj@9sz8umn&7lQN*N4e}A&4jvMevnw3#;RSFZaDcmBp%Io+J`!yg z^pc8`@Ku8T=vP;Wn+f4KA4dgCRrNxR_d9Na zMo;~Eg-GjhNmF{;x>M6&eKP@qyo|zmU7mzTq6VZyvzKco?@HLzkLj$V z8>Qf&9v~R)VaUT#i%Db4$w)!eh97IEoeF~NXZ`Yqx8#}cEY4W!uh}`$mV-}>j#VI+ z=os+Ss}$0ACm?aHr{L~8iq@0N^!azW0Chl$zl~exS$Nt{?TKi{ljst6tR{y^-R_~; zLyCulPFJS>uEws4N6ikl2TYgbDz2=eqb#falCSK#BQ;1)!8dO{ySn=?{`vp>fBwt= z59g*F{Otb22M->+c=k-Q_u=k6Ay1;Cfpmu(X@o>ps@u{}_^DlqyJ;6Ub{_*>#FSMh#yemwP{>JV5SGpVbG-K~t zd)qAJXKt)sd;I+Rop+aKXZOzUKKkO0_5bMN-~Fe*c-qwIYk4-rYh8OCfbo2To7qYd zt4r5(qD@NAgi2I1r%AT!>K>kQNH8zV;S>Bxrlkn1_atp6x8K!mv%b)kE-88VPiLbH zl+2$&D3Mwnvz9su(Dq=8bSqR992ArtTQ*SwP-d=sR`%a`ysp|Y{4c8@cN6m!KUq#D z4MiIpM-LZhibN0ozO*)xf!0^%jvM~34Z{}{i`6hRFzmAqa1R*BY?=bp;!X82+ykfu zND7%`7Zja!M7k6JOnxiKNK>6;-OG%K$d1QP1rU?hh061Q%3Q2LMNGJVh2gGsD zDUGkb_Fqt#u)?6+uBOX$)&(bQ$zV?b5_LAv))fU3^nuEi0ECn}wBi$-%DK-Wc5&(O zWBI20E}wn&`5*l9SD%0Wg@n@N`2X~mfARPK@b}+*^NqD@@Px^`mu`$DPHEiV|K`^Z z@0*3+qr8TG@4P=tRW27(8(KC}t{0Gb-oAMM_NDOD^QJ^or&hP&DVN!7xq_W3B5n~} zUR~X}u%`WO39wP}|BtKt`o8PP4m*#W69E!zk^li_cDK}Ob&#!KyKRl-HS;$A=qx{Z zq?xx_^I%K1EVa~u8AK#<&S^g1T^F?1oZtQ3drqCIoom;waO&7-$U+_T8vwTTu3c!1 z&|6hvKwB5Wqyi6y6;_}I5PNb-IN;%|wMbOhsHDw6o7GCAUQ#`RBHYY}u!Q-7{&b=l zr8MTy`*agu?D)yP^Wi#zBhc)6#27Xehg#fAt>McsF)1YNYx!a_*|z1;GyCeax=BaJ z`^F9(KYsM>xAz=8h`rh{1(n`JTuoL$17H|NU7`RB^Pd_}3am8)jpCTa)CEAawF1Hh z*2$E)`5_#mSq#p4ZbA>O?Mz>goy&VWcRYM}-_yzZKPrz6J#D&w|KX$U+wbT*DaBw| zxE_cNete!idqgg@bil5z2M)gZgSVIpPTjh3-KJ32Z{56Mk(26J)L=9E6-DZwPc zG0m-84tVX^{{20lV=jtUwIJ&Ly+;r3Tzj$Yt1my7Nhi?57+%Bt`WwejzH{QE4?hHk zL0-Fl_2%_!dLPyx(vnV|%&;UZvw!uOELLi&VO$bj^_&~@jW0ABpnnx1ir|n(pdoR; z`B6lw+WpW}nope(O@39Qnn3ktw)B>^(#tUxSC-b(4`TtL1*%2|$}`izk*!IVjA zm9mGhvdt+k9Y&|cH!DvWeKM5^kiO4awpA2prNGKwGt9^bIhxTUF#;#e{qGN_kra*e zH6g!mF5J-)G_n8Y(*m|~$j6_kVL=N%8BcTVVl1w7Ucj_&)bKJNT^u}PaiFAV)l>>d&a+7uk{=!~ez(|3OBl@x;% zhB;P_a+ZDakSvEDNJ;37h9NJT>trq)5eU*`Stn($fHW?HvH}40)*W7qh#>YV!YJ-H zz_GgyK!cBs{Ri`@B1Qgn!`gxGNrrBiuWcHPHfg0YfK-pn7vKfl^&sgyCZxhT!gcN# z^7>9nDJbg5KeL;p^rlR7y9qp8!ImRVB-FZ2W`fC&8I}(S6IeE~u3FqOam-+yoS+r~ z*EX@hutXhhI4%FM$>x_L&@fZNn z2z;|6a$7A@=)E`@)~L7T+5sF-7Ih$aOiq)kezp)|MvGL|nzgB)Q>Pbjg|Jx@u61j} zla>Ijd4Lo(3uSaBEmdg`(bU?L+r>SFll5NeBRKI>XKd_tqn1%nR+A`_P@lAt81b_t z>oTP+O(wIFT}#eSWJ)8{CDGtS!=zOvW9uSoT_(2Wz@D9d^{@Z@fB#?ptryonzH@8u zj%^ptetYl1gLhB7s|mPp@d7f<1Bg!q7B(iicI~nirrONcj~&~+dk@z13hv&!Cx*Cg zM6ERpJqGBR>KZOv;Mwxx;X~C={j;SY>hIjWZJnzpb8w~gCFQf{&ymWhcTW=Si^*YqkVpNtZ`bzJ*WafT=#U02NSD{z&TC zwd^<|Ga+!Yf7exzeiV&R-xFUpHc#610Xiut7ezGo0dNM1n^~%A02LH-_$~H>Iunh8k#ap$LG!p; zXZ5L@-{Dn~4#CH6VJ0U-Rylc6;FmC-u&Wu}3tqT_KLT>T>Em{9idH8^K{8JL8gWL} z__=O701y?TJ9ij_cy$=Qri?SoB@?WHul8n_Vj zYZqM_=CE#uj79IBmwUx=wnUwX$W2YSM>Ua?kv2Ax%Slx+CySB@wqrlyRY?nP^p#iz zVG4fSl>NnvCo0`8UDP)AQHXj+_%p*}nHOk)5`IH3)lMIdnIE$HNcdD&RFv=SAtnSK zdK7x^F0W8H_SW0B{W11mC$2J^;#|8TK*50o%|)oHF6(FCRXPWhxAynj*3`7`zI>m?;R z_*B>+3W8Wlu9HZo$~oz@&#N;A;i@b&-k;x5loFkEtL!VAO9oU;FWdSb|J&*K!Li4( z=PgQQdgCMWdrS&ApYMG^fI^g?89ZGNzP=`Hb7v^2+E&}k<$OSv?G@oucoQcfv=so2 z`e($VOcjQ>oN9&YW#!DK7NN30D;_zspl&`9J=Ct0$d-?*OWp1vvI{C#HfaIm*X>|M zn6!1y&4vbd0e9QC-|txjXk2KeBqzOrFz#8&X=%V?9s{tEi&tsU?Gk_*h;l3m5mopW z4Ztcm;C<)Dv(=$uu3-#s*h4%lJ|0y81l0p1YG9ZOmh6NZ z;}QbFsg9`f%nD^(RfIsIu`5{{sZOEnFkJ#*U`I1XjkY;6Fxr-9Mrnr08vp2DfHPdh za>lG0hN(kHbqviB24Wf>EW@e%@~RV(1jx}asxBesAqbfmB}R>Pwg|Y9zy)HX+ls_P zOw5^Y!VB0sh8S&JYZRS(LX$(XrOF6Y98)*P+UsbjEkQqCiu((LqHZnB;F+arbY@AA zjAeow&s0RjOc>rV4HLVjjm28US43%J8MC61B_LIfG8$LPe#LDNPZwn9rkn9-iyy2i z6=TGa-~;d6D9)?;kNP(bSh%rg1_*IvD(JZ0drVp4VJY zi4jFIuRFz8q#!4}93`592pRx7@CG$5l-I+6I z&i?JU26Q`$TKqp5*r!#^3M4V&^I$v=IWg{Mf<`l z(GDFu#!+y-$4{Pm?rKyKaN+ak|PJ(qi?7;&zG(m>IuQsL)I1i^6Sp=jIHHqyAu%LvRGKi_&JX2`jk#o@FcvDbS&v6JWR zo7a1J*%M>*_wL`hT|Z&Rc6Tky^4RZ<8{d6<_A77&^kz+u9X)0fAR{b#C+JbtR$t=A z^{d}q{_f75JIw-Fxk=7q?rgD#(d(`m=&UD$m0W;HC931AAL>$gPT51@m+oD?`W?q^ z?b)%TmI=M_2S5BH^Fp_7+&F*n+?6YrJ$J6gsiS!@HwQfCYe$$byJ$t8;m#@;w<)18SDVpYhk96m#}KIH-;-Qao1K*d(v4 zK^)$?R`5gt2#a%Mb|63RGMa#`5J%gP&q25#>Xis99W7FZ^6Br)nFIuOBf#ZLC~_5U z#nmd}3cIo_IhXi7u(DSsK{j$4%jNVJ$q7g4gsMtPQ43yJE_d>+;|F$bx02PWTH&H0 zd<*vtJ6W81_{ibLbSIt@?t84zPT@M!TlenEsoA#Tu?G_k0hxhrtkDd&LB~Ufk7zlK zfIZm$AooPVPoLembyH@Yiq{SwRu@s{ImP;1#NXq%+m0SRa`?aj;RN|Td}+DD7R!ka zA3AjCz@Ziky|fw0Bg<+nOtNg@@L|iE_uy-MY4dp^5etDdbWj`6YbfgF3!Y$?r*(Ph ztYvu5NMb%!Q&3G@!Y;8qNv~+!2F!9Hl(~$ckXdA92&WLaaG3M)9BAK+xy1t?8+i6L6~(-WS;wjT2A<+3^e^?{oZi!vMi)dXAzHdcNe6b$#OGuXNt!v7V3$jxfWRq zE#$Ht;xn(X`{d*I-#vYrDSrO@-<>~yPMF1nm`=mr8l(|DgsDFwMz0YuX3-N@TRcK@ z>GEZJ<-GCwaZ60hB((4&q!VfsCo7I23sgKivk&Q3&(&q?RnpJQ$kAL}qfIQl|A@xV zu02Xbx&N7}G#?~SA?JhpL@ww*`o?QsFl-FkgC8uL`<*a@?GPp$6N#%Q^^X|L(?<_H z^ZKZkX8ON6#jrDO#Vnf~Ilizze|y}jLA!S8DWHj}u3f!ieT%o`?tg8|*4_KJ?%HqR z*dWNiri4(Agi6rOP`3(i!7p+`T&rt(Uv7Z{+qPIr$NtdacTS%&XraB2CcFEv+7&7y z*eaH%PtreUo+#@R0ERD0;3BIWn_8#+&%Glz#)p$Br&zYBS}_=T>x}yD;q=@@t+rKZ zEQ7wZ6&wO_XRh+qIpi}9e^?QJ`0x==Iz4&3(}UQLj~vk*YPX-KPrMP>nw$p@9-8qS zcCFWcdh_OWUf0-&*TOLq3m7aHGDLFv^!p!v^znm-4{qGJan+KtYgavo%$m2oe8y(< z_l{-18qSPFiU?{94oC)<4hf7Q+gvhG=0aAwIkRiqi^(ft>3HMHc|_m)7eD zJ7l6~V;NIy-5jUAY^!qd5~|@Puwl~yXsq8*p$>qxj`ab0M@sD=A?3|1Pk8Em0rsU> zTNyyat0WZaj30G7{OKAJ5=ME`pa}%FSHx5q!fq@pi*;`?D78FA()?!hC%sY7xPOg$ zfXDEsNL=_EsRtrhL7?9r#gB|(b>#w2*3Q@Oh8`CK1~<@U6$~Q%1u=30J2~bw5++9( zowzHrAvf6Og-2Ndj%$_Wofb&eF}Gaw@Ritbu!I&Kh2^P*knCQcd~YQK&o!*H1i3Vs zl_sL{%jYs_O`|fL!~%Z&$HT%G@nsoaR}GMBP^gzc>7y`pewj?APalHJ#*gURM27MG zwqU4D&=Z)NJ60g*=^`%{YNX9~UZuaT<3ygqSLH|4qJg2-ZG&D-V%UbIQRGq8v^m2b zTJRT314;HQFPKY|qoiyvqLV-JhlnwWkrfe@-~A*uwi36=El3reUr7TwnGvb*X^gQb zrbD*p*7oc=`R40wB>w1$M$BQ|-2Ov{4D)+0kX80u??2eL_pudpTH?ER?|8%z5Y3yR zD+0Rn@}(z(#gO(L5g0!d;y*Ux6~V)%g$tq8dK>lEo@>d(`+|7r%`G6oh^KZv9Xz*K z`Cje4?!=?VBzN?+*Nz{1-CF-zwcWc4wu=5O&sDp3Qt{)bw!k*|{>UJzxYb%`+mVOj zskp<^UJt4o0yBsb5jjf@Uy2*UOR7EASTFG9&bcD6(IJZKQ5c^t(n|v_TN~pOeL`kq zi`I=bhNEII#8Cxhu7)Cc@nX2Y&WejHCL3?= zZ|X#?BNJ;GL$$>oO$}Z+ZHU^fJzx~a*bKiOfG^1m+`2rtwF7@)1wPwyZy9u zhX#Xa8&KO^G(^~%ar0rRLS6P}{?MbSFWU3RBd463O@;O!Ir!S4!%nO%v7NlIT5f%Y z7L>FY2K`zuBp`S;o_^ttR1ZDw`tw;P7jnWWa9jxG4iqNVnn$<>Fs*>f8nI!`Nr?pO2#uJq6~{SeLfI zd`7YBsx{fAN=DpKqY-&w^M3ivh>Ie;`IuHXc^#Xf9QZ1J6|gp^Y9J!m(GFHd8=vj5 z*OO2iIaYLWZ609eqkK47(=jJcoKTE=ymtM%$9cW;%d~(L+p7@}ro`4B$FarG<%{P~ z%<7IFef^yi?|ks#N4gLfzq@$;+}Uf_tg?L2=EU2#9XWi|VleV$9(V5PUEG_YmUg3} zAX_Nlh#ay;XIl0rkCFG?h4VZ1>^=VG4^Eso`OdqiuU^0Y+s{7x_T0IKcns*b4|i5e zm@l4GamJu?l$l8@DxcX)4Ydt;)gQvT`nzgm@=3+4xJo@`Q;LI)i?%Wb|KyKxd9nlr zN&;t;IbW_f;(U8fb1K9q(q zq6Q56tZEIR1JGBsJxdtGdAVZz0{t41%`9O<~qDFJ7p_Q74ZKL zJmm}g26l2*q~hbJ0!XqEzd%Z$MMxCNKGrbKNOx3I!2#*SF{Kh_oD5+hUo68JBjxH) zY<>JxQ~f|{#r9^mB*5AweK##6hCy#xpF$w*mJ)A~D36#IU@*wL13jnGyd) zBZ}q|GxF?ro3Q@%qpxWaJ&xun-s{(|k-N-VdgG>EOo~xc<}Y4a6|u!`7~pG(+r7|6 z;E&tuu%)ZoNPR8M<}ABqGlCGhF+^xX4C{q@tw4Tfynm|1rN0EvcTr@uKS2{EA(9OP zv=d+XtZtP+qZBRtFrelUzlKZkS##&lCnZd6#Mh{efNSZ3W&M{xV(O-PURuh9y>Ouq zf;%15-6@YxdWxu!tMlzE1;RoYMUd(V5d_fFe+CU2B|t)< zM(oV>przGUm9bI_+*FD>F4YK(RZYg#_0EiqDUU&;o`|v3$kb#eu@N9FBg;H*Li3-C zTLOf|Nyz#y&Ea%TkcY;slmVoxqBXZ;GEOJ6pYU`&_A0(6fM-vQo$q=7)O#O(@PR?; zfBeUPxNzaT?GUXEvDYJEZxHqqx=aCrkaVvAdd;oF4E3`YyZ7#U_uW$;efY_tLr3(= zwmf{YT@yA-O@L~KViFaKE_;%C|k`qX*!4C`eU*aR0uI`fa(r=S545tbZpl#Jus=JBQxT!+de@ zwd03gd)(@qJswx>Pde1(Fe%1`v3Cf%bZ~gx}H4g$!huNbgwm&-fQv{sIR~I!t}uFuiN?a zt$+Eaf2xCY>C&YO7ti0kdGq{*i+UH9JebP04gzFPa4mWA=NM*8txDr}5dPe=AKPB+ zvLjhL8h!QE7oUCk#om1f-#U5fCx7&_Pk;QAFTVNu_rL$#d_|ihsiK0{*!5pSNDde! zWp2kV&YbfzgC?US0~X1I7&qCAPd1`jHDl@+NaGO#H6AL>5q*`|y(R&t;#8NqPj0@~ zp=ia&qef9Gs9f26k=t^jlQ!U@c#c)B?~y~kHgR<1<(NQ@0V4~ZlZ8FVEH*FzE>s(o zS!Z++NXDvpWQ&=x6HO)a?W7UMa+FgSUloHoih~tz`JcuuOCBpMIE}V^>X0|zLn#9<0GUZKliT(Wc;R>e@Cqc7Lc0b2RyE*VV5rBCqiQu$~tJag9R9HW}eHI=EI zM)x&D-g<2pylS*Zlgz;-(-`5`jB4oL&rlP=Y11?yjdX=}X!n)y7m{gZpdytKOHx3> z*j?E-a*h|=WcU9wt9*QB84rPTn!Apb8Bv<^p!6RFemm=H@f(6knhaGaHy358a5~9R z<0Ex`k}gzJcbvFUq$4*8lwp0LH?g89(^c7WOJSNtvC*Pj3CpR;IC+QrrN3@Q=kQC} zuy;Bd8cn(ie5vc`1kBJmt*g+`W(2NkT-H%kxHkfjf7rs_#90+i2m=dKbh-f*)JzpW zo;>GL^6#Q4Jgo(Yf_6EP)8+kAl2Pb&gMQ1B33h!%z$Mv zq%#E(%z~9kr*7ws<1C=DBYmo5#DF$54FRspUpAsUcwqOD1A7}i6H_#co_A_7@5`57 z_~~P{fc)AF&0Pa%vox&LrMA$~JU4V!6zFgN&YgM=+8E)x%A& zKFT4WVgR;-^}&O$9eM5E{d?E1Z@qv2zF9h$&3en@M0kJeR=eYwF%X*Uv65e9-nG*r z1RL8KNPBeeK5ly6ZR-{dCv?snIshUPFHwED&sZRD&}swUZdM22M>K^KLwr+cAxPnC zS|EVxLr&bN0f>a_SgU0w`k%N$??)$6esWJs1DS=SvZ`M3YSw6U+Xht-l$@-(PxNqM zlSY*l6FN^%%PsruHvLRKRDm>iU4y*FEE*fk3i4~*4ob;KvM!lG9(?@-gjtPzu;S=I9QjYDdfvFO+rIieyZCgaT3 zH(&`}*`Q0AczJ@fQo}eBR%r0co`fsltU_~L>ZTsKop8}9XcXCRGuo~{`~LeMeel5t z+V+3?r{A3U_ABoJo5j{e%W*dFqC2h9R@@9px%;WIP-W+8l1k+XoIm`7KeStx5GRsLoDqj^zKIW-+rhicE;8hv>tUm{b6yl<%f%6mo>!rM)*(cB157Z2cXm9T> zTW47mWcQ1wk8j?%djHNXwTDxyfJw3)`UVHA9AfsS(V$SjTS%*=v}5S(12xLpA&&MZ z(DQTwYHY)Al2rbiTB>UI)#TPYID>jc-hSuA zFMjzi^m8trJ!|I%Yl=*{fXsEzqR!p>cke%ZzCRWiph>xQh4I=lc4%8p(;4Pb3VaH*@(EjZoyV&*DC!aO_5b@a*R9Z&IYyu zmG+MmMe-Pd+}s9i3KK%6qJ z(OkCvg4owOa~fQEo%l?t!SqTOq8vB!_;BZB(8R~2f}vq_5gow{;RbKiI?Z$D?K)jT z132kUizIahl_@jSn+#F>iu~Yq44vHoslYCd2G=CT5|Euw?wS_E8LnEG7ILCeph+fK z`^e4fsd`06yilz%T4ImfHKVoMBa;oxVQ0*DNfG;0JTwm_ODB-H8lwEnaG&NUryPOk zAhhHb7ifbnyd$2lb}!b`vOEK9p!@p%hIq>%)}I!Pn0(iQPjQ527rF1!Oipr`~?HEXZO;)BuukU3#T z7{i7PZB$m#6 zCWvT8S!EZcjW=O-`A2udO_`JGa+q7#V+T3eXgPw`-F|`@qrt~Wa3Xh@h@k}Ka5uI# zFefIN+)J0HTHM+zR^M3eAg8=NdL68sap>it1D!z;hARu>Zltrwd`Q84SKh4w~j(rs_1{W zAm5IN+llz0=1EM-lwa&Mj45Q%8PKa}L%oOhJyd6U!Df8T?Q2VIFfv151u4&-?6O~> zeMt1stlB7oY~;@FBUZL!f#`XZtPz##fyOoi2BQuB8ox0$G|Pt)p9qAET5)Yi(jpo0 zL?%pzOn_+lmrW?2T6)(pX($MwUk3K9m;bSv#5`f`E5+yw)kcjM$SAXi4ATA4Xy#U} z%K+S%O5YkYc*Zd+8K~@xMc?9FXTKCH8BYFgsb4lOJ$)L*B z$Zq82Yh)h7kV7;gtsm;IF=mg?o;r2<=`_|8R;jH7P*C3E z3Dq4In57%oD@*IRH2<>zGjv!rYLltq^7qVtmGg!##y8;2`AF zr{DAV^5?((&E@Yd@*tohv_89#y0GM>iI#h}Z=U)3ONhMg_pP_ydH3Y0AAR!a?c2A{ zpFe-;;zgl31`}J7wU3iR5^Uyk>AUYPLGkA6Z+g?*#|I92T-dvTA3uJG3!WSao|&dx z(+vj?9)Pl55xR8+Y!z!UWa!&wI>CzneWOVk?Mm97^N1)xXd2rWB|=cjbVhEfGOt&kFFq9$WF zoExOze&!=fBFRyYC_@qMp=XDIK!K}g3AIk%!)VySD$}_O|EMAxi^EMSMgW~=24bhl z9&8d@V3*}CR~vEk)I)Ikoz%$bBFzKg%C$mK=5%1u z%r<^KN#Jk>Qs@3+AZ8|a_rVwsG73VVm6#?14A6uwnP!>9ibZQ|r-j093XdAcXaXtm z6IYax9AM_q&;v~P{PK47178~ZN8qfi<@tN3W7bm>5GxTJoJ`5|Tw_mcKVBjTxe+Pu z@hvUwCXG;Zr1ZHJ6r{Q3kh?#*CX;j}x;dAMtkb4d21xP*2PA*akh5m|@U9pA%$xL|543F_|E==1Lb|R`kHnRiVsFyWAmd8GP zuv>iJ5@ch3O-F>T2a*m0)uh#StLN0 z6xL`XPu-+#p;&b>g;YygE^kQGKjqu$bG8lWG2L)7GPnpiLz@;Ny+Ul5-2Ur^$(7J- zOl2p{ab{`814|K_N>g-2l}bZtP9l(fbyt4LeRVwyj_Y3K$AB~1spewz>9m^+lT{IX zH4^%4ODmW>z`Yj#)CXt@&@BF$l|O_Kir{CBW04_HOWkU#l{KmHVw(pnbLEztlh>A~ z-mlEnvU3JGbj!Ssp*6{nP4R0rqMsOI4b=$$P{0BcVl_dbW zc>av=(N>Tal2ljmzL2acx!}SJa`*`{I8I@wYTk|o+34le>GzHtIeOX4TCZGTaAwb} z_c-8HRs!37HuUrqy0LA-MuFxbGv>i!CwRjCktDcgg!I$x-2SW`fRwT@#v5>oL^Ut^ z?czEej;A6zAG^nF*}Z4~-UItRmb!Dt9yRpxmCLHC4vBGnL-nFV^e~l#Hy!64_SG|H zuDT^A;;9yI-~DXwLDHs457a-^DcpS!LK}5?{+NF92)Y3B=cHj}HREd3n-|Z8o3o5w z&Z31-5m*LW^-FpxqC=%qbBdvs@^Dq9WISzCQ|LT-WzLZ%n_5sY890MgS~5Zk37Ub( zP0y}G2(zUCY%TrqcI4@3wY1Mm^pf5-Lk~F)`qwjl`GS+76e?tKIs>iAsV8DN|6b31 z*o2>iUs&;!iK8HPW10aJ7q4lx=O#S=&9DC5_3Kx^`26>mE}YXJ-=_dfpQlkYBG`uvM8zWeSXAKg5_te`_<4{s#KmzS2+zs26K zdu$MU;DFb@*ox}TeRBZI=w8|%)mX>ntC!DTxbWJMqaS_r;V*vvGX`_^?D=oco%LQ~ zO&y}E1j}aIeYgeZr?5M;Cd;6!!x+o###_~@vKAeV6@L-z;-0m=78Uoy$0!}7r%1l`Wb{PG$I>A6| z_aX+Sm(MlM&9v~=mcR;cmP092TS3GUHQ9$Rnm0fb4f!Xn4l>FHGJ!J2+F)@pA9<;3 zjH80tM$Z?i<-D4>AmB@#HOoTVnKBpXY*a)&K|;kkhQk!Nk$7cWTQ~|@EJqs_`|x_X zzMN)mRC+eXKf|rypr59_8avt^*#Zzx5~B^E#zLb#5mIy%CB%^U#c&aVBC$*c#51QT z3x&HIc*!Pk8BiQV!iX<)>#zLsMCU-33JU(+AF)s2F*s@5dj>CS(5fuNgr?36NC1_9 zgAlF$F+Ply$lo=`R^1AfAhqFd3q5Q{SDB%*Y4540AkGmXa=V$BOA2Abxp5%<@fKsk z+jl(ifK@w9)zmnsw5CXoQ6$2PB^z9co&=`>Tn`qk%M=&z`A-5Y)l0HsKX{YumVJt`l0bX`5aD=wFrEx|j#@bW>r19#_co?oGp45=t>fhi` zos`wL>~2+!9c-T`4{nY@a51|RRVk(!iI_T#%cARyHeZF3@h(y6R^-b*dC>1JOIgha zL}m65m9NT1ZY=n*_Nb*JRfA!w(f!%^M@Tpw0DXYg{aYE*+G+jRO~Ii(R>sz%0jsOg zBLsAcDexHkhPtdqunEcwZ-SeZmbsdV>Y_@?Y0mPBh%a{lwp=Ud zgch8c8L~3uqmMp5dGgeEm%jVrv;X2TUt8(4fZY9F9}dJ3k-e-MDs|+wW4i7}uvx#i z6zl!6O7X2JGvnU4V?p%}b4=ig-CH|Ej9qiT7DDb8O=7w_Km=KC?O@?AuGufyqM1Wc zWLjV~U1o#p`N~>H#+Ou+eD1d3$p;V0GfhBBb*cK#TxH|^T|IkZ%5BCetITtxg5Xq_ zWbX%*5p7=mV#ni%mcti#RpY`MH=kgUS# zuVn8OP6IAk*_y=WpK@f4uYdTYvr+|JnrR*|TT9{_4xC zS1()bL)d1MLJYkT5St_G06%+i>-O~<*RTBZKmXcD{JW=4{qj%##AXcNe0%2Hxw9VN z)`8-6ZAiO&`+A5Q$+|9&y-n=EA^Wj;VEDKd{I~DiHUUI@%R~|Bpp2Xyf_tDu$78qI zgMEjs9eaHH{=*x$@7}(9SBB{hZNGE>_J95R|7JDOvDc2CI(_=xQztK8x@5~BPc65R z@$TK`imf7Q++<`;SJN{n#LGOXp6I5+j3S~WJx~2vg004+Gz#QvmwTE1R<&{!l4aF7 zhDXDas|er>I5^-ML`)L}u(GjnGh>uFjm%Z{vE8cF0G2UukyyscWo-jrh7z-lp)Si( ztz>T!avIvwt8FS7(rXdP!9|}n1CaUVlZ9fStwQ6!hP92GkBDgY$+LGhzN)WCa>N(+ zk31K-`Vymfgb>TaqonAM7ZjBP$1zv7Ne|1w4`rx^rcl3vIRYv2tMp_K_?!>&ih(y_ zOF)WuqA{Wi9)@CxOQgv;Dk3>q=g=f5UDbFobO@tihYOB1v*st?y{G5k3S!(A@xvQ$RN=X;2fk&4sTQ%J&0PP=z zyf}&jfEP!hRrQ6MJD=B5JB$G3Zzi%1CvL~ZFr+F8 zhUWPtIuWx?%)yMB9mX>=5W7HBV^xo*qb0Jo#fG}3HoO`bZwlD6%QI%Yf#A=0`Glcv z=rGtvMK|No3#<8!MOnemR%D7gIH<;K^b=3)c^=)UWovF+TET#36c~T=+F-cL)olLL z9ou_?In(dOpW36{77OW)Sld82@x;?qTM>iT1ehV@i=j*M1fr>Gtk1a-QTr7=-EOb$ zDs=vA8L@iI$Wuik8^2{#mU>l7+6JG?QwJIaPkG@bj)3Q8`pxrHEfGW)fXy z_$`xw4^`6g9%bkn>&W3as%Q0h$8@Z#&|^1Z%zP4GuQOJG+qDEjoHB5xP7=Ak*8h;{ zmS!?U|JtoaCxsdjsa(_mQ?U_~^GO@*gJ2o~XwYhySm^(>T4XH{N7EW0GRW*f$f*_0 z)2tB~5rOj42jhsU8Y9NUIijMBvr%yilRKdUdyT848{d!@UScg^!v)a}N z#DyD1IRhuXi|#YFNi(WI$lFf7d+J9&`U7yj`R1F8-(9k_Lty#Z(PO{-#h?7*kAJ2r zdSURR2X~Jie$6qGwEK_XmZcFRJeVxHY)x4NE?ZVhnx*5Uu)+RYNR)5SytFkaF{`j= zSFZ`%W0UM{+wb1JZ{K%A+ER}me|^v11D2Kxtj)sd;Xq)Ki+C`2+dMuKD|}@F-AoJ6 z?(i;a>%SOm?-!HNpDc?+!7+nv2cJg|)l!bmGAh#>a<)XTRr}b|$==zFL97u~;!;({ z2Nbv4z9)uyj@drL4}1K_81~(}o?+Gt;DPLl%Gtiy;}syRXp}J$p9UUJ?H)uXnJXMN zW@|`8#SCQMKW;Es8pDh$AX!?k*1jU?T^4QNn(B|Di#5 za{(lVD)vDRMB_;$I}^yL%b^a6q&*FF=hlti{Pv&e#(w{AzIps#{hMFizIFSXZ@#*4 z@vNQ{|f&GVAk=JAOo`eDS#mIvs3>odDSd8~>~%+COWKzzTd z(jEvoapIk~UT(R1?fQ3Du6i3;7S?*JUJYrVP}{_P^2sNE_NV{y%H?Zcef@B zMmV1Bc;N*t>WD@*cg?|7MRAkh`S^sLf%a2P9oy+_t|z3Voia>QQgZhsOeO`YUz-A% zUA7n0%~z*o$O)}9rOUUe|6Pz-MjbQ{$wGLf>KI2fLpnh>gELx(?QMe$NBl1UZ{ko9khUR~O zEIwo;sR1B|JCkdn3PNC{1`xvXC6QP{N6>@oyyybyGDlx|*6ngh8aTcZb~q$B$zeg! zw~{$@S6yhJs^L(|LOR?V&l{>r;sHG_gEAD&_qttj&CmDu2Ky>F)M|@uW(S7H(WkY0 zxqIGjGaF=;SzbOX#rU6=L!<3El|)mBAV5(|n1S+M88SphKOvG~xzb-)OPB%kq4Q>q$am_MkdPl?pDBI@u&=|ci$NGQ?x33btl zK_xN0Q57xqbedWZDn!-5ZRP~RYlg~@mR3m%@=PdL1l4p^^F(bDR18#IOhO&;5kv7= zm!?Y?SHP4;hh#)#O;Zw*k;xKMR8E<(^+2hJGyzekjmY+K5z+-OLu~{@J(VXXNq7BMkmTlsrA=edFIiM7WOoR$4!vCOu$!5s_CLx8U?Vf z1uxPG?G!j=7wL`V^fJ2i+GHb5S=3$>F@7xlfQLYdQ%AB*m&0owBu!;FlSBjTy$y_0 zcjx3)0At+Tw4Wy;#c4grr22M$GRSM$Pr|WI#E`!ko;HVR5X8Hs(ASe4<*kVt0s{D=iT^)ae5<(GqKJqRyYGWI{VMAuD@6h{+VmfvHD?MUBX$Fkd4sZPp~4Th_R zpy13w(WN?u^USnLvaVsx2#_^7=4+%>wP-O`asnXzR5v6fyo5Ht{S08)y4zQIGH=C} z2Q-vt6}tucZ@vBYrAyx#YBlEDIBT5-QBB1-whaX!N(ujlrhZdg|2Ui3<}I%u`@t`M z{>yy_4}I~)muJ5D#uF^+5pI9-lb`9`89bJEbA7!78_77k93Oc7>R_AIvqGSv$9*p+BosRDiD{|UJrn9xsL@R?NE`OMnI zrVjMJOkmmwf$i$F=)-*b?bly_{e^{CCr_SW27mO^KmPXYnJ>Tm-1|(nK6yfEB*sfj z8|AU9Z62oP;vPP{_nY7R!|y-;{n2A@y#K-bfBBdH&SMM!T|9rmCd{}(kJq168+5hp z)O_c*kNwlk2fY3EJBCZVbnbyy7w0N#z=~1XbAuEwZF2UAys7)-T<*~|6y*I(av z;DC9m>o;zarTU@|>6Obk`P={UcNqNegAaf9$3MGv?fS)w7xXU3kJmNb(I}Yl0NxLd z@9WysMOveS>daK3S9N(Rudk}n{7hZ#Vx>&(G*zi;YCl!m)Kv*!HCZ_Urpls10JAS= zGA*w%sezEXXh{7z7&&Y5HuN~_K1g{&6id$?u4B?y-hwR<*xa;SzOxz5wZ2sy0v1Cu#W6_<%sqh?ghKk9r{TK)YU!f!wgq3$- z4ZMXUObOF(V(^y?Af2P)W*hmcjI!eM-arW|t*Pf7^;;mi9t>jOQWlVmxSotB>s z6oNw&Bl9x?$fpy7s=`5ee+m~At*L!@C42|eNdvs4o1t&6G{_81Rco4*^$9R#$6h7K zW_Sh-=4q1@8a-Ax0t4^NZN*IR7pgS2F$35lp-KT&8{AAL zL?+V%FhD{z=r#YTh4e_MfiFWd6)I=~V~joP20vdCU|dDFd=kz2OGcr|mrU`{#Dwe-t8mvEVgCMiJ;}b-BLiy zF+=<9u8A@22gy`ehEasfX0z*IK_Z|dK>Z-`K+g==Bvkj|2|GuAwihWeh8PyZj`Ny6 z%Y0)*2c!Ojy%%jTh^_VxI($U$!`g|}0~o5SFOAYYw;lE_JKyy@fC?c*MMaF~%b0>r z+3d%fpw~*SMZ&nj2!=UT4si+2i=}Z`?>-TX6tmgN`wzJ2y(u|@D;7&5vZguNYLSv? zH$W=N-$t%=39=jioKKN=fqM;XRgh_=vv^lDL5)fP9pz?htVde@N^fmTyKwv7vGs}S z8kDrZ3|HxTo3q;Ikw<=Q^S}1{mpFB--bjpu!V&3--8Jf8r7s$$-NHgNL2Tv$a~8_C zVh8`$PLM`!F#k2NDI4K_$+Cndc@`F&g=*pIbnvbd-^p)K=fj{H`bJ{ald0;mknhkM z$R+3iRMyYBHD;C$y3>txGWRP}>EHx0L3T>E;-2?E_<$At^FRO7-8;8+;~8%}Gxb!h zXztHXhZ4ccQUDq7BZS%m(_GIzV=BCR@4XMc{_>lD{(t^o+w3UH#E-xE_OE{RtK-Ly zfBA*ubDP#5J92pU@#8{&=GLZ~z4gK~Egs0yWu5?Hv_Qaw-I2_Plq7;AcXHQ~B2FyN z>sXQ3!k+f7)pJ*MO=BE*ywpG8d2|m*(h9qlzZ(ee zac4{6A9xT)M`&C}dUv9lF>u`5YSYfVVsY;mn@w{)VCsRLyY}+JNCs^p-bUR>Rg_c!Z+6UREgjzkTiM{d;$80*uM~ch|Jxivv%e+Pl~q zM3x6pidKUnEYXe4)OWKs5k}yH-NQ)EU+xU6Q>1jpY^jdXP5+=p{yC$b2+|irb!!oo z(=`+4afX)acwko##Sl9=RPCx7_AkC`rqrq)Z}aMHio16mI_yp82dw&OyQgPQ1aJbAK9PyhLrTQ{%# z?SH=X>tFAC?}HCM{`lh`|M(@BDI?qqt zJ0cLzy9hb#Z@zt}^^rREs>2Na4Y_5}F7OqNe+H(OTHBqDHUiAGLp~(;HLWrYW^Qqp zNM}T`=Dlgm3H6MIwdtVzEYp3%4B5dNROQWi>dMJbxPZvw{$j2A4z-%?EXj34A-v3l zg_LyV$z2n~@t$6O7E`bDVeMJEU41617RTXC+>~@YWpUD4f#>rVwyM zuht657(}ng4C3MwUVo#6p~S6a8O4*ezUXgAYgU@3R}ZGBs3MFckAmER|?Fsrw@< zFlBeE2kcsey4zE6J9Lw@9l0+9TFOqu(ZIO@Jhbew{j(OJw|+*Lme#G&oUp)~I&HPV zkRy#bifG6HD1nG(-OFO6M{b-)=X93pM;2mss9YGB}P#wxzS;BDO06TH&QOj&w+`XQMOCRq$T6e>rN(z5#b03 zJRFPxc4r+BG#3IvQgeH#qp7K-3Up)K!{i@Jq090iZIM1OWlS=$)Xg}Vi!l0M$J}>k z_B>;~FSoq;#t+_p=fs^mcRXq-^3LF>l)Bt=@d| z?VtYiXNv#L-~3+!J1zi{J^S|mi=X}Cr$7Cp^B2zl$N%_$82GpL+ra9qad~1dm&1o& zJ8n=ScdC2Fcm8B-U^&#r9=H=EnxZx+G`(6W}X4*W=CxG`x4i7-HSMQ^GaPKbD$zPZ&^|mRl z!B+YxoF{o6K6u2-=}3@@YHk9+KU(z0nQ&T4Qhbc3ajPy&28HWEpR}6$@P%m&85+I* z+oZ+)hhAuP`{vzSw;tZRZSPTccyUZ~IgcOSyL+<}=yq8a*mDb6ADLuD#AC)>V`{6) z8S{22ovT7O!9cV}Y*st0kyNEr`WD2a3=zayPif0mj~VUaEHVaMm`Hi-!CPDEn4j>V ztws4pB|J}c&q5kewfDivXs^zP_q}6FccK@lw?o0MH(r1Jtsnf^nXkY8{PW+L5I`Ow zYlR*@xcl|jcfLLQ&6{t&_35WS{PTbHt6Mj&fAPiVmoHxQmK2^1H5)s8(X$yQPJl(2 zVf?SY{QT^<-@bAD_=ywm{`?oeAh~nr&e<|Zw9Ybkay^jURRR9t*^L`F8Ix{B_Rr7n zF@xdJ6cdM{1{`bfv|uSELucdLwisuzn&^Nz1n*TjxPSWt51+6T75e2?FI&y)KKae> zf9EOoci%nv-uv%AHc!**%${%c{~vg z5QX*;^FPHH&cifxI8Jn`9+00N2voGNkym`Ql{1x`MUZor@**1=mUR{6x&p_s%h zC}%e8h9U`S+^X{pFg7E;Vu9RBoD~yPg#eW zh!1!nqwA)Q85+PjAYlU3 zek9O@oP};wRt-bZGDehWnS3Ha+}%PrX}()b91$-86pMI|Plg$siqn32O%7ANrkl52 z3vadoXDxuB0Q1o@6tjkHXZ6BLZVj-(o5VwhkdPHPNx-tDo^9OWQDWn8h+4gS=#nVJ zC@^u7WHJEiQ!#P|qrBYQ)VvfsI!b+$r0oSFmrEXFHU<$2q0?{cA0nnYo>|LFth)aeP-VIa;_InYs>MFh7WD4a)c zKMMw_bm4D4IS;2S*0)>eBn78Avgd8SROgy^dn#-SQJ#LJTd|PTE`?#PQ>C*ogE-tO z6a{jxo6%FQ$&{cr@&P&v>r-0!=XdCxG}kJ2OTSZ4@Z<|ZpqlVR3d`ZX?aFUk%rj@s z+_`&uBFP$K_sZ|Lxlu)c2vH0SK7i7tEbhm@1>acR`OzmIdZ_p8xwBt>@tNRDly7$W z^yv@&;xGO}SNU)L`v0^mys2pqZ|bNrdL!;9Po94Bt+%bn_u^hJs$z$tc1Ac=Cp?`% ziVU3Qn%OeUuANqab8*^^>{0ccrhyPV1R&s^EWC?)T@6-GZ93!89Sx7CUkU?T4I;2b zlDyWC>tI{m?jyfAy!c+&mM8T-^Qxhg4bV4lvb|y3Z+#@_Gz< zOKmaum0JT0OBGe<+DG-O@Z9S@>|!b0zjx>My<4~L-?{1C1yCxdLL0b`ZPIwT#e)Ip z7tQZ^x)1S&1B@K-ASo?TWoE-V|4?Tryk+lsA0*nSzlfg)d(v~gYIDu6IUfRSjQ^3X zGljg?-sDM|zIf?nUia@jHO=Taxc|VFt5awd&h%^&#vFT>v0hS6_jq1 zrEB-@ivB^pdilF!uN`~;{r673`|dye`X9gk>T|n~@h)hz)f&^fa`}=yf-LW`=jck*#6k~T&xBLe533Z%Jhp7{m@@g` zf=roZT}-PAn?!M`tG`os^7GF>zkKDp_fMUE^X<1N&cqRYhE|!WG+A&k$L(YhE}6lk z=6$>CUg1c{D>iv5Qfjq(%Db9V7el9f^IKvhr}hAk2|!U!x zouoMQm1XV?IqS+bF_;11nMeuviBrwp?=S{nf3mIKFS^Ui$PN4L6r=D7br2>Z0D;Q{ zY^*_B02b|uij1i*wI%DC9}94ObMvF9QBWZ!XljNqr|((;6>d-{*TTa(AX9hMiH%`| ztaACPVjU6cd^dK(&n74Tym>1JrcH@T@?li9S?iNQFtf3>X+=r0k)NeHR*>8(m06E$ z3rU>Jd7|4SZ!`ibl6(arUV*FSo9lsura&wRDc5H%FgEr;Oan8d@-18~fC}4qnoc_1 z{M5uk#QCn2NiJoT2mq7Gu$WtE%Mdr^OB<2MC#hBeMKzvuFwNqer=(;&^-C_m5xNSeOf%EiI#g`kgiTR_^vTt={U zkF~jt54C~_1)AQtW>ZY33V^(j5G695M0rPv{WKsaR5chW$~1V_yS)Skiz=niTo1vE zfo*I!u2w77%J!xh3Lva_*JR;~4NRwIDyYUDKzIxN2|%@)?%+7|>RN_VWwdZbzrhpI zd%XHbOde(STbgQ+xFEchiL-LiO_+LAu1>`M#Ta3ai3lC$D%t;577 z9a%eyIaB(?W&@_`QSq^c`ruOsntI-5>l}}lTgpGnwbB4Y`7`X0`h->F)wq=NR19xH z1bpR)HeW;RoICAuZo5xq?x9V-Aa63K&+&~Za|n|y7cfNAYMjB3%^)o_po9WY1SEP; zH%FG|<0b+-Ul&!r;6$|Zh*+d80YuS|G$d5qiCa#>vThcx%eS&FV#|g^nc!haJG@QF zL%v^p{<(!R^RRDZ7@QunBmqaZuT;Q|Z-gg>jqL_p74akJwTS5Bk3MBbfA_b4d;9hc z6IdgeZGZmffAPZ~|48I#agq1f)l4*q>rwL^_TN1I#&OU73e4`@zHQB$c`pOBrXZ@Q zP;SVMdD7^bMW$k3nhORBTCQlI4c&O#Z9J3tqZF9 zf$;dT&3(7os1y}?_7Csf7k?YZuJ7&jx-b+;_Ar2kaUaN*l4SH3%V zz_Y^d{>d-@<*nN{zx?W}TQ_eC*%42KEDigZeg5pe(7zR#^@ZTgn#M&FM^3{<$2@b$ zfZa50!XAke1V5cBV_95&)I#eW_a8p`?29j7KYHv3KX}7a2mEn$wF3Sa;V61l;ekCk zYU|ushFu-YN%WIr3582#A~ACO^_l;b64>Zx8B!}Fb7`HeT9Aw;lMEX`-HYLh&f(g< zaILdYIAa=PG}#@|zdCo3i!VjCx#5ain?wTasv__h5O5L`u@W-%SJfS_7kIP}OR>2m9h{)9Tzqwn$|S(O_z6MrRwdo) zH%NY6rYRSQ8MN{a+y!&psHO$6&#ST#H?aKj<`^ByeI=Nsg&|~xt1^!sphQ#jq}Q~& zpq3K2Fa%MSA+Y?$9w!1aWyOG;6P1fHFQqdy~KP zJsX(3^n`rQ!n1+#ud)!CrVcMlEpw9AT4+?Kn2V_mjQBe4J#1R$i87K5jap<-CDNb z)@UO@gy0wpNMM247+G0xpL`;td=*T1{ZERG?x$Z;&qykpG?tTHRb)b29FN_E9PG@f zgupGz22QLD>Y=L}2{?D`bjl44lh@bqI5o-QS#q1AXjrzS?u%B zvgv6E!h|&d;D#a0PxJ>u6l0Mu9dQEyxvZPlmd0%4SvKZ3kwE43gW<5 zRAIb#kIF$#WJxMuX@1C;XP8IRs1GMYj!sTBLiYY?M{7QLXjmE)<2V2hqQ4Rc9r9L6 zro-SCN@SvKu=<)4oUDv-8e^*}45UR{?Vh2QxfrnS)k~6)>C@)a=rF(|#gSlbBB~fPy4y$s|a+E|<8edmrHLP~Iz$xIMTZ7zYur6G>OxCLD z%8!&#(}?T@CTk9EBuEz{FpR-B?TG5&jkHgXzW#$xKK+s5*v~)vOpt7M6r02!IQZIM z{nh_!ul@h~um9%4rSq&?q=_Fwh`;T1Ki>So4@AT_u3z7`ci;B5r;JaL6)vEi3Q-Eg z$cVuhSZ*pM! zL#z5t1w47=1z}cxKYQNGqIDTO*K0N4h}(ew)BE?PiE9{H$P2qcfyVQ_ zdswST8fKtUb*T#aSCyvkUN3*P3=(XMb$ipFIF)tiihBsC=6V*rY$AH}_t6*WXoU}2 zo<7-Oal6gU_IPEP^)IndzTC@XUTSQTv)FlzFlkj;n=5-tdc--^yS^Vgw3=T%v2^Ou zqx*O7Jg}~xPE|_l#BSfYdHK6bCr+Gr`>o^m@87v_{@ji0yWf24tslH~+&qfyaV}mw zZ;AXHZ@ghv#S49nXkh2jqdUL*?XNuq{PD-1{_5ZUrFVG!{onuH&6`(+B{N7g;(7V< z(ZjpH{ilEY^7o&gICNm2>^D5V%al@Za-s;s)ek-pvi3!r3A*IaI{B2+mN8_msk9DPo^}XsQ%3YU)1wXl* z`A!_K0CY0{u^*a_v{UZ67_myrhfnN+h;HTdR?$ty$s;$-PRbIy)Uwl6GVtB=e9I=n z`~L$nAXoA#P!4P;u#lm=AVD(XCb1vTN8PM1Tbm4#iJuK|;h&r=b2KYgCONG5m57w` zk7aePg#NJ`uOM|^z5y&pc0UpN#D1bf{n=ItxnC$A#KB1-`6z*58-N6mH}xH+XZ3r( z*V6z+eT*eE4OAHD7jWQ%kHnTN_%G1xiMcu!LZu-~=*cUiY{;2_i#2Sb4bUto(g+L- ziHheLc`eOn7H;~0QALeKF<7$LRAbqzZ;%lU7Dx7#8!cn5PgenwD~b8-=xkk=XVQ~7 z@G(_(R~;+s{IcOPPJ=l;%!WxTl42?`k;H;LBp`L+oBMNtO}8Y^e+cJq;P;j3yefm$ zGRO&O3WB1(lswB06Yr>`N*4j;nA~Mdp#E}T149m#IO-E3ZR&dUqN{K~-JK}O36YG)c);;y5^Zpl5xWmf8iP3MZF8AcT*l3@}NHj@w=eUCl)b_%oW`(?q(n9-qcp_SY4g~PD9 zTV~R-rilUTrWFZFOPldppvps}uiAR&s}?03Afqcy&6Q&CPg-2ZwhOK+L~_U$w~3tT zRLEJ`r=qa3tZ>qxu2L32Zwe3r0ml9Uz%e8ymry3#MQw?I#uW;X*mu~6&ym4U7W(8i z8KPo|pTa{D&heP>rKo=-&iPaYIVgOGk;?}boWmlZ3YYY%dQ5Hf0BrIPLr~_gtfl%T zRP=5;aq{%Lr{6n!?%aj*=L~J+EMXShtBf*OVL~Zted}+kq#j1KEzJXAEA?{(?UJTr@Wi79 zUQcE!v5|zSezhUJmKmwp39Y*@y4DniqFF{h$}7sz?P&^AD6N5~x}chan1;KkIS(`S zkyNLe5DpXoa6~+S*cw2ba?EVfquYlLZ4=7dNwfDhZ1)t>K3gYQA_?{HfB)N4_U3%| zUB+|a{P{EAd}Z$9t+(I$=%Ww6`Q~f=1U~J=J12Alu3fv<^U&6x?%F24|8M{GKY#w& z?|=B?AOF>V_}`TJ;;Y}=;E3S!npFd&ai33g70#bK`-8XMdGGXlAAbDFx97gKc+At- zmJ1ia6geakW|AUD+2nvf;~;1|`|X(zKKSqtfASNZQU!0{xg{$ZKw&=PsRd|HDXS^P z3?wiJAnNf@E=YxmlWz6_DiN{tR#n*2^)Yiu@{ZK4l%5u)=*r!`r+@fz-=5vfqiRm+qMhW?N|?{4Z$D7yUiTM$E?4qU+D*W4GJqDib!QsyZo9u2i^TP}exf*h zlOFPQRJ;=1AlqtR>wV|&Q!`akZq#ocyOa6%JtVw>C40B>s}+gbf!Y14#~kXhc#o7v zK#`W?Q6f6#9O~t?0ugPn02u9^DYCUs1!5!rAg|S#;h=}J!4_{NpxthGOA#E%#Z#`7UwKBt8 z-!6u&@(v9ZL2;?_mXtu81fj`uQ21;Idh zIf>_-naTWvnR8}2XC|JLI9@cCEH6?NN7gQpB1IAe0b*;QH|&_t_r9iPe%<}+-|xMu zTX(Blw{G347rzjdbd-Ss%yPa|2?K49mgY*L5tGA1W?-?N3Idfd}V%}x_h(BBMNZVq>GWf;~Ftam!rkh3)GDZPdHRIY-W z-et7n2_P{LuHKzV86a?oI1MC%X({Q1T`cGao)rPGQ!WWcd=!g{v(7muxfLIl);V&= zV7{oZR+}EVCZf`Barbn5hS|y9{`()tDRxO(o%HPSN$IpVmpP(*&QLfZ+-`~k zGwwnx@0cc4RWT957(V_9(!?kE^!T;vOl$i>54jNIQk`;G0t+`|lDoo+I;6|A4hPA` zx`{J?obX{UYdhKG49vkY9D`%WRwS_^TbHT3EWFS2i(^D)2SBL@@?_$0Jm@&<7y(sM zG1TBYUil6K`ku5+U5u@V_c>Yc!0{tTb3ydw*iUx!qV@51+mhpv{iY-c_ApL2 z$uf@BA6&y4#6TcL4-3b7Sml{=X@6abJ4vz2)3{Ee3(!F8OOJb!mE4#COT*KZn6?=4 z6r(Ph%Nwc$unVkcj@KR!*t+=K3p~qT{NfjmOL1wRGM=oA)@TCV~cue2IOHI!-E7Y}Ix0@iY zDK)hi%G6k+IYClPIMFuhLOw1eaDvE3##4xzU$y29Ko;-E+ z>J@VUxjiS2ANP!dgL|BM`RM)2&G(+aaKYio9PVpZE-U}+Gv_Z}cusjw@_P5(U!Pn% z#flx=w*qO49Y$jD`1|+n{QZCZJJ0L-$SWWJ+Bg2`JMX;p%b)$!nhTR-q6BB1_0Zu? z@_Fyln;%?#@7y!zKl0Mc&pmhX*S~)Iop;}6j`Xg@zCa!N^VoD+mQBg-?R~Vhwf@5& zeE-nVl^0%o=?h=@?Ms(0{q!e4*|@V_Kx*H<)zuR$w5kSD-K_W6Xyu3~^qzWZ{k^kF z7+LHqLZru15&G>kAfp%l^N*U*l47n?XGK^Ib0QS~%4w>Mc|V!J6q6>_<7+K^y6yHO#KYYk9Nv*Rdqu3FK zvucE)8jX&AXWE5;L1;=Y7y+uAjPVTfx?7xBxK&|1&`uwk$q%z;fF4NkCcaEfMZxc- zx!Rl?-WV2S*Iych!$~o146yzMfHflFX?d(|I94l4@_M}C#qu^^=)iof**;9cnyFoF zDb%+`*sxtV77N`n?!h~KUuYbp6;CY&8y93+uxo)0n3X=`5JhJ+5AqV&lj4s?mkJCH zgIpdrh6N`oGXwca%p_GsU_+)7?56wq*NB7hlnqKMRC{poTF~)tB+@tJvEeHFR4en# z`l3)fim9&aHi#$A3Pf;kL#(eFCy8X;-TEI^EF!7n#pwYAujz=e72`{Ci6b~+Tp1j! zTMvxBYA3dbiBjFEDW6B~?o}GzN`~4r?E5sQ*4zt$K7?G&_&>~Hx*6?UvqqnajPJ3a z!8@q)RW;?SxArY;y674ht}H!zZYUb$TLHd&C1_%pn_(OlQ4eRW#ffQnG^8(~*zjC$@8Xmm z3_}?7A*lZgB;>*&76vVN{!P8njNxKfxeD+8FL+n}!)ig^?d($$K}rQS9IHBx5@!+d zcDgz?$Ap9|J#@!iJ?4SguQ5Q7za{ui4#70#>pP5B%$3)Rr!v194|uJ<>?WHS53~lI zX-QojOsZ4HNGT!zw6CACfH|zf<_2k2(%@?-InpU2u=Ua<4A2sC3?agDMG%E*b5lB` z99JJ473jaODe@)CrFQ|1GX^1Z3wOB(fN`T5!TOZR5=+b`GS@APg=!Wdi_zX+dkO6| z7K@elz(0dIaZ-P;k3<_*9Qp1XO zQl}BIF|lpu+8Ve_IBS&0nPaM;d2bZt_06+jqL_t(!o0dQ*-{;Xo=dZW% zM4uo!V5fFX!CaSuNXpO;FxHq(2twpwYHGD83^axnAw_5*6l(rY ziMqVXan+SN5;Wo(GJ05qZ%>^)cj4mmj##nd6CbfxLdxk=PyOESef62ApZ(Tf{y#te z*^g~PUgetFskdCdmc3zz}DgDGMgHsr<&j9g=mE-x)u}QI1LL_Iv=D{drYbOClQ<5<7%)xv2 zZ{EE5)Kh0po;-2u=FRP`&123~+B$Oo-nQ)q8oF`)ik+{|Jp0V6AA9xJZ@;y%x$*9$ zw`F#{^x}&$2_4IGXV3bfjETbqI+zGU%_~IvEHiFx-TrUi|L)my=U)BztIs_1%p&!BH`qj%fu6V z4pIQ5r2o4=`N zVtix|xznuX7O1as!{W76M(|~i&e=M7$BNm;Zph!>Ets=rKNQVEe=_O z95(ek^b7)_w5uil#n|`{EkH*viXS(LP%JQm-O$shniz-U>MqiZyyZ&*s`GdO3>QFx zynA7kd_KEuU>L4JwNf>NjOKnCMc5PuYA^_d-+JGGRw0YaHUxk7Vt2IZm9(%Mmx^Qi z)57qidjM8p)Kl+|8C@78yFOP!1tx5{P80Lsl9nyhM{yNpaE zDYziuyZd#tE&$Mdtr=FhL#dzkrw_)2a3Zh45d7L2ID=~zv*NfN{JFg_C|VXL5%lyl z$|~eTI8E<_r#f}|1f`K6QzRvmA~d>|W+i{O@&SThjIXB)@ejGn5672Rp+|IO(d0CX zA>}}E;0PKB4E98at()msLd!>d~p>#Xu(oEa8Ya+sOZcT zXkwSUhVe8+3lp&+AO{1Q|K9#FJO)gC1yvmsiv45up?C;5-JP}{b2Vz$kAdQh6K0=t4G5JBI zR>y=gD+PBza#`1kY!5uD(=8qf+Osc^iH!u?)S}{O-~Z@v-7VL@LwYKeHO2K-J;t?N zXeQ{H+@?_Jd9F(^3T+ft?5?N>rYR3G+z_NRwMwiFV+fznuU5@30!l?>sW~;=c~#Cm z~{WoPFtAQKcYc1P%h=Wu#Fj?S!T!4VqdDWbRoRz#AE2ih^#>B;&ehOCCMD1CNmga z?{H!XN2uu~5adL=;wktKAmrU^ZiOlRT{q~nX z{_$7eedq1}2xxH( zIjnefbk*TDM|>oLP7FSJh#WO;j4ccO;6x^Ij6=E4!@S#NG=fGtts={21;%5c`3dh2 z2HK|gTE*iDO{e^IxpY;DkaBaFxLETZt5MEj9nCYlQ`m&66}_mOpGA4AmqhA{0M$U6 z7O>QQQ6FkFKW>GMc6N$U^ixO7AhA7nw>NLxcyj90X{Qr;cmkbwvi}EHx9ag}!d?pw zZ~ywOvuDqJ^yQb|z4Wd_@vmNepCq4u{zX0W>id`NQD7f5j>EK)j$^s5LAJDG{nm~D z^1bgm3G|~czw+G0=ihk!m$z@-U_b;QpoIrW*mH<8xCf|f*REZB{`pUQ;^Ui}7d=X1 zeSKXMN&ps{>sQto^iolhBg6(d{B%_RcmMWpo;vr;$6o!IjmU4j@#f|C-*dtb!l~t5 z-N#d0k_@7oU*Q_lVEl%#?}Xd{qGvNJi6;{pFN@misS{_Yk|l0BX+<~ITOZi_ps58= z@jqdD1(PXCB*|mG&5eN!eplz;6!5hkOmaDR9#9A5GveAnLP=^-SU;K3LtLJ z%k6>h%Vd#{N{@4-ApwG7hi^Uk{k_8 zx0UWKtC``%W~!8|do3usF(0(Gd#SI3X=v45$IGJjkz4x}T#z;}V8d@9MaV*Y$~_2; zcQE9tQJ**7sIss?>CtP|Tr=v0L69Q^uC*1;U7mJ^w>7^sJEbeD;KHH4ghf(QxM8yk z@uL5UX89gvVbk}q;sMF|8ZOK`n7gmw`8ZcS#oCaa3GuJqotqF_L1V*XNSp?Yx1_;P zXw=EI2%DVL(UHVUs^i4OEO@vvzvBSMj^V~M?QI}7RPb8x*3ldt(AP)==)9*7J`A1J z8oqdoRNopk^BUMM6Z+(qSzj!dGJOtuT+j_tEqHNBA|QUnghd0V6^#a2&+NwN+!#~T zO<`<}1NBg}!LkxRg$F9G3gR&Qr;at|#{zT7u#uI{FS?nVE}wN3KLuyr7N6 zX@#mP!+;FHCA&u=(F`o1!+&Am5L_o3B|y%60DTJd*81#p;Rw_Jm6%WUDF+qBjIfNL zMcTt-t;0~Wko8QQ2T%51y?%XTYx}`TO6So!_dM4UYFwX7KRe-=qear!=U?IY|iW8mHz=y288deZw!0;B-t8M0QFpxX07v zJ#KjDDqn>l&^>l$GG=2<3Y6An26E-nRwvi*bB3w2%sqP39)n|tN&%`jjsUP#`REvz z0nDl0xdJqfquX2ok4i}x>C)U051HW3jRpeZ< z)TB;zLcAkQJ+ki5p_L;md~nXXNC%HlZS6JWj$)ZPaAVCNW_F@wQ>1JK)?rOa9l4L1 zOD)pI$VYwM)Ubbq`+vv;AfFqatdRmUp< z?0iDv6p1w3Oe@MGk>Q;yvw$w`UTw}$N{o4M&SB4GZ*H`BGsTkYtr;tjMUSZ~FWEz4 zrHgBtqqBFx3RhX zfz;{Sck+$GV8Ig4!j>FPojQADW#!W4_n-qdG#vn`Kvus*^SfX9otHlHkstl&hfe#! zch_TkChdt#?yvsdKSuCh{Mny51rn>A5rh?P=|uLv^3_0xPqVfMa?s_&9OQ*#3b)Ss zSE@&4MqAUYhKxn$un+HtLZ1KuWi-6;FEOY8fzA%3uQfaB5ohTGRrZQaz4n;o$%&fh z+WJQPwCC`OrxhPAt6$1KH9qK3iV)aiODPLx;+TDOMW;6v=(JGnQ5`=_1tp>c54uy@ zj*@26mCq_c55Rsh@}mnWXZCJriIfm;wnGUJPDz3e6H1w)P+OTZlKwEEJK6gIP?ad0 zX?3V8}~h7gHj&cXE!aCKXc~P{VSfKwteQzX`3Si z{$rP;Jg4^c>sLJ!_u|Eicx=zWt(!Ms`0^{Scs#jh{qFaK(n(U&NpxYliP&C}6b>CE z{i|2ryK()-g^SO<_UX@DzxKg9Z@=Y0q?DGPHkj5>sSx`fz5Ujk78XA8(nr4V`7gZt z-g~0^@*$yVSYdThh-h;*hQ?+HYV&Y=%QW0$0Y2yfUsukbzxdIQy+Tp{;U9kJh#{=C zCBQIf_N!NWn(+4BR(6Q3NlOYf#uBALo(tikSENvB&D>|SWz=~H3!3or{v#>1OnS-& z3VIob#vd^97pJm+UG|==y;6sOcYXJ5?xzANpz|3+=f6^nBi2*Z~jZ*(zRuXa6cw2sAZE# zv!T2H11<5+_1bpdw_+8Vn)?f~1EU7q(141H77oY=3=270BMK!U(d{9oveWw9^rp38 z^HdEn^^^n#Ty+yDiys||0<^U1GIY=q2Ynv<>B8uUi+!BG(CI_3u&D7qx&~5%sD56V z%uP+st*JW|l+Dv_u#H+(K=H5Q0U9d%u-sa}?>%s2#gtd*t`k?Vb73gZS(T;CKnG=9 z0{_SI48O7agC3sw zo(w=6xDc@E%11jtB5mG9G$hDlSuXm;hOpr!x}e@&)+fd_c5cxyhCR3<7G)rmb;dE#!t zuAW>n;ueT5uOL=u{tak;%W0KkfbsBy#KLx&eSjBE^war!b|m~6m<*Y{U%>w>zW?c#-%zx4yAsH z2AhT#^%$4JI0nR=j1k$h%0ZN&&sRuIqWq4PctJn&5Wo&k(#M8StAYkI&(VaN%bbKt-b{!<9=**gyl zOH`AvqciYdpLE>0B#50_10TGxY8dsJcK)b=uANG!Qy`7Aue;b~_>2@%c^&}n4L z{!5rRkTerW1v+a>cL1`PWQ8#UmK7tNUR$^whqvd-{33Tl%rl!S218i zR$*wuhr5|&ZeXLC{DJfaKYP_7HCJnaUs&eRM@@RG3ngr#nMg2yfVRh9*vGl|*vcWt zDOpQw0YQqz6TLx)f|r@h1n0%onimBea<<|`ppN=j+Bip;M4NfEP{b)4htdSkHcXGE zqMW5N_iQ*NQOi@KAxWCdnA9xzAU`rhPQrwJuX+R~D{Ic+;3}y{<`Ykj9>yQmg5Q60 zbN1XfzW#Mu{>^Xx`TLjOjltz$ z5G+a$eDaf@`mHZ~{?esO-~RTu%(7+t4=$bO$fPg{P3Htzoo#OtvUB&~A^A7fSdNKi zdE62>@eg$~hrz4*o(8DdLW?kK_Bc~Pmm5Co`@N+y8G@mjQYNT`sB{JyshNmuJ2lL7 z)KfEO=4{n=+ibjNN;wsj_g^!sGAQm4-X&j zZX9A)Tyb5%qTwu_B4yTR;D(>x8~w%_`kxUT{DZmAOs}OLFbc@BcGd&xNTsu}`}NGK zR;QmnYOV2+exg?0ZXj&9Hw=tzNyHTPd_6O=*Ep34B~}n zFTC-q*ExmQ9E-#*_U+x+Sm%O2fAKk1?V2FV#>N}3|LV0*y=Kn->tDToxJcy4ZAhNE zZ*@zrnW-`x@T*__;>wlPmtJ`BOJDk(Uvu=|d)LgKNX7xBoAHHNG9{Eh=0OCny!z_d zr=I%xPk-jI2?Z=#aH9cK=CS1)gtB;+?65#9G8;+a(tDS#T>0Ro7hnF}-~H<^x|m8hI&qzA*z_ao@xBTz7Knlv1rjP0v|?dW|%r7=i~c(@9!|D z?5uznK)58SPTbylVs#-Fi+Ec=nQRsyeicf|9Rfd55OOW$vE+_h3fYl6a zP5ZyJw0l>@A*G`3gj=Jgd93=OGB9@AnJWQWh)zXj?Clby^c?Kot>eQ2U^-NvwLf;6 z_%M=AWf{`(&{|$m$wTw47BIpWX~IKI0M#8$7`XMe*ii9Z8m8v%g)R;t-&G9R3PNQ5 zfExB!2K>c{;ZOIchRawq>NeLR088r8yPzt@Dhv{?Y3}h8Uc?dOfnjWVOZ|Mbr?zM6 zR%=M}Lz2}p=xZ{^ia3JIXfReruc8%Wvc`fdnlD=OaP}>^4>BqtMzAAAZ)Xft3gmGB zidg5O4e%~c-`5BjZG0T%7-aNS^JZ<@kw!fd)Z8`d_S*|$oxFtu{SGQH>bH*07n-OF zW4iEkVK1)4(@edO{zq>Aqm?BMyoE@$fpm40A3N%{QKC74cU2dKDHyUaA>L&drsqLu z;WXx^6N`b3q~8;Z^bRi5@Aw#p$W>iFkwjX zZ8S=)F*Amc0x|T~viaWqhea5LFq~a!z_2;BLs~c(={65M5$$SjNedYl5i_i5(y>h^FPHkp@4B z7?Rj0d}V0|Ip_<~EQpR@q_!|E=V@n)1F%y`^pue@(PBr_lzK5S6&8fMFtUIl#xeMO zpdL8s@H^xXTsZI|xff5G5&K{Opypg9F+6FkhxWht?78Es$FVRk^TGWa>l^&1Sl5$%N7OyfTX4{c6#F~__aEKgy0?9M zQw|B$(v)7%LJ8gB*6RVBaxrSgL&)9QLzPU`9&B!RB6yKzbDo1oxuIA>?#LTV*h5ZV z%LjPiy8olPb=u5_fD1*>d1Y$KINBkYLV!>B0>IoB38w4rgGbxjn|KA0 zyxxot$QcZ(OfjStDUMZlJu%fDk0(!#%0`zKZ`1B$2Pw7{M1SHys7z9VahM`O!J{oa zb`>(^{f~5#$f*5@fc%Fr?V&?=Ys3t|XlY9__38tM)i(8zxO4F2G0)FkJ$&fx%UADg zZI@z_Xu0Bt$kI%n(V|zby!z@Vzx?IjmXrOhZ~vu3A7*eJLpETd;HiU#>tasc$r(C8t#_sXrO=aB$ zmfvNTonOCbq?Bp|^c zXZ^!^`E?N~(43O0;E}PdvKqHvsMQ8iu1IBvoT?6maY?~SC+Vu1L|Xi-ZGXErD-Ts~ zIb*qHnJ$R*RhhijJ_4m9viwf(xxc=C^W^E%TbqY14haxE^X#)%-+!-qA%O*q@F7`3 z?_PSx;Xr`hxOv?p&3^R5AAR<7zePxI{`yS{T1$;okcW93)SNeJkAju`hs*(g_OqX! zK6U1$mtHz|=Ik54dSi2wb03$%jA0B^y0d3%^Ue?c{=Z(h`26QT_t~o-eDLEe*=GvM4sd1J(UOvv0`JF`(+plJHUGV8o5+iqdNMt9ZCZyL&Nh7w_5} zm;S9uJ>P?^-5h$^ z^J)j7@bD~F;xm>Eslz}hWgLNQsMNTwQQ$wGyV#hJDV*qdvf+`jBu)PE+A5~jGB=oZ zm0%!^X>H(1Zgn;FSij~XJZL9FINM;Ens60IO|;5DdfB*yq_Bw_ea!2QBO#tlVw^AY zeK64hV4bR$qlVcgbF*nKazZq8$YA^#CVPKw{KW9E-9@KX!<~=M(PYit8ie-T z+1R*#<0iIn^qmTKe0AlZ(@JWX`|*&^PAw+wGQ5DbzK>sS#r9noi)xlBVMs!(Vs4?( zm%KL9xQLG|ddS1@gd4QM7w^Kn$wuOb6}i#ExHI!~+Qat>EKLB;F3Q}>TpAE40g-Uz z;8OIFirn3qCr}oYP(otP;iJbs@rlpyXf*)`veulrENpTjE$MRb#aSl>B|$uX!~vx? zQZdk+3rG3)@AYgTuD?ckVx{rhzkC0_;53PxSUVw(NTkMs5>B2xIduF~2`?OaDI#(^ z9F@0awR-=_6Klx@K$C=-Qvs+`a_M#+J?7_b3IK2MyzOJ&bKw3HHz@dt6DNoa+hkG~ z{N*|G%gq5~b#HFj%ZL#qyJANk>6D1ewMwn=lLOj6T7+5nZ@PE4!|c33DtSc{7~YJl zuC3>p`P|vu;HK&m$5&SB{$2nXbPIwTI9pXf9^^49@YAKRA1iO)*|>9OlfYne?C8qc z+S;kLW7G*)gZI+84ZzlnusU!4@KLQ}CO46$bjtYIGpnoiaNBtL(3I@XW?Zq3k1bln z*xdRIGj z^_l0l3k(!nR&O(NNAoPk`pXKE%<&!HA_~ zHD{GLHusSaCaLk>5PgYdQFQXWe>um-?+9+IRlUDHiJT)$2Dlx9?dbWE@in(787c zj~+e#`7eI)<&V7dv!DL-2S50G$#D?UHy}a%xbQ^=Jpa1kvZr!|2k#$==D!Z{? zfVG%{$ITMGr~=6yeX{?grwN?DC?RX}&X#U?cs2+ld<^yUZk5MmV{l-6b%FLb7}2<8 zz;0l}nd$hAD0QSP3oz-%#QLYWai+^E`pvZHo+N}bbXLha#{s#Dv-P5yxI)6XrhLqg zq98P+1K$BR?#&Spj@`-37Jm1E0SA}v6YoQdfW>l#s-a-8g9K)wB03g@ldjlc+9@^D zByDNYQBb_5UFF#GeP%_j31$=~_=WUhShP(`qtaQ%n%p(BACeqZ^NK!ys;4}z_EBLf z;foR6^T|j`py%>?Tkl_e|D_jS+}Rd+zb)JyN@%yO_Rs-qWAD9p>Egxbam8Vvvkmu0 zpZm<`7{0gPehX@2^`Q#0Or;m`_yTS8IfYET>eej_2)C^>eD1TKldu2IJ8w&C?xjzC=F@LGesuk|hFEosh#_rtft?4pH_$ToPp-nU=Z9e;&N0fp3WdG|L-!3nyG&zRtfUSpo9b^;i$eG^mc&U4`#J^})-{#Z#Y36`!A} zGFT4)UqfkN52Y#sK1}GAp8-@i3AsvFzA6b}zV!w_w=yAAwnnD87}KYvLge~P7qSFI zGfjoUhmSi>n^-Cgxv zArCypQHw5ToHBr2?OQ>*M?F`>h}sZ7Nj4f~UdEheWdqliKnAJLwO+z2b}~t!=YG#p zG-Ng*WJB1@y#Fi+C!lG79yC(N$hwyA;;`-Dw|_D%DV_sJgDThmfo1W7Cs)?QYmXjY zE#$`2Jj5H2@(~&FXMhY!rO9q@?-)CI5{5by#iMc26a;==ECka&sKi%SbHYnoFa}OP zHO6bI>v9&QAg`>jeFjqqRKaXzieYYSAE&kvokPVo;YWPy6tg^tkrZ~ZaoPnJ)D>~` zfdCeR*uYa1xJ7^T3~mk5Py-2(h7Ue+EHxs{h$yXs!4lu~=0IukDB#TXZ*O9uK){T$ zVEV?(A*RN*j8S}EoDo;Bh_r>{Edm@p)Z-G16VDK(oWkm?h@J0juiw6H{&RQj<*|zLN=pvfxl18P zfwGGeQfTgqVImcGp2Gvhw%lS-XKK+9wJeOth%iMV?7Zv7p`K!U?D#R#XtdN)&&x~{ z^pJ#(9y)UV^cfx#(UB(So69*97#*{N>+~^T|Kr6 z@W$pQDNr_)nKfy`XNbuw71-7R$1UA4d0btOdy>sf1LzFuA%wl)J3o7S!`{ogTU*=Q z%5!JWj(+ZujdOHGF23h{-@kqPwv)p_hN1@LwGd{C?RfG@#yxxHDRsH~HZv6bLbTaS z3mSXd*Lk-@3(By)ce?>J1#R|~3&pKy&K7c`T6$HvH>pYe4S*8xMKDFPe&l_)0+H)@hm=$MNIH5%>eXwY`1o_NMpvw*TPf;cauI_4T7< z1%E40EQ#)-MbzmZO;ZjW0(QH(%w10MiQ}tVcki7!brL?h4R*-r+C({h+uP|$Bwifj zzWfb_LB-kv3LsWMobT4{+xCHIp*i!NElx0;8Lh<{#DU3)XATjf7iP>YS?m)UJW?05 zh9G;)*)4o){J}5tE+{Iye=j80@7!5gSvNJ3J~A@^cg*5S>`BMWPwY~(Yn&Y}232lEZKg?0x#w34QtT1&0YAcSJ+v?hylr);d#&ew0sHJx{!ey7dlWbuF zf!9&E%nS(>QYhZ3iEKVXrXmq4&?u#l*Nl6~m3_^Mw_lSiCJ-@|8{=C=)Rk&7J50a> zWeH4VmhcgvnhiSrMt(nBu19v|!UHxbj!hakP*6tq;ZcICj$+{%rcG>w-hoaA}0>K6?Gp}DKXM=Q}F|$`2t%8LbG40 zrh!aj-3QIiJbnH3UwYi|m;cf4{Lvr%(SQ1r|NO!AYq~=tnGM8n#Ko9{`e8tC3)+dD zXj7_0mDEsz(=kFX_K>*6BVt!r1R$*3q=_T$0Q#F+y{CfxPjlw#t|&2KzIv-l&A9JN zrPBNw-@WGQ;!Q0f-Knrtsb-1>-Q9+WeE<_hKx%gR0I-r(!``6`zM(nI*O6)?G?tH3 zYb=rqjg?b0BM=mDs~HGpnY#Io18JK^7fytKe|S?J7khah-VW~}SO2eD3?4%^a`=gv}Zz%LvP8OTQLREe8Y7!uuZH(cjt;68+#^{LN?&w}sFow~YI z(!AHM31WQLfBM-|Cr^uS@w&sU90zdIqTKZy%5nqh)+n9Mn9Z5vz1VdV2A;>bm7~$a_C&39 z9Thw@t2#*(x#wJsM-MmdYyhH#21v^*J>0REf}P38vSLCz=i2LL5N2VZ(4?WM1qcuM%(LllZrr){!F3zE$~jJMwn;rCkx7d!>7N|oh&eEaDf&G_$5*Ss zrVWG1Jt69IXP;U-v1TUK1Y@?RP|R6&FvRWW$G`K=yH~GY1NO|?+WDuSUORTYDD}Yu zuB_i)$9a>##nKof@QurBDl#YfhbNt%n7T7^dXmm~k;229+(EjJ+S@xa_V2H)9!IsU zuGqrO5>FU0p7M zZ_PQ0F?!(w!CZeCAu|V70GH_e%#TaNLI6Sr$33^#H|RL`G&RygywCs4Rgy$qx|Gib%bVVwC{Xomzye^B>U4(eYS~dzD^3y)|ywC08kHo6^rahFeSnuebNMHYrm&2F=6>Tg;Gq>TmLb= z$=(!hq!wfAbo<9oY-h~MusH^Gr3V2WRImYR{=9NY4<6oF;bP>fMn%15XmZ5_QjU#_VtRnc@|U_pj?iJS&_yat7r_mO;t#2j6}tx?B?(J&>B4kF4Js4M z>AyvQJai4os%e7<%roC69zK~x2`0>TZRP7<)Ju~ysYad&@U888j6DNf(*Ga)Q~K~% z-}>__?_ZjwChgM-(!1aLyT8FO4(c4EnQkSOv;gcI09!}fxX&}>M|>m>ncgDSzZjz#|8izynmC8C(=Is)VXKRU%33=!GC*-Qti{M86r$7uGrcT8`)n-6I zBmI$o6>4Xi+J!RG*UAWTgbpV&=|1~Lk8l92-ieJdJik*boj31 zLr*g*&fUI_Fz0`D`kJaoz*5M6G*oJ??6PQSf`y2S0|Qz^ZoFFl@-7EYl|i1X!^LSM zPT*Vh_?dqv`D8dEdT z-ip;s7YrP2JTqL2dxod-UL0&=4kx>g){>o7MHI1Mr6P7MV#Nq_m=Guf#HTdHPiP7m zwcy>*YqObgCBt9dW~@;F8Bl>Rb0AvLEdAc&b8An(_UfmeJO9G5qbC3oUgLt?ziUmr z01eat(C&n?%E(8Eu`Jj8QJa<^Z{2{~B_6OFcl3z58icmCU^~x! z^n{ZP^XB$_NeT+)uXR>kKX^f|1}#RR7VR%=YvZupCdSDISyFe5n{X5qyt#gpn{w>f zF?k*1ygp$HvH8Y0cBKL{zt@Y(Z2n{jdI657BXiG9ElSQ5YO`Hu@*x( zb2o3@x^nFU;YL8@Io!H+8-bd8>daFuub((V3;2NUYL{Gjt~*=Xq{4w~Y)wz>fZY&U zWjQ94>$&&x8SZn0NQMK4(-=+)d|e3`97}HP&WSaiH`cA+S{Io{K-xFC49>>p6+QFJ zvtr}|`|Fz-J_@8apRXoLoI7HZMYc`1pwvJRy!CLhe1hM;Ewh8yh^`Cp(Ak z-Pzcp5?sbdJ9k<3+qQWfK6qB?qesV;@wl3tT`n;#q@y?Y?_XcPLs4|N0;{uDa`fmi zg68kiVqPQv-?roJ{rAM}N1>_2eb-S+Ud@7K`nJzZhEwr2l9t`Chm615?1F;=F`hcS z&B@5)>HkoTubL$@3Pu6JGQx__yA|Za%zt*sKXYL zHV3HlQj9GtFw)jdCD;sR25;$=mIiDoRi-W;BD8k=xL!c1Bs(qrSZDSj9@M=kwT?NV z|N3UuCF^7kCh4dwEHXFcr)&Fp{0^%zS83v8B!?Ebwbzi>lwOdpqjlPHD&t@6u!Y-h zyr51uT07Q!h_AX1Ev}z{m?;X#%oOvs+`l!I|Qqvx#hvxlQ+q)tXirh+;`d+$9WPRU1hj?LB?!>>vE0M?1a! z-S7U5Wd+DTcxdX?U=jZfeZVZJ6>p{w7#ex~?yHuC=SP@D*(J7M(La$`ys=DY~y4d9LL+1y`222-_WiA>+qo$wap>pqo z6lh5cU4u^`srA?&jcsn+vD9e~fZllN@}-Y{^W2D z^I!PF>%V;c#`O>EGJ`Sk!7GXHEyw1b9SHBdbNbBb3m2bz@4a_#8x{kZ=FN*R_E~XO zhsm~}f9=|J2Z<_UN%!x6@ZYXpzx9Xzctzy>D$42^OCC6jLx6yH)XJV}yUvl0hgA#GaSWX5&8yE|#H>MAcfZlolx%6Q z7Uxs)@~6RIB-0S^Gd)ESKN-W`2kQ_H?tGapv)uZy6)l6eP#C3)qiDJd2H>^jZ`y%& z5smq-3C&b(z}3oJ89EeSG|hc8%A!m`>ffl!+>DTlEibogZ~9YZqac)hp6&>KZIR;i ziZJb_KbQZ%-k@F=ruPjrSPrGT&4j0y3NxXvY+Q78xiNRU7T{guQ$y95!c~|S#%B29 ztJ_n1j7;4}^+3!Ygo6e~ABJtQq&h>8Hj7 z19~p17dzo9bIXM2p3S%o2RN%z3#GxM$Soc7`pV$&%j&tU4>#TzGW6`Klu(YKW)z(& zkSy|{qelx=y>)=$pV3Nz^pXXt7&JW>!*;b!n`omgODFzIpjmacC2Pe8+V|xC!A_##Ke{#t>G!-w>0tpi7^(M|*t;@T&5gc)LP9y~2^<^+Cl zefWZMr+r30?sCzzBY+vpu-I-Y05g9yt6(YT>BR99$)2fdPJ_Zj-0^oVUwZr9OPYX^ z!G3#nhoso$gEtS|ytQt2fj)lTa~CgMxNvc6XZ!YTQ<_JX2{gjz;Ep0cZ)|&;{>;hq za))x9iPl3OpT2j_P{8TGV{_lu9T8+9#v=M#n^&(~g#;ggS*HZz%Sc|oeW#$x1BYkt zm#Tj9aoU7W~{yTTB?Y}PH#2O0$5RL9zy3wJ-Mk8aa z+!!;f+~B2YI``n7ev2D3(m9b`pUSH*NF$;k{4aavq0FnhrVdKm35h-WGeMgOBY$o0 z3CZbj=E9Pb)ErZnlEM}@r(vp)W(I4|9v$lH^_#R;6?_ZMBv>JUPrR&)D5meQ7_*13 zIW`MHA7L1+4APt+V?EAvYt?>+l@_v4+Uke347|s(v1=~mIkpmKx{!65?G0n9P;F(v z6HjM-AO3!o)%Hw|Sh<=a8ArE-Lp_lXKx$#0S3bVv(;22BriEZz_88KU^EXl20 zl1V+H9;jldnZ`p$RvbJFA^wNTVQ}aO8xwMr#KxGTkz-RW^IJK_-`uZN-Hf?=2%`f{ z|Li<&nUO_jq4ykL6BDrv^W^N)&wb&`zq_?__beb`N*DjoY z=JXl)Z-)VNE}1bahrw?~b$LN%Mw}6IyaE?ID&AbU>7n~M-81cJ9QD_DmD{uny^1u78;FOJk22c8}D-lrD$3Y5Eoo>+Ax)p>0TZQVE zj%f}ML)_>UdvhLw5qU^XWBuhBvX z>Vi&@(B|0w^Uq#bT|FkoWK6sN@ZK+e@$;8oei?tSU%RT4)IZ`hYDSp5pLD=FojHql z7%^z3`9J>A56?e){^PH{dh6V|U;XNJ5kU)_-HwX!(I)No>eZ_jk)Az&@yVV`x7ODc zD2J(TiY7~4_AhO3-o9cEPu2aT4Yij#_`xG2#G#~)Ps6V<1^O?w@B9rf3aWd=KtFBGuwv7f z7+SqvGmRMdqYeW}AvY5H$awjfM20P?@BED7Py^j%^nG*`N}-i@KWfBesK!J$xonE2G#u8+)` zI4x<5VA1J(s0Ir)fzr4~oEUk;ytp@bDcYn3bm)X^8cL+;jP=dxD4?Y}U6rLGikz7P z7{3l$KnNLjje%-fVoiW^a%SbsH~!HdeDb58+S=H@cKs$sHq`BNOhidI#(QHE)OBT> z^h>KKU%Z)w)OB5TRtF;4a1-IJwxS9aovT1x@(|v?dhPPnE9C%`sj;jI#`dNG zz**x@ojrSU?Ie^hzkkI904JxLSDACPvbw2V2#dxUTv0A(0pmFFCiCCSAO=gD+eE0MW6ww`mA^V-9D@Cry&&fij#)D|@z}EYMnSH zkwl98lkLq-@_{xzVOK^>j{i?s^q02#FpB%QP^go3_U)r57}&;Mv*P`df^6;OXh|`$ z(e1uT%ibsFxzVPn$5&R>y|cB+9&^H_xnR4s9%qS5I%yy&_nHg%;LhD05>K%mUeOa| zptI+$jqmp&m$_sVY^tm0cOc0@%~6vv2XaNq6Jb$_&zu93sp-v|>r%ofJ%oANkf*Jr zU?L_J3eU|S###a2h)yn6J|4@I(RarY=Q0?!c!2`QBw1JDU*?>8)ZjSo^=F$nnG0<$ z=XwxDxiccT51NcaWYXVGdjw@bDZky43FN>m3adz@>iOXYp?*zo}Vv<8>D4zfop z2w1?inTca4&0%fZH4AVw7E`;|R%@LbN$Rq5^@2I!iX+Y^fk0yzmvUsP+X3Q$xozLS z002M$Nkld}f---qWEce!37z)l3kw zs@c}Vk_+5)560;$U-_!p z;NO1d+Y&p84CfI%qh+dGcO&d#Nqa6{c;WZ|@SmJMbMBX~|4N>>u zUDDAKL4NaS6Y#MO+);$x+T*JYoJzx@(&`WRk@vvjI=F*!ya-*=DS`$g((%3Wjg!@$ zOSK#f|NJeTDD}t`xZx+75nZkMM%3!M+|*@}GiK=(abi3xEypJfM6+@WnTe$~dYe`& zqm4LGiy5k#OAl+g=BBb!eed1%%n-|ythr6VvO0O&5`mW4I+<0pnX+?TA6~h7<@D*( z?1@+gLv6zQ`s=^^=qs;4!}OqxHpE2EFr*Jr>Pl0)CUo+s!hmSedi8_%ZQp<4g%>{i z`7i$Jjn{A7xRy>)lnx;0Vq{5lX7;^+lZASK;E=fFU%gIvyw*4vCAew!MOYInYCyB`PcuuS6}$}_MM&0^=-Zl z_eH$hS#th^3!KpxO^or8gZms7!P%t~Mr^nV4jYeoejHXoW6p#hp+runCOMdpGKAjR z*tjO2#1R7dj-dgLV4}dkdV5yxwAc%ZDfG` z^|l65oq8`%f&(7a;)_Q*^HH2EJ1U`Qvolo;-xBNj>jZf3;awD&bx;I|HfHJ11&@Qk zm&UB4D~FG*oaBkg%e4_u1p3zcEn}_|dY*my*^{SEamwU-aH`aWU0Kjh8PC*txQ@L6 z+YmZ;?wr++JGbvpj2zef`y9LBx$DP{9~bAlbLTcubD3C09_{9a1e4Uq7E14uW7eKJ zqZLT6Z)|e;6h?75{deynnM15i%;BT0mfa!ZzB=dT-?w24U+fvs5IG8yo0=9Xre=(( zO#-W{_Gg+VF$ffaU1kD!nX5-_GuSIR-Zl*8xam!~OlvH!s^$YOQljPzJq5`e?7loL zZE^2N49~3q$X1}yG*OuoYm$!aAa@8fzqYxp-oBi6&%2-%%?Wl2hZ~}jR^1>w$w`Et z{^scG+nbxFgjAyGN@++(j!~(j?P^uaRFZi!%fcLSQ%t4>CWj`N(7nB9o%A)ey1JrQ zZr$B>M@P{oEr5^-B&^1mwYbGwBTDVsMr?7!+(2QND@;d8m(B3C;9prG_HF01P4_Oz zgYerfJPA&S<>|VJ2(CH}=X6pLlkptRaSXJ5ptalwTNKs&_5M8o84NriEOUm*I%|ES z(6y6Ho3nD?0&8T|2{xQ_hmkk+9aEk~%4U!IVoc%a zmtP=EEE@0fdk;?+;Q`s&9akCogx zMhc@r#7o@hC*J@U;p;z5!Uh+qNhHLW#=^eoW!`)p-d3nm{x#fAp1``p( zdB&LD;9jldEA^&YZ4TIMyYEDYbTrnsX7l!@idvOj(1G!&VH&SM2y3eLxqni?#53(*jE9(Tg( zm?RuNQhd(xCB0*0-g@&_FMZ@?SbAnKD@D$A1)Zz*TBg9BkPF440FckO#}Z0u8R3_| zIQ!JOkACzO&nJH4jn|ny{85Ri$fD!q{sUIp(R$|08QnyTMo#4;<1O3emECE1){yu8 z?|;uTtzZ7=%d(m-U3#x!{qtY^Y<0yzgVHtwkbFZ# zJc=3AbuffpYfp{7Q63kK?d@(9_nZtM(T1h~K(rTxyeD!QmqsL0G2(q7%M>^A|Jr%5(@_~bZ;{hND5_MKomRC&j~mt|-QPGoLs+Z8**exfgD>Q#BOBHGMqXy& z0OS=7YV}mukDVCws%h*T=eT-8#r2bXb@d!OXA+&`GF2QnL(}*wLU93Xc5&?J+V6km z_g{MUBX>9NA33~w?&%c+9mxH?G}!@5=j}V_7Lzwzc<@^UaB5j8;)hG?zBkb_y1>4AxovzY?D z*x?4oA~6O?8zA$cM*xjA-JyA2xZjEL#^fedE{Xw){n~R226uVR^)BH=l|q^rrsCLL zr}+6yDR_IQY$SESIp>O>%6K#~wWv}gyloIm=Jy^6@mi9ZfnJSj8(@MCYC_5p5$Ruy ze*y8WO>?163psf3x<|AK|9gMy_u?^O$3(L)kLK| z&6E1w4602CM8-9=-Qf1EbyT|&6sn`oj5iUTox)Tg7DI}Q4p;?9AFZ zf$9|tJ~BNzf45p3r(>%t$1PiUBG{UA)z*KwO6bKCgP0YV)X(Y|&7FMMA^>xC4Ov?n zN^^Jf#yhxObE-ku9LdK;Vde8P3Z`v<5JyTVO=U8X^rjjsiz#mm;YF+GFi_Aqvs73I zF@lmijxY9|TY31H4KvMkH}`MbTdVJTc#-Z0da-uX2uJjYGBRAISdTk-6RH{~BT;=@ zKIqP3T8OyPITRVWnI^RI19>z2yLZQTPkZgDrR?^M_5!xy{Xq^W=NPuET^+c51FcqT zRG0oVvjrbOuj2jVrqhe0UN>Nw@NRtk#nWxb0U^z2;>VW=|9Hyy3QqJWw<4qZIfaR3 z0E|F$zoVQP=Nt!=8AU6WDPhY9^3Z|Rl{W6yn7rQiWm@F_G?PKl3>$aNqv+w;n#&AuC-rGc;2- z04#8sKiX6+d+YbU`n6BI_Q_xT{FmST{(s>_e)02Pdhvx9l@Yr!>2;=QZbAZE^saZU zG-mFaUFt9c=6rP{U#k1G)SX_75o6nnBUjH`Q@hYrHOEGA%R()|N9SicO!v^k>O8&eieQP}%}NKh71 zl&e!NxVzY7!_D1-iZ2CRfmrfzWml#R1q=Css9g;C|ZxHg4bf$jcx7%x6CP%U}Nd z;r+V?{TX9m#96IfqmnLX6M`PPxzX{UcJ6?$_Zg(dq4yAwCs(dr75=|);lfKVym0x- zRa@F1@L#_FJqr|H`N~%$MtZWfPD({P?+H4FyBZ;6Qsv_yaHXfp&>d5#YI7~I6~zotfLBQZts zj$Sq!$e6obg9{uV2IKJWgL~EO^PoN@n{P*`JG)E=YS=W~4vSD%B^4T4fyfeu(y(Re z7QPMOg~xrH@9GTZIIw_SuGU6H{0|}EQ~nRpoIB7~Al2=s|GkD2yUdTCaF`~7Ub%`a z;;Lztl5W?1?B7yB&0q%KmEDS=MbXY>C6r*e0X=GWT*(!!Hdn zWN>`dcW2ReABeB8s((-(EK2(Usrm4pm3@c5_^B^F|ICXUx15`?s(T5^;QxV+qwz1 z^V~$lsknoK#Qy0k=KvZe6r6r-7MJY&{*GVdwc|Y5YD--vCdVkBDbI-~a{bmV9?ZG3 zr!`Lhx0^e)8Ng^!9#;?dz3+e{Zo2rW2?)r%f*{_{jl%>8^VIp6ScOq?#|)O*GUafX z4k_~^bt1#ch~qFe&gq~#y3F)n`zR#R@ALp0&kB$pHa&rJs4F~u=Iq*;<9qEgIA$8L za^u!5E*h**xJ7I4tm{yxPCF&P4F+bZI-A$(%BpEp=ApzG`x^+M6%YI&U*oQQ+AVC@ zWW2}8QEr=VrZK11+7_=*>-cmMdL|b-8X!U_OIW?F*jt;X8(P$ZG-__a(Jydd0AM~k z#=iI&lS1Ly2cA7mDCtNjn~%=quqmBIlHA~DUnc?SYn%Cj= zXtqWvHZ8?{@*-Rl5X&h}yj{P43usb8h(kQ;hn>~G?)M!K4eY9lMwKYPQoMh&RM-^cSisH{SUC-6^zFy z6G)xMXd1b4$oJ_r^6BZanVq@xj@%dnEe%n9Y~Vw25t!-bs3KVzVl5Dv!Quh`tl7|h zM#KaYu5^Zt%_#BX7VV$q9fwn`tQ#b9#PQLbuPd@+La}c|A;|2?Fdcf_+ zI;FRS1=xX(Y<85(Af{RyvTpn=2m{R@cmep|YR9pSz@5YS7J`FxX3?ze< z#n>7ur$iT`IL}GO)Ocqm&D9O$g8uSNsT!x>fvJt|Qo_N_mnl7Wp&qGG(DhpLmW~h8 zhKi`0-)e+vZ}6Wtz#HvcQNy!P4j-kgF4$Rj>h%3HXP>%#(+Otq;d4}5W7N?NWD63RQ9}+Vy{6r0SCJ-XZ{}j?(yFMl&4vg6;HN+N;f3d4 z`0Qst`_`}C&bd!S}aO1GEyA`^|L?L#*D3R{OV0> z<14F27&o0o$FwCtxBA`R{l}KEfAII;#}y_wT+rSe9y+DH>mZ3qo(zABY2HKBc== zQJyKRFdkN#kZaqeDdO5le34uQ^rs;9wqTl*I8EgYj&r&cYwA$*%iDri1D{&yi??a5 zMOeTx3fZWU?{xN2l8RI`AX8%@u^tS1eRp0e;BY({w3llr34K+gEgV5uu0@rfkn1&^ z_z+sQN8WdJIqidTLL*7Q)`hD$)q zqqs18_pS6Gse3!=8ZYX=0h?2ux5HPJJh$G*uk|hcKL< zJ-H#WpL2kkbu167I_@-~R(S3>Eotv#>O#uLSC1S!w(3Mz&o1?#2z)@xcvp`Fq)jYw zjWdela%rx9aP8*$`kdiI_u6)fOR#V*(1QmjSC4Yo%1K#f92Cb_Y}G;0h(J?;x_Bqv z=-bWofYu`B6tdMn?DWCq0jzhj0Uc0!pWadl@)dYcmtkM`An+mW?<}UzxY1>ZFNcSiwag1C#SrWT^U5qKKAmvN<(PaRk3#6y~ z-RU0i7I8NV$)1bgnJcs_qwpTR@mK{tXAH;8JIq%)xU;kq7(hiW_ffy|WQPJ{+)N(& z$Kbf)7DP!Qrl_lijmFAuCbxnd@T=8y!i(3&K=O78x5J(oIF^ zs^&_jDh~)P86)?pI_ykN^rkF1bIRl#IhwnnR`#YjP@51)*#sb-@MAqcZXPQJC9_sc zfP*o-IY@eFV~-d|DR51vTRQIf0QAu>HESxOfu>U87Y{Z!HmFJcr^(KqwG$^l@$pao z_$NQUb@LhuDWfJQxPi@M;bcGV+Pua?G?x@I}Or(nwm-!Xuig-jaYpJBH&@r*1hj7?(+e0IB^xt<%5@uB7^ zscYEyVUgfxM0TP3KUb!NlG9*}kM&*_VFt#G&<*-tYSa0?Xnp!XrI*ey%|%eoOl0wB z#w17Iw7L+KAGM$mj&4#BZA7xi;)HX*UQ6X^-X*v%T2-d{@D0mC*64segN90TVfVeS z^P)j?YKY&(cWHUyK>PtPR0i#l0W||ZcF;~4kMeAbD?gD>fBwRGnT&e%f~U4SE(B4@ zkZrtcoHk?M?zrY=a@!jl@4kEKm6u=s#p}N)(ARGD1_&<|sx%BYgr@$xoEnUBFM-{2 z_~^m6fBmNIZLhriG5Ld+-@61|HmxU{H#^OBv;TPI@ne${aX=|``pc6J;IRB@gKkge z!2`oJ`Q2K-Y3OXRYsOYfw(nnl|C`_Z^Dle-&hPx_M?Z4Bn`1|qf5SOO4Ru%p7KSD4 zMM2Vt`O~lEJQ1s))GTQ#5@tCQg;lbT5(7Kn8X80?l)K)t0{;`YioE_{|cG#!Txb77z^I=BeBUO3g$ zJ9I2uNLLY;=%>x17Vf*$)s({D6!1ZjE`+3SQ5HH~2CkVyF&x0tQWt&x@U3qP`>JTo zB4B=J=}##{v>1b%+WJjh1%L|r(uNPq$F9y|;b0LC(~z3IrVOQ;>Qw=-hL%E_s1GlQ zx>VEi~sx5f^tcJRk>D%u{fy!D5Eg zhBxE`G#y+&_sCw;Z-Xu`bGDyZqL%tm%8HTciJiGle|ge_pLhZDnx2~#g3P?kj$6vk zY*=y-DjM^#$00Z^x$&7Ky$AwN;*Ft*%W~iVkCq|AHIIKR2Q>^(!RPb#c+c6R$3Ok@ zC(f*%I=D}0`S{AQ69!=`e8%3j<7=nRo zY!4Uwu<&*f&6Q)P*G{-hgj>!h@5yNuSPJB`^Y)VJCDSS5mEWb z?}r^1{e(w# z_E_nVCnAl2KMfZ+$o$|lIF_*$2$@LMUOh0Cwe8fKdzU3_UcF*iKP{?!+6IF|Yo~HF zksyzPfh*Btj=nL;pqivs_|n-*Rt#jUKRLmB*t~Oxx8t;VIw9MH*J5&IOYx2M+q{P~ zv_90~NK4RHc-U;lc8*+FhozP_Ks)xIKl{{k=g;GeV@x^6a(So!3LJnc&ub1^;hYhiBq${v zUU6vWu$eO<@+!r~2?qm6TegbRN`GT4C!OP-rF_p>kbd6WZe|F1jgFJBj&$spbrNcM zF>9u2Y`}UpntsQF}PkOci z-D?BG32H4!uSF!=x-IiVHyy8iET&L2dPJ5WooW@j)`95DXq9Yo2Azmeil`+DGoZ=iYPP zVed8XwbveBF0*+B$wh+2Mc14+U-pjI<$CunMtiD9+2W#gDEuckvK`Q3NFWG&{okAGc5 zBbT6qN83A#DjaT^(@XEX`;|Za{X-*;g5zNUxa zO7`q*F$My<*eIw25REY{qSAR%?F$)(o7>zIUFB`Az&R$(cw$54q`K-H4S*gd6KO5? zhc4@av#`BP0QKu^YGrOLAL5?pl2&UCTh+aVCM$VXS2k+u2EI2MlcgsRZYp)B==PT# z$GBgw~Wh(-n^k}}Xs@yJi0fUiHrZO^Tn zSO4kf_kQcG-?9w-!;d~tYVvCBc2Fp1NDp3v4~D-#J%3)0B~NTb;+fuw-j>ngKF|`I zb#(d}j0{gD-YkOMyRP53{+ECC7hn6z`~U4v|Esw`AAImDhKlB@31x=(NIyAcB8eZ( z**&95c?O!neXNcuSV`DB>4IfQrbu< zd3fMP$ios{lJ!8sv5-pp5^9&hEk!ujxw~^q)zR)OW*-{27!9C3B65|bf12&4Cy9JD z7Fq#|BoPBUDsgnK7HtU&2Zo?fIf>;AW3dxCld7Cj4AqGPAN|CW9BwNr5#9W)T)#WZ zM3>;E(jO7HHmtTJ8qMVAKNAPicd+6OUKi*fe|I*>6_rT=^9%V)isTMLlq{dINJ2!= zd>l=P$x6pDpai-o9R@?R{8SQ3Ltscb9aa#@MXr$$V<{5Ap>h$1?H71rfs+k!htxL1 z*85$>I@MxSEF~u}AB5>bjL%EP+ycq@f17X;3r-&kV39{tR z9x3XE8i1=pQ8jS}QAUbFqUsBEYqPFb50)y%!x|Y=qvlU%?mh0gALGZzPaQsU;*__1 z+t7Xg{#ShALfn7@9k(}c-n?w(|D_8i{CxJ{p5ga{BG%n|tGN|Dg~)D+pBJ357rUrkSWLDQ z)EGdkOAQ%6y{%5Ze3^LYjjZanm&b}#>r*G3xOL~&zN;-6)Lz3Hv(`IE^5M2ceJ@UZ zWHR~GPVIzac=X`G1KKti6wDfuplSe-!b{rmQzOKN&~;1?wHg~z=Cw5xpPD0NOQL$~ z9TVb6WyLTLvWrZ%b{Qi`Kq51eK4>^mOl+OBY;N+B+U&y|@AVlnQasvOZ|dn>j1 zEAbZnrr{P0FxFO3>@^v>mBCaBnO7@Zdt!r~+u|{nNf8PK6^I5B*bGv8rUkn^3#XWdMRG;WWaJdNIRIi{p&2lF)myI&yHPVfcU7D? zJ^kz-8AcDarth2OG!El-QC00#-~|88UbEM&u{hzLav8Sh04Imkn^B40hP~VDzj=3d zr|&r}yV;6kPJ|&YA{lJ1#|lv;&G1}v@Um(LFEL`PJ(J^P3W_6Xj^87Ui5}}9kX{jk zcK06dkK1E>1|toDJ~65NrPC*0J$>pVzCOKhLDvGjGEVn=DlG1(HJlJc^~NzY7#f6O z$~l`YM%USc`x3F5xzEq2R7lqfLXlAR674~W6-=F~d$rUBcvU}j)~Fk*inzx{Ecm;IpjLe|Ak`*1 z3wEOk<4OUVjbCX}tX4#~*&o+A^$ZJP7*DpIzWvs3n@RZNAN@mhOpD8^j=)-Voj{$2 zzwyoA|He1I@#BB^(NBN=Q!H!M!>EW|yL!b|t~*XM>21b_i9A#_?D`JO&=|04T{LCF z(*s|}PxnSF;C1RjaxBrr1#UXaD(Rs(p5V=)lQjKS@pqC@O6GQTUTECdMD`=G>RECp z%lV&Zf@I4u_7>`}ilVy(KnQ!*Z}n95uRGD2XdpI*CAuQgvh)Z6>st|Jz|5m|L1e{| zAghUA=nR$oh@iZ93h}y!cn?D!tm$6rANnDq7V|XU?3l=G;Wrp(I(4KBkl0@VR9JO&n^Hq{1X9QBs(J zqV)P}uU)usK^xg4){6*)4R&;-DtCB*ZcQ;l#Sq>N&Q0s$7yt3^Z@&4~yYK$)uYUO- zd^0-pa^CGAUVP~#HR;jg$F;ma{^Vm0R_BJY&zW%fBacup_UGI|tM>`UURj@;JM`NA z{nxKt{rS&-=H15czyH1+jeh#GpJ87q=XwnJksTj^p}d5zh}K?NS|u0d*B!1ci|94b zDo#O&?LlX05>TA>Zdy_sDYgJOO%IbRYa83?dlcUj%9^h30^(ASP0j7Hm25yr!;l@5 zWNwj{r)74xXA4nc3|5!yCGF2rWj9CpzQCQtWp$Na4fQy!tc=@nYue%i8C`|q~dLt984a~{q`u#}b4xr&`K_G~3srK3A@Ag;s`ZD4MGdKkki znIHLpcj+5JkzQp|UJpy{JdUQ7Jxs*F;;}|KXoF-DChNH&&twi&y2vpRZg|#Tr8dMp z+tA}QiTd)-9s;`|U>2!pmWkz@n8pyte3)I^hhz=RyD<5I5r7s; zR{ql`}(DHDe(c)L)Z_;#d8;vXX1vA8^Vqj zb2#?QFgEgbK0#O*k}=AkJwJTl&~7XAbvl|0tNY(ZpF18MJ?2eTuYz^!<_%s-aC!L1 zVRKq59WTA4|NhdIE4J}z#)a75%dPYPPhI}`ymL%fp$gvK`QtTOwP{JKQmT&8l_;FD zQKLhQqbPdG>ZAM5ZPI`E@FDcqmprSL0f)#-dayi3H2ZaMUOH~Qw@R!$br9h-OXt~c z852OJl*)^_z3HY&`_<}}1Wui=Z%;?HRuoN$_7W*cko}!|#yLJyI!-*tR|n#ZlQ2ji|TmXEv~Ku1N@pRGf{hrvXIl(N%W+e z^5^|ZpFO>Kv$+BH+gaa>$M{V_F#UTC6&o0vYETIpsCY?Wj~ui=&{1J2YeGHkOO__Q zg`jy>wb!c{##`%9phS-Z)4K3=AvcSo`Y)ST8&Qh(836pT`QeNZ7vex^9zU?(rG+gv z3~hO0%La^I6%#cN$`}DHI}z`W@zo8AP`<}Ox!Q_|DSjrknKc1vlN&T>4jrN_;WHiJ zfJN)tR5oaaM)+mBB31awv#Id)-&04?MApy@@-mfP6?+jbE2~XnE~sJQaKnafEPW1^Nqk~b;$#YyI#g68- zSn+E}YDNEyTudmM%uaR}BRB|K8oW=;Ax8sK6oi1<50;!Fvq>?As(L$B<~LPro#*)z zp?U2vy4JJ?N8ugSaRpDb(`Zct22nmZDl{OvNmqg^fa0*Nk zH7Te`Q2UJ7NS=;bfwSwhH6n+PSgaz1Wek-d(#s2WxuPT#G60sa*5Tle>e#&d$iN~S zQ-||@_#WlESKSeJ^-fhNfVGIx44?JE)Cjo3wZ@2qL=_FSYXz#Qn%dG>;J*_c)1FEB zh4Cyf9cK6vXYu)bNv7uxS$!C5jMFk|+{nQAt36wcEQ3go#iA)6OfQwT1J&%HMS65z zVl*|Q17LZt}b@XU^9*`1wSbQa;izmXG}47NIXza$O5Gs`l2^%1HJcD*Ok$l?$X^^lxRvC6=zw@Tn`2>! zTYqpSO~eqnel9%GTh*j{3%bWJ$n}(+qt$oiYzzcxf%T{C^jkUvPZ-{q2l&&6zlb1b z50R><;VZM)WDgKheBh0oj0Xe$W4tOK_#SSK@(stKt#;0JcShVji=6piFcmBf6lplR z&)3XhRSEB)DQh;Dn)=cuEnz0SEHaMcwT(bp1ib&111~>&`TjGV2W@7>WqS+6>DOOB zdFtfVD_2t?!yJ(SKyET)8E&PLdg*Ld;U>wUH@W*)vOf9bqf@6&fA#&Z{^A$^?#=P5 zL7e7PzB`^=y?o{LsZ*r!s}DXnV0?CO-H2G&sGD!Vv~hI^^dG4=>GAsYGpA3VedEm4 z%U7>lx%`74+IddEKDGoKRQG;j_O>T+J9sDP2(rMak$})9; z*?FS~b=scIh7cs#OBY>nvhk z07{*XNS&VZ2y{8_q6Z6>ANtA$6FRtQAC{bw7`8XS%2P*CT&^b}33#d+prHnUF69Y? zPe;*n=RO{cuCvv$^{`CCZ8p$}8{vn2z;?R9TCT7I)vZZB=f##A>|rU{$~-@%Q$wrZ z%iMWT5RfPu>Bxx;6=PI$H!zbdCWrKuZEi0@J)ab-y)jw}GX#Mp%zVEQi1JHG0*HR z(c?DVuSGHMlL88U!qz0~q`fV4tE}x)w(0;eQ7Zq`#zgHoO24C;=TCA)$g?)fGxvmI z#mc*TrI)E7vP>`bOzc);kxNIrMUj%7*zK(;%Ct(E?WyYu{laH4u2rYm#xh@))TmNn zh+_8~OuCrhrI0ndsOQfb!H4-Sin)1%Uo)rXi0wN~2imt^zrH}Q4noa?M~AIpY`t&x zS7b>+c6>9!V28JOW3R=XDDtqKq(qGeUfF+ipfz`A{nbjEDJzPC)kc9? zytMa_H`6?PaK~OvRzX-EXPLsi`|2|dM*ABM7&1@(^p=#Bo~Yx70_^fG3&QdjweRF>&Z6b zSH`p+2{GUkn^fawW{5T5WR6#^T26y3FrQ}M=M&8VXf{Mw4#8b_>@!>A?y<==hp z>J^dE#W6OVur479w(8KaS-^L`^QXF9zxO}?$E8c3f=BGxQiip|qWa%?`?u}D2$Y$b zoJcK`J(hCqi6i1lOMOS-Mva{g#VLM?Z6X9}p)uN=4#+e&RqoVWNr)u!25N|!+#C@R zRQ`$!6DBdF1E8xZoi}-7Zk5*&TZ_BwtSdRvLZ1-y&xon?ClTt$I2W0pd zEvM&5C7M0UIYdT)g}TKFFET7C%tn0f8JtLj^f43QQx~y%UQ>ZwjA`tcH#W_vu~cYY4w#WR~4qE5>yowMqX~v36ntSXxBAD zD*S<`u@?Pqoj-TZa+-JEefPr;Kh$T>a$d6e?~ESYyZ_mTAAR)JTW@pMzx?-q_3m{J zhZ$#S`u%h`7`aspY*b2SVw>kX)GQx;^6_h@PMtjYs<%~Wwf^vjKY07C-}?5qzx|V+ z{`AtNOWc53r!ES$&j>X>(Vb-bX;zw|W9CD{1@)lJvdGVM|8IBA0-P9`qR||z76N^0 z6XaC>ll(V^vDD!q$QgXhC*LGiaj9K40#ATF8;y;Kd~JTS+EM2`o#KerP1?y&iDO5e zX91xCyRX8U|3u^}rf0HCfB8gqAQkjuNEF0q+dX5!)nDhYVkf+94Vk z;_UESkqbvp+?WLMkYT7ix*H}o5}w$a0+YF*Z-VgYDv)znk73f7vACgPpf1Z#P*E3o z{nE_j!9$qmQBa&@jdRoWNQWaq+afUibXDzKJ-)I>UBref_f(2_OFP3>*@w?C;eO%v z>^~hcOR_yRjWB)ccAfTmF*1U;Ug|qp?&@IDSfFXukZ?5HEJzL1!O#~P4_>0 z@R?pcc&}c&E~6~cV6aZ{U0#i}LrK1M>ozRj<5~y(u3e%od;hD~(FM=#JD-XCg((eK zKYwl)nkNsR=$ynL$5p%HiDf)HY?>~<*YSv?9eXH9k$}r)j?V>s1qy<{9Xl*_)$B(D zwv&}!cma@5Kr~#i)-Io1bFDb99BzX3e!fZ1z3pSHThv-0IJNuDU5!*2oTXvBmm%C$MtA^Y=~H9w?|qqi)i;0g^0V?QEA=t$2oF^Q&%PSbWE znYxf&>})>tZEKOS+^+CCTS=)(%WD*CIjFeBaoYxOc5NSOf12j!*jQTs`(5J#3%vg5 z*%7p61Hs~b@9jOdf;PtEbp%#6GXi70`6@P8&6rLA-?RImCI=R(rY7bSVIv#SQ9x}9 zOVyAZSfB{^f)Z7kH%)3vPnhhsc7tUv#x6Q`YHu5c)?NjC??XdR?P0nmj>BR)UT+6b zB~~5Q+EQjrtQTrSnhjTx8#9JN#E|cTcMrTZQ3K%n}AzWv6(VHn!;oZXmOR&NQ-TG+D-4kRPR+|+?d%YF zh~1mXhU?Uxzsz-EaAN3sFf-P12-0hunisaLnx&}?XAx1o+X|_|K=SxvMYHABOGVQP zUX62m1T;r3&U#OfD|FQc$MHl&h0S(Kx#56VR^GBns5M&8$$shJHX%h*xLrDiQ5vI-u7i`S{v=_ z(Kp3D4pRsraRy{4_Cs54LC_iG0pyGup5n?36`zbQVfsWw(f&MG`KENUXW*(?K+6<} zNv_IlwF@NGX4Y;^XU-#ilHm+@^-%RiHBm;PKS>M0w2GH4`L;Jv3*xC1?1$bC(5vg} zUJ(YF1@Y?-e#KhW*po0Od#evcFth7B-}w_eF#fOa{m)xB%tb_iVnZ#3;AYId`PSR7 zzkY@zdVKF5W{I1WjVHFv5>2U&sViZuQdi}o+MeM9Y?8qgyo5*U6Bm-Pk|)AS%Nx}I zkbO^DawiIwI5N&ldz8p31tWw+Ts%C2ff%F?vSgyj9yqd$P^-KVGReE6%+~8qFZDtx ztu+9&iv5*-s#UnEDCuFQOF0X=7JY&nRzVzM<9gsPwUiAFiyBqTP~y_f%AG&>&;S>a z`jsXNh#bX{sf7#hz{gbv5rQjHF!M z8^*DZ9zSv6{JAi=HHao2bDDRmr);ma}Zc`R* z#~=}JV(Zu=$-w4@(Kt$2%uv2yWQWS{=8YT1jNdr>2Jd8}p^>b@l?zcXn8*L_yI=ao zH~!#9|M2%}a`l~!BBsDKwmI6Zm>Y4PdP-m3lS>ybzWv6Vr%%6j?))dlq<{VEUzy_g z&2N0;fB)oX=Pq7gux9>hJg5=`M2K2dOFI3XFXHb0Ylaz(2wJ>BG-v~ZOsPQw!|j>z%B`DAc8xVs|#z_<~@ zp2#Lw9ckqcTp&7N9C&$Erv=rXP0o>h$T2#tC8j;JuM8!3UDDbPnaEgvq8$he$A&Wy zl1DTT8mC^sDW+FZn7cBCT?wuP$(Kjq5AtbhF)^4*RoyYp1g^j!DiJ_7j-^mXj+w%M z{Tz(dYT~w*3{0%854=W2+FrKktr`k&*p<-kXg#B%Gz?E5YOU~NcTCIBqixQ$)qCjNW#52#r4H|4MV|^o;3ifB zMrI3z3oNEcNM8cSD`K{S^YFn#in7R0TzlopwVQX_fiza^0a8DG zZzF1Hqb~d>J9Ky3Fvm^1zc--3AI$_6Rd(U#Y}$ya-lEjZQu((nOCSy7 zxv%WCv%GF$ignb{UV`PH3YyziNUw7nMS9CM(>6 zcW8msQUm+Oq#YHgN^D|rruFn5dm%bwMDq#&?5TM=I86O{v5G@~ST)B39H<`|7b_pt zt<6TBiC|<@tFSsqaKR*TKwlV9xUIur$6{_GlEC;w0ia5BI)%t8EX2XJhmK*na`?@o z3xU!^r3TS7QpYy4cO#!|J<053>TOoqf3iUl*TN4z^n5=8oR2EjR zTfL1(TO+#QD*oPm z{1!%QcbTZ8n@jhwdD~6KYQnl^iOzUyFhDQ3H}THv_7cBPtYEhww^4}#JNiL+0SXyu z)oYSH%Sh6CZm9rwYRSw{J4tbRO8@{s07*naRPmYhyaZ=#LF`?WT;=xop~fL|eRywQ z@nZUzAt3IKShEwOh-sx+Qb?s&xTLIFN;P5(4`~6aVx)jhfBZ9B0KdEq8Z-awvpX`X zuF;M_TqxOlp6{m1SW`V`*i)K{If;)IrSqiR5ML4C(gLJP?Wj8&gI)e_+Z-Bt-0Dr`Q$; z-}%mW_2z!>|M?%cZ(rx@dYT0MCW$j=_4dB0GGfc@NYfid+Usfog-BT}-B4d_5Nkv4 zWFK@@mkBhe4%F#O-WZY8Ayf!5?;z5(-I~||Z_486AUP89NHy9L6|yDA1=QUuql*f` z3{WSj?oNT109rT_AkVm5(tBd3I2UH zsZ%f89H4FjYyHlgIm24jm^@Vc4uz>@C?^8K95NRrpv_gp49P6!WgCA#{OIG4&YnGM zG1YHA{E!kE1*_(`%tHG4&;RLne&?M({KG%|`@jDO>y-?!^HDq!NR6WLWG3j>HRo0I zNfSPsXS{J{+m^MToI6MQSFT?9@jw3KSHJqc{`wC;`9x;Gxh{ID4Lb7bYyOjNDxMbX z7>8ta#IzKOp%ynVW&4R)99$n>TbUR)Er=tsIL8}c6@%=?)u>}Va3ef9A|e6SQHdy( zugMyW$vL!mHaFHY@J57-*pN5|WDrU_$h+$#8V0cB(?pwbf{OUmi}Sc-izJr-^xdDY zKz3_#uZymr@rkI?dJe}OOG9K%)^9YgX0RCO?!y6Gg$ zbIcghoDJ^8n$keuP!$u7vUhirt;dUqqI>Rn10qifqra6)8mHV5y+bV+jh<2r<8v=F zsD#Ag2GCf3R}0SE571H!0Fb)gHiJP8n;Rl1is{QA=YSq*^6_=c>5N))gjxrQ2cF0Z zqlT-6kkYY#E~X{hRZ8_5Qfv-JjVm5%1#teRJ3}Rcl#eQq&vmX?D&P^PP5)()c9c)(1 z(L;w6WgEHJnRs>51FII)&7W*Atxz_xVr#$N>0akGZaF@2*4AFS1Js@M0}-z7Lt6n( z$F;4#cbqtG!x#ajoi`Dd9pqbjpOm${Makyv+fuNuft`DI?h#?>I3;)Bxdr+d=bqaZ zNf3*ahF)P&t#I!R!mJCC(| zP_7HwcVH&g7e7cZAV&7$U1G+d4Ymrcmqb6>Im;5%LWVziqT;A*3|e`J*NmvO!xC7C z;SC3ZX)upKwV2ul;UZBaA+mZ(To{X{LDae%VY}=Y>srpjX_KA(i8G3{s6ia_dC_p| z=UD0#wF3GXEG7gHfR_LT9-G!?J`27u%c$HJO38#-yZxx<2pt>6Lna`&pC49S63zxt zZYme7x7=MRwLj^%kj#4|S^%I9vc|X9*{U_}e0Klbh4a_1-`u`;Pnd4}LB|*dg}2$j z%kxSB?;SH~bi%tWlSQ?-+Ulxza84Ph*@i?F7(`C!gP~S;pfdp~Y8pCT2*853Mc-zq zSwNBc{NoM#&B0Ub%>a4&$Vijr>CKhbPODuM&w^4e#0r6e4}!LqPw8vAjG&xj$pef& zC@85W?4Xax5(wOcmrHHTiu4K=+-HfT%08r~!uHe3qY9CpZ+)KUV$w`b3zi|8c2=Wk z8b8JYw8>itFp(ULMC6eMP8?B4#@2k77a?&rPiQr*`AT6+6W$cPmo;WA3c`gG4o4|s zex6p4s-`>8n(;e`b!N+e-_5OBaqcuH$o*KzL9$pvUm|f61U4`&)=Z%SIR3F3Ee%Y* zY%!lef}TpYJn(e7fYz9sMmuNKU+@32n^AD)+1h1VTyx{u@2BU6qJC6Ff@1$$cYmt%#OHx>EfzjYLyhcr%ns+|L~8# z``YQ({^Eaq@9y0j+CC}|2(tiU*zxSulc(N)?`wKb9zVWw_Usw3$6}g>G@Ubz+)7-@ z&ulVLkP}EhbD!jU(0#WuwNh6ndg5fgoCpiGkp}0NeAk-~y*kl&Zy~N8k7fyjQiPgVZ9L3@Q5A2j1Vg0U~NjRdS z3+NaQmEVYAu9<-Ira=MIn2h>s`&j8(#BS$V?>}~gvUDP9YV5A0E z8VmDmM91okAx~hW%fnLL2lo0qb@J4CFXioh=aiUA9FYkCA|s^pbx=ffWgI<>ON7z# z^QRyF=EJvt>+RqEop(O?;DhH+yq#Ln<fNu`NT+ct(y`qI7^f-1Ufa^$kk(N?|!J`p(DavbjmnfjTr_yZr6iy z&EuXgf>2WV@4+nHAJPWO23F^?`+;#DY!Hs{JnS(=c%D4tWF>>Rd*a>lhhQEq5M4)0 zU!IN}J+;}+2kl48?Ks-TdDJZg4MUD4TS=42f zTtL-4=xi=c=C^XpXlel#L+m~Xs{v4tN&witKf$)YIdV9QE{M6p28|R`zDS){vS}>r zGQDy<&qM};JyAFaAmTgW3hE+^YBhfZ3i_+(g#I(J*Z}&Yn|JP9zIxewn4-F7)U?4i z`c1!4LiLdDd%hPmft?xE3b2Vi+t246v90NREDU_vmMXfHHm?*>-^D@m(06YO#dUAD zyQr68wJqd5cU4YGxVqk+gxqOUWv2I?h*kJz`%L}n6{@|{N~Ear$ioK@no%u*Sjsk^gwO>01N${ubUr@2|ICCV zGHdZ+^>9WBtlm+h=jvpp2CP0&Dj?StC}cO&sgF8ppUA@`j{30~tNJ`koFkig-qswJ zK-h_+WoBG-ot-afZfO@01Ch~|-~l(XL!pw&@Cv62P39Crf~Cm@n4ANIP7x%@3&$zi zvJGPkfQu`^nQD!!TF61^7?-7~-E2=GsyG!|_AT5u&iC z&U0emSU^Ur8mUR9B33w!uvdy#_o5$V+J&=^i_tc{9uH-G>e^&0cO`f60nx%49hf`B4_xnlh%z%Ko#r znfywynwKP}M#nx>@(_`)sg}sLq5(_ptT<*CD%4mm67@lzVlwV=(iVBMBts|#@w($I z+pARU)Ap$w9(T%7ZCY)TykmtznRXB$9-$Ev`#7YB9Kc}GQ6;86OX?L6Hw?$f(dr4W zmCe=c$*R#CBp49SS%DLw<#*)JdC-xc^wr35$C3${#GvC&%PuI%^EEb7>b|l!AaaUX zZZ%3NH+@j@FBMl23p+9fH*J7Kk(ItUQ0$Keyqf^#%C8b&qpHUih|!8W`5wCY0zxIMq3nJ z>UviJL6j~fz>S%K(=^Xurxl z=@|ObJyJXmuTu%qa&>Jm#<_q*#}GtqROQ<0gp*}J^08wiY0$$QGMF4)U1KE~qYGkx ziHYfFWz17&vQCY9c`%0l(D=?ERaVguMs##CpOuO@AR+nbwl0md9{G|yxg>$%0V);; zrF$=J38~MVeZyv1dMGM-G5f;Mr40E_9dG)WIGO9$XLq=W+fj{t=dNbF@uhPgpKI); zp(n`Vs{_vh=P(Z!(Spht&3`S9nn@5268`mXe)Ia7GjG5B_D3K6#;mVea@+N72{=Ca z_#^eC0cyopl(*!tgy5N|LJ>Fa_@w=U`<~VJ-R8^((VH z8ye?PLhU*6te?pu+p_n;0|av9%1895XZ3%lita7JGYudX_=;md*Y_xUO5bp}G&l*V z9?A7U=z&*E!Rj-XR;Msa^i-+LJ)lB2@}izG3I?#KJDYj=7THc zKA=leO{0R$PQhE`4bL;)Sm801`SFw8t=Ck3?684aA(yT~v8k$j6)r4|Y(VJcsa4yk z);34Al=>e{+t7{lpjRqA^cvTX&!4|^`Jxyd2qxRcA=_RD+YXo$5)nj(OIlhes1xw{ zZC4zPX??(UH@AGy>wa|{wnIyc=R|B~`Vy}N4b*YcVzd*-nGnw^AUS^G*w}k^k~Aw^ z=3!et;bo|WeMKr!tLNMDQ;^L1XX77v>O5(E!gV?3+!;+fE?`6obaTDqBTL zR-z|2a3OXQ<*$BmijfF9WRtn4I!^WWYAlKFr$-L&GjqS*`r)rmyF)bVBn!xkY_I*mQ z32hdK=;uZZZU_)|Z5Fy2sR3XwP>#{D?RT#6S?Ib$N9_d zh(h$_3943^P$1?5c`j}Gau!9UtiJT=L6$Jx`BiV@T*N|}0l^K(@X@FT6KP8BILzU1 z9tWP%Q0uXOm*vdkLx+y#f9HM#m2;B7os8vjcjQndu~-X$NAf!xJ`0T+-6hC%AOi!V zFCZ4w#7$*^PxOfh-e7^owz`>jCM>5>OAO>sVU}V&dHnior_Y>u{rtsC=gxn6<=PFt z!uAY7n*$bV!hg{_QMcj>J9xu&Abl|waci;W;`(J(kum1Y4@7%slR~n$?M#*4Il3ic=(w8Cs#V>wo zZiazAwp?k6*0Iur<$5D7I*9p=972P?@FlR3lj|ieE#quo=FHBglu4GckiF!Dk;|H( zwmqYmP^jEf@3wxK*3`-xGgD=jJ4#YWD;=G5WwRD*+Fb=Xn=tLNRtCY6p=!2N3esXE zGacs-s6q-M6?*JyfcI6}_ z!-0SFN7?}2|MbFnTWaT^nk!eo1B~>tx)PG#jJRIBaQ@_})8?(IQk00R*RDMKw;k`j z_uj)tKf8AGMj{_8B$@08PnBk3%|EdVS4kmpj8hpc3f&6iu!22Wq7cD?J<_dT)=`p$ zW@2@al*pHa6qlCf7`D?!3QIwAIu?;!OFNYK3iCQZ0XK4(y*#Chg2{mp)K2A5>YjCl z+yE>fpaypC%r`v@5ZD7WB!ZtAO_mpA%GB+M+`#TS5d8#W+`ID#SSW%wKucHg>ub?F zP$nr%r7lyr0$G$K&KK)L<4_m2 z*yyK|kb^PTsjhMlv9Lhc4m|Dp&cOK)s;CXUU@Rts;6E6M2+%+t32lg4T!`Lt5lDxz zTS|2HYcR5GdKQc@<)HF~Y;^UobYB0JGA5c5K4=5z?nBRP+cEY_I9Vey)g-}pndXN7 zBoU)^jI<8kq+MGK?Rs2*w)ODows{Mt|7tiCRLv}&Xow9`@Wb=gEz~}b1+DNrw=3nN zYd3D%^WID)ang<3x4mJ?+F%n}%v7#NPz}A)2B%QK9wpCV96@WS)2e z!KQFeH>&Y{oA@3%a71sjm&jhed`ZZr)Aa2(&i+Br0ySOE`c^#v-y~Xr zlaNmZS@bDR>=hhOO-8bJjehb6RtRiwx4V7YjP~}>0S>SAzD3z1QcbBXE0GzCx0sV{ z!v(^`R%JaAPmA0%atzBNU$t$3MI^olo!+H2>nv%73Jir9o|u7xykO)!eQYKOyt)hT zH@(8#kEd2+`_MZTIK&LJmlfow1ovZN#prd}3yZQjW?5ysbM{}g1^ccVDo?#}-GT=% z&E0$K=us;d2;VErIAV=F+U2RIU(jm``5XeFZFDvD8X)QciaiQ*H;nv#-u1 zBsb>G@SrDWEg_EDl#*9$kluo0jS~vV+NF_zpnxKr0A-}QdD<-2W>4fhQ%^h)LgTre z)#WOv$$hy$UZ_|+c_fu#C_p_Vh7^*pR^iNAJ9C2)w9M)Soy*DYK6eotE+L3O1=634 z?&tr~TFAyD2KbAkmT_5dAd2l{5ahl zY=mUs74KzO&~1FGr`gyB6 zmYlzIC(%bxx3}`C0q#LG*2_E`RsrXXNlh$**i3&Vimah6@W3WHl=K052|KZ^TOo>> zXe~cUZ|lBnrHbwY)^+UIar={;JAdI5&44Rcd00#oB;a%kCjR5g?q_Kuwh^UFGG!2n zRWnhye$uGL(9l&TI$a3L8Nx;V_N$8iiQki=cNf0=vU%QKO+5<4L?6!ieaO#?uOSa;d%UR-&R;i8Ao;S8K*fWi zS+ou!sEr}ncfb4H zzy7PgzI64HfvBzBy7nFm5~k0IxJa;aHTZj<=Cx~A5A1)%RufO3-8XCY(&bC4$@jkc z-cNq=Gt-Zkgvd8?SZeJQ1v$!5oc(4?=v|C-yQXQy(a~%@O5)zl#?=$I>zM|p`k@Y$ zrm-!rk{|;S!~JwtIh+%e`rR1t#(_ja_+*%zosGZA3Z0H-q=*_vMnc+QsmdkaIdM_) zNk6EISYrx{mBJ1D@@Ezj_jqy)l67O<1U#%ESr#VQxECM?Lxc`F!z)m3Oqq#_o~*}l z6vVE9%B5DUCi(i3w!8x$UmcU)U1^t`CRYS3$UO;7N%Isk05Wn7lcPtzH;7|nlcfh` z9YD!v3S34iO|8frFG#Z5nGwk`l`D(NNz`|jwYmcqH_k^=^B{ZdDx|?pjyXHtGNfZt3fadc5mAtEW0i$5@tH4cM-?6g{2|a*}+Co|}Jv;qv8A&Y#mo;T6GWPoMhjx85Y9bLT!Wux@x=d!gygt*P+Nf*tzv zbzg1oHe|lF*C=6uvdDG1b*I*xwud;i{&R^5%z)s8gtQ^~C~?e3u!3L$RL6RWR8OB3 z?t3edxgt&JdHm4yDehiPXqN41yJ2%btyc}t`RhdE^Q4YxPL&PhDW#>fq>eH6aoBs&CN2)j3x7jJ{Y?zf3q`43fu zMmHvAf{Y9OF}SYu*cUwF+D3_Jt8x5w6#q?UNFD|d0feJ!NI}ZB(>m^8_p3A*cu`l@ znO)VS!P!$}A{tO1B@7%nR1BV$Y-EBF%H3eC+$e2j2DyJ_)E5KyP0JNuRQI0{6sw9C zdUGL_(Q!LD3d3$B=58Fwif&kPTM8|kkQ7YWwZ-}Ua#vC^?8|nVfOt=-k8L%E9!I z*NoQuQmEn#!l)neUl5)kMnWZsl3eR}x$^NdOERcg*(A=?(^kxleuEsQ$-a;UCt$Of zgP&+QR%}ofb)_x;1!)5$7S|Dlt$=F&%s>bV=soo$y9!d6{LbIp=uWqSnu{)Y8y(6( zB#E!&aq2|sf`Gz_&ixMf^$pKhOp;t|PrM~XFh-t!LOAJo>YdMRgQbOUl*!CJ8|pKT ztJiLDOtTdYr_f}5%52DFwO?tuwU!DiTeEkeiG_J7j*fV$Gyt!NWYnp*g@{}F2>3N4 zmAk}`t)o^dd+%Pa47bcWZ75^OK<|pF|G)gjFFyY8*IKdYKx?I|5ht50etzQQYrp>o z-}=QbesSaab^X5Pewgs1+;eFw?)L_gcKxawzN8xVQIdpx5_f#W z4Q1pTrT|v$r^&c7l%4w+4U9p^G0c|)*fa-bf*9e$xXQt3vzjq9LrIRNTzLbH2PJ|q zCXgfg(Ya)O!k^BWu~3y6OtM5b2hjp!2*09-c~n%66GNq84w8(;xHC$x->Yp*{jqJe zTIRiJUnt(#&ujwfYb>kl*vspd*4=D;0s-6#0vg6gTYU#^42P469M$`2W3dDdh%}oA zld}8q=(n&S`88K_+we7yVAtLo*RR#K#>$k$&F>I_QQQjgf~^Ifm~V6P z_S&$O0e1fV7KNM-grwru0?7V}X_y6|7#q&%OgSEdb6*BIj&YmA*_xe9#e=pBL=quH9r(xSL;Qlw2BSVs6n?C(r7puoS4 zAXtO_v7`e8Fy@DW0J`WdRm~G$FA{lFQU}V%rpKcpDOIhEox_Zlaau&@KGcbqEyaK% zd5)h;jTwC)r@Z)uWAPyoxbtk$( zqGS~YU)W~3f%Ie`FWzMHd_YR@0vc9^ik?jN0Y0#C)vfBxSV}p`wz~s)BsmC^@X{Fi zOH!&5>23El=pu70vB$~S7r|f_{>m+-7DkUD_qEyugR9Cup63Wq01y=I3tAX~92Uvf zp7%R-dAX%kA%#LooXij=4xiCTOg2gn2UB-B!>$15Y81~r|Ma0(_~MSO0MlP38%)XD z<^u5TM}`!hYHT>qcO?bda_I2k z0|yQtIJjfa-UG*9y>aulNiW_TRbVG>7HewBDA;POC}{tiZQVzQ4jfec+;84?k!~vo zxH3U2f~(l71a!!l7xDD&ox9g>-Y}uatA$lF@BH=|{$TH(UDo^RX0Yqcyq)Wb?f-T5 z@7-+&q2nix9~Xw2v#lMn;OAV(J6+^}`!SITPusKQpI!Xt=0;KDtB9B`$b_Kk*xfRZ ze3v!B`qQVj+o5r?cmF;E7dLO+(i@?l{;fCPEc!S5eCH0H^v!!`MCW{B>cf1cK2>G` z;|8gSmK}L6@qPWqW@K!p390mL*mC61A;ENH*Jc<##(uz1sl#sN0}VAR=+P75xb4_< z^{M|099Rf87?J^5uA%K{L=tr-F8woYRYz_zvtvH^jk3@XSTlKM>ETF1Bcmok^%m{u ziz!N(LQyFox&e^1Dz*i!If&u8I2O;<;0r@_eQ`!5rZSRE)>37F^#}ngK?a^G!d?=9 zLy-l}4XUZgMvX`AR&b~|GaZwM!%>l(TUbn^qTX>72J)EcWRtH#=+T5lvJVdfY(d3d zI1z&!q)0jVa<{SRzweHO17C$#GXKcsE-Hlr;zI2gJ~EWS>KiYM>&UOw?x-3IM38G% zluY9J0esTTWdd;nKgmR*0g{srdilu|RrpT_QY0UQHTkvs%VVXph;cKv0_j-N8Zuph zv}ki)nJW3Du5=Mege0dFOT!bKVnLl0-Hr&B&;O9sv&j-}8Yy7v;c%jBl-Ujy|NmD& z9Eml4Hn@V(MF{*Z8oNf}!0+yyR5?;FFHS7HU-euTo6tO3=#_v4X&hu`mR7?sx2q5a zPi_{tu?3Hk;DAj58D1Gs&?jY>l&|0@fAx(hKqg=Eepj#Gtp9>zde+RnU0HrXzb3YC zO!h}{3f=UK=mcOP1~a^6nuT}>XNE_}6)Wj;D@25hX~J8B@{(=s;2Z@r9TdXCl%Q$;XB#9{rjgoMgO}HF;27=AR&QQA``5vVXAfP7u zs7Age<)xyDfx;z2YStZ{xki!G1#(69Q04+_nU6myQ8vGN?;^0dLvphjg7z`=IBZOz zjOrAmm0dx~3of?CRwxM{6~sU^ZFb|vKlc`%@#lr`Pa9!S*Jhf?2Yi?SI7@EcLAlUX zp&DsH5+acGAQrsDX_=%4V?ppeqD<%&ekPGyDq#|TQ36XI)8$-Jb2R}Km&u* zOsqQS2!2?q0(28C+jlzmD^rZx7`3$ouf6u#Z+`OunJkZwT(4ci+*(1!@?HstSBQdDM|> z!1vr4up_ETo1I8r!K)GZBu*HJT^-xh%-(n2#HTHQDQ9$sXaA$b4d@_uZ;2yga`bHJ zF({+m7aC>;JrAv3kuFM*=(^>|RNc#^LD~6S1!QOqble1ZaFqEA)dnvbP_Q8*Lm86X zA1f&rVE((En~RKr<*cq6c*p&Pxbu#X$}>NyKenZOG3c;>4v;)3;LiOx)U+jb0hRON zd5{jsAXe$V0M8AU_NhkKiatWr;}SR$UVL;c5e#)h;oM%b=&?*xSJpV>oqV`bgegq$GNft+5(HDQ*-Z0y{_nm#?Eq&PPlF`(>NrykPO;8h4TK3oU z{Ra;)tA${?2w|!PZLiW)!bgvEC-87paBVkYf5V5j@7}q3-P@&Z-?H=ny}P1Y(eT;V z&zOkM8=5e!N4yO&1={lmk9x>{Ji=w?vKch6&KD<5eaB|5fh{(PX`zj=ECjSbLh~)x8FE(!#fOa-SO%)!&nk=tq?UmPVv`VT!gFC z0#)0R=Mv_;B*0KL6ita_p3T-_{0ye53iSv)`F!T(Q7@Bg<|>zLx&^^Y1jv^x}O7;^gdxBuAr_kZxWoFeCkw3+kc&F&^-m?_j~ z+{|7SU>4_lXtFms zu_Z<5Fl7**)e}l$*qFq)@|7!!y6G$QQIZAbh3n$d^VC#=jF8h5{r6IgvGSq$1smK- zj-?#{>Z?Iq(WKhO8QHK&<*JRvq6Me$_;iGNU3cz*!WktI<3SSMB<-o|98i_i&by!8 zG097n=xv9bbh3%ao3j{T8K=fXRC9?=9jr7&g#1i*|G@FucF(g?go+WfTNtQs$Tppi| zn8G$G+=DUMIp$jmX&13&a&sEJcD8bUF~=%fY!?1k7|3nRCkp>^u&t~wojlm|q!KTbbqOaEgn0@cO&a12ntJq}s| zk0*yRDP5JIQBAeV=XW2}+lMr_pWDRq(Y^ck?37|Tu*wX(mcy#$Y&tnM%6^&)u7U&A z=H<8U-1e%l>bTwwq_xntB-YZ-bbxfpGuR&f%xkad4p$i%b`<(+9e8cH4s(R*e&QWW z(KqeHT$uZx<(y9(wNKB%qeqSyF|bs?)cl!KwAxV705qYiZB3II+T9usJjn~O5Crx= zy7S1dK9Ph`((%E*p6#&d58c)FPyH!i$%%9uX7Q*Xp4duNn@wf}3<4TtCDI^y{d4k} zkA9Y-t0?HGZZcJ47EiiDit0zg3AqJdag-Ko3{2<7poq|n5a?lf=mynY{&6spd%pox zw?s%r5dT>es)6lDDDokb?PyMJ#846zmG&mo7#X>jm5zT5@=&`9SKJI(G*xpaw#n;e zLhwIm2h*fP7JHL&s)j@=X~wcL1{_sIIwc=;=$9z>G2A4+ut!G3z!DyCm+&`UV+wto zm#cJxMDjTSgv64S&k(?yAi;U8MAO6sayrN@K9lBLb>6)!W9~?uI0sv|5i@tv1={(KW%K$l+L;=cpCyf+* zlnA4I^+04}@qz)~aCnKYr2r@a8%~zohl?RH=pGE&5gfWCEkxEVPYlZi7Hlm0fLSm$ z6h>ge)>W~`R2Kt&-f?Xo1Cx*~tv8u*x52(;uJ}muRqAMR0%-wP8q9zQf-nP@NSfzk z9m_6j$4{O-@yVy3dYyF4PgKCcJ+5p%*MAYc^A8HXIXdkFl9PrwY9*L_+)E5iMC=70 zgbXEJ7{;CIqj6U?iKSk5?%kg@M&xP!oH_H>*T3=2zx>O;xOvmu=$G+~Ty+F$@q7Q_ zlP|ybb#I3H?|=4Z0<4#I9RS*n0|Z;I%JZ$g`*nDD6@WHCS|ICeWtdw9X7N4jqqLH< z4zgIulSp?9O)FZfz;xY*{5=GE6}CKuwCR%el4T!}@r2_4#$vEFd z2HhlUm6k$OJp`R}T9o8Hens(9nlrgx3Ib*W|I=O;$rT)hu|DKTN(KVk*eNqL4=Dp~ z+||&MyOV&?iQqLV;LQ?y@r05lCx+HoPG^C2E6yy*C`6PBImwnyv$LKIfWYG{BQNF}^(Da5CE)Zt3$TtP|!oAT(| zyDxX|erUPd{fF(>F|S}JK6d!=M<20$tJoj??}v8zWNZ1N;*okPosTIg?4#UflTvy# zn8)UkrDhi|UbdTs0HSgD7{LlrX3tLMdrGD|aX%tRbV*`TF46Q5l4Loum_U*2>ijQ- zvO@n@CoSv-trq@>Bv>#dhxzt3WF(`7=moOWI_eC0v@PV|cQSU|Nhl}9oR%YV2*}C| zyc~0kiB1(O*O=jkgjj53p7K%~fJ0n3qYuz!OQ5(Y^?(d)z`O7}r-OZ-mUe^|MM3Co@m(<=VA7ckbFC$ihxxy=ucg>r)KvAl9l-+f$f=Z(^(Iu=v}z zfA8zZk8>YKj~v!nk03Q4yyo{~_u=CQRs-L^_dx5yoYZE#YXG!Y&K};?29C#%9(`5R zf9#mQR}Q^mUv}~;givWF7o|Q<37!L`c`n#Xk$MCNYWhnR%PF( zxuS|tVU@pl_K7H2qA5bh0Y+nmd!7U2;c!u=;Pw+YzGg(k&A@n)5fz_Yl@XIAgVj4MH-laj$kPA<{0$N*SEw-{l}I~1~Vt&g*AID+S&wgh`iwM2#SlskeVJb4el&@#Nb<7YX| z%P*T{cly++H_pChs-Zp2R$aq~28iMIJFpf*mDFXW@!3lf@Jm}}TatpCVnYrRN;W(^3Sk8E4AQ}_}&<|`&LJq>NZ zE6A`@8Q*GO-HZUG2M!$kqd)%6zx@24KKjkCy?h+odYrItcot5ZRUSWn`YZ2!^>6<6 zZ*ShZzVE<(JBJ#rHPmX+)+}gx#h#3OyNpvBF}w;a^%We$l38sOGzHjIqQ+CwN$yDH z@m$Zv=5~N8?i3LSW}DLvdZ{Xx44}+iC1(HmX(!YeCO68_gGrX(@MyvqrK86nH0`I( z{IF5DpJ_poT1llDjwnxTKEQIn$`=5U96^~Af;bCcm`5sYoI&e5PL& zQLS!OPX)>F4XUkswR2dzGAm!Jl`&o}rqEtv1g5RV0bJ1MY(rm3kl9Wmkv^91XBCXT zQ5Pq$B#90uCCt07>(nuiyE~<>GW}F{o#83zm;x`fvPP%WqcB#gY2bi*Tyovoic|@* zv5;QAa^>{t)2vi?ab1b=$ovXuGKfa5Qyn7QT7%f2@Jtorfg4R z-FBw3dcTI10=o7-y>Njevt^J;ZYB`asH?y#9DdW9dj8RlndF@AHFA{h#4tNi(9#aG z5=;=#p5|@*WqFA{W-_U& zC2E2=Uz14EQIhYZsXi)=e@Mr0OFBYcCxd!~85NEpI%G~W=2Fi|M29Nt%X&*gx!JU1 zT1Atf6eV3muiAZcN$Jt+n9wj}r5e3uAmtz7bm9nm3U*>;SU~&je|@i$C2V(Bo<--VZ+J>tnR5vOL^KQYAnm*x z>&I`|M;McKJzxs%dg9*D=1loraw-2nlqA^DZSOZ}U3}k05j&q)%3}?sjr_dLv5l@R zDi!jJa+?pT#uMgNtZUo?x#Sze$S8~$+~5*h(i;I(`SzfVBW#hlx5+3VZL`}g+~Gci z)er7J@*bUg_dny-Ou5lic=gp2hYugJur(uTlua4m@yg3DzkcfE>653-6jD#`?>$hM zBp%#g8bz6DE^TZBr#1O?4&=!cW;Y6$#;KLUE&Azpu>(_Gjk?1*+V4&8C zJUtCPoSvJ^5QUt>;H0`7N?<~`$u@@06bDZvWF1h{X3;cRHZ-P0N|`_|D8=UGB^=9w zahe3O3RoprKCa>wJ|N36WJF3fxba>&JCtB^XdEF8)VUG#j?lmwR9_yc?$n$RMe2OA zq|WkGK#g@~W3D22ic~H$Vf(n(F)8M%bUyNVHYAcNs$+qiohFjcz=O%Adq^aVfW24pJGPI0x$N>&UUq;-JyKPiz`dPJHwSq!%l4IKj|#~4qt6IfwT5~`|B}2q z61;y5n?sbT+q?eY_y3POw{HLJCqFR-6;-p|cvn*yxmv3jcklhZZ+`PXe)-D}e*J-- zU)|5@XW+xD58Ix|LQ1QwOo|!I1N(8IE)WoDB@{5o4uypkzGbV#aV%u2gldOP|i=4(~)X66(MwFF3nX z%m-v0MM){rr1T6(r6oIV5{~X<0a;Oc{BBx(c@Gdo%Ed8bT@>u##*@hK5GC0s1CuqY z$_+|1t`u!lIVj4CxXqE2(4k;VILW0X3&bR!IFk-&yT3{|Vv@IK$ux-7RvqPVL|$I8 z+%mEpjOSe>qv#6sj%Q{89Xx!{tec1Ty;m`)OJ562Ji3b#p4_9LP5UQc1KPbY)Z4f3 zo;Y#x>b0v7lXZ4bs8_Zmn^mM#UZu{25d>cdq<9CZu%@2AD&|_JNHG#;|7IP91{QX| z{QT(yyZ>2@xA)?ui~3C7eDkdjKm3gXK{;*1XJ>PySf?U*dTd!s;u8o_ccv~<2Y)=I z_Vcr52hn6z+6!@wZQ&nHi8?`Sga=R!xt=WHMDP*^eK>)N#SEZc`cpEtV_K`x>N@{c zdZNHus_SG>48*k5NhVX)5m%?~mLn3QWen~G-kx%Xsb%ZfxZ~&*;jsQ*OzPFLk@qihnBW6)Q&w@A# zOoULemO4v1_dP{NjJ zPARu7wKzNWY}NOF$CRH(4<0|d zb^G?6&pv}xAEqK`-PkJ!4uYWn9|d@^iM31n*GOQ^nUp2;=Gx~^^)TJhOt7`*ojaE< zT{6i>mg&hdKXs3Vw&Hq39k0~xH2weHom=AkeS7xWZd09k`oxL1&b)r~*ilUgkjW_n z$;DunIF>oZi<_xhLo4a!iuy}!D>`>o5}#bxn5m1BPL)FNb0mx6H~B_Kn+T3Q zph0%C2c7qe+1$LkJ^&HFkmZ32hgD<~sW0NB2x}mM1_}?(sn&|+y#cJ|lZ_JGqlgJ2 zO6)H>LXZT4Guc)pSMV*3r%^t_TFJUOEIC;@Lpuo)3JUrwfb&5yl_dZGKmbWZK~zi7 z=o}y~5-lnRcld)JgQ;4s5yiOIImfa5aafHsj0RnX4H&0kgW&}m@f0q&HxD+zS;e4m z8Dao}QR4ne#P&1eCeRE&L%$9Nz}KOSEHsEQ^U#r}$>BI8s@kq%o=5o(*QAJ&q1}yz ztGKn)FabY=MU132zl!CW`nU#SGL*ctD3mM%&-24LHwy)Zlp#AvQIOvNaXz0v>^iTs{U_a_ZddLf2s^cFKU0{I{9 zIL6{kO?+kV-ufG(NWc+VOj@2~g);sy$zKH7fL+q_H@F|om> zJmoetkl*2;xp7XFKeW%|gGW!_`^tMqjvoE1zx;Du8Ri%nkK{SeEc=%MCyz`Q*cg!nM){LLkd)0Cf%1rd27Cm z4D;z}RK###>Lv!Nz%C+Y0$^lx(KqFfnKnT>&w@h~5^0^Mh2+tWYVUUUEX5CR-Svn1 zCT-O%Rl=YGGDY6tgGv80AFcf#3Pi^!tA`EpL?Mm>Fl;56Cdruhqh#QC7)^?m1miv^ zoyq!z&P$s^fpb?CD#>CP8ROpuLxMuD|003F&}Ysx&h6v(;DOO~8{DC+bg3ZCkZuBx zh?-A51=WR1rM-ls?NNdfv;V+8`^M^DCy5A1EoF}y0;Y~pG&(hC-Ddw8No92pTQ6vC zlqB2mRSR5mR>Pd%SEr$5y3;Ga+uK)fTw~h?!!CS!P5>nyPzHodkomVMmI(&7B(D;| zwUslVrs&GNZhHoxKC5bs#38kKXXKU}$n(@pOZ@X+n(Qb6jC|&LZq&4BTth>Va_CPf zA?9{kQl`Cg$CK%31S#Vtk{2=Sm`rj_kRXn+4h9#1j~EwS(d16wW2%*ilBf5j2~d`! z9R^~h0<)lu^o9r~5JQ2=s#9<@qyn&6rs4 zh9DNt(T^UOCZ+f*jW^4wVnsCB!uW_B>A0Iv3=<xWw*Vwz|1;{l>@V&f8Dd?Eky>@7HiqXY6<`rP=@YKD(D>!A!Y3AMWxJ z!OgTAE;fOBPq4k@4exnvj%>g=g`iNP#)dSO_Fd!hAt~G-2XS;X5UO|Se zx8Ho@t6%=zqlXUb8kkZf3l>KkMZu%UvW*>%g;o}rQ%@2UO8gX}%DU*yl-ry<-HeWb zB-oLBGjW@@^vIixO}D}E;O%Jw15M@I_IPa#ux?P=AR2sQEuQ3zJZJ01f<(|$N^(xt zC~Ir+%0Q)moR%~VA*0$v)ycn*r58CjV@S$Ei?NQ#%^>g?KM_;{zk`vlcawaU*H6j* zD{s-5VZ##qxeFp#9e(o0rO$ZghAfjyf1;dO`lPS^tXP+2Wk8GsZhT>(P?7M!i8uND z;4woBM_JuZQBz<6T$IARXp3iVF8xyNMvr;W&JpT%;-Xl%;wKh)t8pfA+C-^xmS37m z4&ct6dK7Ee%JKP(*IH~0B|-1g-;m^PIW{-uV&=_6n1t*thT=44lb?U5WsxFq(K{#b z2OGE%x0F2_=$Y=E)y#`kVY? zI`dj@s{l8}`O7`K|H}S-$B!KGhG4~g3{2716W=C~M!g(mUEC{Ig zzV=^Uef8D9`J3=6^FMa%SoK(g@Ui8XpVb9bHHV7&>McyK{h7p26Z`?1Z`j@i zz}oSHofDqYVNJV=(AGW5CgK6rrFh}fPrv^4uX{t;omuncrHY3hE#Tz{Cyf0jj8&1&WwzSW zQkQPh4hu=w=eecrMpZ$Maorjkm*Lk0}*Kd1ASrh^w7?L6-#fcdmA6ey`_$2?Z6DK}aEK9Z} zg~cRDkN^o_QMUJ9^E|(IKFBBY?ath}_q}hwb?)5x>d`mfKH+os^wb%=z!ogie$ZeY zA04nFD1&g2jr6h zHW>E%P`r#cGd;mFB*&Xq9k&-3Kl<mDN$;ldW<^4^{8alx9x=lxQXCG zf-cNeoaF=%AX?=7)IWJ@)FNOBLr80(RhmvGZ*;*TpqivZqQsgrH76q4j+;obNlGB< zHcl|2bE9g$nGlAvhZQVJLAOLbhCgC$pCKrhK}EcbQ!t%})Wg)2QIQ-D!I3B^DMM9D zNTC2yrlu#d2qcC=Xct~LBAK7vZ^`&CWy!%v(+DeF-~yyAnHctHciX{4$2tf=VMi_z zM$0x4{MiF4KTwZ#6o0^(zrfR}qefChWTTMNokY#H4LTHz00%a za%|8?4k;E@*{FjnSkmJlLKd)5qbmYN84DPZNdjlA3o2Er%3|*(d%%U3JFb3{qB(VMlk@4o+@iTPo+AmA$*eT){W z_V1vRv9Y!F^3@xb>o9M&yt2CV>ebk2YieTLiF`S!rD?2B=;DRqNmkgjS}bxzsf;&{ zj(9+tVE|>{nt>W#-;ke_b<3_eJn6|?JmGK`d^($F78hqeoaH^^uAw2{vvQWpuQGq| z;fF7tJzIJ6x{nnSSc$FNfoJm5HKG@{Z{53j^X5PQ{$Hrre)Q2tY`lWDd0fDr4eW77 zPmXUq@qog~Uh`(|-vL{ddu4JQ!Bw5dHHya+c(fX8nI7h07~W|YWM7Hs4e0~)c){eD zz<|mUqRFnU$wX<)Xw7)ru2>Q%ok$$^=dc}Rh7K>FVz5_5q15vVp$!EEtI;6soJE{x z5R9ydMw!hs+sNVsn~jE@i?>}So|izA4>Y8F;{U)vDBQ-E)K*$$Enw)OmM?ASf$S;SW^lXhlr?zy6 zB@^P~(6wzqIJ^_3YCkzX!o0p2;>HjvAC^!Yimjh|jJ-5Q#i35QQomG&Hc9ku`2b-+ zp1%O)@i7k*c$klK=(Uj1PF8e4q1JdfUk!qo@XY1I2Au%_o&P%XlVo@xPv4xYI_Ep@ zy?p9%cYlxN1~MCQX)2NF)vK2bP=EESU;pMef4{%C!wO@Vr%>2Dp_Vj#tgr4p#Z;D- zMoqSaQHh$-IrvY{QNh#7sC)J8y+S}Y7rhYkDpJxNUP3DlC@gJ*7dt<0^~82tPP;fs zu3g-)i)=W07yx|@0n&u&!!-Z$|knK6Xr8VXt+PF1ul#f>CGM)_)Dt1xnoC`r>J%>cg0 zP?-}UHu;weWff8v42N988z+ktKjcHh#kW~xu*8smF@_R2D@8E{0ymD3bw|-uK%-l8 z=?YwfzyMO>{IXUIdrluB!cyMlLbmy-`QXru(E-5l#7P;@6w=T`W(L7#n2vr&FCVjG zV^AKe_VP#l&4LhGbdxD7=GC+6C~MjCkV#4w9XdTS=xK>jjFr37YJ$Lubs*0k^H_=| zrOQP;SFV6(>AxLfZZ57);f<$>?mpCYM!jhjoQJk;=YUp8M{7t&oX6ZO_u1h4zE@XM zlhe!%QAD?PnaAI|yxd`5P!?EDQMi`{?lU>0ieNtff6Y+dJIj*#I>rPsnL>ZQj@nySwY|?OQ+n$&Y#O zHv=Qp9{Wlv%fFc1rwO1gCqWcgucs#*fvy5JWq zgfz6E1c0WVq&gMEB%WkB`+`^WUNNDlqgmn5h5$z}!aF_4fvX0iLBrAs)g(DS)>i^( zQQ%e;c<89M3Xv&M%2u>0IX5_F4jKdT29-sz`gtlM{;xEa9e#zJSJ8D>8zcevQ@}_a zWC-A``5;H9otl&> z4c=aI$zgvlJ8vGIGR?pB?Bz?|J7Nmoa;!O13*ae!v50vP>T(y_vkx*)3sIcXYfpK= zb^0&+*tlSNYI1gZhNlWVLZv=tQFH@gcLC0*mCT0k;+pIn2STaYG^i6bfL>Ev0ISyk zXh0kT_+@eYhz#8~Z{1-Bqt8G4%9R0HW26 zp8$ia0c07)RK+Q=r7N_<%fHr;ksd}GqQF@lsYg@?cX|RCGJ@dVRzqpyc7%jU7cB7H zf>FtmC#p3_CKtEy(%{h0(cyvK>Om3~Pj$qN{9^%btHv`-GD0Y-+EQ*5O`AEywjP7S z?C%G(9LT7EX79e@;~4B`i~@03vf>OlEs~s^F$U;hs*&&9;CLe=BNWfWeRmn5moPp) ze!RBko>UqW3_Jwr;!T@x9)Haz>i_0%{`=qm-T!35aB^bI*BL>?SOcrk*P`EW+ShaH zQaT$AC63LI2)49<*7KiiCaE8U6s%E};w2m}R~CQ+P*ztZYAP;TsoH{j9|E|1K_Y5O z{WH3A{O7(<@`BQ!q|M|(08WNvSP1{fD9yJ?2Fz64rqUuY*$Q0dEtXM6j37e2L7B*g z9l;>>9EM2Zl&d}HVn;TLi$n;=!HGGF)P2!}BLMJ8$fI}(DWL!tM~>hM4-TcEW;RnA z0gVYo*)&Otg((FHJpoJ<_A?`z=S~X%s+Z<3d6G}eL0-x9aruRqDL81>ZJ~RI6iE_X zqXcDp;(^?76Np$Imh5@x^ztJv$(1Z{6Om-bD0rlh`RcFHD8#C?x06?2S-L*R@--LN zGecv)B7G5cAS!;;K$%*u2U2n+F=@#?4ygiciIP_rj28lBZNob#>FV=|3YG&gaNr$a z#Gnqga|1(62+?Yr7swQD_MWL05s45l)P!Jbys7btg=@F&+`Z2hbO)!WkH7owPoI6d zzPU*!ig(sJJNwuwkL^nt#ISOd9y{w)7|=6c<`;qK?7LRSttZ$8h>jGa0(gJ{u(KqQ zcL}yOHrZwrio68Z>T9u?z(D^Hs^RtDVYX1U3A$ppppQTL(fsTz@;f!2QmnqB7&T%5 z%YYXUj|jraI$Pt$bInOw+Ks(~J)V8PezUT+vB56gJP)PokAl!GGyUp-VG=B|441N- zhL}kjZPCOh(2KjT2DUYfM2v8S0V^yL`9-*Km3sUTkYPX<49*QZo^SI5$Xh z3aR;FMnL!nHWCUc6^pCHj0a-^P?JAS{NsK)j&7pU#OCEttT(_Kl?6a@rIs*0E<`4s5i9;Y;ZH|Z*ts_1> zG}NLGgoY6-I4;tWEA)rkU}9A*RWPgh*c=x$3||14F_{4ngEpd=Er}+p z0L3Gj+n57FQo}kz1(>_S6tyIur-r9w78X9DDv~l=8Jf%=upkbJ>^DfOK+;m_!^&1J zV!+;q%d4z}t#-mwT-|(es}fgK4Wkv*z0?u$#rdDJ+D4G8vZGTC6F^WZZE;k}0jY?r z5#&cCa#2H@Xucr}c+EGXZ{Xw0n!+w53?nb+^u^k zZn7Ma9(@Xe13VT5v5DE{?jF5xCeE1zWpF8G+u8C1EI=9P7#!%Fp1it!YmrG!wgf#m zWK*?ESJ#k8kzf>}Ewl{tK{}kTHF9+>ie8$tqRtPbig05yzM<~?5dpX+)pgGo7c9lT zcklk!-+Z&bw_Q`WJf@TxYCPoZ=y~|?L%y~2B3TRa}!|8d@6c5nHv*<~?;HMHPm%l#4ae zOEr^4)EYqrh#B050Tyl1E0h|{9axOBX1BJGHERfJX>;iw!2`uY78esP_`@2COh=3d zthVVS@K69vQT1nEHGotXc*w?C=GbrB)ywi12^E@KOFLI5QdoLJ$AWPk8LhS9gdzc) zy+Zv(+-cLTa;#cH0KZgBrs{yC0iyJAt7Q7AEO3_xCtyrb@_k&oB#Hk|Mfo@cn=QtM)=HUIm6k->grxrz4;cf-8HyH z5HyaS97RX`h+$BQ@{Mw2R~Zr4nkBxeAfm@t)PRRl*b+wAoJ@9{n>YH-m{heQwRrT< z4|8!Ci!-|W@`@?54t!1w0u*qFl$vD%GEp!W{%}W8zz>m_nXqCQBR6;`XW(*{YjS~u z`c0P;5`;zv)YRbjQsPR5!*XO|5LAq@0TM#u-Wr$E>m9U3qVYYa(#1f5DMIEG8?eeM zlKjUV8lD@83QT1Sin#D%UI_UoZ4J@zEwsr1p}&BVMcMgJ-HrkMCBFOptd9`CNLJ$?F=UNeyk3v)Bm zGwkP+KD`A4sX)$JT5|eq!*_k0WZ><(v&(ZnB~Jrz#-bV)|DT*}?e4Lk7f&I1u7^fO zTO&-o(cW}-K~3E-gg{N&3Pt7BQo8HLt5#+!0G1BRN$#Wxg?p(kK(P zFIrHEbP_qQZ*6_~)z{BoykG>v6MyFISb(V1nwAgDsUiF!O)sKZru?dzQ^o38%Qk9P zMs zMFW-zaFAaE&{5JTYDA)9=3-(9SU*BCYKz^#X>QF*h^k*@rogrtlS7)%FeeSI?2SCi zk|#4;Bz>YLB{d^nldA$2i8mYtk+VLh*NLfeAiFqnk%+k>dbABVU^t~T&;|fFPyQ7P z7$YN)fVh}m%Fz`0hV!-r>4T(l zcPY9oL07g*q4K~TbCZN{jw`y?6Jz5Gvop)=aJ;>>zOlKvy-jz&mobA3Dq^cO#M`~E zSJvsR1%nl3bn7U+hmr+ zx85y;Rb;FKw1*p{^H9>gU0&s=8i*HzuiaYQ+S+{ea)~m_lVuW_%TLdi zJy*t^Sj-{J--nsky#q(FMmU{a7*}IuC%-SRxrJ1<}W?Ptip^v32MM1Vm z%xGvUl!o9@Rn%TiAy4w!m|edJaP&Z)bP(uhJC98ouisfBVganBih#5sLUQc{l+o5? zq!3{S>2ixZ$IyWX83j$krmA&V4nE>d)Xs@beWs`3grJkVcLidLXnx1wyIu14Yg-K#JpLeV|M4Wtxu}mYpYwH~diR}oCNC~{Ccx?dUQ(wX1Bg<|!vfl9 zo;UpQ(?7Dj;IDuA>;L_?f6F_C{cP%m2ry5VLUsG#-L@{z?^OdTLksCNqJ>6PdX8#> zGjjQpcaKC{tyt{1lTYI`A9jP3cGU^U5Ve@wrbl=JZn(k=0X>&J7|d3jnMJ;|(ZiR6 zL38TUR_0^1zi*{Pzu;HbMUb%87Foz9T){*N zv89sn!yUSbIg7I=`J26m~#x}k(qw4`i8PRZFYUTP|?)mVJFpu`hy zf~gDuR>_9PxGiCQP6Zi@!UzdMywnsT%z9}Ho>&a8WNj!g>k3w(!C4qY0J>mKIziAi z%}G>j4TZb~9}z@l{e+0JLY(T0-Grj9#^%f-vtpBba@UkkW_`*(?Q)S<_HyDXW0owg zt_x95>NpZ8uL>w?vP&?BrB&m>36!lSWd}SA34G_CXMT9BLju8J#eXa&cqXhV>kug= zBBirVU}qe0u(!9ev9-RvxAgk;-rg?rUTh&eKRd^I9$c4odt2LEtloh|gp~`ixPTb? zjKd|@*)cpaJTNfKu%1tVvBY+5eSCL<5vYQKtmGr?y59^r%5& zl@Py$6S6S4p`ya*)*kViAmSM*Sp4v3P~%ca4+wX9dMKof7>?97rm?X=j!cOy-IE}C z;A98dfdm*~0<4lvKwUnV;zvxNf;)JL5TiAShzeZVDmIcfIihAxij%Y?gIQw~Bq$nI zz$4-`^T8PgWhPW|u_g}5RM&51g6DX*D8fZ40v8r%{!wRGz?--isnuFl(WIyd%O~6; z1@$<$Wf~9!gP09rH|eHpA#mg-BZ>Y4feg(}tYf{D1Oh0+Jl+xsdx2FKksd9{LIXte z;TpC`R=9&!-Gix-s!cc&$+F584$~EtfX9v^XDZZ)n2y;+1!^QnuEfAi(bH_Cs!|fj zR@_A*B2D4fZl8pfd(}gVNsWIRY=8?(+z|v|MV}T&!SjqdEJe$tQfj3xoaqLw5A$A< z;YbYwl!(v<&W;c`gff>IRfIRHUgeNj7B)f%(AXg(1O-SMt2;nJ*Cn%wUM((w+` zz|;hr_>XMvZfBt&vlUK7I+1w663PxX?&NDE6BEm98N{1H2S3TD%fP_k;;PUSiVWQK?KdqGNwiF94P* ztqo!Wii84yLn^AoEOWO;L?KFJ?kFEKL<^WCRYs8j05htN!WMNVTa23oJs><-&PaNE`KBYw^}j@N`~y=YHpplvzS0~d#I5YUrgil&P>JDN&Y z^jc-YgPsg6K=?&$2nR7PXFtAFr(U2XWEFBn12uVY$i9I=qUn%{5MUf4tUZGHRMecgoAw1XmSBB8TA*Y$fr5p5+OV9Q)?Ki{P|{>mL1Wh3aF)E~4wk4a$nQv)5^mMp)a*ox+&aQb zZi+o{E1y+e4=d!I0cUeD`2Zm&&i*{Oj|5WJnvj4sNV+hC3#hz?Ur<6wd5cVxBvug{ z6}`boH6B#mmSyV^`AHaZwQQ0kyqKDpNe2=g7eFiWK*7(0TvCLXdSbuMi;iwy&)wYE z=5vz+gG5syuqDnBt?02Uf=C=0Zt=?RgM0VJCnl`uVx5Q^4^m!*MqD0t zSn_ycK;cstY7HREBe&d>TktMEF#grokC$G)vNH0xVr260`h#0H7yae{FD2MZM`dn& z=cIUQc5qssMovAT^rY442hi9@Ea>SH#b+G>SMnfL7!vu56pCf- zfG_}{|0vDH0EI%5ViK+4aKb%@cS+`~f_!dU%qAgBfYFRH90-Z%E%FpyDZ3MasUxIB zEEPggNU=?re9nqXQbiPfA2?MI6vbe;Rc1Q_b~ZFaM(l-=bV-TE|6mjvq@|Hc#}{mH z(2%T%RCu5DCo0AzhQ(TpN7b3ah`W>(Va>)t8Bqbs?0f`tifi%8K=>9gKwOM4%^8i9 zu^3OC_y2QA3{aa5|5pW2T2X#^;tX8coyZwyl#OTOGtM#h8kL?B70U4;I;0juz zzq5-^k&lm#yjfjc+gP6%YuSNMqY^_|co8cTs%B*Chf&rr4-C;yT3DE63F6YrH@r>7 z$4@BPE;ywAVg-Z-u)kbfA`kl`*ZhQY4)SO}$uQG1eprAiw>3RCySl!?hgh6be#-P+oQQJzw>i!%hM z%pLC7RG?E`O=F?0_!AN}`i;dJOD0`)15aI{-q8Wm>eWsXuZbED;>JJB$u&7qpI!r* z_yd^$NSxY0GHljFy)~K{lUpaz<|!${OUc|)6)I67fP5)_;mjH{7PUo$q=gZ!vWSBT zm@E;pU!UA^Zzl(Ai2$kHJsGg5$UekeSCnUo=Av@89cKcb%y2`Y0T}h>4kuv_AP)_A zMH!{>@Lv8&uJQ`3l{H*JDNPf8t{`uvlr|a!JdB81?ED}hEJ3-mOjvdLuR>_*7$q|K z#aE)wYXCrWav_S;y;CfJBeJNHUqi@}Mpb6$+pe>-AMH772eNe4?8egzzw&c-#5%`+ za>o9h_8g9n&MvNa@;^R42}A69g$Gh(V5{2)ZPr*knBvhH-vB-KO+2G)3HUJr!GPU8 zzyJL|fAW`~y!ZZxpZ@6&?Dx%U`e>X&NKukC`IdjO(S)~crkj)(GRTe<4gcboQ#ZAgMe=EUG~LjP(&D;Ah*5g7FaIFIKg36@t^zvOk!Wj) zS2PGoKm}xLctlIqIFi=%Z22BS$?Xt`p|eoI)!T zr9_Lf%70>ff~Br>E!owSjxil=3bOlnozihXqq|QjJ`sVd(Lx+oMTB*3Lmy&*`!zj*$pz6Qqb;%Ot>K=26#?BOJRtHjnIlt%&UgNO;A~>=t;HyBG}MNVPs9_l>eDue_K`eZriw8WrJ8 ziW({%L^2kThzF!D5=me>Pym61>}_k1Di6Ub*A!!w9yN=~qAq!I(CcG4es2%&8nU*( zUTtF5oLUGfz8eZI+XuSlOosHcd3nz%W1(&y66z1s+L6R+HSMTXBcU< zW~ZjF%`ZHAvGl|97wob?4JLi+WI*V6*N-ki4=eszw!>_xvlG^>7^deOglDWv4N>96#God&2SU!q!OS@6AT^#RIeKKN!BRyBIQrY-W zT28Q8bpaLvnj=_HZm!ZIZ_(4rf&s#R_SA_H6A`M(4+ME7L{*cbLP%#51XTPd2j|F= z(H8ZI9MV2Ab1QQ&5PluJ!h^=N<|4nDBq63$-~?$3lmjXBAFzO$>{b&_l@ixxoIwQ< z{)JC8qbp%52VA!KloC7djJqfX`luz+SpI3uQ#7KkXfi{7xX{t~;n#S8+HNq&r=3f$ zspZ?+34px%iRiL(X-vj+R{#r*nlx{0Lx`(YbQ&E!%pnU0D~*|Szq(=rA4UTD9+^~B zbT1`l{1}(AyC547b+ISaA=_^B@S)$)sp&cVo0sxVx0#ybZ3uLyC$p_4)BirEVDZl> zxN$1AOk>SM0@e=jdcZ&Z({F$N^I!aj4?q6=(?9iJ`PCmx&H#hKO)ndwv)CEYaA1_m1;A> z%>3&xa1~TSn50jpEe9%Y;R%xd)UC#jh4I-?mLe%ct6I>A9z2R`24kQq5G44QM3|J{ zQW+7+C7j5;n8=TN5l9!g{R&@LZvdo#ZpxLCNAwt2_l!+E)t@XCwyJx$6EoEG#T)@i ztQO->kic-Z!N9C;%PZGthmJ0c;UxBiiF~V(V1}%ms5md)HR2M-VSCcUc=wX^<-CpC z?S?$8Igz3Kbd{1B;`)Op9Xm&Y`U#Aw1+szDs;8JH^#30wXx&nvCXo-sAj)dP^(1JH z$!uIrjYL zjT?56Xti3ML&H40b109Kb7f+mvAti|VRN7p(9*n2^4|XF7_LvP$3ug?qixDO)?kL3 zH>3x~M%Ylmrw1dgn>TLIQo_EKnS+n6z1#nLHwbn6iO7%4%-JY%|FH82lN69p$%lMPtHA~S+mnS!iPgzK72@?CN{A~9d25WEs}%p5^z%@myg zRajC9xu>LHW-B`-HFUiPt<1GaDT=qCMh6v;Q*NrSulz!T&u56{(*{QQZ6C);k(q`W zB&wX;xmo+(wY-KlWuNhhKbN61c_9B{{bO+CH5=0yMdkgxNy8SwEj(;U8z7y$bj zEMjJ>a^^UO2B`jkg(+uk!b>VGY80Au2PZ1S2eMQTY_Q-L5_woU$<{p6)89V*{>96e zdwU1)NPSD^n^_OoutlIp#CHIAVc>)(s8(kzk5E3fbVc>sYF)M%Jg|EDENc3f$B*1ZsBd6wam`cq&^& zMu7$42&>6Z*3wah_EIGzh&tx=1Tr9sdgV+}ka684hV(|pJt`pfqL)^^3@+8T+=PH2 zfs}beo&wOQv(p}(_%W6i zj7`v>vnA2~!3o&02n7cSAB(YlkQ;kat(kz95rn=OYZMq0Q$GIjxBvL7U;gsmgNKhF zf6YRmvwRQh<4#Ttr_YsqqGmWRR_ z0C9EeT;F!^R)+}>&QhXd*~$1C&B=-P`PWxLAeKOB^wLXYC=KCIMkhm1DO!Uj4%?8# z?esT2u8w4nI?P~SfC|WZueg&}(7*u~&Dii%tS6s7DuZ$K0)PbfmU`}?0BZtq7JY@2 zqn%D>(^M-OmOcH`11vHN7BG2lyv%sg_9_RE^pm->iy-ZyPcE_=p&uk4cLa#CUSF#Y8|A4GQa}it(zfB@MQ2fKq~@U?NL+2m5ZO5k!=8QOr}GMEcFGI=b(rTB zoWs+rXRnsOd-`n#ydOH5s5l@xQ4 z#!$oz1nV$MDxE_17O;4>um&fW2rMp+=GIV~JrIG%Uk{cQ>8{#JDP{g<|n93h+6{&q2J;rJut!prZM5;ZBA7p zmm&~rE=HIjV1VQB-sD#p0udb}7!J;f@z zF4ZU+bxkhy&Vie!KB>GZAVw>S{#Zwp7pAlX8C9r3eF`_`S;uV3++CT~13P0L7wCa1fHH5lVFGxJl^GjzikmcYGJ zJUn}Vn8t#(jXA`rE(pX}Mz@c&d`(2uAN`pc8(N2}V?;m|@sYZVA(a71Vo<42d>hQM zI#Fo5G=S&{i`eo&&M>1=?Xr4LZYeZa=ZnmXzd^t4!>se?*Qf-VfvS0)ShT}C~_v7yzDEtJ}}%1i4WCZS$(kmh@A7}?E}==q4B zRAuuJgI5O#-l8J^V;^m*IpKuZsF6v5jDb{x^Op`PX^u}>g%4=@{uT=TiX8NMhkN@j z`g!KTint@VWO|P00mCDsx9;3sT6(d$zDh4i`jCeT^i`a#72nuBzs#&de2HGQx;sDR zIlwP|@eB46|L)1-q1FJ8@Y01!l_VM$2Bq%Mh*-~x%a%U=gJ3$Rde!>Pi5Q70Ey|G# zWYH&tgCDe^)-ZhPka!ny6i`TUI5|Ti7hKDBgG7nThrNsrCM% zjkr~sBtrtX#Xzbk0m2+tNu|MqEJf7rCohUfQ}(K41cj26PL!vpDO&u32yv$PBvpw6 z!xf&RAkT!G6m>(eZV4K#R#Ye|)g{9v=m-Y7fW?qpC8Cm&5bcYlglW^a44OAEB8=86 zF62qGSxyXmLOaN%NXS#jQ;R@D)hlCACO#RR1=BhpQnjfEV%QXR`IBz4fw2*Yh=QpK zKr(+uOsIzs@VaV6(WsM|$l+@&S3R^Fx}?z6RKgRF8vt;6$1_JU&V}4Lv=2AM1`NbM zn4tb|dbtLgGX+hqc-*7O2!m|tyD;)*$A;Zn;Kq)>6js9$rN zy<#-5o*@imBro4U##%9*$N)nfI&Y}9ov)osPIabGE>};O?ui-YR_-MRQ$FL4&o+FL z)-WJD^3fdZi-+jtZNw)<_bOE=qnN{tk%$C(Tp^V$DWh4+m7sSB_g_u9`h~BYG z9zt3*rJ9eixuF2$iQHg*Uah&9;028wLgPRvK+0 zp~m}_(XLPd^;Cgs7i?NeGiJ2F=ThG4Sqv~?-6#oE(b>Vm10HPhXpt(JUCUm+S)QI4 zqi@4|ie#Vy0Qykw;P9|(H3Q+oDb~LOip|mh9&&- zb8L%z!G?xUpFFqrqYx8fCnqNNT>#Gc$#vr>$`1Bzvo!@JiJn&4#Z&<|)k8 zCErq{hu1kXKhKT}E32zBvkN@Fc=GsbGSOjTqGWw@i`D+KvkTX+-=yo0MC{0%c88Hq zAJo85g|bSln!%nq#|XtvRGzsh{SKw8*rwp&SDH>NdIbN1m@idUtf(WIZlpo8M_l!o zU~ZFR>=q*@Y{!M#;qI*kWYq!6-Lx3sl`g3X0XzMz5?D))2*%@!xEs!!5if#d|&hI zgpLFo5VfF-3BPCt zDU?2@EV~1N75NcKl3jNx5-wk*O+Zzqp+k||c~#>zTL1IlLcH2a3TW?)?A}qYqZbZN zy}TjHw-We7f-c|5)Fn#;z?IfW--9?dZeAQ4Tj$_NW6 zN?@B<_~H^j4&%|AMM-Z14_U>6ticcQWNaK1AR>hDkkf|86cvlA07*j zl>m+#d1cE^fBOH``zL5@c$Tc_I5eskWrw%$MXdPmWEfVI_*$t*Owu#-x$t@d6tPP0!EsNprYEU+xvaM}_zH*GQZ-v@0r^BA!$83uLGx zy5w?r7^gYa{vh2281WYh5-woYLF9Z!QtU9HdcPHdv}v9inn&yBARVI?TNi*hH&%@JQ(b0 zTwd~8!r`G6Go0{B6`MBDorqo(h@&+$!neJ?_~Hv9A3l7zzP7ftvq{ZP8$`EdczA4P zc5Z(D8vML^y+nU$v^C1>0<7e8Il7gavscuZP-dyJ%+PY7TsS~k@)orbh*lL%dmyV$ zlVoaFG^`I;GE6s95NLzsFMU^}sE7|{O-r7}{*@w8(@83FV2!mCt(L-@j9EE2Fe%?q z+vE)87#uk2Wf_VoOoLCG5dtXNB1L*J5WtiKbA^1NQUEYm^NU`-$?V}>c!|K8MLKE$ z8Is8?uIRB$rK#b`yfBWR?4Wxt>4A~BB-EnTb;2LzO0J~PU{;zLi_s)Ga~>NJC7054 zZdo2MimBW%0HCO<;>0Dk8cl#CW&;YUQaH1u5Y+x^NeOY}bp9>pJoBQqq(Nk=-!3;} z3wi}~B!IM@^s9?}xVqbBn#IEx7YwjrVt@aT<-WW`cyMsQq65~*(Cx-D8_k{m>4RoJA>6>+0UvT>s*W&wupMkN@HS{KLWV0WTCYN^l{C z92B?kD*RiTx2Wj*zJhaT5 zWSqnWl-d*x)S+@H%P8owFJ=OsDkHTdd1aw-B-QP$v7Y!t6Ck#9@+1M>GmDFa=4fMx zTrv@OF@Z9s))8W1Fy;V8)igAs1M`Kfu`0@I{ecs4b^9EpFe%rqXYZ06EhalB;;Z-21MkgT9^FhNUE)!{s8DMho$d92glo;T$%Kf~D*8$k1 z?UZ%*BBG}Y6{UbddnEkhtdy9r99IQlbO8$qSsWv_qDzvyh$FrP5(Rha7Oh2hc2O0^ z4AD6{BnT7K6EH^^;VCT!KpZB1kp~^831-vGCqFDbUwU(VdVb@^jR*JdvnlWL@*6hi z8n-}p9MT4!?q(i==N&yf2cKu9>EtAP_)vqn_H_9*%la5>(5}$wXQ;5T$r2s50BgQg zZ=d7xin%o29O5YmZRd?U*MItxpIlp5K(lNhjBz3z3gY=9+tb-6!-rXt%T{`?*Ee{} znhJ!+e@A?7?6B+2o8_I|U1siZYed1c>+2h9Yi_!SewmfHHaEkT-i!oLCM6{fgLa}= z1doO+r@CC?WdPz)^6oFzoxRL_fnlq6ap zRXRCK!G;8a<4{e%z;vCuIfRdNcG^i&r4oXAw5bL_;V(mn;!J&MFS+yxN$$kdT)0LB zdOgt@D64?t#@-e+5W-gzDiT*D6fXI;1L`}I#!1D0FQS!yxD)>0C3)kIycH|8k{$-Y zMln-sskzCVqv0pnqKZVINY=`;42T}O3vO7baK>&V_KE$7>BZ*O?zhigyk1$M7sXu3 z<0n5HA05)eS(uxfo1Wp(0ZaUc`UkSif{!Zt<(Cw46&(6el)XH^`$dI$Yw;#^*Wtkd zs)u<7Cl(ehl5Vt`j=^CpGc|s4)UPSIcr@QJ#3igGkyxp(x@YI-+4qb$jIUk4&MKiV zKKqpAMRdys7^JY`pD&m7W`{SHXm;@}vr#yvr{!Wu+akz9AZpZbVAlAhZqG#Ec5$^* za+!|e9;mdszUHV5z~{&n{!u%z^g!or z7=XOcxl1=kZD>ET3J-UltQlPQABDIO1hA^e!~!Mdv7?Qz*i6HYK%896Xno)d14FmAcMy|ub}#G+ir+pZC^4eOknUx4kGOG{TB zY`=TPdgOu5u4@bPyrjOixpA6`5R14e#sw>f*<0_5$5zi@JY!z+gAYFZ{U3f0QOH@4 zWL$IbUuLPRu~ctbo@8Yzs5Nw`xnB$K%mpe}7~VnR|>smrzs3P3Z9sH%JE5~F1~ zb5d_E2ehQ>FU$(_jH8ct_dx55z?M43H+BcqeXtdhAdpE}RY~oC3Zl5-ghrlW03ij1 zt1vFCp;*~G$(obqlrf2GZ*s-LlJ$l>rW2|^F6yj`p{n2);Q=+JGRd}*nl;%?@zCO8 zGC)p}^?tDc2X*nPX0zr_I}02h!d-(wD|jCoupgurjucohs#$*kilB*R_9HwRD^7@L z1o7m#7KpGo2Sv-ld6Xe;6+yw&i!r`=JxXC=Nr4b18Ay?D(6iKEnds_eD~;Yx9$WKF;rr(=|M15@^$!d(v>^4Q>y+l2?+aK6 z$bZFijI%c@D@+~uPQ&Th@W>GF7Q?FAs&+zM=ZJRjBJ~!llL7;zsYBIVr9&?zJAN+T z8XJ8Hqw@08dgQc+B=S_?;zSlqXZA$tH6#}sAP6Ipk{*ch22EOI=?EE)MGZpXEitKq z9w(0dH7PMf=?`%Z>|7#*Wlj+=a}7;6*dv$p$z(kr6)6`%iKia7fV)u}vGN*9GE+B2 zz_OwKiEGpnh8tZ1Omm0mKoPETga1V>WTA>BK^@5^7vLfhN$_Asb8?`))XYw_rhU`^ zc|eB0SVIFqRx5)-FvIEs)JbN?B0CHIZ@-A8K6_Bdhpw@}fQfAdM((Djp*u zZc-FD3d$8+q9sayAx&{j5mrYquS|I(T>kY9fxLkbds^r`#GxwVf+;b{OcEIaUra`g zVH^rxB^K`R3si?1C8}6uCeB6_;}VGC8b5Bt6H5fsP}~G@0zp(%bP;6<4nQCWZf_M^ z<t_FK~BshGN4pmM4&njr)FiEJ#6x51APDofz%yxf&nGmu&FZ; zAyH{^5b_!#GLab0sD=-J@I7+&En~ZJ)+;j*U><~T2OghaZtw3sdi>?}JOxMYJF#jb9)6MgsIo$K>+yis5^E4A>Wo}o;h01eka%OUBiv8KxjK%du%rC2c z^)F48)n*uNS|gVKfuJdfCadLepyBazYA?obmLQcKY*mbj1`eRas}RwMH=wwcH{7P| z3ael105-lzN##z4)K2E)M?%^O)G1a`^P@vV=Ay$riOz%IfW$)#K=1tHfO+mXH~ll6 z&9)1uFd8)wCQk`Xx>#(vU0;&+#)dp71w@KM3{i&(m1jk)A7DD)rj{3WSq!Z^#5R(k zpX*+NJS<~VkXNr>ZERlhD$Cx^4pX3vy6HU3%+8`f`t)oUh<^EkC}Q!Ux{0Ytrt{V| zH*39sI%H)lkJ~V5cjuGuzx(*3pDfV^b6PLy3B$zb4XXbr)(nT z;Z^5uJ61d?wv$+9kTpHci?y~0NU2F`@&U=cMv)0T5}NSirio&ZRQQ)hB!s1M$Jki} zsqnZqAPQdAG%rEVk_Edrh8kUxBlznc21LdoU^vqsiPA5lEue)+v6G!rS@N{0lo~@B z$yBk>GGF|Mf?3;v4}r3j_JUk^)~&9>PWi7^}eyn$)`5Mub)-GHfMOcp(cd zZ#;n(gL9+jh=4;f%56R`f`ZmXaD-_G*awKG0ldl1x)Zc)QRRIWEiNie#F3(F-ta*} zG^o-Pomgs(#$u2b3$gM_~7_63I}-? zJ?xPi3pXCV+qbuW$S|I%Cf3r73=UpR&us1P(BY;w0ikmWXUrwLuLTdxp~UOHI~O~Q zcl?0P{ysd9jIetKyIb&xphYjAc_uy&!17hz4@OFAfw|dPcKlqJpC26`w^ynb<&d^0 zhRLIR_@~x5KD}U%=8jID&huWuvzM>+cKBsT(Zg6};P$OsDAFVd{e-?hmOniI(tO)pZy`&BdLCkfg?|;`5qE2Z>c=p0J)w zd@vI4o*)rTq~$4cp}oN-2O3@R!A`YGH5t#eIwBr{Oj9n|^guhMED^#^HVPFr`6FL^ zLwo{zilY_{ohnfkkYkzUkOO+ZTy(9_!k-z^CI@VQ2ID6)7yTY6^ET4kl#)GAT zSP(Y1;XB}*l)I_!qAv0Vy_fv^%sYGmY@o@QIPTPgSW1Ih6f!AX@=8_#DANR+(&VL) z-l8sZ9$fAHL`{}5F;@E^h$-SiO%a3p60BiI-cvHUrgkw|92RaBQd%;vswO3D5RaG& zOk~3f!~sxR*$g3EBK(t?c06#6++hYYNv$yx)7!m4q1x4sUZA=RcDr-+>jG}tB8Om(gY7; zi*TVW2=a*AYFe7P4HR+oLr|!3(XEgvnkc1M(dP2wU#Tv%NDfnq*kC1!3Z*qXa`W2t z-MvGmZ`d!Nr48RNy<#EItK~P(mR_-#feptV-oMZ11F2!CVs`frS#{9c-PbXNQm&R) zR(LsTYMfO=-3%nyey5Y~0;|x5fs!&_(?M7K`oiqSty>5CJXUp~C{HOb*xWe%1Uqjy zL_EZ=lu+4a006tp;vO9FF0u&RN>Q;B#Tx? z^C?*Y)?gf(BW8|C65_snqk$BM`v7Wi6lb3LSBpuPtklb>F)2YRXCkG|Rrj>gPa;@R z1G%|PItd2rFB$}v`mV?)4_V9>WagAnu|Y-9=ss8dJ8eyXOCfTeUUimtTgu^_@txV_ zg@2Wv=9gG*BXyjdtc(jl)^owA7kEf3DZYwR*aSNd#~6S&hHw=tGf>4_gtLg|QJQ1O z3;=|bH~o+RkP#;0sYWs`O+G2hzLPVWEDeF;7+XuH?Os@wfDJO^i(BL9-csFkFC5`pkT>QFtGpMU<@dmnzdzP6~rBc>Wi7vj8rm=c3Mo+ft$*6Ev5D2d3M5PJ|s!Z;BShAo#V)XPQO+%l=s@EtI zLQ-=xn2a6d{FE{NxgYt~LVi3T`C{fmAFQze%xE!NR&sjB>}x2J2WrGQP=w{!HU3SC z?h;^ugtWK9p{ID&J#D}Zh=!0#!PM1;s~?%t=J^ve;Eln}($*chL}W^MGDVlqOA5hm z?FfL*lm;`QsvlxSCLTm0k0b?4U`1@Z7)svAf|uwN5c>4_6dOMZ!-r9?{ZOLxr;ECI48KY7e3;(a!SW6D!6L@wgO;I`Na(oH{KGqpa2|! z$(hGl3<3x_E~Nx@v3~A|vU_^8zkAi&H%LQnGsGa^?1W)E+87-kz8D-jJv%=*J_HY~ zk(2+dmhEO^X*S$`{g0PaBFCOcAEXL*$8iRY?P;a zfNCLmgfVz=1{^!dI%1-L9%kkEjCKd(5dg+oBlM7wn#T}32YXbz_usjD_no`!qsN4Z z8xH2ff%^vsWRc14&TDh?e7*-dIk2+^RvT(PgefUmm(XZS2(1kanZlcJFlJ|r3OD=` zidSszZcOq_jLs^P?pVaF0Exc6lBWeYQzM;;x3)YM2a*jQ0$Lr${Nlu9y1 zL_^_UbSg8NG*)pC4pwX@DZko`sI=&EBSBGq2W9%Nnfm_YDosy z*7T8qm?#$j)l5T`6nRwlVPCv)2U{91UvA zG;n1eF;zm)i!6j1ZOENiN+H|{(A=0cX;$7kt6K5`#xTJhHFq+3DO3g`KUtFo+TOlF*FLjnh@2)KKrkI?l<8Q3PN=2Dd2c;UR6u;SmE5Un8S& zu&{Uc&dklNti0ae-??$)`rg6r!Qmb=JYa;n)v=EOs1bP=lkV97uVhj8sxsV{72GV9 zOo8BY2o76GVt$3{H*DhxIz#kan$%1PL$uZujYZ3Xzz{<(%(lB`K;cohl&~2dSKfj6 zsi2UU{F+MZabq42HN?y_%_p(3yrwcX83f?Xh7=0V=Jtj>00+prk$}C$I5!zloXCWf z6?^T**b*j}JNE@;+k*fIE2aX%KV=j#I78AfO?@VIEk4kdH2h#(4zy+spM`YJ&1T0R z9UrZgZcGHiBI_keA)OoNog5Ofe;)805LW@;FM#XH9C|Iu-9aP;i?4|D@~?0-miMZdM(Md25D z4vsXsX(YvLKBx1o!m<~{BRZN%LxITAxaP!?q}r~>oZ1sDqrzy1>H|{1(Ced~#Tw{^ z9Ap6ne-@jD)WTf@w|D zT%17*JF>}{evTtT+(b+38zlnv6Qq5ZS}tDv!7KGV_;)Z2L*ZZMoBH;aR-uR86Q06@-wK9hIx@0>KIETL8JYqfllh zRUqC0z`elkdsNQN^5HCuXClnF{btuW6X)b5lId14#ocr&RVS#pRY1t%aENF%Vn`6{ z9a%vHH3k_ir|u-F9Y~^Gz^Pa(YHyhvfH5hF!H-NlQ|{npdp=V@5>KBm-P-OPX~6^; z-q&?%p}(s|f7;!Ocx_!CJB}!lggXcQ-0uL3d03&t7+`9=4~{ANv@v}73na%!N9-s{ z$tO1jA1$)w|Kj59JGX9)wuS+V6(d9HLZpQwg0g5I3cc_^iyEYR#J#Jv8}G^e?7#da zNdNfx7rdP(lB;vZ61-`^^a53)is|Uw+1X)xBEsl+czEFY^?BATscQdYCV~)(vZH6> zy@!Xm08r%#8bl``)>FKI1$WWEF5i@QEoCGa^?oJOf<9&9K*1cAS`w1SwxY1~CpC!NFt(PbHE*4UGSyBOxc+6NO5_te{}Q z1(&ddk~Iv_NE_+Aqs$dTi2)!#aDn>G%C}jhM2JlADJSG+#?+CkTlBI4i(iALIJuh4 zvK9z($)H)o1h+5%4I`mqLWosuh^FNb^p+zi7b5}#31Kc~A(;y>_y@O9fA}=5s3n@z zbeL2KEGx6}2+F_ZoseqqDhbLtd6do$xa~@4;bHY*JH>LpJuwKz2tXJxH-m4H%UjAw zy3963{*$6gB~hHdzP06L>RwdBV1jj* zpaB7!mJYSt^phPJK#Gr(UHo*B>AS046w8jJv;#UO^I0$nN71XSMV*{dZD{~gjEo8q z&T?Z8FmWeXVTi++F)oyXlbQ~I*vusJHFxN%oEl_u9_xsS8y!hBE^e@Y2-bD^?e1Q?cAZ(kS`L69Y!GxlIE{+MB z;DFPa6fcU8vngKjTh(qAj?|L!uJ4K_Kmv-MgAk-do?jRjjSvLrc|&DTGYnFAj;+InKiBVZmE?opbApbBS0{}u6 zM>)g(7cA?PJvy&EcX9y0axb@6KfXi^oO(dya-rQnf=tvruT z&Neo8==!t2FU$-K4WeZV+W5ry)Z~=U;G8wOV65NK)76@qVsp>U%`LVC1qplo&{H27 z92y@T;j14zd%H{y&Q8sI_`&;&i?`S^NYU_W^{FY44l^jT9P$^_A-CIZTt4~kDG~47 zz3r;UuFl14*MI%&fu3tQV{`&6XgS)rs6M&bY<|ey~9v_2^HkVHt z&(F^CxYW|3t4K#@rFdev{70`QsRS5kN1|6tjQ3Wn0Z%q);3KgFS|>paS|8hMYamgp zxHP>enjrNNAi6k*-l}sDWwODk_6Z!}XieZQ&A+A=UVa5pai%E@A3SyTeQ{*8^S%%Su4f(Zr@%cf# zg(V!DySp3A8XX*-*kbivl#>g{&;PKx$N~o%XmAVL)`p%p- zjZwI4R8}jd*vpzCrXU_S0!2ElaBT_413X*WL(miD7<3A27Vyc9;s-|A2+5O_fzro> zAu?h}t&)p(t2=5V8Pf1UD}arXaS=I%JJ=mCp$Zs*oaknos@Kd}fr# zMkr&<92q(~Sg$zEH?xLY^ih^q*4aSw>TVMsjl71mzO_kxf}vrs=p?FI5Zyh~^+wdX1AM08HqB11vI)U#2oaHLe!G zK}AJ=#}o+w1)9|5yWW-hab_UGxUYaX%k%0tCZ+ibu zCL&Q#={nI+02mi5E;c6?TWF6M(iS(R0%R%-K23&A^`@D$jEXR+B0U?-#UU(IscNYx z`8Fu<3{X)QlAXISn2Bv)2xjS9J~4rYQ3D-`R&R^-X4~7FtkLCB0IPs_i-z}h8MvPH z^DW^69^+3+u~`+IqXlKnvWUQjRlk`Hxs9G*l*3JBa$ z5_p05_T6`;CMK5G*5y~_!=sL=R0z(A$ke@6(;8=V*^|WBLnDg7Dl|$!UoNtVA@QGiq`dU#ZU(NQ6VZ|XDQt$}4j4eK2gLmd1>OX3`-_Uc8!%8HP7-7V_W131W~ zm184MK9+;8XdH)vfwCA8K9K-iDqc09T36KXJ#5K55}0KJI=N0hKfDM3-fp;01=r&?4CO1OXs6p)617)Vn`k_rwhR^xI? zVQ$|EY`GT+zz)p0FbyA}xD${w7Y&&H6Z&I1#ljC^IZ97Nm5{gK(YVC}o8el*$qjg* z1PF7|4lEk3i1HyrqA0@-+qGB^JHOJ1b!S=~EIurQIlTphDaN@(4Y+VDW^_XfdLC5k zXwIb}FlZN^0?41QA5g3q9Ool34geUeE2PYVPD!{@vp5e=s^6pYg$2Js$-@9R+1}lM z_TmM7cOK#K#yt&>J5TrZvbhAWj`j8pQ!G{4$S`wu9ILDAOG`_qXD6)wqii!rWz)pB zkk65^kAM6zgMLt@sAv||W?f|U6Y*mMVoBm3enH*Q@xzN3-+cY}ryu|Lz4zW7 z8y!7AI^@ChuYd7#K2z}cyC<8Q>r1a*egEv)gLm%O7M@?Q7KsiW%^Z0tC^J(NR2C_T znka;j&sr-#>uk%We@2ofTOO+E>db=3P#&$pr+nkhT#9?HO*`e#UJnz6xI{pzV1edJ zbo^6O=vPz(mbapV(&K6=8Kf6mCJSKSDyRic21`v;SL|{okG;+u=1`os^&6QqKF7sb z)`PyS^-JdPK~}uON-Lo7fPEyK;*k(xgbQhCz>+A-aO5F#_zSiqo4aZLsJ0NlK}Iey zRo#egxQl3L%PP#76LFW!(U;m`RYLj&Fj`80hW;cK$f!Gb101gcNtn}OF^K9f$W+0* zhsRH!FMa#Ni&v|w?D)QSu#dWKEX@7&C;$23ojcu~ez_thOw~=8f70RDCPRCqA=?j+j^}eyNJ- zRK>U{)2UrpM8QKFJqUi?^nyrbk9bc^P16XoxSl#%t6X`YfkuT5W1S93!z|I((OZ#4Ldh zr}9z<8YK{e5~(+kR5BBOY{K!4hEN0M;D>W&Wn}F*5a}AmpxG8dZnG5hu8|1TRfhBsD^JT6{eAl%!49mW1X`aA8{}Ee!l$Mc5q!Bs z#|A}kqd?58xT}po;~30m1x+~!+p=L_L;AjApfDWfH;1R7v}AGLK0uc`kb5`@m#~KD zc0OG0fXI$oPr1Z>jOsz`@A^J zo59zvUF%v|-dJBh80=3khMqp}uJrQ-y|ETA+YhtBcTe{w&)|3p;pC=Y!9!D*Z{ED& zI~eQRyNrI3ABhw!Crv^);^8DZM8aHSqkt+aGU-364#g)o8L4xv*+k%>uF*|QM;^2< z&N!rc7kP{yU^Pl*0CWZ`@JrTHl*|LaQf4YO8REMM&`&L0v}9gN*wt=`UV_D}Mp?!M zCP^Z^-H`Ajyob2-KSBhgt&`;-Ol6c?-A>>J4oXpwTp2J&)Fe}CODV%nNQfKMG8ha( zc}P@cgtqi6V6=F`ji7|gfLa4%nnZ#yai+o|5x8-fku8FW(VpNCKG{+{^t#*uW2KOT zfNX1%WHw_FNXX<=%_yTxFH$Oq)Y+EgqGYU0(nqnWR04UEAolRu#RDAchrn~M*=)vH zsp8Ie4EFI=d=yc&h~d*SD1t%2g9rCM z`Q($idDr=+*3ju1ed#1IP5cpo5zN7mgC?Z)!<^TyU;FIyum0{Ie#2HAzx?aJLJzd8 z_wU|jXMpvM%^#jU=L?oRZRdd?b6McqKk}J_CgH(iUpG7SF<}so(DdMkpy&LdMR{V1 z0)^UV)JkMwoawqT8A?s*(=#kg2t+{V(`4kK_ATF&BFQ<+k640T z@6x!*Sq>Nvm&4FB4qWl7(wCTF*c|Xf(A>fkWijA{i|HdKG`O`f8F4QliXe0fw@|h; z1x>nhN|hNV@}(hsEKtRk4>5}#a-t2+*cJRDQ8;k{K$Ml~|6}S*pCd`m`;7Z;TmuXS zz}$P1D^k>M&@wGK>HWo5}$iLpFcO>$_7WgtK322(`-u_fn5jk?!+1>zq%_^-03;FXw` zHDg09bq3tEn&zm6Syb+5^Uu!vPoKa1=SN>Zf3x*=?*KpVUG!*0p1t1Uvo#Z=W1rl= z$9|A8GD7Pl6yON2lq&1XO$vlEa)nvcFlWL}A5%L!4VEir?==u59eI-F!HV^au~bB0 z9S0bQ`sdLsqdxkl0+?=F9~PQjd<6hGRs=0=LMTK)1@#3F(Lt4dM3ht5$;#Ro3c=z= z`*H%*iLAk}!x4szYa9V=RFAY~i9#uAgp8^}gc^!TJ6a&1mTGCCA|olZf9}%MNguxf zJcktEp1&4`Ed3@=p{>LofgXC_N*J0JYc9^5Vk1+neup z_NXQHkC@sR<-z90+8R5U9d^6){@HMh9?;?8UT1dtoOhwlPG_g5*ae7v&evETw7kT- zQcQ@LGwP-N)l+d4EW*^l8wbo@X*?8QIU~i?@}h}+n#jZz3o7hP*}{NHeQS{wXlQ17 zj#_u?-8S#p@txFtnp*@&RpD1(N)mAfFhco{xeJ)? zL}?Yt1}dJRnMjkpXoNgf{*{wj$~$}CTe=tH$uAsJ#=rgga}OR6Jf z7Fd=v*@U%(QHj*t7zGWDB`qCshycyf0T*}B=&>SP1(bt}UJ21MAh#IUjwSgy%IX<} z)A)Et;NcQrsmIMNZNNIJFu|X+6c(8|3y_|MDkP07VztKtfcdxpmj+ehu#_p}h$NuB zI3uPFgVDvQSSb%q=f6&24;ELe1BY!-sSMClXWYvP4x|HxV^#M?FuoeZnE9gTC#+}Z zxMd0P!s5dBPrjq`!dt;ScH<3TmiupS?_&@44uLU~iIQSxzg`apFsFAiJ3D-Hb%iYf z-)!@#?%mVwDWAGSPSye)9UgV&7jAE?KY99;uX02M4Xak@RR+L3s*JIkF*S_ViS>lx zN4UmR4W?WWL9g0ey-a)RJM^~tB-9}#oE} zCoBR@JDP;C8hAmsx9V{-+yX0D?I&^no(ubyxNP=z6!5ExddM5(i~d&*fo)PB%( zt757yBu9QIa+HXX3?u;*IF#BcB+GV^m7e$lL8zc}BUglt+(Js7i?uyhtBwJ57*x9a z$%*tSjN!=GsyLYnFgXzvdq`N*ywM!wp*o0WJlC9%4cl5?6%j=gK;Y);>gS(-zJ6;RO&F`Inas4Ewl1l-i3d0$VK|dg#NtA#Fm9N*TED&d&2PW` zo4@;q`wt%c=#x))%LwskU_bis!yleK=W{%--fq3#emCE_PYv<>)vK4U-tgoB&tdj% zaej{c#8lMs@n2YL#AK^NAdXXY#=04{vLGM!GSa-wVbM!|p@r^aI0@zZI+}$|#o~1V z2ojBp4h=`$4SKTRuyzR{5jcX#nSop?ivefjQyLO5tS>@Lo}gv2&_E-m;88<9a+N%| zvAP3)OOz&&JpiT(6KG46uEHUbvD_L$cxx^FITy1D5~4;FqwSVhN~|QoO$vD|ExJf8 zBE&7%s=z-;$nIYxs^kb28H$RrNC8BZK;=qNRWt=sC&Jhc3H2~;D4e9g8umCbmLNo7 zpv)W`b-(%U>5DBsP;q*EdVcD6Y9KVixb>UI->MVk#K1l?%$$ z!*UF}q+wE7b10YrJHjeD;_ATClZc8dAf?2*b_%qI1nP+NF+2CjAErk7Bc`$Sf(1(3 z#+0=5$;<@l3k_1NKA@g>vw@!qS_+eC!YV3~V*%reh(;jZ8p?DjYPQhLRa(i>0009h zH58JY5S(bierSouIIht`=cHpSg*9`-Gwz8G%Yw*oLxR*#nmBq{km%QOXVyfP!d8Yb z<~AN_a>*Mh(Lj0QQQ>Na3Sum$NX1)*qbqDx{NNtDXFmS^8GRf&&No+<7v|>}IWjZC zn@%hWIy%^2n4ev{vBLL8c;vXig#5z%qPv~UyYx9TEYc1G_?kc5TdKISaDrOMg*a4q zbjfMdN9i*89)OiRU44P5x2$YqK_%3ct>1@+78aKniZJotSy*87(CxAho9j92Nh~%w zp_g=a3UM|7g%LD(THEiP*9>@ots3mQT8CfKkWf*W*%3ANw=iHoREXiVmYoEnw6oDd zLxFJS^#n}zRc;a~j_Fj2T+oRqVq~MY`Lo8*G$CnT7*QZ$DJ*}nqy%(HPLi%$vlQ}n zHq2YE^ggW7fx)v{&BH>g$jS7*5a@v)L5{FABPh9pOq`7HNWaTs&!QKs&w5?}sDoib zR^^6_ku6*hs@jdnhJ4z}d`A!$(4Zy8k{>BCp8@9a;D@;CWTEz54fzs@KV&e_41l6a zo_vYXC9i^j(sit702X?hcwNwvn1{)5urI2jo}+N&AVSx-@*t8lZeUwWifR1W%S0h; z{@MK7!!!MR91)N5sgw)FBwy)obS=>=u{dJP{S5!^m|{@@Z<-|6EG|4BB&w$VP_;H z^d+kvfKoOAGNhg@qX7rj%E~@>V~G$>FmU3J;a@VG9Zrw~uTVBm<|xI_=Y!l(4&xuN zFi{+?5N=Hk&#mn8AzL*@<{SM|g%MDAPl}Ps+iE%Wp7DT?j-b(5={!h5;Kb)1ouvg$ z^@tIaMmmvAmH8u4dRj>TDk_qJYBWP(3x_Zw5Ge9NOiO8uYI;b=8`(5SFPN2>Fp(1E z%>>H}RpkInN2W{~YpG5^k^)8`6&d_wrm_sOn?7X*I*n)yKLxUO;11SSlF3pFeI-d5 zz3Kfb7rs%2Ee;_YD6`t$E;5yU>`^8oPbjL?xD;`#4E{MZU2Kqx?qEDIoUH@w&!%Cp zX<*q8YUfOMaF~GyPvhwLQ|N$20{m3J)M2d}tg4b#QiEp5n{^a#njBu>;njH-(ma3h zoKM~`YNu^tLW;>GHVIu@yT#lVN*o<`?R=kf!O_1t*cz12(;OaRD|f4ynqdCxi$D7O z_QobWkXMyTYeH$sv}!waP4fy$)Q(mtl#`L;?#ZLCzNSNebN$xK=P&;DAO6vuEJjBu z`K+5FFf%vH`$pTld;j#SU!flh5P$XSueRRp@Y2HM(3qd)nVh5{4de9abw4LZj6r8# zj)Gw_7-6soL2fN7aSzXWbMsqA^0dZa2Cf02hh9-f^UaVnKZzB#QIi}|L|8l&p$HL^ zpfoHkNjJ4Hg*8cJ5rb(p5+wRR#K;6TGBn@|%8osPmqh#~$_uIwa4BG*5kr?IV0#x1 zGS9fdRI{eQk{aVh7(fIwO|z64v{{Lce$3RMO zuuFBN0F!cL71>TksvK@Kfq>SB{FEkjDT1i9eMDu9?*a)O#mY~l%4>;AhPeW$NYy)= z4x@#OD9X!0aPfDJ!`=v!JL9_1e39WGA0D{3U8ax5io;(Ezn>Kz&m@e3S;LO zA9T}w_Vr&lUF~o0>}+jysmkVNrWZQ1w{NZOyxTeH%-Tb{aCZlGBUoc|%^S-sujdf~ zZ$?pZ+N%zKWK~qDa?+si!3>DYtCQ?2%4EEg_G7Gd_LBfK7qkorU>(l<5$h-zIc2UD zWfc;#jCn`K*>{0`gi0`~e!j?_3VWPRJmA#pJo2I>vOW{?cnRNj+}OCyhNoD;u{dIJ zj^b9l)Qgo3v5&?zXZ1ESho6j7LcK4Av^EYX2xod)_VhE*?ZR5>fa1h~Y0MuAAbo6y1YB*%h@e1|n!nI32HJ*y0CilI$Ive5 zf}(bXk@^NTnxaienmz3Ij1XuDmv|B~5){omOwK?F(xwBEp^NX|y6cD0jEi83#1{V? zNB~ug5@Ezv_c{b64Toa#-Sb)Rlx}~95{lwH;Ke!d!qC}2@9`2KP6%fm7UN%CGv=%T zunhr5Wn=W-DT{ngyGPHC_IW3eZT{c9e#xuBJWjA);@R8S2t5txK!e-MOJ2L@tJi04 zjtlzv#mVV8&-`DFyk$)gds@v-&mHq>&IMcYqTVngk{_Nw-`w0hK00yTk=aUCw#-PA zlH5*>Gf+0g3CnR{Voj1f(6m}pu;jLOGJA-ypO1xvrhp=PigY1_=O80O6nm$jOOA{r zo1p_1I+0EXr5?NimF!C%$9d#iqc>9`m9n%0M&>`%c@+rX)IAi6C|xvi>lZm>P2T8$ zx{Iz*q?9m+@J5BKu9y;KQD|hP+d#Y&edV;1jqCv<m9p7zhazS)$y`W@4|?P&QlwgF$C9G0QS4;wKa{E4dU>m{}+!m9S7C01b9HcC>+=P?X7PzeSqU1s43U(341 zw^XVELv&OQQDId_+0&8GqLu^VBSW=w5U|nXfN2sF17buXRwB|N5rI6HZRD-lB`6M@ zw1Q?9%Op@u;e>;WZA1w{P04oAZ??BjFNaRgFWLQ;mV@asI!bsiZ4MhgzIyZavyVQY zw^U5cL@h|kwZ%3puUJ5Stvjx;7B(|Xm?%qTN<{F;sIEXz{1hG-Zb>qhvuFYu2m-Z> zcoo)gbQt&(tsr+i?>9BRfzOlLe)Bpxj948;*fqujAH`d?U`1Uv9HE;! zm5`ZZD%avCOmmm_SP~D}T%l%!gj0y5poA$l$mz~2e6YO7Zk{h*y?MR8!xBiA>C+YC z-C1_izqN9s``{im6&s?_!-57C zh=zoZ#=BdD6saNOLXx@Cw%%%E|DAWN*0=e zPZdPs0xOv+Fe;xs35ew)83ehIZe;K=)3tpdEE$b?imh&|jDb-W&1PXA5TR`;IW(Mg zCj)l_p7aTTH6k`+2F%(kt8|F8=Wi7q4I5eLJEwqgg9wx>d+gr%@j2{6&(<^&!XKWL zH%E)y8YMOLNL{C6(WXRJq5`0z(#BV*lL6o!8O%%msu!1n4kJhEF$@?srNe4hLx3l)Md7c%um z^E@>)Ngsms!xWRr>nrvIWT@Qj9?{mabq|lB@C|zQ0P_MnEa@uH3L{*v$4-8IcFJ;W z;^)J9OzpdvzrIW4N&%m$(9cO(a7`7D-7eX2(Dlp|L7zMDz~SVq=WO1@l)b0&Sk)e3 zhurOV?|6g%)$7;rHLo;y)HYjG3l|}l+|wfX2GM3)V68%QAr3PpHDPe#n9fphm>0lM z7R#E_w0n~rZ5h)1FL#n8d%*H>j^3!df&sNYp3v{Jk)ly%DNK1CfQ*~~%EyP6b@`Q4 zBu3C=yuw-^v{jmE`9P;#!ZkHuQPba|j^yD{< zHqrqUZTwgR#fJ(+Qh+6ENVNMOSpsLdW@Cm?J{eY|MApA7;Hmt%Hh+Z@G*OoQ`x!>^ z#thHKIP>CfpE47Z$Q1E$CO&HqQx-P30wM!!d-|KUDuW(T zKIBb|U?!%o*lK%WFb`5;rIbY=$$iAfjm;LNPmU91cJi+($w<#?soxt?RSUcq>>Ik4 zc1ud|D4_{ddm?aIs%mBlCs}J*p;t6*D^*B@LZJA))&-TGoVA3bSmg69b8hk`Nr@_hG^i#vT9gJHBz=vD z%6`|EA2l4?ekyh z>Ei6j?A*)QDZMEdS9FgNo5tYs@{HP;SCwM}mccp0L%#6BC0|uqTe-<22TCjpI~-Ne z&Gq|`P)N;(^F(IFOxAIS?#w z$}}eD#%mZ0WtzCn>v93@qEFDjGz0$dM@v8K>R0FCUaz_fvd8g>FmP9foC z$C4%jI?AtNV{8Y+2OpRnqoQI@ymafe02*+}l~a_orqfp-OFux!vV<0ih!)4vk$Z>l z5(zhzw7&v(4d@{T-h&uCcc-2_K0SgTU(CQwSR~8@JyOyiV8I)Gdi;tG0?!I!2|ofr zubl^5d^dx-*Bw9mBXr`AkM?1&LQoU&;N3!zz)d_iSx&U#QPv@FUpR;Mhmzt+L=QF)Bglapqis*nC40ho%xJgWeYJieVM2r6(&eelUF^P1PMy4>Wln zs+c;LnYUiK#-$5THg!QvwB1*xnlQUvIAtmXt(Pg6bH$aNn-Al$+}6 zq%bkQ@5I~1;Lq-uenAZ@4j2}70GUVN;-dyOfZz*U^to|ZUMX}&2N2&OV1SSJWVSIz zcKinxJd}t!ghrd)fE_K?L=sJ`GY{-JIy!U#ONQ4RKB{Z=7;9O4 zm!Jj^mK~L1#;bSE4tHDId*40zo&}@JD=YM;d6kwA_eOkQ6{9U+O2C18pQjF&6HL{Q z4D*zkO?(dyj#w0k2)yFMG?+9ci_~ln$9r2BU)00Qv z{O)&8p8nC#{_yRat(UJ~FD@+6>^*z_oQj5L+03r-h`@J)t}Z|M_!C}3=5?YjD_l>y zA3l8WqmMp*^ym>A^j(~vu}GI?xOVBVER#5``J+Y~SX!SzrmEQo4wY<0Mr%iGZfh8~ z1~QpoBcKBMovPuC(Mj#{r^6DcJH5wpP68EH{liWktv! zngpnF@XLg-vQ(-WFCCo)@f8}n{;Esr&QJ1}NhN{jQT#cFV+2vS#z<|d=xl+)xZ)J8LU`Dq9}x@m>OAT{&9pjl|Hx8Bl73o zB&bV?NGj{r|D_0FPTY<{F+{uJ<0>8+`{E02o8`dr!HXc*S92?i?2!UC1th{pDh0AV zwSf)sc#plJ5}Rxw)mbv+#7>vaa44U65!jYBL@E+A6)L;N64nFptlUvpK$C_**q?UD zVuRhe3u;&b0S}89o72*k+1Ch#k;~H9x7hmFCct7eu6$n}QuicUX+=%QcV%8mJ z5CBsO69c9VK(c_F7v=yhIkwdqmB460i=MWFrDToIfnPP);0r6UVh~7C_{p>V5G6Lz zL8yr$jW|V8CMA`^iJC_x2}%oh5HDm)Al1-maJ`5t9VZ~^;H!K(05h6AX{Cx|XAnG$ zT8q96g8`R9#t6rBK(eb!zp|1BMy2 z%ztqI;pX}Vi1lw*Xjv$Auqr;3OnX1Bok^3W;E_j{Q8S%Sb@)}t zNOM`^nvYTrC`%f!OpDxF!?DF*8C87)S@%=ptXVq!%X;lu8Y}{Cfvc>dBB! z^+XcLicyrI0FXZhoV1lN_Fe+=kT+NU{7=D9nvh5-F<)I`6bzmP0%CRuk6sa&B=sal zFMu@yrmzAW1`;F!I8xFeh-x`#HA@oW!~k$WkG})8Ns^Y`K&nl`&NKRG-*^oGYehpK zRL~Ic=fuAN$km9GP7USj2lUav5*w9+rjUrH`y;$8b&16A1bqS+RJHLK8|io1E4({- zHtrXHv-uudV9_t9bB`nAL#)xK^#?}2bX0n03=rtIA0M6ZrU2imkS-H3TU*&b>H4I5 zOc$R=2$ax?scBT=NgEUVmJbSr?tB3beGhW1-q?yAXdvhaI2=~eEMzdVFv#`w{@y;z z@EA6w45^a&6C0tv7Q*IOMh%HKj7;Vjf0>!E2apydUABU-6;^gjuQ_U0ku$$gDf$^e z0Uw%)G7r)%Hd4ke$AUmr0`?E|H;bAqhnda|a=eo{DH7SPMOU>jm%~9&vGF`&GyST{VPu2we$jv#^j=@Ck$oF!iqoh#X>CNdx4Fz=RY+Lc$}O z&)Npy!l^O#4pdPwC|^@0%e@Zb14fuxuo{3Za-?OB#&J32lnE8auyV1S;U(4sfKG2^0R^Xb{mflnQp&PMkX4a*Dt5Q|zq;%(0KVoO86KOdmCprj zKD=0G;JNw@PYd|kH0_g~OH(2dsO0xrQVqU-^LBTChkE4Rox41;=*&#Ndhvpc^^adF)u8{ zBN0d~VYCnS>rWg@q_F%2xH)T)m41E$z>+79Tv#HL^DRCGPfG(gX`2Lp#&6X7$RRUf zwNUA(0QqYfC87csDDb$6ap)q+C>%;Em4A{$89~ulTHGp8)E2~haiN-Y99mihVF2x8 z90e=*K`OU$u?kiZQ^~T#5g;~pl^z@gkwgSm2J9cozUIz05p&bi$GsyrXc?a1ak4uw z(!7m~;O!sWyMJe6gJEamgS?>1Fg(e|u#RmBQ@mBnTob_{3Q)Z5hdB?Q2Hms}7}S8) zHP9(qNwmg8T&tBtV@v)xN^rl+>52osr@{g4*1-0~-`y&b)$)Py;ygyT)t z?I_DVb^Jo7^VyF+VJTzxwEy(Q%inzaFRTRmyMOv;mibdzE-fy6G^B-zpSimruP(0*Yq)VYYPfCGx-@(Xa<#X zp)eIRUv?##dYGD9eU;1Msll(K^s3Bnu1)hb~UU*Q)B2IiJpR~C7qbgc0ouK501}4Fu z6D?+j`AUFuDIuEbY4P@lh~5feLJEwzBq8H9hsDei08Xx9LxPzJzWFK(X)A(%#OfoF zq2U=VHz`7!ASP5mxDdfj4xtl4Txf?H%HuJLW1KDm9H)$9!q|!?B}6>`VSp20%2Hmk z%?wfi0O73cfe*NKF#sdJDd^}xBGhlMcGZ zqyQ&WUFa}c+iMD()13sZTN#y(^0?ox5)aevPqPrn^ovV&lj5_| zyTik{Ft-!qzI4Zav$Jy@7Fse<2-~aR?UJ;K+Pd#0=ZNDL3}Bp>x731txp>fB^DgWHcH_x8nAU4z?4$P zk*5D?EV)AxJt_hu0t#%R=MV}YnA|sImDqfhQ;HJqpj9xlAOxMudk-%mTq+b&@FIq% zmg!)p$jQ#_;M7RkvO~H&xTqntMN7Uw4h9v$(?Am;)#fT1C}7*-;30*GSh z*CTjQ%K{D*BS99-IecK^g-$zhJSK1fPx|(X9(r^uubK5fzwGmrmv#3n1hTtL% zLFfL27K&v!7kp=zKj!#XS8lL_v=6)K54ccoh9XKM#a_L<`^%$nS(+zSL2^g+WU94~ znpi`TH}tpRNGmlnJ^SFngR|4KAAa~ASoW*_$&Y`$zIOBA@c4iHufKlw^2N@(ZHmtF z;v&xq*nscu#a(8<-n`k`xV6TjTb6NetlwH&UZN+*e3&ZoeGcjrDq(AixLjNn^@*e` zF)-4Ru{crIdM0CZ#l=U=T7n6Ct$wY(00IP!%aKCOJ@E;a@9JP=HL*Hm{(zPTeh-C) z96l9>G=c!{xq`rR9~q<;toWopqkEYc@l9@!!dcBX7E*8&7SB)7Ko|`#fX)M^)yxqc%43SU)0BZ+=DPm&4 zH5A7IITM3YF}>v=&}!nHMt@IA(+0UHX$8{F?H23B%v9_FbNNO3#ux#8AUGfkc!_3} zqL|!~q1q#Len>m4c9d%w0#O@bb5fD=fm|&~U#fkXQ zDm^!%AOIl3V6F@X)!(8#K)06hCJX@QT8`VCL1t9rLE=R3pkbP&*3Y)>Q5-YyLKJ-( zp0%AZ#q7oztDs%R!;~iz^A`n8!FGDn0v-Wr4Cu}Kwm^D-Y#PKQ z@YpE5E52igof&vAD9CdJ>mAqu04B8?L&n9z)+FKOX_8_4?!gK|5gmMM1W?Rn``aVhb+0Vs`1lCW^w92ACDM^Q<>$#$+1a_-$lLq@_i1f#3= z3w>L8rDkGC90v7>WiH4L(fMgsn|ZR-Qc9$TfVw)G9Jy?SWGRX#MG=g*N&*qlDAI@_ zI{@ves2TUkYptP#MOX_k9(73Xr$MJ##)I4SdZ#C+VDY5==FOF(L%RM>RGwY2p#SRT z>djX#pS|7N$!mC*Y_E)eQ)XDP##i$W509Y61RuK0%`Na=${W62-9P8U*_4CxDdQO@ z@Sww`#| z6C0WJI#L|hP~DUs5K%t!HwJVF9U7oQHp=!lDdt`vq!h>0+98Y}pn+=U2(ABP zawjbKk%kVM9&sBzy8U)eDnm^g8m9=NUz<*mOgl<;y;1g%FdzpR!P1J%&a6dQFG!j} z?l~uOl`V1=O=3Y)ASs} zFS5oH_#(Y~M>9AuH|S(rqQP^Rm`{DV@`vxaGxvW=_r>{k8pZ2zs*rI%ChZen{-nU@ zfgym^1|UjF{Z3IS0pSck-4}U%hV7&1?D9o#pK1P+(^KY0QH~8g`O^If{r{_rV|M@P zc4ud2`GWn8#l`8VNnm*5#+z|$Hb-GwTx1)rC7t=h3iWe z2+qthTA!1JRvzx0oP>R6-y*e`@mOytMPUnji;BT!)j*_LxJ>=A}Bct*W z_E2gB4Q>i-bTPYC4(dq^wB=4m;((%r7@1pL1ES(2+ksSo*da8H78qen(_M6)(8hQo zX)nQq$vwH+IMG)m^NK4M{SZt&9`_2Ja04xoSM(uj*iu@6C{_djk!xgC2#y%R0KkDv zF!~PU$EOrE0!A~9Eec13&b7&)16m`2VvKOus9D3CvXW`mk48vSN={e-p?(_36s%$l z_=!pJ3NW$K#Rwu8s@AYHwK&~eT=?_<@Tcr>_22)`-ya;ac=~#Jj5j+c9^Sq4XaD|B ze)8$3%-^bX23K%~5!|hd6axa0#TFp(GSYj>6v_=>bt%ZS`q0oYC^Ybf*io6ZL^yDX z0u{<9Lppy@Cl*B0AS{H3u@%To!_)||5m?|$hlp@S#~MHKs4}_)C7C2; z86b8L)R}~c;120dN?3=wc`mUov5uIc!45dtri@ihI}j3hyR<4KCIW+MTSx%_G7+Sr zu@tK7%0M7`QKsfkHfSS4l3V^MZm32Ct(piTL%~k= z@Hmvmu4X4$RXoM_Mq~`ju~D8<^7OKIa>{2-scjAp_o*2vVtiq1ZFQC1%9j@B7w6}x zpxo1tBg|k?#S8(NunZfykq8LYCcPQ%vjoUjfsroN$Oy0Oj^_a{0sxB{&-zyk8(0iV z*NVOoRX4Ntkg^ls=z=xAw7kOH|Jusx!TuiKQf0G6zFNSOPafqC59c!l@dP?>%(R9b zjR{tS1V5}-9hifX9+l1FrM zSqvdV9l4pT1?s7U6ux>vEsbZ}*VzU-QXcWYKfSSnGeX^>*vd-OUFN9(4I8 z&k!F3nBd{SIiCh%lzef?YwNr*$Rqu;Gd2TdL#?^7yLai`?;Y+T5bn;vmF;kcCS0e; zvLe{A_=i5m?%qBn&Ox!3a9t2z)TII0z3nD%NENt#3 zm!N{M8eT`a9JUlHBZP1~Ug~Fr>Evqi-g0ElDNxZ<6g?qA5G&D$MZB3FhGuSlMLHr< z9wU5+fld2BkSP9>k^o_8D~wp2>zY7tG)!y>r*eE5w+;-_G<@}L#1%bqLCosTdz*pc z4r(a11fu!ZoQVx9yOsp!OI`J-UsHBtb%O91Yg05q_`R#w8w zL{Q_{ZHI4G`Kw;U14N(j&W5r4N>;@1uSnp~n@~}A$ztOl2lraX2^OjWg_nx{fD-AW zLDr?9SeC7T83ppLFd!H7Y+VcL8^G)##j~{D1tV_a*|mbV28PEjX{yM5evATQ1_Wz0 zM=XsB@|HcqQGO68^@w`&?oRiZcO%(3&V9Gge1dfn&XV>S!?W(6S9Vym11QhC_x5%# z`@PAr2^Iw%?C-NTD0PJGUuT{)4?o9+9)+AF@1u+*sfE(|>o)mKV#*OSjfHzkB+G0mJ|N z>%XC*`1rxy^_#1geulvf#mu@?yIQ~;fgMqvNZ?*08?ARC89}mJwfXlRgkXpeu@g(Y z(4up1&U-)sN|qD;thb#_p`sF#TY%8nW+UOc4IV>;Acm;@c8LN9XXW80B$beit?C6t z;pELv?;Nyl3c?TktYlI`Kn9VH1tYF5+xpoi8&~>36X!@LoG1kgB#N2tNinRaY@Jv1z60f4brw5 zsT`bAol2#k0mUj{(aqXiTm3J8@#nX0u73IGH|%yrukPNR%|HI)i_bpzV1cnH9U=GD zvS76QwC+ryYhEC1i;%stK}Qx< z3>c`0x2Xxmst!i6@+=IRDFZmlnxKYP&(c0{F_%I_B72AfGnC};vo|Y4=|VeI%52t&mHn0rn|A_Cp0FS-ldoe8Qwh-rVxk5Ok57BYCw>x` z%9h>_k(h->p!@Th>P2TD>uIYZYB^a-O5Om^?=Dg-ad)xP^p| zbolq6Hezg75^FI)O-V?1;IC|qz`-F^Ylmbkmw4n8p#4K2_W}koGzb>GkkHXIkNG3V zAh(-viX7sTHNXm{QEiG7@{}K)c;fKaL`d8KiUm{|&?CQafs>8{M#xl>f*NfJ#t44P zhoPG};n>$Q?k3wJ0aP1>Nth6YONoN*CC0YPS`>9Qpj>k5VapF0;(T1ozf$YTHl`N& zoGqm0T3KA$`~le*)jcvIHl$&-ZAX`BKm* z3j%t4>la{V8s?)01xlOC`u!zsvIK}lzOuj7j@+b;8)0|WIKg^&*i{|G zqG4VGU`Gl}!oWOPih)Su2&xJbAvY)~K*|p{dJXsBC`Q%fx|vamDFxjigrpG#>ujnd zw44WLFj6h5lrbS|y;}E1BCZ&c08};cR&)z5;sR<*jEGQFGDQgqDq1B_I%x{Zk6=nd zu_TG~NEWF%DHss$*b7<5gO)n`4&?^gSYFYY%kV6_)6A6Yk#U!e@^xWGjB#L)RaEU_MU8^fqID{> zRA{IVyYvEBY&SVH%m-{H&M)a0@j!qc4r;6GgY*8G<6JNs2IuAGOk(G@=j?q^)Rz#A3l7zwY~H9-7W)*gS~@qfA=kB z0GuXlMx;rM8=!$-p1YJdj| zu)_C!iRgltpqWH*T~E0VhSJ!aQsfjQ`HK0CAw$h#pp%tE=>+_FkNFKJCdp(v5J1O< zcl<}gr#NwlIx zl5fuTZX39?tM-8u1J7cP!s^d|L!p4+)nSBWiL7KmdEMNtj$qx40%;B2s36#$l+>^(U6QPJ8p5s% zJVCO)QA2HKRVcso*MTyOB9Rm{Iq@@Iu^^$@j3dJQ9O%O>DI_-xm_!VbeF(~dvgrqf zV#KhTY@-3mMrgG#tPUJyW^W@!vYKyo4O7wag}B@6YY^o%r;YXXwHr%xN+`gGN5?Gq z;mt>$nKB!~I-IlK86PBhv%T}|#Y^@y`}iR}oZC1U-`<6?JrzqB0<&w%D3s^;Nbk%H z0{}(>OnET=9AVeQSdcuUS40cKOfa1rDqDrT^ldZhT)3g_I2)ym40lh@%^U`}cI%l> z;d71?{mcI2r;iVI_n0`^Si9Nj%&@TvtBV+B(2e8U1Fqs^>Lnj7@R6vCfwVC;F_Y+k z4Ac40K4lTIz)5fOv9iTn(r}uuN-ROrj7b(&IEitc_lXwsG{j(<&wvEgE?gp#HFYYI zn9Y1HxIvzUKDJa0B{Ae5GIC%E^(ZMrx1d7CY#>oAWYM`eQgsto-bKBHIa2X(u~pG? zq3KLCw&s>A6f8IrqSz{-L^HeQCCzt)OCN_+#T4GnJ~Atl&OKNb1)uywovuk%f0OqN z>YxD(sGL``dL?o;Nl5XS&hA`dO_1Fa~ zj)!R1C;F`W>9gC%KG1v-h^@eTi;IT`s!7sGm(k-j>v(zLcYlAE&B^@|#>fauaiEI3 zVks&C2e#~5!gA&Dab6l^*&IgXMZ~$e8I}W4vUnY4cXyXZTRb%2bHG#6Y$)Phw>07_ z%gb!q#+JOhhey5!Ph;c4B}l#r z9yn}MmgqI?TXIa~z_5iWqi zi>nao2%=>`*9jG41Q6en#4Ga;M4a)=~SyUWM zvqXd`u36sfo!!~l+j6_~w-}BRXp=HF+L`aLH4t5D&?d(xSdB9~HGv!ZIlzm{h2A_% zgP5HHSnVzF*hR}*$2N~Co20}j7>o5MKmX}ZH#atac=7Vl*T3C({S-`O9Bos&{)k3!VA>y`ODtZbI?syTIiSlP6;^9|sOy14r{wt75noudfpX zTEaSksl0Zqhg%H7NGrr66+z3uM0cvPrbC55oJcaH#8OpCS`mv#gH*pJ4>OK9ZKxVi z(({$LDC(WneyN9C0yJ1KE#eB~O0<9=Eaklb5KsRwLb_30u{Dqpv`nfbpLiRf62zd0 z7AI6TlcNx4?qfZvDJwAqg5z{8Tg^eN#wdDuBdZH)wCm})$i*p5ll{h zM52+Mli9AEl~++NzG2HvJ%XSJ!4Jtji)c*F-a!h&xKBu8(c}YA?NCrLVm#4AkgEyc zn3L=d5C^NA2!bF%k}e>*mpB+c7w|}C8$sn1r%yzc^5PLF44RD9U@{HsBEWqdD!Ebd z5Yodbq)|H)7_f*bLat6q$00?9Ll_H5tdV??3(-=mp^^}aX2*)SRN*m z#+xI;OVqL<23s+V&&-Z8l~3<}g@NMe81DfuEwBeMACwwdz0qel!0Kf;-sg45lheJu z1HLu!Zg-Ed!SM-GCx_kB(~loM;9J32`mB4AclPo5Y$D$@Vs4-H&B;thpsa4Nc0w2H zFbAZz7$fY8h@#&wHwzRNZ^mHm4F{#m3%*e7ml>Iu;TggWmj29l z$VcXGMmvkVrcAe=8CF}O6j^M*Kjut%)ScyRy;iQRBhr8*AIn&V;|$oUa>3gcmy9wm z%E*;hW*Y|clsbzrzNjmR6-S7RkvEWpwHZT#Fv?OeVka#ZPLj1Hb$Cmlm{J@GN=BhM zQe6w`41{U_PKvR2mNGmVV=c2 zr>Em=^+}0P>npwz;I5r)#J9WCW4H^)tk$O;)orvXa0)8Hg?2JtgPds*;kAMx+g4QAX{+^M9h)} z8z1hpU%s`gYG}ewb-AVv(?>W|*r3J=9GFIvpx22ciWV_vDj&J({&)~c)*E`ifH9!) znjA66MdQo9VHethk~PVMnioMWc!~%MX0XvDm^@~pHYDdjk&dH9JgO<7P$0mJOI((K zy}t4oFvvnVk_VE=K@|T|<19AG(GWIviRGs9#&K%)2rC5UP>e{MA}ZLxQ6!2YLLi8B zzoEp+vZWZZ=H*(Tjb^k6gGd|6q5v5#q)B0}S~SwIurM@2C@nJ%jgm1Q&`lb|l0rH) zfZ3P{jgned3t}|{f=@CzQp=KnfMt{BH)gZ-VJgKEDKMOvJIMllm-yqC0t#)2GclN! z)%^BC`FGo89=9@}%aBg)%F<9Qq+;Pmwty}j(hRf;O#jovrMuo|zo^S$_IYLt9Y*Lp z9XcOnC*3P{*>Yab@?bN7m$rjqP8FH$`J4i>cF%ieK9F=UO_`zWVbk4r@4kEbJ>St_ zYfh)LNCaeXwOPSqS1s%Ars5a$L8FC$T=79he3=v_b!Tl!pYM-;@ATL6}rI4D*F zH?g5KS|LD<}q5xLxktv%8!ug za7ioZMN2LaiK!4dLa7*NASql}SlAGUJDtvTkKQ3#gcBxSXiVf0nSY&uB>W>>*q(tw zz|_4vch*)&XUUBdoOH@!dFzh6(Mva+y%G+OkG6Jh{_yKbWQPqXzu8@f@4#`6#;OsSS#na_3xyel;_#wsGWLx;VOshG6E zffX7tG6~}>4k&7>b3+Y{eQ9`VVrG7>!xmVKE|^1Q^A=E0&iNvzjV|0SkvUWrQPa0& zHwxEcj1BvNVTT{T}x}yw#Vz~44_MRJKC|4A0?x;S1Ktw3a|*5*ilKNAp)|HC_#0PRmH+| zP&5VjGBVi6WbvySQ3&re6aPR;M$>{FViW{+q2U&<7@wG=cix$2g4*4y&d&H60NwZV z(-WS(Gke0=l3^wd^1)i%=t$;8C~`$#LX?hQC7p?)#fOW z_^pEH&S1^LP?k(pJKc~=tb~qc57~)5;&x>;&53U@p0M8z5q$#?f;^i81A#Pd5rPP< z2Tx9f3}f;Z4&f69v8`=ijP3g|W(E|cg4~+@+M&FEFm{9)Skv7k1{aa7oYep@@WM`Q za6`L{Byz-K3KmrZ>RroU(-m9_oON4xWKJj&MVt_iSahkZG;ElJS7W=-aBhMS;B94z zTSr+!8I^Kx(#l`$;YV~Rv}R#qll$ahLyiml|- zPtjh6eP4;wE@2s^Wb8q-S_MORMUsW$SpyIj9#n^ZF)+0wqH?%X5M>ac5xVk1Z`Nb>`-$XZxe$ zlk1zC#8V0J#rx;4U%q_vhDAZ62Zzj5Aqp)QJMA#mx277pW`$7qxJygLYx_5rmf$6u zGMO5oSJo3|_^((k^$#U$0-49^pW=VyQZ#ZP%d$Sw9rPK$kUWAUuF z(LL?$>>tqj@e;ys`FPlOPx(&SgZmHun8TQ)De#6suz0# z$&Ex)_=rfMPezdiRs$KX=`-q)3nBC%Y+O#zNs=|JB*@s1;38N~Ndr0{G7K1EfDz;f zXWkOQN#f#yZ9T^oQSeuONUj8-MM{L>RuI%BI-Z;edDGfOph77+@WH{X6cx~tDP1W* z;5}uD%Pmb4P?D4{wNHXo`B`E-%w9<-Jn7Pdr8aEgBmg4`p_4#>P;rl#P^h9vGy+-; z8)aJARs#{ovr`+pr1E6PYYa!Aa#kUvT{h9K-pAponYm-ym9oLi6hMlEKtd^3D%?=Y zKu7oSNKToBK%=Y0R^wW5=>n-HiAfHEaHb-H7{!LGyflWCaw~L0^a8k+1kGY04J*ubz5YE%4Ocz$k%3W|)4+u;w6A?S0vtxC_O z5fWw>2=^zTO6LKuiyiPY>RlG~FmI2T>~c7Ef)68>kbJiho1`j6=1goUuZF@fJkecW zzqPrx#@A6I(zFePoXAt!HWzzbjq}#7EkUt?Qr)tN`t*qAmnmh)8Rm$+utKI+<%l}FoEmXeRwzouCgrNER}Fc2FrD=ok&${@x; zLf4z1>O|(J$YdIM8};>txExT@0}L+)w8#M)cL$cV@(`0SA;U1$3Y{=64HU@~ShaV9_a8EI#~bJ5Eff5G-W;GC_Td`Q-3 z+u19=8N`e7o%wkN0}Mjh*T)6_<2dp(oeqZ3EF5AuOrM0e{ul-vcTau$@Z^*(8A8+N z;fcuj=*094qMWb@0ba@3n6A{MqMuztT7c2#sAY96`MKllOYR6Y;^=o^T!a~wtd5n)pFPuiRwPSKeTz%8&ZAS7# z!>saPB~=Ci%3>yrjgsdK1JsIyP}3=7uPntOtP+j}hSXQS(9mOnjBXGxfc2tqh-eNI zBc(7uG$LA%)flqKkPdYRN0DC}EXA zmeTG6Y-E)n8!+bgm?_0XN)K+xR1i8uLmPu2tY@p!xQJnDckZm~loT05!|{xLIx-xB zV!LY)JO8)`D0}bY@~p+dX~-h06eoPxRr-ompJOZ_g~qrY2>gjD9R&rZPy z@;seALz!@3jUdk-SWZX-$s2fo`B#6%bAZj281783xY7?VHWm);r zgyU8L#Lgws#fORtR+FdZz)575JN!aQB|SzJUk02kGT|gD$ei@fY8@gd#I%kW$l{hUbadHUxaF+>INnA2m{K zYXAU107*naRHPWFjZj<^zu=H9u+mJ5RcH)aFvv#TmyU+fU;?RjF(EW0YYeOi@E_(1 zEU!SRj;JK(-g;QUNd{Y)NeM;Z;Ab(H5I+@&tf3}3r4S$>Ldhh0CowY$DEa#645CZf1SKZZ6cE6FgDYAF&0ZWZn$rX;sU#(OC^D z;1yGqaLxKHu4N3Y0H7jGw0T0n8-v<{f6b{INffh$wiyzp^aYY#m8`%}6HWq65HlqH zZX7-dF1g2z;ss7!bSop}Y9TUFjLUfys@Z!g2E*z z^7UW7;abQjfcUn*P;ssn8q_5~QG-ZsltsnXie>;kj4}*K(u?3s6(DPBn~GC%EDBc$ z1da$Y5Hbo15NRpogKo^Pe$OsH6K4;Ik8_ZAp$5%Q z5u&7K7Pw?9Iz}l9m(-P^&q+>|qPYWQ0NoXU%8VOX3EB>%($DSx%iR?InkFD(afu>U zWHj>!xG|UHGpiptL8$ab2rQMrE9D5ULYBPZ$uW|83QuVx54!-VSxPxkK<256+d&G6 z#laLHiFA-@_N-4?d~A_demRubRFo{b;O!Y?2gF*3jxyhh^aTKX-;D9FBJ+$tWD( z+%DaJ1`3pX`p7g949!{Xg9Gy*oN~zAKYZD>i+9~n%NKnxh%$r=ZJZPNS_s*V)Je@A zrXpDC;J}PGh%Nh+cE=CE!;_9a>eFRkTAKglgNKXDH*VZm-90?`?&%M#Q`*|ugCgs0 zpFMwmI5)dAGhqeeBXg{a;b5SOhGZTe@{LPd7MnDU*h;YwfS5&w6gu)Hpph_w1Uach zOY@qNLH6s6wp79362=gdCO1UqLPm$Z#;BcKrt0w}qP6i82q z%mAe5d4mRc001j-fK^dxtv_+WiuYoXp=l5%kd{C~)^Ld_d5Mvj z454brVd=xjeIalpchKL!x5ij+X2+@mxDpgKu zM=KF9E3Y(wBBqK-m^39;j0jT##EmS;A&dK`Ojp&YYs;HIa}x+}4WTQhm#LyF2JgvO zV+zn3G%0eZffs%t%3FF33yorCZv`TuNCuXAGsPOT2n401a_^kC?74@ev^8gJA2LxA z@JJmV;0UkO0E;IM|3R*+!Gu7kE(gfxuQQ`-!^@Xf4|xHAEs~i(VTq$#6qGbFms-@4 zE%)hnIb3BCPUakJ$@0iLl{ntPY>vht)_Fth%qsxgoS(D*0eN8_veJlN5!ER>DK9TA zBiFMZen4?P)pC22H<@q1lseZ+oUTH)2Iiib9Tt-^02m*ia;cGB2d%0HA^{Kgl{fyG z8jo9tzI8-=a#n>r5&u3*23ccL;4yA|&ze(Di?w5M^8w7z)Q5oQM&foefJ=V~OX)Jk z5Y*b*&emAwvgY2O#H>+F@IDc_6`ebqSU^qXA8r^ID-s={6Dej(1ybvCvUL(3dw?P7 z&^KLfMbT__ev88keQuw%ry+y|i$v1sL!}h83~RyYg;p@~2_St$OTIu37ovpB-V2vn zQ(5FH7$mG|B8HmSHkl|0@_CYr2+C$kuHITY5YffKw4;0?kNkJ44f0mQsnA9_-JgLu>#FI1%qKg|FXw6;*6S|>Lf2y zIMhdjVNb4BH|QA4VhFRnvQEW?Ft3wUv4hq!7AWs+(YYG56*8~5!y7fULCtIS%Gtug zMFi3+8?Zb#a4FZ}s+k1~5UF1AZz7XTV3KTxAuJBVTN9yX={CRoS~o8PuT1 zMTF-phBxLy3b?xF(6nSZay2_tmW4%t#7<&PQb-daJ*lV1YZu-6&9bu zUE>D@f+i6bN}};7jgWjRIn842z(48ep(-S43KjXimVc?~>e)XL8Y+ZuBmvSPR~R=>vt z4M+l4i6h%wQhRkx%Lyc-e?AfL;O>3)8bfh*K)QS9&hm|wfBfe!54v3_GZis6H~YZ{ zAIx_;G+gG7S(^i5Sh!V+n2CGXwj-Gfw8lqs67p}j&3fV#!eUMH$$VBV&fF!#q6H9U z0y>65%Sj)z8l+~bdc|Dx2pykVRrcg{MNnQmS+U>&3sAVxt$ zlXMU*BNd6WX!wXpS||z{I2DLAmL#obb07oQhfD6HD9yCVL@R@Z)bRqr$Y9RmYKWAO zT(ui~G)QcxN@>^Op>D9S_Rny4DKm5KDy-F>r(AnZy`LqH_uF<7QR0aEd5f^^^;gLLQW=R-q4H zl{3XZtiqpp8cfd3j5Koqg_N-AEr}&Eyd`1AGnt`yP8v$oUmb|2<6D4B9p zXG(;KE`%s?N|6)NLq}^8<4y78l5i0sRy3LEL4+YE(Uz@{ zXraJ^9Cpb%Vj0dI78SfAptJ@0ieuqC&p9E1%S_t z5~)65lOU*!&1RxV#XrH+8=y!d%v-`NO1!9}=Ut7XHeTF#UvaKC#3G&Gh4V zO0dY5YICy_W8=K6#q(1hP0;(NLj^}jvDOIDSn$a<5A3z>?f0#6D%J>bHZ0i8LUT%Kq_mEnK(yJq$3qo-IpDo-&vr z(By0i{0YSih=!ux6jqXwgo*qJqbuQxmIyVW94rROKT9*!+5iLq>6Jt_R}K#JA;GPp z3O{k0Lhg;1t*rbC9C)w=$Jk`4GZC0X%@9LFcw^TC0Qf5;T-ii|KAxT3dhA+)1y&Io zGf>ouEHa&-9U(hFO&+s?w9Ix9rwmmoz#Ji>eh{h69Gs}2Yk$dpS0{~*_E>Oxc7Af) z_rZ~$Oc`gPfxW7c;r*inw*TRUKzxZ6J}hu!Ne^3iqXL^&9-JJq?w@@=SuMQ0w0LlI zhyzDN7Ps`-50sCH@m(-n6ua{sMx8BT-FQOChDK1ZD~Wbh ztZl_W5?ufX&_!bG#wk4I6``cBuPTbvxrLCiWP^Pz0yv6Az(e23Bqfjbyz%vxIM_Mc z!ed{!b8FF5STmIa4CzNgg(Sr=2jPew@f%1vt!O$zlSX0)pFtbfT)QV?*OflL&Q*@tq`=!3LXE|g5pJVfR#F+SqU>ta9t=zFc*pn(MUudNm-c4qEsAuo491VwR5s0 zIA$T(@)n1`x=NFK64b^5&oy51UO9nbVT#Z)<38U=1Qj0Dun{e;RAj`TUah)F2BvL^ z(B-ax}d}Ei%hCGC;7|rwpJ+i3|sg5o0Rq`{>APPJ{*eI9Xg~BeoTol02-Ja;i^VzgyI=!+20PX=h_r=*M`%mE>Ar7i;o=E#*;p<<3AO%mQx zafqa0d_ba2C||VYf_w}q@kX(L3LM+>Vzq)@<*7cIVc>UTQy500;>$rcF&*< zhFD(Ts}N&ZWrZ(kX0i~2EFzR#hBx&>)=`%SKsRjFlCUy)1|AX^oXC_0`^T0rNeY>A z>FOGG`J|)6>VK*>YBSziWOjAT(Sc+Dh6qzA?5x0(R9wq;)a^@kx>WvjvRZfkM(*vUdwbJ$z#Z=>X z7z2hYCUAHJ06n@==*{dcV64fWm>@4E*^QFEm(>)8Nv3kIuffWh4Q+zB_=lD^FNo4W z8X-FB4+OA|m7X>dA#G4l1lO=!Nz;V*OB@sTj60A9j@6MS*61495b)Ng+zx&%K#Fw| z5YQ#91XhTEDrH27D};O;$&eD18sLjd3b`u^XQ?t6kcwKQrBg2nIn|>Hp+TzBi7zWv z!!Bvifk45Wfu$)o%~Hc9yw%7e;|m%%8KJ0&)&vkn%IX7lG&C4we4eb4NG4A==?vE`8X59Pa1g^Bb{BGv%H5dUEsJ*&og6**dy16hx=P^ zUef}Q)1CQ+yZ7$Be({1~K$Zbt9P?zLtLnVM!tj76Q*0HeHEE6cqOV)}EU^~IL2m!T zFXoNT^v3!l>>||bv*oDkoDo60yPkl359#i+4+@*#6035UPJ23E|c>ou^OfgC7bOOj} z!F8L7- z+6Q}LZv};~UOiwKw@`Q&RVlmOpjeHYQj7rrBYV5%EabfMtsTpmP$H`Z0>iG}-NBNG z>!M2KBRMY@SYbM;BM+4$+UlX)hA0fk?qKYQz(X%QfUmjOBtw}tLa}HwcompgBAX)C z3vsq`q*bt4ppLTNqYq)R>X++qzSIj7-s5**d3Dv9nb};sdBl>wqhkb^o9iG7c)WVc zA7vF_%0H!+hT>%S%#}Nn6D%`@6SGHr>pN|sS)c)P#yjAlS(AATk_cn0r&ky^Z535t2lDZqk=;-t3&)BH_(~m#8bLTFV2yz}D9`e~6obM1@(@LXdonsd+R># zz!0)HICBSp90pEiVIVVgQ7v3#o8yaJ_~txDXpSVs{K8B5#go~f3`hnpOvizR=ctX! z$QxFn16#5R2NskcHwax>5%EaW6?KB3h7r)JNtux?C`jho0&^=xByFX4nXXXHSV1JE z;)j?J4#}7-f<**rL=>oU^^ErkZN7l1t*jsHGDHF+%S2(2aHK2xuu^9qb)t=_@-H!r zE`ve@vg%I~E>*H}}Wx&epF1tJXL=vdH02Y@>QX+_FwTXBF8OjlI;b(e$N&W~E1qqboR&Yiu zaOJ9*U<8zPn<6`~TuUL{5+M=2M;IV2BIXbAkji5gNg;|5jZnaeDj+n|=3-(^Tp+3f z%6$;Lcq_g};}80f=R`bxCSKa2NFxxp{`>0iH2WLQ(l{_7+CAyAg9_avTFl_6FFs^D z&T1SHlJWF!a6mdT+2tX2BERhDf3Is4r2^P)vA?Qn7O#H#P-d-i{A0u zBPM7j@7-blG}ivm=VyL}hDI|%mn}r;XN_Djq03yWUlCxVmeC0$LnIjZ^os5V;L)m; zFZdgw812v=Mk1PJx{gR0SxK^VVIPh{mVo`Xib*pg)6o6B3k*1LB~{?S1S=8=3X<8* zk94ayr%nxGAL6AM_h>Ffo|(^pkY?_sz7|2^W^5?^f2#iUSCb>l?>rfdoe?C1Op>`t zwWv!al%#23&}hwI`u~43m}$%a4G680sB%#-}~q9lonoFw*7fU%w+hfIlyCI;($4(OO172+4LoLb+SVOAZk{}_p)N&D;J zj||ImzIthCTVdk!cXz({;`{Hu`QeA}P}L9c*1GY+&p-S0+4XB*fBp5tAHLU9`Q=x? z`t@&qYemnerW3z>0nVEP1bqAande+O3HR>ZB>_z}>)3tz>8G9m>{(OmiHpo;6}Tb3 z_3oNaj#^5{J#eCuJQHb;+?Xg9z1Ya|^PT+)Y6ff&Z84IL%(!TT$R2YPv+O+U6?Y;4 z=`3Bgyv*s)$qwsIcd3w`^lsjRme}1O3l=ES>Yn+p`{wG{{p0{vhD7dk*B)!?#sX4! zHX1-jkYverSU~56J_e0RNk9t3E<_W333i%6ffiP|*iV_-ri<9e!MjMf#Eo0>nD_8D|L)OjhF>SU^(^KTw0}<7PSYe&+>&)}iQJr+zpb(he@Xr&%dj8o5 zuZ&_VP$41FM7X~Ci=tYto~-I>e#O8FLGkY%nMtdtF2%e`zPsa7c#f$-j%_ar=sLF^ zg?n=SaHB<@6~9zCTYmcbHC_Lko-2QJPygGaI^r}$5b+OujkvU%H@x=Q}Rc7qBIOQ>U-H-);s(ml#=;wWsis;}HCKmy#4E+4&vZ$u~3Q zDZx?tBn^0gmy?B$>K~W>DH?^Eo#Y0wDe1aOeUffcE(-I2weUQTVCy8p)_3njXt-A40i|RQi}q!byAp zaakbHI5HTYbwZ?ud6G^EI8uw2U0~wDOveM??E_*1qJ7LnMUypG78rGy?7-x*}L%+i19yrH3&V4s?_$(D>g|B*_LX zU7IuJj?lqSB0fUt#k1!=^ktImFMjd)ts5VH|HHS>o<4f~==<;9yuNq;zQqH7{i|QH z#P{EQ_lH0Hldivc|A6)6wbVZKCEhUAozTTMS>UFRRNua71s#xs<-L=rBu>!fz}0#>r!&-tIt=Lt_7c7Xyq3Z@|)u+M@-#A zV|jBYN(Vf+WH?p5(AS#>+?VYkHY_t|NMXczyJ6D&$r)ykFw?r;$G$< z0@GBuSo4!l?tS*)HxEAk_=_(-_eFqvzCWNUxvqVD|Mc-=-w3{U=jPx3Pk;5Ve*KmE z-w#y}u^xjHBlE-YR`tYOHAf=Tdj8kZdWI?%egtVM-m2b%y&@)}Ti* zwTkxSu9Xpc^3)W?08#v5I298(ImRHctIB9)wA_mnrXyaOpZalCeIv6PBm65Nq6e)< zgT3)4{<@4s#`9O*7Hby;#)|1=kzC;_I~m0g84CNbMsxsKDZFSdHESjeqOvkzwN42E zG`xyfc0lHEI6}-+C2BhC{h@;E#ny?49xq)?F}b8RUt)O}#MS@U!4!>M23ZkHg6?k< zE1=yRAN>M3aErUKieqBdNh_iHc3aq^d_*X3x{YlQ<#rjps5+c@fZ~K4N3f9!rs7bT zl^?0OZnYPXNu9t`F7Svt90N|1nya)7ynh$75!|cOH7s*bfH;&B!q`Rf{KJnC%t&Pc zTUW!1p2Ue+YK}e5s*_Wu+%#x^%zUv;ycP)wD!1x)(6JllM50OPfkI^VY*D2+r6!bo zh;5GLT*;n7iBF})&?szGfBQbefuLH@};I+mC z*(xYv=9r`VgJZ#@+1c!gv7GE-n~xH)%WOWeb4Lng%i~GsQHcDiUKp9YNFH(yfr?7k zhwapPE-nL`12FFr;tN* zfCN)mK(lB05eOA2$g5Ui4kySbx zowrI<@0_$q&6CHEo<4c3vwrjDb)N{(*Z1Oy)%@?=b;oP%z*}E_@g@ES6rVnQVy$q4 zi=zGgNAFDkr?PcFAAI`x zAwg&M83g@m?m3J|T$1WEX%KYowR1f)@kZudXz?1@j8=-Yu3S3oVg~JUi5Q(x^n``tI!)q zBeAyt3IXH|e2`FI@uFkmhGO3~AeY42RS7cyURO|qxG5>f87sECDQ?1E}v?#V9UlI|F^>}IU!z$um4+8?~c ztY#s{Y#>Nh{?-+)kTvL8XG~bY==Qcv^7R|@?5()3 zJeu6o7{W$6EJU3?bRf31?moHwk%I2^weHiK7qoV+`*#LjDcBusx)lwm-EB4GdUhptXa%57c|b1`mGui(mZq*S|9A;F|#6J(9&jcDLU39^1q3 zzkl-Rk#z?C=CA(p%s#@C3`N&US*N+PKTI7n!VybSi3<}l=D^G|pl}Mkgf1m2?fnj| z$dtaIjl{GQ5zBBHyQNmmXcIyAf7*)CvpX)0G1vm8g|UUMwf9qCZ0oDTT#9=_B56AXmWXf}6Ai*D%ggyxE+zI6zq+3*;gTvjS*MSaQCiYkS4QAji6=}Ul{cA3F?gsIM>0o#T|lBO za_~4s=ECrVr_=$e+>&FKrcF^)j#|sa#na7-P8<5`hqe69#@|g0dTYo-dU?q8)YVCG z83JioNaLV-bPxW@YHv@Uzt@*C#X`VEmh6!SwqGrsm~;D~KE31Z-s?A7!}*Q3I^AN3 zmf9Q7SJkDyVXV=)Z7}wZs?>nayVozDe(?I%^;;$a-9J+Fkuo2gbq{Hv*f^Kbszh5A zQD@)9?uVruF;0~Rq6k#388zcnz$GT-dO!+l>Rl-SW;RIoEYVhxce)(8e#MvHiLevd zClnCHw{FyN$ta(YAJ>T2_1b`Ktx?`8I4gFp>hUYU7YjNAnY3jYt#iT;?JwZ&x|^56 zB4HN8lXv`?%T*l3!i8$fvqrVo_wdX{RmDTG!wy9D*#tdcu z2}_JPhwGSLot=DHhWCB=@U7~o^n&U`ANo~ZjsD~4N{pBl{Ml!pKKbFD={)(>5Z{ll zA3y$H<7}nQ2M<0mTkOTV=hSdJ`tXMzjL-`wYk2yiQ19`+dhKrE&9hfOm>FPgz|VV1 zpilF|_dopk&tL1#6G~K?%lzx#{N~Z4ht?oW!$qrnr zJV24P*gGm!LoJ3G)tR1taLk?uE(jFA zcl9=lpjvkHE8* zv=ebNaM|-sJc{tP;A+xBg@?{0h+28bCi38Cx@E*U9&;PaBNysX8^t7(zB^h8UAk}v zCIuJ#DSFB}xh}|&8r$SRpPVu4*JZBg=Q#66t(wX&z60tc9MP8O)>hWpo8l!#rZDQ+ zY?6oABfuI)n|+=Jpt4#US>$e%$HbR&55VEa-04aK2X3wc($E{e=6`c(dh1{$6ps~* z-gCOq4V6Q_kZ;hOAcO3?=|U%meJ`-i{#=RbY_=vl-6PoH)}(uADG z7L|L?_0)8I2jJaX@BaJjpKnkq?e6J~9{{@VRkJ&HXYAQd?U+aKUEhaI+@-wx>Gyy9 z^Z)n%_|K0YTP^_Y5ANQ+`PCO+`0(#12C0y^+fu;32mZz&L|PPf$QnPw((n)Qj9q><^Mm|K)&(}xBv^Fr|)A`sqi1bNw= zALcnyZSn!@h9r{ajEjuDuB~;SZMvcYfaNY*+!Z?(ZR2OSmgHFkl(^-Nbnle!?fC8@@ z6LOB8R}v#Q`60_(^d-ab&=G~`{T&N+dVo)a4g_jUWg*=(;0sU+)iIJe8Fr$e%-RKA zn`$xVAK8LE>9a>RO|r62fh#v_Eaec5$Wh;3(gZEk#}t9I9FYZ#X*bE0%lsO-|&)r>TI~^_qo|XsoIJEbkQaW^- zVc8^&u@)qQ`K50Q|La6r>(fUAiror+K+GHBF;v2x^vq#-2h{h%>@Vl)wp1eau*e&oOZheBbaRx zQr*671+J4tTp0d~ngkfRh;J50Y{4zsK~L78y{chm))9U8D*2Ns(+xu6;U~WeG__Ba zI{VYJ3T4dllX*i`scbk&2VZhFidP5!7iShE$BRo(V336_`D$yL@`3SmuA0cO5sKre z=w|>3MonKhdb>kpwUmc&g9p|Wug1-a1NWj{!1Es9Pi{$HJ%9J54|;w6;N^3(cpj@(A6wOm;IWb%CBvDBIuuHPix?bL{l6hAZgZ=EXEh1arUYM`!ve+Mosf1*_@p~KxL<7+XLbUctj zu)4P9h~s(Ie>3I#sJuIY-luIuM@xaWniwO&*TkUQi!^!R*C&ifHIZkyeSo)!qoyQ9 z_xA0NZr^_O)mOiMf9?D49zHg@f8*92iW~b;^bOK0poR%Dl|>{h$7ww&AKm6U_{o)r7EH<3N z*~Kf)`g(+RTayJ!Twsyb;KWy7e);8BU*W=NP)LPYipDO$PGMw=cJpB-PK=zjlcJ(4 zKWzJD8)?N(E{Wc`RJvf+t4CTnh20uLdzvfrz>htk#W5e-k2M#|C8E zB(^7oA9e-Vv?(h~1apO5O3BF)La@oV>t$U$07B0?(@gsS3x+9&O-3Kug@|02j1ofX zEH^eug|OdffSUZSN*F|_si6vTs~U7yNu8*8){d3b&fno&)`^p{k^>NSU@w%Jg!vHP z$YVg|C?%^bhm}I}-x$c9yh$rzmxt@g7#ljq282$Gwj=_m9O4rbCosigx)wrUYD4Oy zb-p6x>?+t(3ehy(4NU}!ml*AIHV8^E=ijA|=gA(~jMy)W0ZCZ=i{V@VB^HAiQaiyi zuR$n+Ak2&o$3z+P&>7c+pF~p_)Wu9=f59w*JDsBQR9BH>fFoAE%8LQElU`L>?OEKP zTByJepe4itnD`-f57Ix0(rOdSm7qg3gtKLbXg4UIn%Fblbi@ao0(*&s7)|1ms~ob_ zQFVuG00$2kZCAP!IT3?1$?8G?Ml0+J$+dm${YXTftY|w%l`W~ND{G~EM3g_(x-p%z zRAKB2+0|>N=$Lr(%*sFwsi;_)!3@ueBCykEaN*MRUZgT#=zS}RzIzYL<5^w0nKg}P zh2%X-RCc}fd-L{ZpFVi-;B(UTO{EWi@+_4&UbNKKkk>a{KED64LE%;ceeuGB+!^@R z-Fpv=z^coeN%ZCwsy^jv@Si9oYM_gf#6#(aeRs1N{MdUkk%ugPiIz)d7+K|B zS;~0^Oq*4GAptlRmWkUk!?+~I=XC>ycd8dwzcyYIW#^ZhT~eF6ixPvm=;vVD6(FYp z@bB*13xrHMnC+Z$TnPQK8h5;h<+#LaRQ|-4_MVBo1 zdD!EwyEJ8{SsbB5jxF^5l0jpGT46Sj1MQsnmwf2p(3I47%ZGImUO7VUTfbt1;T^ioslU`2l@ z$X$sdzxq1~rs8nAkPZuX_#}4B9Y?IILrjYHK}Moq!W8P~3Q9m33}lM!uLHax!!Q%*II)lT(*|UARp`iyLP`W$wKuS2bpwz$xPl`Tyvs@aYtM9v(P*lgt8Qn{u=A0u za1|ic4m!^b65Tv8GYeqxVS^Cn3&pQyuYP>@0GDos6IVL_iis~B6EWNAZ~K(Qg0%Kl zEtRc>X-nh?L<05v9W0hY__`0*|JA)u{?$*{e)aIlrw=}R{Op2~?e{-?{mr);KW+g0 zKmYrG|G)h|{*y&Wi|gK4@M4p3gV z_ZLDr=i;LiLzu$xHyQ=WrEIL82v63G5b=r#67gRNOr{0?g+S~z2m*o3D$|iJ5sMg1 zzA2DsQWLKLkMAgD26+=pIfd0}?M%iH5|Nl7VrW?Kjs-4ziSDD7EZP{&Lj zT9{pm`6>Pq7<-bf`<66`9zt|Fo}Hi>d8q)h!iCX=@8$Za7*5QG8l9EoXr+tv{vy|Y zoCt@v>*Sx4Tm;P)1w`iXi_WFog}YcSjwW+*LD5_t!lM@04C|PcA~epC&c0)SWvF#U z62N{GSu3o9?OZ*>1CC2IHvGro8A#R?4*tnwm-89t74bpi)j^7 zEjDuXb`pU?IXECS0P2@rIdM3G29dj0D0XtnErYK}v&u}OQpeo+%-Wki$6TYGu~3Z% zrp|gC-qF@sRYC~>P6)?s0CovEHG+vkd!S+}5&I>VW7k10WO`SZ3)&Y2yRwh#WC1rL ztS$BwR#gvj0j_?ol2hA2ckb(3UXE;8e{+GbQ)Qew(*tF<3@w&`bL|Z3P(5poH!ZQ? zBd)Le(EE?j?%lifk&n7tZ|(K#x9?wjiNSN-x-i7>6-sY=(cYrZS`FrI7;14^c6Ohg z@b7GqT4XTL{Fx<#@*l`Cv5VT|@`e{KbDtFFTqi?HdP&YLb92xsoxRuHQZV{eG9x|E z(HcjBcCm0Mis#bc+s450TurnAELxi4R4tT?=kxeFOMu){EV-Bo4&(CmXrp6S>`Gul z!Q9}T3{PIoTq%h;G6+hoyvtR3M%%4Gl(|cP@ zRz}JU*@f}5aTZF))AP?eGufpM%35qu$cAGV0Xoxw9 zxZ)lAwYt)}ylzfNFyrN4yW=f*RmlS+PTM`^?ix`$QsGvCZx9Z+HmKVBlSL2^Sf^G*Q9_2b*nZP0KoH0Os-~_KW>E3=p zC%uQBIdc7g8o#4%v?7-61b6D!W>*Z{n^^w2C)3``xpDp0!^ck%w{W%5pr779SNVH^ z%$I=O09Y7kqenh4hDB;AnY**Gn6z8?SQsa;fRI)yCgQ)=w5Y{fgJv=7fBDm&JuXy_ zm=*h1fAQP@<^TFWfA;BpkC-)!2PIjF=B0{;9QbLHGft=TO(rO{rtfUWnM+BoY~^n3 zST;#(g7WwXAkv92t)h)kbXaDQ<2)l33pQ*6A0nZu7Nol@uU~P};VLK8GH2%sbg&Zv zWXU5n5@S6gE*IvpLUHJoVwp9aiQb`tlJ|&{e3CLrDv%-Q$yr9*3gM)w6PqilFjFZ# z(OoYR;L(wzkOda?Qj_G!6It6ak{mycg z2YVf#+_S1HE|JN69imzcMPgZbtDMkNWW*ETqU;gk05eh0R{{(%OyCI)#Z^k}E~cEr zTu^1@WkA-`J^im!J>_8|42!#L0{3Eg%WjQ1vfn=5;XpA;Phl~v@lmj{)n3DNAa#RPx&1-Je<)o&?PhA zHbKRhVXivS%hwc?rj{}6(LiUTe9_C2G;)&c$IkPVG?mSs2g|S6kP_o-Z_xPhC!HkI z_WL|)(31dAA$6RcAkM(5x1YcCc)?ijPj_zW*5CW=vj=*2?j=aqq@P{@t+FJVe^MCD zHRx*~^SQ-Y-u-mz&PN}8{HfMKb7g*zZ_&FSAenXwi53&m6Jg&1J|Pn)y;3gYRNR(G z(qorN3^cIzu0!Zt($Rm~f=Q8!P9r^(4aChH5GptOF_>r^#q1?RP*lRBmBUmb7gBNR zP78Q)*owlK=emhgu$&S#tT{;9TuRjwl}N8bi^5JDZK3T73TjE05YrJ0r%9P}iFisv z~&cJVf`|H;)pMUf1H=bEp z9!LO~g8Sm-Yv1hw=|$l&^UUeG#{#~0;pRdJ851-{_v+P)$B*18UiW0+^Iv>^@9w?F zPo94F{rA87$N%b~)4%(>|LE%hPhULy^Vi?FCs6@#jQ4%5N>w*Xkk7UvVfr^EfZkb= znR$)LxsSektkQi@ZQ=AUlkPtU5CT=%`MIupW}GPl@h?@1Sl)jv5<_P5IN@?aC0Jzh z;#`_ba?R4mE8iM#wV1BkCv{R3+0p!2LMNj(CRqZcgu0cPpx^Wx)1%W!CBPANX9ExL z7;VOz31&ke2mhG6D-1FjrE4W3mKb}Io_Lct*-O2WFoLDjC+INct_Uw_+$Z>IP(Tr= zWe@;s5!7u}uOnO)9NX+Gfj)@#JM7D-RCe9}xY{iRS|KMR3TyW|JB4c+{B)yU?2oOg zrY61H|lfL_ztlrH?FruhClb`g({doc3*glzjU(X5v-nhAS{idej@uSDz zKYnOYP;--RY`SPs<@@thAFtg!K=2MgY43e3+;Z2?`{Mm<9%HVMM}TV|{_yb8zx?S> z9y@9UJ!1abzxg-6`0P^+i^e65PJufCu+e>n&ZAk;$>~8=SH~+aMLCrsuFPqba$si1 zby)V7o3LY6iANSt#l{JSF2-Wa5RZ05M>mq$af!2kuHuIy!h&j&0H*?P1;5znWO9tr zd3GN8vA_k<3cfCrv|x}=;x^Q)&qjICT5+0Q%Px1S7s`$H#Hr0?DG@lGs|uVo#7>Qn zQrRcmbd=`-Q>PMFW(Yp~phkpPwO0jf8KS4G=y>9}`+(A;b>a`;(IFnuoo*>rG0mq{ zu~AQ-s<|+DH|WCyAVbVbSE;nkS+=@+PDWx>>ifnELig`Mkk0c87bgxPtKXFmD+R~G zD%qHfRN_=`=yVfq)RMDgDd5mLMK%&$MLdYk2Y0NpgzV7CR3F4~?E#%y7w9OgLOkJA z9Uo>D_vHqah-<8d8`1g;SkgoFbk)r#+OQOP$F<@9H=yz7@%Ww`3p}lZ|9nk4yq3{n%z<)D*`JohtbY>_ccgFR~FL zq>+s6mP#)aSTG{h*jG|xr!-`*D!oG>`%5nXl2IU&VuhJu=sYX$L}D1$rNy!FNBzxeHMe~X!yEZ^KVPvbU8*eZ5D1!HWshq>2o-!<*?12+z^Z++x_0gJO; z`>{I?9XcR4Z)wA;=Eg6bDtt|ML@i6zI37cjDX*;9l>tMvVp+y^i9P$f$~D`c`~b_m zN{2SDJ2Z`TYOa(JVKrd5&MV%o)nv(56gY{nG?1s5l~DN^e6>QgP(>tbiQKVW+BL=G}be___?D2{=BwBU)I&hMqY$Zb7-}jpb-xKQ~1E_VvqW zPafX5_RiX!pML!D`yXFL_l?g>n`?GmaDM$SU;pmEy)=aXt6%-n8+!lvkN^1fw@)x> zp`Lw*r&VO%eBcEFx3&Vqje!L{uvmn9S*7Lw#d%gtfLCuVG<)aHXP*v4t^6&ocKivEH!GHPt|MR==zExTj{?$#7VF6KCD8hooTcwS_o#dNi&d#3V~%rEy>5J33ez+ zh|MJoCyd?HElN}|H6R`;*zGpJO@%)y)hr@Wd~EHZYJ)~(Ati1#TG=C6IhlS^N>5G0 zA?gRXaj88t^cI`e0s^C65P&WWAgL(K!Ly8)Y;6|532WsUc@M9R8#Pe%p@)Ds#Gx!! zMkmhEm}wVvq@jiHeLZ~my?2PIb?@%|Uw!!nYg4bN3uhY4vv$Cka78Ai=?DJxSSJGU)+f!k~!uebedN(_E4*jMZl~o$Hk#^ z<=_)f2dDhurv-|&O<`6QjKX1sy2P~unIMXl@X;6?O?FHIkm*9%ONpHxLCpCvtSU*q z=*Ax9RFB!{A625G?CE2C_R$g$$nisn{G(#*lzi7#)ycyZk!?uKWmR#jHhw*l4KGk#@Os~H9+u{ znXz!@uVNI=9FkdQ*eZ+!QD4QUsKmXP_n-BGs|3*0)rqv+nzB)cwLlm>Nr^n9(FMO5 zJ;7Bmpc-anf&s&?CQTaYhh4M>7w`GM=c16dMBN&(L63Pt3^|sQXN9)ZL`FUNkgE^d zkN^#8&8LjqI8P*?S$+DR(k`W2(~`1d1cSUNxCBXwSo(1D0<`F!v=|g{{fW1=D}>8`)xPA7FeiNXkLg)eWjP={X#FM?rEwFT>DK45@V>QoZoYkGCH|^+ z!$*j+cCQ6w$Z0ZpRISFBJc+eT3TS#(2_ys&X1-*pN_P+%xvUSp-!v}kB`(uEe)#_3 zt!_Q6w1~MSn^&PHkVzRg5m6mm^YO!*AKd%+{+)YwUB7((+)|9D4)q zd~<5)({A=b+G4yKA~%wygoJZG8Be_#cAjdKs*{6vybxSSyj!F@33Zk4s4Y(t*GNC6 z?mTrxZXyc8|IJq>&qllLM3 zjOZZ4jXH^qL-Z)G4n_@Du#r{6x8r4?@m(bnp8T>dfFmxn<)C|*S5J-vGR?1fA39X@>c-9P{FpOuX-fBDP*^*{ZmKm74u z{^|Gs^uyyPjD|?c&{>tRDYY(|@n9Nr^W9T+rX+^6uKHVbsQ+Ch3$u1_@vvZ7O)q&D z*{a6s6qQVeh>Db{Q5>~weld2;symB0hg6iA1pZ}lrJ}G10zmif0X!>Q$`qtN5YfuOv^NRfpvKw`B9$gTIJjE|$ zkz$J3E0X?emtwgKJ2DrSse6Qxebh~xnRf?gyAE%qwk&jx&cgr@E-7M=tV_rUN33IX z8eHsh#GQy)ZbU)}wSv6JKxmm)MROf z4i?BICgz8krp$u}&Jqd+?UIky`sFh8q^rwS8GZVWg7!@SRNZzj%e1GG!{9hj4E6Bf z-D$~HFd8AtYR>X-J4JD-xY~Z9z8eP?%ti-&z|{W{I%=%?%w+8y`|&c5W?VqvgcFw%C?8c zw(B>1HdtHY34qF|k4;)V;_kif2WGr%R>4<+UGJ^`$B$GJ8yooW^Uogm4v6Wm;=czS z1!_nM90oHNdk!Q$VIy}5>m3{2Qq8mX80FzS26-tX1h|WQk|P;8cQIp;h-|-NqC?i5 zwW74xI)|BWG{M=lc4(3(4nvW4>7qpGG*(dzj0ft&tThgwJ*HaO#m1522*WIaWQb^= zI{B0~zeXu_BPM0!}I?b_HAPNkbD5 z3S}z65)s`OFc>O7{~|$(Q5wCNF0PY5+Nm(&L75V=UaVxlq_;a0CQzQ+Q7=<&LS3D^ zBv0+dan&4sqB)J?VVA^*S3GA7hVEoyB-ciQR1*~9AX&*ihGnwncP^RYhb>ijH3(TqUYccwywOMB=KHjrb>uc+{Np$spjw^skxmL+ zX2g{?B}my7r5SC1Y~B+$;30(zP*zuKjR<+F=Xplfj0{eZ>p9EiJNGBzW~GAo+=ui8 zC@K8X$1$onw0DY%ZB?nOt^+|iDRO=leQkBQNGzjsDmO1)mjboM2{p|Io&B!ef^5$H z2Ug&EXL`?vR?U9&?Vs=8|LEg;_bj#Xy5)a-<@d)=Kfd?zM|UmeV_?qWo_8%6b?cT{ zK_b!s^P@*U=*fKi@jbVt-UGHa&nFK)^EKbk-#vKx{JHmcyH|WCF7)Ir5`JC_{LqXb zq28D6jP$*KZ&olAQkyWK>u(;CnwF)XS$6!Hd&?WoU%W=`pZ@pZ!%LV* zlMiE_;9q83(W}I+USZaS@*E*;9C&-V-I58!g-QHuRaa3Jqk@`5g*J}~*g~OZreZHz zEGN*{IA&%{h>2R>PMVac#1yCM&q-L!2ZvgK`1X=vZ(_c?ZgPiV<%KQDT)$k0v!8g4YI_bbdQy`ty7#zId77V554QH(;2I+@$hw;VvC8S~$rq6K~5THDwCHwipiQ0p6;Qg{97n{S^z zf2OCt#{Kon7W%J_*J4@u$GInZMw6-u-N={4FKW~cP$dGf`0B$~@7{Xu$cm8OU4Nre zXuM&btlzSL$@NJ02r)6Giig6!Xed}kUnXsHS$BZCQ1ejraMmsu4?@vVrCw>4lAX^n zK(^_k^hD+lyJA_|X1Ro1+|P<*A6i@j^xtuWP!++$jnpah>g~oT0L>wTUZFiw@FM3d z$dDn%#7TZUirEQPR!lNO0#50PJ9ru;*tCd2rgnWvi2|>mb~P%2Dl#jBXJ4xS$aENe zRu)jCB-1K1rv;boac!dcPg-=`TgDNQjky9&J#u>Ax*ZZmQj+)pV8^4>=(7UbfnsaEp=`3!eOQu=PZ;gxh364ucQQ~_@sk)?s1P}PwF6xh$<*TRIl}3-;1_-u+ayAQgH5WDOt3Cx2np0DTY&1`5yLw(RY%X>;a@bB zzOoj2ilm+U7>6hZm)W?j$`P;%g4O#JfL@XQ>yadC^<4x)fkIv#`ThC?j8S^6L32~G zC@v@^X1tFeQI_K-V^cN_(0VZY-m8MQ6(Q?0dYenHfmlD+JypoJZoYY;3Fv0hy8&i_ zJoE`+UugO0Q%?_W-!U(!PK>n&DKstfIADI_E65`NAwD)FbBLI zSKPah7gn-I%}EB#lrmko?Q+;71)`z;cK82)3#*^rynl7`0|R&0EYxAWJacsU)D#>~ zfd2WP{`Bpi|M-QsKtBEW)r)6>>Bpbm`iz&}x=(F-8sPO`jsAmAKNFNz1adp!v%Y+8 z3XceRbF)wRdYGVzcf!w}7?>F5ae}&{c|TrvU2m0K&Mge&4#ek#z2*MN2lqez^baQhq{5C7XobLG z=UpCAM%%fpVZ^QtOIOt zqwm|{?{4T1iVH!#WwO&TaxpxTL_OjT%W^u(m5Vm}{;J%|I2C^xLB)56&ipDd;Z)#H z(Q2QmieZ^q4BX5Zc;~LB70g3zAf?Ss@^_4IJQz=@OQ!1ClnsB>1G%Vi3Vg+Lxe33l zvv)K>kk8&Nx8tFJFYd>mN$W?lEey<6K$1w9Dx?yb8q_s-32 zW92mwBc5(L-%?cuq{kILEj$s(0d1KbTz!3jKmse0;S`b|Uh~w?0D{fxUF35c?}35O zf!Xlapn!8L;$AaXqZT3e@XueLw|;+muS5UI{d>=zJ=LJy$rkeQJkIdV$#=d(Nf|vk z#627I)Nv&)Paro9vMh1CTk3ijGd`uVs4S+*Gx-$p^h*s!(X*?xb!W%`d5W!02qnms z)^&+xbuZt2mDQSz5cIf7a;|6LOgusGL8DTW z?Sh@mX%UfQq9Fz)>MDif=zxL%NrGVwMdn(ucsq;ua(8w_h$y3#8x(4(G39q1xi}X- z2R)gyWE0$iK6Z)*;!~h(09_9_yomNMn!1>tsSv^uAt%J`l6Ul&yL*EvbeL**SC%S_ zE8d6ANG7}!!p(4pd8cI{I!2+5{n28YX1Y~q7qbKz^#CJx2<}4o(fGt*ir_CjvE$Hq zZeWFct|T!_4kVJxvnzYVQF2k_RK{&cVw=cEuM`aU_+SKFy73Hlij3B9W5Alx3?t)2 zP?(Qz$t6UaMoR-+cwaik^mIX6Y_Dra`p7pmiR~g{aiI z8}F1gOE)PNG{HqTJ!K`tm3*`XU+N&1m23ms?TEAF$caB|nnV|35e=J|<*)t9H!=6C zfI6>u22zE)%Hl#bHpvr-ophGQQ6k091-DD-A_G8WC7kL3T3*my97-o4YgF>V)h`R& zH4)yQucrLy^CkkFN84&IU7J-6Tp=*n(`d`YA-amtGh+{1RpnMj_^_qfGz&&>Yc>RJ z+O4R?9IGr+U5c?P`0Xp>q>X+VNi}5q(`TQ3_2pOQ9=SV6!VS$dQ-sl`8N)w$^634O z4-Nc3_~MtJe){q4J8n7>-K~dZo{_G7TA-Oa+Cy#r>t3b;w04%%E>C?x!EBRxRAI9R zm`i2iB4VM<@teSHq7a1Rc=%B#YuN2H?O~b z@mf>+)fd0`;~#$ihu{Ch(@#HX(f>Cuz2<9K&lfMA!#2y`Fyn@w1F z_Tq&OdKNOestd0-O+-!j2bK=^h0fbX`Zd-jwzXvJZY+himj&H@x^vj`s@IPm`Oeoh z-TM0mQqGOa)yCubX`P-9uj@x_#@el?i#w6AV!%+FwK9 zzU;6T+H3eZ?Ba`9gliwz zdhZ39HUi-$5X9TK1U^%KUztSg8=G0{B#NJsI&ty)v!XZ)z zgiY!jUVneXeLznGEZNSgdndT&EYga?N;-_hme5&bkafB%tXD2lt*~5TYW8=fa@TE> zmxTs4a-9{%J?E?BgueQ3C!`n^Sr-a%jjZ$wuhC7~{$aI`Om?_xOU`Bcp*rb^4#-N# zL2x{eek31BN9TAtIkI%!tJnZY<*0PX(%B@zszEPBb9UT_#DS3~*3vjy!4vYvJ8Pz? zLzD507`R@0=%jf4NwdB z!hj+d?Ib2U)Ho+pf9vuk;OeQ1)QKY0!1oiTI`wYWEfj6uzomU&zV1tYH*eoT%l*f# z+qduCefz|+zOVa|i2HoL)zenIn>g4*WXty|zYY7iiE%GlVbr=KT{}>hQ#@Rhr=ZEE z-VHLH^gIHu(p)n=M;~`^z&w_DVdu&uUR64!=r9#?)I4#F4B!O9bd!_u^PjD%HK#;q zHpy6}1&fV}zRWIj(FIG2$MC9b$r`wX2pnQ!Ijz(>r}6?Y>j#TmHOz>EAjayOvI3}d z)~oCy(NW556F_PDgv3lKK+DQP9Q$*41{RjT$yLy&=d2`e-IFr=3#@$sIL--pQZGRgOUyVabG4;* zl#|8QF1vQp`FsJx4=GTVZZD(j;J1|xe7+0p)FB>j!UitA$q=;~Qc@XA0TWuPmogTI zC$3|+G*af@Ax%iO>^dz=9lZf$w3I9VUCxO9WClH(hIX;?Nl)#wg>){GFAGJwb0mn6 z&HKk;LDVUBb=fI7&;a%HfE2^WgO&Pn)z8waTEm-&MZt(KB0EpuBg5d*E?1&3jE)&b zMP9Q2R!jZ*^(!ybzI}Hmg$a{{KErCt&220C=~p+#-ktCn=`};g2(n#Z$+o@SvKJ%E$ncpl^Hk-Kl8_QQb4=w^86C(;~3y=LEDbkiu;8 zT%6>DnsAYBFLEj=sJbGyIy%GPjywClQ?)>&wzx@0E^6=63-#qx_<2Ns!KYsh_lTSYN4c~9R`Q}f5 z{KKbGz32PUT@2n>i}UqMuZ(-P|LWz-hWz!ve|+cr0A6G9$sp5y2eRx271J z#{=F^*Z*fSH#qIR)|_?cyc*F=%0T^P=d4z3P|dY%mS;EVph4w_qO;0?1Aw*5vP4^% zmQfyJE9_JoajfHYcnqT&bMXMU?psu1oXT+sl{q0qqL5FfSG?4)3pU{Kb_+#t)<`iLH=0;s8Q~owmIQVpV6|peBvz7LP7=ajVTDlp z3^K>mS-YGCoCL>a5$)nyQ-E}N_5Ks&!Xf8R^((NI-E{FsczoPc>)Z<;A6u0CR!u4{ zsJ6bep0(m*Mf@Lq8%sWm>3gejD*P&givlD%&6utr@1vY$RBjjXD?aqiXm$EBQA)Iz zWyp^3kDop<^84|}AAfZJlUF~gR^EO8=#i#JF~-lGfZ?lZgq3{qs|&P9*R>TNnNwPm zcE{~NV@Wa5Tr}}%2F*YeTop+&kWP&>0;Ol}Zo7=@=h<0wbL3Nak+8r>_S27^JppEe zdBYA|nrB5nTvks|5qtmn4>qP4()+~C&QO^tf-7%9=puxZmcfR8Rami+G=bXXuuKUm zILgFu0=f)$V(f6JW~AaQCh=sR?w{cV*+h&(XbG~%rKHP7{7hGwz&YiEz=@?19n}gR zfRK*L3+n=J(JOh(j-D5~d8JcGM4;Imk^+(wmlQqvXpaIK=g~tSrK*1Eyj^%AbcfZk{!G^cI zNj#Lp_F^ZfNmWICzz$r($&xk>G3%nlB|C&tzSDBjsmo+KMoOwuvmYx*a+yoD8~NFG z)&Y7+D}dvE1rn75EtThS7`p)a&9H;6o)o?(AS64Pc+G1hruT?a0R?xw$I`WzNv=2p z6*is13hD?S=VCD79pcx)RXdI(t;|!PvW3JW-V`3~q-=|Pw6D&F7|D@xRnFydlJ`AJCVq1+WSVPqm1Owft}QX~++CiC(^b zZw7&;NZtC@3!NVQSJNE1;xJPas+w2N16DKl^~i4Cu*S=mUs}lJ6J!562fow9BHq!A zb$$C=KfS&4`uYp3?|*db<4-;@b;v65&%Ls#D*Eu|ZJ!Ug`JfSHvC%}^Uge^IZw17s zYF}Bc+tHIJLBJ51RA>vKw$)OOu4Y8GKOzUoZ`IaWGRrw0m;!dmtW}P^>zUdaS07p< z8{gFeuC^5h1waJAWvEc&@;{nv;t7-Wl!^l~<3hV+SZ;;%2wTn~EDqI9nz4HSjlU;c zoDgRxBiey>2gF;*NM-}KNrSo4Wx~WT>g=SSGoVytIFc6-Oi-I>oJXb~l|e`WzF1AOM|dp`f^#~=Oh{Wli<_CCNjU;pXFlgFPt z_{3zJKmN-=F7vRif8f3E##FCcrnQK)_f(*#Sx}}5 zQf)3GOUgo(P+iG#81-W&V8lD2*!d6!DH4wpLXfPLsH04hoO6LKZx^UY!?uW}UDk8F z+6LAxTZMEw3m}M z$g1T{xDHujTaq5@0CNE>Z<6u?kVZJu>?bb)=Wj#zu#bcaMw&xPb&)W!Mm(zx1K09ditsCsKd&_%BA=_(Iy2D>Z9Dbu2VqP)?UMVk&%66ce| zY44AKqn-yZiQRUwk4Ms9ef8zN+jm~ReDma~h32010?wnyhO7jeg1t|Cd#V3!O@%^T zG0jpW)0VjJQ4`#_cTb3Vg~;~Nz56}bb3a5Krn9QRYeQe@;zojEs;k*6t7VM|NRh0m z_{`5xMe%XS)TC1SE`sbRJrgtDDSC3_C(0@HO+ z0oN1I4iMT|DBz!Ii0(t6vBE4Q>Ge;tS{AE>CuD=Eex$4+?Q5e)`v!i*u# ztAWcp0w0&AlYc=$mLe&32V^#hMkF&!ydpQuot)Is1E$#`U>K*18%lM_lwy;%kRusW zCzsriB6tiK2R1l25wVEt@*`E=AeiOYGoIUA53YN`;0{Cv$*MpE$6!YwI;lFGJ(oCO z!DNA`fp&HRmReo4x1GGp$x&umK{0yhGb3o{#l_E9aXk_+inw>43`$>W#~J9r+nrmA zQqE_h51GL6=o$Dbb!^L$ne_ub8cf76hJBIYTvR?ohF7t^*nD~U^MbLE zNR#3wFzwkyIC}|mQ%O>?Mn!a?J*LD|zT_NZMI?`xgavyD@Jq9-i{IJiKh-_((&UYM zcW$w3V<3VrUYiveBkZgL?u=;H}vzlNDuP))5mTw zOhx+8cY*KU@_MN08F>0olZhB=dxKVLdz<-0x*IXhCM^WVQTdTTRnHecTn=RvOi3wo zhpV$L%A5Qrv$|rKF}Q57BC>r+%PCT&o=8|?&XVG zk$vg~SSv#zF9sB(YRo$SmkvdGr}nmhV@sqS11Jx53JU}Yfw`F9MizA7V~qj$I!t&` zbCK4`bSW)z9j{&UQD>vQZ{EH#|5u`C_b;D6ee=q9-(KFn{*#Y#{pt7LKYR48;k!34 zU;XXh{yU>}-~QS8^G`nc_?!3d{`}3K|Mc}AfAgzf`(E!K{_y*!j~{vd=eEwEyFR?d zK;K#3f-!Q>s%!3n14+#v{>6ij%?Q#pc=OhZAwS;v|<;i)e*kd$+Sey-b4^yYM zY{=_4lvzy9B8L**rIa&rAwCLy97S?(tC?!*^V?6J8Te^@ z$k@{zJ^gFfjPgJD{PTU4yJ@W2Ad^&I`}ORCWUXCc5U5vrZ{IO}$hW|}@-Ne$8leFC z@TVt_pBaZJ%|#RBDqo6z@9}0YfAd?0FY=3;We`0{^G7lhH8Y1{S0}hs>Bo~{?WXFY zyc`yfMwZX$<1~{TJOFvL|HEhmNdRRcxyF03e^?E#(pZa^!(xAl9w*T&=?Lx$R+O)} zOz5s_4Tc!}s*zv}uN_q(ZgND()kKqhc^P9-B6ZS>@D8Hg#gz78B*)MWQ&W4@!(} zDe1rfx?8Fi3}kEzKykZVqHxjiDjRkY7KNO6^hKh0JTAgFCI@e5xa2`Y9MT~MadAq7 zM(ns@vX+vzZEM0y=r5*MCQpp=llK_L<)jTY$TFR52RoVKIti9NvPDq5FDa*;4_M-r z9zU1eG|xD4u*y(L6|)W!oW>RcHfQ<%ICin@ohDsi2~!oFM6=z@Qe79h#9hvi!wx5X zLqDuL*?%w>fE_L(O}yZfd9a|8H|UX+oY2$MAv%+EMJro{OvX+|XH+7VeR%zhT^)x3 zLG?%vxUv-Gp~Fbk&1;$u4tj_h_5%XPwx<&2S015awHxOQ;1@|o$W&aBG7~y2=#;{T z>7Zk8k2En;PDT(pR%v!tLV0s&NxC&xf6eD< zg*3?(xVk$Z-MRDe{f`<_A;Pr}jEr7?^%@ZtZ8|B}Y8I|t)Bo;9K`~!99y`7N>u;}{ z@;w1#ysfguhU;$T71Go&UGsSszZX8|^7QFr(}4cwum6oVEGvlFP5JMe?sx8fqzLHt z=`LDb&YS|z20SG&F~mslTQ$4s+dq8>&QPR@$8KeO8h|du)QYX3Bm6UCatGD8D>eMqtS2GsuJO7(F?=wp-RNoi7Fs@Bt_`0_qo641+i8GD0Fg~_G4`uBwlE4VlT2UusHiuW zbjjILXi1a6;MN-9)pXRp*e0bA5lT02n|*LiTp0Z|^Xr{Yc|Cjl;@P9;Pkwmw>S+_S z-dunA{KdClfAhuX4?chJnWp)>@4oxw7*28!*e?pFGX zp(+NcSUCeWi@4raby%E4J*xHq_A~&_pzPExEziCsVlS)qp)BxXHQO!LQre}O?-;<9 zr-tOsoT)9ht7PdWrela*hHAso#T0jD%ls=)#K>mdBAsgS6p&CrA$4&HTXsEyDN{9` zVH`&<5iXWm&X|=ssbUr=R_=yYB*KH?&|hGOcx-3vl;w}$7$t!GaB+9Q!5{Ppg|)l1 zIr!*HH&&)gY_d;g>ILk>vpv9tOFld*&X5tAL=9#Undn}ueN#JU=tNjV4SPzv(s@=q zlmQ>e<2@z$hWm`>z)02okR`2n&mZ^J|-Ds*%%o6o*un^pv zzPGP=vn}ShdYi}qR8RD7=0$?LorzXDS3+3KrDJkdqC`#xl0e67GvCbYC}b$H8u{fZ zg-)V3j?;MMxB|Pn-(EPjLvThKCy?9Fs$(o4yBtdjY<|AzvV??-PlDHy!`Ou!ZY6K@ z0V$yFbc%n(RuZrwW6uP0M)<*%zk0+)>;tcO#5>!$zWR?>|Ni2+9g-C5hHAO(QrI!L zrLiJW!^v;|k!1P6Q#vAiN6n5LX3Mf98ljOAx zB12rZAkc~9fY0s}8K}v`<8i*CJq#BUB@q~;Vj<$ersdD1!Zql|8YHW5fU3=kYJQ0qK6wjc83kk<{J3J?jj!wIa$LeTO|d?C&N{+GXJp;B zfQTYG)$WIH2An)cv}DTW0jdG)f6rfyhzv~a(yZ6zjAAgF5dbkx5el)ON zi8Sq4?@keEf_*FNSkC{3R|Q_*zVqT%J*OKz=uax*mk~SLN>UfO=|E11C|35Qk*8ofH0JBUAt5R!LCzx32~m6e zQYRZsDxi_~yu6tol`3~}My<2O=pU0*Y9|=9On@xJk;7Kj#BNs@JJy9<1!^il-{m!0 z-zS}GC*NC2iJXM28A~@ z-YRI~nA5afPMH*T=LPHKSNx0*cA2}T2Rc1-N>z~D<~P#4JCA&38{gy##GBzkxk#8z za}owc!?<3slAa_AI3a=81v$J;eR9K(s>^89)!Vrk=@GA91eaqONi|jLL%A1-0`2?( zC%uRRSv)1x5fg>9s`MO@FNYD^dCgoa% z!*vs!!9W6=UK`|*YlPiA8jALgC2S3od*%vCA1gm_DqzT@9+BM^VaTC&fnkg z=9=2)x;ueNEJYD6@_R-c?~6HvW4Kx!*47Bph843;3QzKI=co>mQL|*o%qxzU)K#Xe zhfrRl$eq*_YG@TA+a*?a4gti0oiyPuWd)l_0ys{uAG50?6xX{tKAxZzs7xD*usD(* zePh6ul{YpY=hZGdo3yEUwyK4&M%;d9Wl<~tch>G896Xd`P)61uMo<58W6?jEe$I{$ zm4iE!L~xz1Q3`8k6LT6#h{&l(@-qE4k5&3I-@(nG@YQE!C8^YGWtkP~oJ2DFACS zN29!ufF3C()=^&t98JKJ?vT%pvnbsSo%O|(?f@!o{tL7HHQ~Wv`knh9-F$yKrm0-+b_al@->>(4kyV!o{L)Z{OT#nH3jaHZG`}Vr0gMrv<(j@ag?~UJ`im?CIkt z&oo4y1!#pTpgo-K;e*xTObV?xvMqCyr^ZHnHLA6v=x~({NmBd}r=CuBJ&0M>M%Zc- zSAbT{C;c3k(JnCm;~^B+i?7nV>=)q^j94nD6O1*k{J8kOz#k-8yhZ-#^`DC)7320u zw=KZQlmnEc0Rm7O0r!^+Dz?sR=t8Nkc&oNZ-xi!lF>Ml)m!e)7?0Tu4N73d8pZsVY z;%GJzjHAFdOT`zobMQ4U?!A8g_{GylAH03t{iAs}KIZVodm68-{`u=)e)Y|t{`~vj z{llGG=IY+DJ}0pr-2cQ1R2B|;`sC5qfAYb&JD$k%w%I;n&t$YuKKaCW|8w^MKmBAX z8b7{yYdz36_gc~C%bqTn@$>pst02CA|Mc0@J0E@Yt6%-vl!1Txr{DYT=cC6@zWVB` zk1Zzk$*0$zJ%94_>BA>Kbo1*QVYlyGfBv#D&e?&TSV@Ss%9XSh7i#e<29m=vN^8}l znqOp{!dqAOV#?e)`(>n!!(_a+W6`{osSYimi$cc?J&$5E)WxK;8mJyVooWJ%8UxI6 z5?Fl?SBHI`Iv#QsV6;RC>B?F2+IRw18?Kj}JTg*<GNU_HbnB-noJ zxtHn5$!^jDO8UVcyy>wZ9wAn*o0n2V8V1ulP3W}4IdYlpa&R#lQo-3 zszR7i6LP;quS46hDu#pgmal9)CiNLs)+SC;9p!2(rE_q#`4N%T^9xoJ!>%?pm^B>r z^}A^+0*J}@ADWg=xFRf!sMk2TMr9t(Yytnw7HS#|zY$r0RVs05)n(R~1L8wu_dblp%gor3mPHVK_j^ zGko^IKJerm^l%~`M9}LPA197ExE0sLmrMu*wx#*;{J7YSEnn8fO1yq&TU6D%m$bgr zmAct2`L3W3@obM%H!9U0tN&b2wz)S^CQm!Tk{lQ6St)YI2!xL$qvQqj=7xA~PyudQ z%4w@Q4a(Kv2f6a#9j+BF!BzGQ(A(lSx&1be{A_)L@nd(#b)PkiN&SStnbsq@+(>>y7+x z$3Lf8I^!S)&U?YPrvqH$u}`yAUD=iES4Operx0VXN2U=0c7Zvu z2Ew(RAss5rSVCI@FE7&)BNBPS@z5_(o9%M0yExE!mp}dgsJhc9&$9He>$PN6W@TpW zRo&CwvkJ+ckuca{j2*(@AWWE7@RA6ExBOqcAp!!z5QGg5qp%1h*_JI!mLYQqwG+o|4eaDA-jULR6c6V!Rvwd{*TnGE1O16J(=fG>{w%%^vxM}9z zQ=&Ka_Kx&;YXtB-MG;gH2(cW>Xh zdE>eX{^R2#zT9|rYs;g?)`oeg*i8X94NM4nM&$VT_`w$syn62bz4r+Fqo4fvrIkZJ z`}y@7&-VBC4HR$Ry}Pr2>ZN2B8QUe)X8*6>K5utgl2v8$`Loef;(7*hwie9XmW-wn z|Bms8Qs*dTou%Qdf%OuyRSvYeC`>Yx^;oU=F>*an;W3F9121Fk+! zpS*q)uktk{%3{F08E(w#IO1#$EChw()LOo>iCl;QT2&!-#`YshJg7h8Fo}n>84u+R z#aYTDq`>G*4g^M-LPf7!DlTs423oa+|8Hbau=ZRTGzS zEZ=ucQCAh8qE0p_B-Wsyn=OX$SWZDnQ*}2uV|EZ{FIbCO;qE+d7-lLH3rsm9&m~a+ zDF{e%v{&t7Ol(aZicePNOGAt}Q7>KOII$5OP>wmxkI)=zO2DlgDWijal+gb18Z?aO z0OBj}EhjR0xpO2MypDNC9tIpTp|_ti1Uan5`@%`H9zB)+>g3{8<*wYkxqESI+qTaS zA3gTO4{qBB$gaN^?Am+gxwPy1*L!dL@GMlAjc}ZvoM6Xvf@MIpXI#A4Ujs;DZJ-6O zUJ|aE$rm_r&D$)x*eGQ6FG}+%SX9CgGqgvx60mz#Isdo?$}Ze-SFZ>n2}KGjRp*W{fzWZzzyN~UY2+lIA&ML4O`QNn{&k1CJ=xC06+jqL_t(( zHLjRq^ia`_NI1H~(e+gF3 zqQ6lY(Plwf0?ubtJ{aJtP~Yst0P@3&$J}NtJrhEM7V^J%YH(Icn3s&8 zs4|X5uo(p}hBo}>7ouzBLaOwf&v#A}VZ}%7ks2vV$yiw*{-j&tb24OmmSy;etndDqDKk zwX3+)B+yo#2($PY^u8TMAm^5YcsH5W&&`I9j`*Gn)Ha;$gt>e$K$fz)e}PFv&C zRAHQ04DHF=BMbiR6n)Vme%&_@Wfs1pHI1e-R7MCtd4zpesuKEZo*!;wKt1)FH*S6N z8{fEd@2(EL8DC4e?G10S#|7JcVkGx5qrK%xfS$!9=$qFMJu&JDuk>`;qsWDQYolJD z@g9=&tfXIO(XM^0R&uKDRb8c zb%lpS4h%$7W&uuFCXy2sKS02{T9#e_Amx;a?*W>xNLI88IDlh5n#M&^c8qVtCKQgv z?jkrySJ#PM+%aiJ+aHY}qZ&FNicyYi?l<(p8){Lrc}%oIT}_jk!|50U7$&jo*}793 z_`H1n)VAHW>SPgv0BiqWK7Z_Wp08dV+`a29TDKm4@%i2jQ|-6w^1nKAGw0%)v%{Ci zFP=Zsb^P$756$)6xqa*O>tFrk({qD8hdUMS>NLlPi)-6%O+9(?&?+^Hemp#2>Hq23 z2{0aVe*O%5w{G9r+uik^V9Z^+w&wwWk3asH=pTLb)vG(ZAAkIl$IqTTd-2ldKW$27 zwUT>Yd%Fj(PM*Cu;NTV;&6}-Tzcl(HBzb-F4k_7#mPFpp9Wr9A)??MsiO4zOL^D|# zD8MN?6=cPLu4RuerAYu$0z7sL#n{w$wwmXBCoycWm(QohGWGq#*B^p)WkQ}e3_mTE zCLKJPLi9as%-0AT-98<`0j?JJMR>>9#`n+%GkS+! z)taN9P4)r@SeIf5MKY#J8+?N~E+O0y074q-8~#Gd66ofFQI`ad-PFDiYO(0LLi^7hoc_IbPfkb?gZih z!TX}|5yDls5W|5PggnXRL@jN1-#2ehFTC-1m*@U%_GXQqSM^z*fA9YLo&$UI_>sw{ z1|}^4G$Gk;5xzaeKXqda*{r~&y)FCtlQ!$kj0NrpIVMj*f{M2b?{qWVt!EO>JX4As zVcTxvO-*yUD#=s?Z<+FWkJLWtIt4l>?wpj$c9SY1wB0A_{X?zce41 z(v|?f#4}Lp2?1oTrZv#uQb0%L_|D7cU!W3g&Z6SRXqL?bHcpSz&Ke@l_-#DlU0y zUk}@i+qK#;${nK;HYsTmjonu9ff#oIwm4FHh>QRUmAD?s+!EyErJM1BkuNc6FX2Xc z`9U0d2B(UMu&_{xTt9&^m$NNiXe2BFPU{}$RqEhkeU z2Whe7yGWf*Do9f)yO0-FClXgUmLSC?=^~z1i#jEY76M0Y4| z^6=(zL$1u2+DJm5UB~Qzca9@rdXNu0v(&Zsk|$x@d1N}C5totzhJ$2%hjnybE;K85 z!P9r1I1aHm8VvwXSs+6K8+$|=p|qD2P`?9#awSEt01y$mm0$dmU`(5S2C3wMQEn-% z)XB?c@Kh3G8}h?cX$o2NZ0gi2QCS5*E1$1kwZuX%KP|(_6Xsr~*Beqh~(eTTmO;Rl`rE+Ds*a&%)$ zqU~!Q?m9WP=KtA?7tb|im$$CEH*s}m+X@iw=AkqSqdi*&U8xQi$rBEBO(bMaF{5ju z64P9@I?WGvHD1+@>@Wc*C3U`R6`nFTU;U=C(R<~~${1-mHN*N2SW->@TvB-hwqWw~ z{(O}(CJ7N{Q{1FiakY!F4Wt8}byjhwgSTQsd87&&nksgsrznUFRuhPqZj$(spcvIy zt44n-KrN>m$%i>Isf+CdJtwwbN;>RMuM8d=g=F;5F}8<^-(J0Z;nl&j<7aDc?#oxV zE*-sibol(C=G_+l4Csw-&z?Mb@bJO6zV*%RE4$Age&NP}XFRT6etYxAu0>w%26*{{ zIk}I%^4^=XSI=KO@nWAH*>A09@O{xPAQx}HefHEso~N%)U)`{C)O+`i4ttq2q4Wsc z9kOEhgRgvOPSEzCLc9W>m!UVL^KrB*P=Rzao-QHKji za5{OZH00Dj9a=(E@c^&7Txgn_H;h#%8yV-X`9n7Sv~TTT919$Yp_{~E~7prCueBjxQkK1Vqs`V13YXgTcLb% z_;flWn}vQP?`t9ULmIrOHsRe|J6#t@%t<0pG}>Rn#YJF*canE@&Q)L@Rz?+5%}0`W z4$ZJPzo|tXjX2neAOZNu{^?|Jp$@C1vJlupoepdB?s*x?01XEel z;#6$PBa_4CLQ`6zdRW|zndtR{aQwIw=SIxHA?s_5))WGi(OFECh09;2Z2VP%YE#+C zY++Q4)Dl9Rr^HJ-aT#3keWGKKcpT`kI74KV7s-{^Wvl{-bkcRcVFqp{=zFkKYS^v! zKR7BhN#Pz_56D9Jj&|l5uB${H@GLYH-C$CB+)w#6o=YL4^yySiT^uDQat(0Ga~^jo z5~_DpONhdvihs>b9t#9;B1vrkN;RttM?+= zpQjC!>z`TCf9ZnTv^6wSF`?_|N8oL3k=wDBYMdOM9)f|#9bFG};=<(c?VDfq5-97q zTXxg~guQ^$Z33)&eONQhXP)xXs(CgTwVfTS!^P@WhL};bol7f0v>oufuJtn|e96nj z=s~2nN1Znoj4BdwQx$X3)RN@>S2kUAP8TKj*<5zwbWD&?*(g*YbPE00X>zJp{RN(} zodE2-vp5+{kqFB&^Ot2bYqd&YpMrxjY~vHSfD6+@u10$5FkE3%W(+>Lt9(sdeUXe4 zEIi%!CigfEIwd7=ZS9& zoh`evo*g~c$ZqX!zdkuQd-W30^D{5_Jv5{LmG`~*>-(R6^7F5K_*FON-`E4zMtxVe z?%cWw%sYS&oG3ze_06xmch4BZjRNgB+t%AHBIamb>g%rl%NK_)oNElV83DKyHICx7D; z)FHLYbe_GktYf^SBB6?cQ`FnT0vrQ0|i zCaFoZX;LS<+k>ksU+Ky9U43wK3v^nba8b^N}9Tp+EOjW5`aCM zMsH3FN28dqhk3LPe-v>)Qio4OOO86ql^96^QYeWq6z^lU(&j(qcbekIZ#qrbLZtv1 z7G87KP-ip0Hr6);6Jk{FmJv-)BLqWf2*qYm%0Sbhk4Hj`=WP_dSO~`mIq*|D z!B~1MuyMC;U8EzOAUdQeI`Su$`(}ehmKi*w^U0^5+5-sthsVb+jt&{evL6is!x&}t zdJhl&icc-zkN+fS_+=n|(H_J0sknFh#`EVdJj`#kssV;aGw`iRIzF-6z=6%^Y+anq z$B!^XIu-=N%c~B$UUbmJT*~O5aLZD2m4f95mH5tm(%4ee*dx|tr<#?OiPW@uJ=1xZ z!qq$;vplkt+Ut-pC#OW}4)Av|P@tpFqLas=6=zdJ9BzQZ>Efx$8zF^oDR-k^gL z?BMBbpPdfwsT(EZszK6}A1uxmRE{Oz-?FyTNDi_beVDb+Y-I;5;9+_Id_aT0*do@X z%SLxoP=3hD7z5K|M_5zOXd=*=!D#R0Aj5=*F~r<#ngaEMrr|9Q0KgQG8u*-!w(K#} zDj$WBcl5EKLme}?8}mjh9ZdRYYtSawNDJSbjF2fdnj;x&x~dk1a7o6lUh~Q2C{i8DEIJz=U{EYYIp9lehIeR| zGAipbBn#UnwM35QKZ>HuFbXD28(2x5;VI~K-gox!lUk5SmgyP_REanQ*3V#G8XXKF zpA+8^hisNOigmSWvFRTYI0uzPxVzKtt?KsD7eVV#ZF_RlThh&+&lo`<1vgJyT918( zYEMRL6D>>Pq(f(edVD5bV&Ju>xmsA%#P(#rdHF(|=2!Rc-uvp;zUE0y&jIVZx9iI3 zsYfy`{Ah@rGMcj4^&a5eeWL<`@Dw7PogRC~h+dJ69nD~!y*lztn0s|z6=j2WMN*r^ z=!tQ0%WQX!CoRoo3g@me#m2L9NEahR4!@D5>I9^}6bb5=ik-qF0vxC1l{vFw(_lZg_+qEtDE$e@%>=GQ*n zy92q&W8c4D{QQ%9?|DP?ZTF#g!?f~OSTd%Yp5M8D<({_P&LH;p)bM*U@W##C?o(OV zE1!fG0H4}NM9Y5u_QlHsPYCbqzWm9@AG>+*;YT0s-@N&qZ-3|F4}a*n;MN-S;COeB z38WvJrZE;5 z80!wu`dZGRU}3UcA4rOgRn_UCD>a8kJ9lM1s}%}WQ`_cu#t03L>rbuzSCzYU7Mx*t zuKDqSQFkXggB6)rNQnApgpPrHZqo9_m~jyo^2zDiDvc%|ji1a&_xe5kU{F;Ism6^e zyXX0h;95P_j*H$3HleMqmZL6Row{7LQ(E7$}) z+$&OGo1SMBkjZePRo@fL@MKg_QqcWWPmB4OQk!i*9yK_KPn8ZntveO#IoP;lwHSH8Yantc|mGcQA{YPSp4$v z>)Y~8Xakf$3xCbi3AB`o%=qAO!`^Zh3j?6PsG%=NN15dmq;L=YC zVT5a8nM}IK)I(@cjoW2|3^l{!r5-4YFKyP8YGJAd4Am}HH5o=CASo~zMn1#K)S1Ok zn4&O8@gR{lR?7*4ACWT?)13s~E4}baA zusY)UQZ}d*H+(?Fd%fw-p0k3GH+Ygg4spFqJ8WN*)>Vaq%uFoQ}GH<^6SP zD+!^@g=276adNfI*DwWVT8DUwq;Fup7Thr9D;7dn2-?6-QWG5TnJJWvIY~b=OdYv_Y0FglCD4YiAsU&L9=8kSga2JVc7(=ILaw#1cte{k>GhQ&X_}O=9)K9@Iov7k zXS_r@cy2A17N+H^px$)*o;ix#LZRSUUl|{wU|~>*JhTAlW%73bW-kXa;1%1*Oxwi(xaa5Q}uK3Ay+>Nl7i5F~#=Q_Ivm5eeJ7X-QV4FGr?L8 z4<4Mp-g|ptH_S^T-8`5H|EpIO!l30OC#T2IUbwt{`Kmdf9nIU(!3!+gIcWd-EiZsq z+S(G22B;5h2P-?r3HUXsZ}X+*zq_d}ksGCaW;|~_*0T#1bZlLG`Q(vd zm?ts18*_4OzTfcc(gpi|dhr)uIW*U6(;f?eSl1%1tMeoP&Jv)E;1kvH zp7P>tY!h7R8D*g4hIsNDR%OyiY)WMjT!bNA^eZ1j=*~@!*bUcYC3`KfOn(L(qO-uENzM;yOCrh)!6=946ja$2Yn}Dv zMb1&=sj1O_YKc+?fu4{WkT^`v#@aV;-Sj5l)7R%O55(N$bA6a4B*CUTtnulkK9*-|sK&hv}Xr*@C_U1X-zxvH;gJd`>*=*!_^ zsHVF?EPbgkOALtQfs~vCrYQOzU01>ZOQTQkKF8;#RLN1F+&Qmar~C$~VIu!zCv@e* zq;T}-)Q`UuR>Xt6X@sUQt3bByYg zi?BTPsW1@}{wFc?Wx^OCfAO;TSr8@qAD9jI>2tLUNanvFN4KdaeI;MtrAH0&Q4NWM zl=hCLku9DUcyPmOoMfID8U(}H;7hm>VQ7|Kii3VeS<_|>?NIq^{2-@K`Uy+kQEEvm zJ2y#laaLP5%_q=cVlX;`!S9^apsi})GR7ALOej30QshMKq;gcSvq&W8CUpb=xiTpt z*2bMyk!X)-!$^E|sX(mpNfsFN5DUA2u|@K6xM={UoXSc<)k&e{#N4kQuZ!gAe*kN@ z1j2gPMee@Z6Uen5^EAM3-W;DCd+EB@FN(d>QhE^AodA7k6{g0SfiR#;Au?@ zG<*#zAV*^pBrm)u18$7;zamy|b+oTOF6F{o7HEZv( zJzbB2Fc{y7mFiPt9#M%Z0PbtH$CLPkS_~j$7ifpYAl1=P{D~>o@cbtVw(R+`j&& zo-$!~f!XurUNtav>C(OT?t7ikftv{}@_BQ7_S!2XE6Ql zNWy8HR__Z-$S%4l1E@ncLK||p(HApy3@W0>58H4Wvr1J%;wq}ME@SF3;1X+}#12~r z*h$n(tB#j@O8M}nUKCa^T6o6rfD0O>g}mx9Qm3|JRwb9(r~BVRsS$-h!Qu?Q(P$OQ zUQ$rGPsoqRnCk~!Ba|=-jD;8df?muFbxzT9@TeyII7DhL?o`{9juHQT=D;vKX<@== z$&ncQj*o~S6E-7VARl1GgThvp30vl=Y&ACuUAkWZxr^yML>;qs%B?`9g2_e-7+EFY zG$%4DMasrA3MEfNzNy~I@?yrM)hL4WMn3o3fGM=lAxwCBbg11$-*^i_gneZg0@S&J zjf7E3;3*TNBK$0Q1$0wL!K?VH`b9P3-1I{8ik2(s%wK-HWBsA2f`bFI{&tBVxMzcG z==_HJqFF6^dExp)k-C>cKP_5gCEMi)RPn7pA3%QDOG8(YA8ZCd?-7{pnh zk#Lxc%LGbnZ1xrXSdPKP<|-FXWrpYBFFeV-KqnANS`1G8O+fg=6>OAaV_Xqn{y&VS zOa%bH6MT@A-KSAk;64dKlZ;hWNT-l5PC*V4oxEINMh~f``;x;!>{m88WfoJU=bF9~ zeM1DM`s{EZ$+ZICl#-+5sBqZdC_}=erTvjA>Z5+&IrcjVR_SFKneeC6CDYPm)oCg9 z%M5&%5}QMX8i(hUILCNQ^jsImq8u}7i1X3Ja3?@S%70!lH~8@821S4j~h)~i_js#^(TFnutv0s2$2w3!55;^s1nkOY97|lpb z1DD@vO0l12(uFgbi&n}^IWbS;QTK@@DR7KUs_`J>q&?%CT|aq zKEd40z1>^a_oz}MjRAykQ-7qD{eazSS8lEH)^BUy&v8Y&6umll{^~&Q|H{tYo45A& zy|lo1;HBFaZSQhzr)42#s&km6Cj6H3H0wcP6^QiXnG6qi*{s!({d{e;Fb zVYOUR<%jKHvn)0RFw`4F<|SiAx(){q31)PfLoVb!CD;BfSGECJQR7v4l445&Zbk<1cpiJW{;3e|>jv|JLohjb<-g{KdmZFONM>$RA^O z=g^zN?!5p0{ps~J-%%J%T{aXd2v%Xr=N@C6}IYhgfH7D>VjFO_{O9~2L6zL<`x^c&a*fE zgt)4*tHD45{RZ}2F#wb%Q7_C;-Z;vrIJ#mBX6U&QUXNln#!sXXKT7z3coQ8Ina*NV z3q1OHA*m%GRHQ-L5#DT+2w>!t1TwP~v^sm1{@7fMbUC8>IP;{*Fw?Hn4GE~ z(^ivZuoPW81||7prt~?Ytu9Qc4j`7Kz@@OJbly@`LfD5ibPy^qgD;v7G5z}=rHw8o z)}$0g0U!sCk_#OA6eSflL-CFiAT%F9yOKO%NndA)V z$c%u)60yOkJU#r^xT;ZLonwi^YX#Rw#w2Cv5i4TDfW=@<;rOJ?y1HsYCaL0KkAjU< z1P;A)n{;_Ky&!)7z4ys`aB%qW@gqTD{Snl>lYnB2i4>QHhU;^0j%+|d?47ILCqES{ zH{~ID(Rdb8nND+T>%P~GGr8>h#`V{cvmLg zvvvMc!8(To45J(+`WjgB?jj)z&RJ&GSos~Sz`#TlF3xT{Ex_uHfOLIqOc8~BuzhHt?k5Y!GB6bQrR@F zywjL-n$9@>@HeX73@ao@%zu+-W57kJVu?jrq^w+KhHy?dDg+-ve~AM{04`Tlah@nC zs|8(^h0OQpZO+0Ld&NPRd={-N#Jz+zZVD#g7_E{9Eonp!e2h?fBnpLe25Y?^=R+%+ zWg<|L9-Z3coYu1Eq}RYn#3(`=>kZQS)xjw^+y?LCrQm}u0jfJAtna#%aW#TvoH;PP z85~QG3#IzS!Jr+@Q9{ZgTLP?8K$U<`LZ>Jz6Vi1lXF*4@9DSmIgF~lOh8uNJQR*O$ zEK$x0p?&LDG_SbAg;JMvP0a~VVA@tpoJp)aYv`sl{}LQS*ZJZL!NuK(xw)FG*ifRL zrb3IoM600m)y5RSMhc0H7{5bBkr?3^OP)$N00cik%!kEV2)u3bH$Nd$vX;Kv0V}&o zow;(LX~!62gq`L@Z*QkfTCS{~j9ao-&Ms})@w-W;i%pMh6q_#iTGL^p6{T(2Zmw6C z%?6{}J6G@Ay?y=qzK(}Kt2kfvUa_~=c6Pm4hMSTWUtVC=0&fep>t|MzG#Xa^936W} z{PxY;cdl(;IXOPCKFFS>SFgEQd=m+@ak1(EuCDa*p=NW&PGcWj(gBcF8OAt_QgnHD zo+0bd&?p>fBm2xJ6+27UD|uZ`KGj8i;HZ7nu-q&>icDRt_3-mQXE;QWRE*izps zXQyv0{omjBin^p2!r0^BfK zc>VPF;OON1j89rxe0Y3fjX*C3yK&pZqsIQkjV-68LMGFpj#Y!H3 za9%|#pU%~>G+d37$~I;jlqiH~W+=L@j&IYqQ?bI za}6gGA}6)`fV5U_?3YIbA*{$fE;7Ar!vLNCo~Onu9^3H{Ys%IFHaZ&S3iXkR=o({A zU98DLJ26uQv^jSmB$!1<7#p^|K15Q{3`U3geMZc>es5VfE*++MhZ7;xbA5OuCgge7(3B4OhqkkHVfb8U64pwg)gsB$r_z?4kWIYI6a z!0@s`JC}jFj<-xQZGUW_Rcl?BcKC~zoT+*6MhfE+J!%_WIxvT^@S~x08k}xK*@HeN zv6N6*C#PFq45U*<+|`8e+!?*83UwvqMhcC7rVAM)W?b{Mj&a4*kp;W2g7$0}#X#AB zP{U8}1GdHQ;mO70S3N!E?%-v&_MX0Y@r%#C_{HbH5azk>SMI;JcYW7bmJp;j@ifB* z{r)tKA6{+($kr0?-M_DrPa{oM;lB}#U4sNV$$|7lU&~IzzcR2Upek{sfJfNO_{dP?~EgAq%aO`5$y@ zKeFRrvcYG*s#0u)al%ZrCtFA&Yvm<2;MWHzLW zjM)XHWv7&|L|Ps_BF1RKI>ra4@Qek@3H`_vM0_P<>8!9y4en5U$&NDgzql5B;hllk zXl5yJ0hERFsbJwubE_^hHU%9WpIUtQ#Eye6UOs#I(iEfhCmIZq$6JlOknsMkTleqW z+_xWPbId$-Bgx`%P$t08NstlVWt>)(Cf?^NOF?l{BDg6t_;TRXE0vX4jFP5d3RGm( zQS%y&b>u-UqjD-fXQ; zivfD`uZC6^5FzV5JrKIRZCpWCgL~DuprqsP)XMMe?YB1kgz)s0U0cpQ;&pXr&wYZ! zmllCo|8Zew&stF_6Zh~el=lK)d|@Y%X`^YLb%3{X%RKPrWwqdm1?^jX)T<|{!a zCXE`V+$hFnt-ut`UWMlY85NJ}WiwK!`_A3spidXoqMc^iQ zKRtT-x!qjcV7Icr-CqwLefsRPHz)7!?rfbLUNV>O(F+RpO92Ra+_>W$9cGm|J(^O-+|}G;eo8H8a!|Fg&W6}GAKw!00R>PX@Y&K#ZD_d7!xQ& z%E`}lHm9X&Fmrx|wI`Q?%HTWkQ7q z%%|YsM@iKVU8PakM&c^Q=rVT}kJ`xAj8sh?mrwunTP;sa|HC`s;YCbBVK$E%zH6e> zbp$J?i&wCt0Ct+uU@U~Gqd6Q!&1kLbNTindKzQ`ib7)dBnv$~|r=A%J%dW1EXk^@= z1uGrGB~@e!MH4n$$ke5sf;#yeVK&5Euywzy{>ir zp^=x&y&jlm*58eVi&yeX3-3+HIME1oGde0A(>OICUCQa#k(zrzoDC?`_3H~;-6iU# zTdJusTw2!MKRP*jaeVUgPrrEl{AKI*4E&D{DL`y6fkYQ@_RRk!uk^MG@JNhk&^9Ak z{Sc`MP-fqbI%I9EQCC#YeA54!9IS#|dTG@wD6k)vh*p%491FUH$N& zR}lzDx^)FAb{j(UJq|iXp8A&*BMDQ%40I8cRvjMR6v6mtV)&``ao3%7hr?0G($R=Y zuIvjm-DPY3=CY1RH&kF9LW3)QhTGw+XcwykED17yxbSu!-b=nZ zfY>X7V=so2Qi=Rig0tWwwFD@)L7s#}H4ChCkX^{kGzHS&#==;cWIicL4q^GYAQC^J z8IQs4S&QUz#wqI+)M>)N<7dwwKebPahh;s6>t4>=d$(_X`(r|TuI-SmCM-?k2O}f0{Ej7flBe=QeQ_%r zj3YTAI9=qqlG&unoD(S=&dTxK8PIGp61vkM8~{dtQ!XUwVg6Ee0D;LJ{e-$lDgfjW z^9@n#3^Txk&#TmVFtR3IN$^*_A9_yHR){KX{6*BjGk-#m9n#{!xP($tzPx~{Dlw)s zg_V!IeRJ^S!TI*q>!TMJPBg#IUvFJHes!cdwaWkL!OI7qp4f=<&aLYQhX-%ZPqnue z3sLaiwH?DZ7umU z8r-;X!`Q&x`Q)juap~sGTQ=-}`1rB?sVwi&){{}~(1|x@K*bmUNzVCpd zUb3=XmLRDE`KsUQ_md3bT5M2^PJ#f2X}00Mxd-oFhbb}-e9&@JKGB*nK#oR>7_$5e z$2xC%f(s)@RexhH-;FIb1yy!aP|bt-w0gcaZPm!}>?OkBPdNc^bm1p%YQOO5z(`!Jit8lh3s1v^pP@}%LvSsZ zAHtlZ#K4mh(kfXEP119Z>F*_^S{CRA2hT{pY?8(bgo69>3e$G{L^6(t%?PX-yRIvf zJ|=VV1b4lXQEz?!iF{g!Mua8HAB!%1Z)vrq3~>OB4_ar_#OBCCg1YTY-FS)#b>nM) z$iz(V$TGhsh}bEv%D~(`ZL6YKqCYp1+)FYx_g4&SR5aMQ1boxdYfAPif$(i|t?HzB0^i;EXxGih^w7E8pc=r71 z%a;dDF2G!E@b&iK%9ba>Y#nFYh-XjVKl|t;opHb_;KoXp!KKCN_a|{mZN{uyouu_w zRqZ>4Ou4FF*D>leU1rfaPrRU+)U52pv22)yLl$iRk`Utaaz?9z8dQKB)n6V|i8MbB z9g1#UjoR`=6)}hna&TcR0mYj7MKe_VLT;Xr3lE9d2>mE3g{`aDbO4tUCDm5d zh2B97Q@p~cM*A5WvGz0e7hej3|HY8s zy!{1g}z2^s4dqyWEkmW2Xb z4SAA5GF5b3pxLjpx0jwC9DVQOpZ)G1{hL4e?vEZne{SAEsjSV8?;5$Ny?XQH_~4T# zFTVG)Pygw^{wR1bztM2?VVW5Lr#W^N{@4;eQMQbvbUr4 z5-g+prKwK8g)d;^RO6``YCK1&c^mM>Cr_Wf`s{NPFPaK{HOs;8-MMpZf7k0Eb@-4p zfq%TW=Qco`rzJHH-X8xn!CP=O&wuu6M6-SY^Yi@COLllxzVY_G(0tYX|ozM$`9Z2*6r4+4Is&>WvJ8L={;B_6AG1S-smGLRX8==PaP#AIY zY?$)KNThI0P45A?a*0JApI$h<)Y=VCcN}@2j&(to-2uC?XWyS@^v~aT7W~cYt7oTY z`p^3KdTm~@$-ZvMGqBe+>jv0W3UP1VWR<5+pY2(kfBS~Hf46;>lsRtw@3wO0Ow8=P z5VDVsUzzO3uwgF%LqNTX+c)pnDd^$D2NxbazH|5PtvmOw-`JzMFCIPm>CZlKgJ^GW z|Ng!E`?qcxE1W)f(u;>*+hNK8;FYOoFBEdaDRDa}(s<#5Yg2%AfYe-^ zNjscYXvvn_5Q(BmM|bHmA_Ft38WAqEgrjO( zrevcbbcm3Z-VFfasB)+wzNDutZp%2zlOx5Z$t)K1NV&8W`7Tpq3TUKRPN@?lsYiZg zsvca>SJoe!4Ah3bh2z~A54I45QoT&Nb)aAI++)@4&;?ugDIlUA!Eyb}=iqm(4io*E zbgZ3eM`IgPMhlNDsYv}JqGGsSVk0vLUI4_6hk#{JWxZ;Diynm)_44TH`lD>#zHWM( zF6+o=t+2I7DgsUNr0)3D-ET-b9er`QcE)oC2HTrq031dD#DF0D{u=|d`eIR(ns#%H zICmg|rmc#Q0+aP=EXBxkulL6Obz4T=eEjU@!QsC7xz}&J5$x&nr(mz&xb6<186ZUbPOd+oSk^nxUwDs4sn!N7I&UTvhP4#Q@l3s%GGTTjp4gV z63rS$#x&I)!{sWoBP+9lVnugQ&6+Fg0jnG;o;=ywT@e4U&ou0l?D-8sy+Ng^`Pva`BrJ zv^Yvic%@WvrhWt;%-GAX>u`Wzxj@Fb-U$toqO>j8R4i?P9XA|OBaI|=H+=@RC$G% z0%}DnRLNm~xl89Ib+v@mdq_%5sQGh(c~Iyj1V+cx(sNWtR2iyjIs(i22Un$%FMkV9 zz(ahpsSM3`3XZeMAJ8%Olu6bxV@2s?2=Tv3j{LAz$y74<`|4AuU#IA#&T+J{=b{z1 zX)~tEjEPzeBzeQ@C=`xO;W+^uk;Z5zgO$}1 z>s?;ib3UnQPhGUq<;lVE@BHDP{9nKOFP=&OEw(ma4@qTbT`Ulwri48W z+*7~EPaGTPT&#>-DLxkX4Wy=P45~tyO08eI+9r}$uN=O5b@=n2UlXD~`tbd`cN2y* zNVyMCVNnkjOjZ1QFTdGCiQ+C$Ui-&BeGF}Z2Hu#pPW zV+tad^7zRRZJ$x1{=}GZwA$AMmK?0YUHYkMt=AK;ki=?KM*iYlkbzC!!<&;Qk3ZY2j{7?Je@R&#N~EynTD;)@>t%Cr=;Wx^>4Yvm-nDF)=%~Z%jS$-JNT0 zaB0mg*}rr5-izli_2E(3y7Ja$K3k_JR;X#p-@E_5;lU3-{_&@uf8qJSuYcp4ckkW* zwaZ%vN5`LkC6a|dYt7*T7nh-=#aYdru1+6KrgmxK8^3vXnr>x<{UFh z<3ph+c8x`V*DK1;T&so4RmNbDFN2{S9aq(uj5oH+uaFo_5n{B-%I5X<^C4gU`{DCp z=MC~;ku)r+TDj*t+6$xi66~?Y&5N34R;2dvi z(?=A8jsrzggr020v-(mUa{@69}$dj3-qY)^LX~rGePg ziaAfb(kjeItP~SnI{f;J(7;H@WGg3YvO--_*?|Q=MREu-6nHRn^33OC@BI)<+VP`sw%jy$EZ$wrKLfA>EUBwS*kBP%~&1u{%MlpLq?Qk9A#d03u|g5T|irQS{b!0 z7T)Q+Oa%;#P>hYbW2Fs!xv?5Y(#EtTsR^RO0h{u9B#1zKGGCY+l2Z_&mjDBuFaa#3 zX0XnppMX=U=s6hDe|!OOI>nSMD|o3J#F%t1_UQEd&wl#z|Ly<&?SJ{F-#s|HaOujn z2M!5<#Hty}bTZW_N=cl6qMmo#>ybV@Jo&BP|0Az0|F8b@zx`K#?OQj!imKKjH4M!u z)ECl@jg(F@aSX-QFx5|_QXn`J?QaiG&;I1kfBf73{Er?Up4#lPiS%@ZBloszyV-KM zN#d{%vd+R>R9dK_mfA?ww+5o#|1{TyVo4m=J98gDtwM5N*nV}lB zg3cD0GO=BpZ=5jd8Fl6ADLuOkTwE9DCS^jx804KyQ71$NrV{1RbAgjD&n5B3;F^%D zDwm=J)Rf_>Pl@p_E0U$n&oX}BYRnvC!$5&d-N*r9?}J~=F63AmUefX8O%wo8>#acT z12i!(2trstuB5zx7C9dvBS9lJihOf={6>{9HoJPY{rfI@Lbn&}ctx}}w|RR{VPEqE za74HYGxG&iXm?My|9Jezl)U+ai`ME|#P#7Z^E^4AK6@Cnm&PI!)JvlC^&CJTgn^Zrh&j8tQd@OGg~$S8hFk z?X>2TA16`szxCI06#P=rmMvIne^n`pi)7Gnwq48gg8ZoatAAGAK)_%81vwlpK3%|C zPq^5hFhTfkW?3W?9a2i4O*yBn!Nz>E5qM?^f+WS1PRQN~CjpNHn;vz-3`mGZCCTd? zg7l2vT4~}d@{ZZzGu(g_?q=^$Vf>Yv5<@5-cQMJ2oTdytm-^mK`VMh95rkYcjO%c1 zOy_jif~N8VWs*oDmT*(tqL)TC+M|kP8(Gn=Pmjg0$|+?shL?1oIg?mRQ2!p3&bvDR zt=XJGS9KZ$s0j^Ysqwu!@{P|;z;u}G%G{8o#25N0KYKoKh*ml z3%s?A5E3)>2wF{ee0cctgBLqH*Y4cD-3C5yu~bX3b!qpFM}c;(?h8D2g1Ne#7Tspb zaLZqF3C|$WhWQNY!}&5HrRS+tz&fCnaG+}NoNeSdU#S^0YnI6)_}bKaC__6k1+G8} zRFTGhU4n+cO9kgAw@eQb7WUq)$F*R|Z5)G0i5W@`(o*7cFk&V+QjFV`2C>2$BCfhl zBEJh?T;o%@m9yg82%T3|Jp7cyN8@@$rZ43PS1?!xobQzosI%0OAI$YoNHW(D1F0g- zu_-iVN{E+4%7f~J1wSm#MG$j6C+wQsgDE+pP9VB$sEgD}3LqxYT=ZQ|xoL$0_|df3}qAag9uyBt@`J(zd`_Gkit5-(3*|H%(O{$KymZ~f8te|#dyZSSO5 zb{Tv*afDDsW-OM8;?V&mhD1Q-dv)RbkACo@!^6X~zxSX2$G`EHZth-Nee+-{J?a`M zmJo?In5TZn$qf@fcCP7{^DwHI4holVcVzR;*(>X_6#!jTy|T^Ehau!1oqTR++BSp8})C!2OT0*jd<_Q*iPfD-S4BjqA+GxWo{uISRB)EGUdt*_aqT z7dPasP8}Fs>ala=V|3gjZYqY{BCE*oA2xlGB}<`VWT`1gI=NM|M)r+sq*TN5k^-WY zLzR?*FKSq=5jtvzJ|~Y_oQ#|sVdKFOTWfnE;XZ^BhX0MB0K{Mr7kM|G{yE7ID34sD zDK<396ooL!e|0;n?p=l{-h#zX@e5ax&Q8`m30L(3DK$E^HzD^K)`rrQZ$LF&)G}(F z$P}ObI#TtTpfo0$=Bt~KQOr!!_gk&QU`8VJWlxhsf$l7gT>-<3LrYBJJ44mVHV&AG zG};-3#9Xd0=FB>En~O??uKML6$4tf|SUaU*Nn&CA%aIJwcyH3bp}StcyRhp}%cuG5 z0V~UGCNx~=t{v?pa0*-43&1@9V9+cs%+_7zV{$2&Gx1^yOH-@`Afp=939i2J)9|Ew zC2TZz<||jpQGp*)k>l3QecMoY4EWhW`*ZDF+197`oR`tM8`VW0w-(P%U%q&eCeF{h z$@f}^K3ju?#I3#K<3l@4nBJt1?0`>doJl0#CJDy=Mb34M(wDtQmei<3VS93dK)M&ZUUNp-aB0;|C zyu?<#MOV4ea9YT`GO8bFrY1nR;RAkgfF?}jgggX^Q(}Uk5Z551Q^`ItvGm7s^)qW| zV*^-RB@+QQE#MSQFlpf%O>~vW%c=`w6gFoOpW&Z`snIc0B!u5-9LITrM|sLcAx7(c zR43#N&ti7E*WmSvdX(@eKdiISTZ^Sc@=zuaz2VM7Et z-~Z(EzyDkRkG|1g|Mg$HWpnhbrcewodxbw-Nj#qIa_l}Dj7F@dO&wNqnGc>H{?5Pp zH-GrOAD-EDx#(J6C_>cZTJ>ycK!HhJ8XoSU%?w{2?@p`Gc~?S3I(CZ4^^*0~EZ4SP zLuhT)=j^?yW7I66u%TUP)#sPKm`9?2d_paAEVmx5^%afbNF(hxI^^z-F)D|jeLlP^ z+oO^DyVt!?NXc38OR=yc88e%JI3}o6@ijLI zxXXMpyN9~3ff*ZyZ4oP?rXk5wY>l7C$^uzds=V*^QQ3vvE89%Qjwrr8 zI^w9F&#=;8_Z|BB7EpUi#4<9xc^|2lO1t@DsgKR~F4(W>V0YJBfcIa#e4$&fneGYQ zSI2J;FIvq@)R(6GZ2!N#YuEw60AN6$zb-%yON?w9%hzR*K_}(2&pvBkdG{Z7cI@+` zlvf6Q^4S-+Z{Ob8y|!lo+u@P^z!*(uCH*V!#E>E_dr-Bt4A|WsS#|j+piT*uY_(yk zopB6D>j;vxdRez55lO-3nWe}mnR>I&#K|!zu+cRw``A8h`TyVnpI6Xlo}7)=|JW^S zLggou&edEFm6a(2qg7xw4Yc-%T=C;Cii<+VE_We_(Oe=+$8sHUm%6UW;Z~k*6TEQ~ zQU^@7FC96BUf71Ue0xr-xc29*i;|dmc@<>-HRrip1;R+B<}+qc4zFO*RsX}NK43hK zrzUhHz5% zbERzlN75>kL&^Mk5YFS=>A$F~7dh1}s%_4IxeyJh!RcBl*y~j!iuNee4XOmkem)|n z(2Ns|4r!pFYD8VZY1|~JbKKAuKM_JZaOu)1B{`pQkoo^)8phM1E7R=SgsM+#hdRsuWaI~kaI2)v{;m?M z$yQ8=!`wi<)!3-X8h{qr$xCG$_9`>>Lu6-n`+e*D_fIZ7-16w#wab_HuJ79?4#>6L z6vj`qR^kG+;V{F&3~fGk(S|{GB)EP1j{B8k`&b)(`mP~MyR}}gPERo8!Kd0WJPVx5 z@dqu28t7qg1>=PB?{D}XIj2@+!rX_JW=sYyl_Qcrq0LbwI3r;m84mxsZ@8*7(GT+& z*Vyh#-7~O;SY=oJRufz}uMs15bvSPtJShTX%L9z@w)sc5G`(1{pk9??(Dd}mfa>x! z(Y$x^#V906Jsu>*^2sB*wK>4+H?ORw6w!^Y>Lt8xF(oz6OpjBSd8nh;^_VoeVlNKk zH1Bfv_Q(CbYuCN!-P9jexp3UZbPOkaH7cDuXvNn`U72IBsdELpG&>r=EEJM7@u@2< z;2+i{$rOW-7nEKNj`)y^(2&n%vw~zt1{|c$6{>!PI+i&1fS4#s$%8W$M>FWLX9%!Z zKPfu7a9@#@5CRNLS}>ymY=e{$7uh&uRep&+5sK+KSvbm)5Yy{S;o&V6`3#lHE|x}~ z$)DVHIVd{~QN%l@Ws6y5H<)Vo-XR?2Y;Xk8%uR4{i0atDWfx)E;C6+rAu^JI4`{yP zbQ1$emDT$ib2iQ@nX0gCa6v8Wy@hFi$8gyB8mH3QW74fB$En{U5*m zJAD4HeCumAAXC?t!A4J^p2N?02h6EHfgs7M{&D-(9oX~7FOUA<-+uRZ|J5JAe08e( z%FC5P60}&0yCADfzG7YBx@lpKJKvG%(}~~t0*_jmR4E4|j2`)M0SRSOPhy#uzrO$8 z{cn8q)%Wk;m+^})9_a9W<^Fxsy&4M(67(&n;e=7~v0subJCE8h+6Eq}sGSj{ z1X7mvLSg$nDyu&?5sm^16)e~UCUps1-Eu8FlG6$FmD^}1g2|gIM?$p56cWii{H(7{ z$udh$8MQ^rBAdiXOOlBetF*LcG}B*Ya&-)ae0p|r`h0Kix@sxw?6uiG!@^ewFAqSmw*5fa#slR~pFh!UWn6ukTeohWK6?E5 z=bsx7?CtFthXZzl;^dX-zl+<~wvqNs3;b8F*h19Dh$XvwjTr&0e%HXwbA<5Jv;U}_v^7XNA4f8CT2EFAqXx&rHy z9=mpA77Wxkto6>&=<`?SbxcZ#rQCin09!s3(L6)J98+t_Hag!rW1!76>#*htbWz8Q zt(s6U%@LE8fO(uL0^zJ`g(HAQ*i-Mij)xcpHuMrldCC5!T9wNwE9ieBz!v?3<~+G| zl%iU1C}q^6m_on@ZK%_c?D7?!n*d5+CGMv4CW9WjnzPYjn5*aT(1+Xw(3-#qR)_mj z2t)&+uA+j4>78V|hD-v=xPd~hstKJ`4JF5It!8pYdoi$B33W_|)t9^g=PP}3H-uQr z{gn_K2;dedrQl%#W)hj+&Y~d4Khmy~u2hxAA-eB;+AS3#0<3e5)bxw=YI#dDyKap& z`LEY+pl#PTV}PE2ZR>QeVrjkxL8FZNdqqbKQEet7K8`q_K7XD53QZs=x9ns|%ycGao$G?eFQQv)^{N!HZi1I6=a&#XstqvX-^!)GrKKl#OH4<0|hw!3%x z^i+GKYTDx;*F>Q<)z(lRL2bshof|U;5g|e9+Eq6V_K9jT3(ZkA_A?P3XZYd4!vlJ; zmLy7rCkcrb$aIGr^Mt?FV)UN-xE?Q(pzWtLihz~oCc?@@q^D`~b-M?$ks8;tM`Kvn zMopp3h$}fCSq05<8P*)y4y_7VwM;XtnpYJIG;CG`-@AET{F*tg_?hCv+FLud9sb@Q z{@#NpFYK@9wVPI3)`hpM*htKMljEbm`3p!pXlpZh#e_*>VRe5U`Uox#F){K3JL=0Zn%SF^VcNQK!9S-7%6=|Ou- zadRGb)tzBWA95iF`Jbb3sCH-`vGKH4-`SHoW`N7Bk_;-K*(OVEoAN5Y%Uz&iX>L=Y7`~qk87)DajSS-!@jC+j5Q*@fn4wAWa!H#2F0M*tDd-w0$ z-g~8=*W_TIA>5O@T5gcJMel>JS+QJH94|0v)JK{jKl(^5bQ6*4O_ts0))H0gFB?oq zzM$)7TSn_To_Wga;J)bzH|4^?Uog3fVM0|Wjzj-V8czEEc1*k#wKal5eGD4 zR;hT^U6dxJZ5~taAB!Sc9X|cg76Wt}p-n|I-IQbEQ6r`@@6gmnM0nJYKp2P+Y-kcp zQeurdT_mjY++@(YniLmOt(S(~ECTb(01D zd4SOD6dzEcXI|iVbL!#YGi^bJJ3rsq?a{y#ap|(Bd!>-WeHc7?4}elIm%%r~6J_k=I0|HXlK0UzrN*d^Q+R<V)5LN*XGGe}u1m<~+N zXQL9pfObBzylefkRmY(BV|Y8y2xI; zJgf{$Q#`@3YbY-OU-jUPQL(4^duzJYHFe>14Xx{U!UcrOGcWswWrkNHTs*~eh~j(v zklV$v^s*z8VQNL;i{&nh%G^a)QtyAohddxf`Ob9f$exBL)*3pF)bH0XW^Ul@@bJiO zz8BW^Rj+y%K}!X9EE9F(Sl}=>(JR9wHBcUB(6o5zR{DXh=;MsFwv8^^# zUzM|PZaT;){>@kJ-1_hT+yBN+*K>2g+Xb#cqc;g7!e!(+VJQX@~CTkLfJ9F2AiE{Tv! z;Y=Z^5%1mwyT3Yr^YZBR!^3AkdHD2S{rM07&)@lHzw+S+f9tRPhkxg9{YSs@wGR!* zY|J3)^h{&UGcF*8ATzNtULw;4dkQ1~W|08~R~1hJ)h0C*beB_+C#A;baGEH< zn&1h%5`!<4Baf5Lr_B0^1F(JjS+uI4A?{0N6JUB3Q=6$UAyiMWWe>JGTCpOvw5ol= z4T=p&*yV|hLw8*TJXn@*RK;>ZlqW*S${&Su9L2vcSf^0PR?vvCa8`QJ=r*{skhn#i z{>#8Kz=c4`x=*p_;0BT@c@kWV!3uz#&Z=iQWEw|acrwZ=K6sP`41n(AKW4jj0%~y+ zkr6k(hQlDsOE%|aFK2O%);bbyp-4Ni=*vX~u!fTIA@@J4* z`gy>X3gVy$y2cU1Mi?n?xRp+#jcq38Wu3YUo^LkE;Qpd#eNPXbzrYy9esKTZzxT~= zh}F0;8LJn+b=mFc_%z`()hTo_%47wcDpAc4I_UN=f|&~8u-<0g+Pxi}J589`Cw#J> zDPZy4hLZUR++&x_$-pc2NQ!3S^hHMBoBGRQ=HSK+5@>8o;MAXIZVFU1%*#isuH!p6 zkoqnNj19x@e!K4T#fFuASE_Sht-Noj~cjnFwb0_$DetibynLGEM)8Wni_S=2>^x55QuWzig z%_#l22Gea*ll?uh1V#Y_^z2+RHZsD{w6(U$Jpxt(Q-GUq-CB+QvVW+DjRFBRG3I-w z5&GcZ@WQbY+=crcfMt8-@WAcOfVEX)~C1O$;}U|4)9 z{-^C(`s8^!Vyx>S9j$A9UcZQOqQff@!gNS{duqJK6OV{#TA5rXCmQ?^&xEH*B| z5dmMJPyF{0+zU<7&&58R1#;+=$BJ_YC_Ju zf-4>~m~E`0TDZnfc26qN9+eK61pa*(z-;YBD$9?w>q9LHp%qa;hZ_+WM98%=zG9}{`z*>1ej+LxMU46Db-GjKHXZLjfc?Pmgpn*d$l3`PUHWD?HT>#jHJ$Q5ewr^fc6nH^Tq6 z5B6Taev={1^DqAR)1Umrul@2fi)<#%iZEXXgH9O!LZVCxM}y)tasZ^KMdD;Ih92O%Bm550X$8XxRFvLRhAYJIh6qcq)aOE3G#v)7;W8sCL3Wzetp$ay7 z(phn+0DlVek^(`Z6@fNcuu)BRNKXznz$p?WSAs3%m*2@y+g!+%r;5)np-ECzj1jh; z$*WRaUx0q4!B!K&MdZ3fX%}sv1W>?Q7aB;FYl#FRYIe8*bCs^(tO$aFp-^d2KJcuT zx`j$nDW2Oa>(71fgMvH8YMgQYtUoH;%Az*z!n>Z)S}rA%bEqKaS6NDD&z#9_~NYxlzS8_z!fgV*1E zkF^U-c2da-atTwh1py-e$|OQcfPg8W{83giA`W#Mfl!po5Mm=aI*(m)As)YV+kEtx zEbq(wio3q2ws*DAAt>Ae4qxUiKgRL7Ph^(p@>LX}27y;-;oBh5C?b^b$S)3&_bZQ$ zpE!Q}@rTaecj^=pZnZmXKIuG`@DMbZX|s+|Fd=WnA{jy;7Z1b^7%tcW?C#@fiT)T` z$g-L;7GWs6M{RIYiKR`907^SUW9he)XUL9$NvcD765&I&RyL+gRH3FO0?K(J%hoTy zkzL71^&(aB6h~l!Q~AVZNXCZ=Qv8C+xxW=S35f2=0Yve0cK({Ac3BQ;K#c8zdgNL* zcB3-YCOn#Ci!xswDP8j_O;6ZrHR(^lg`URx$FXYGA5eE|YnzFD9?3-tr3%Y- zP%VQrzKQ*rN<-p=D3AgsjAC(dX=}T&y~U%zyWry0a;;{EwLxSNrZX;>7#;WBpguRu zM42OJ?p_e#uylYm54?_mk;C%FI`5;!zM!3UgoSfA3k$|LjTfPeRNa*#X8RmEQ-G@# z7`LbNOI>k*z;tC>E-Z##NT_ltp~U&Yop4G8#U_%}K&$4Gr4CEIxM=+e*XY=93_9PC z&1l4di5T)>fUh|T4KO*)iOp2|ga@rlDlJBe%xHWW9Pjb65#H>8Mx7BYjs-Aap+Fpc z09#)00Koteine37Corf&Y!DeU6`72N8tO${T$o@0tjrK=Ui=Fi@C6u?6+D;b$2z&N z8cDKOnKi$HLPe-5wE~Y~7AQ3kujQQ_(+7B7gcI0-X@asSc_0pXp;_S7r9@d7YN3mN z{*VJyBUw%W5J7~-w7QB~>ywjKr^W+OT!=XO0KNLLI!tNGp^(H9z(31 zV^qHzAM-O4fPhE?V=PM)obEz#jfk_uZJYMKUm++^##Kh49q;SEB<`e&@QAxLSmQ%b& zz1e!6F5 zfCv^X&vtGNy@K5xyDW6zN0_!eYQ-%fUFDB9D9Yd$Urqz0Ia3ZWfR}65L@J9g2WDyp zRAv@&jJVHaQOAZ_;;Paz0pMVwGCqHcT+%><1ma?yX)Z32a@E-Yi?!7L$^ zDD1=vMMxDWlPyJId$u~}8BjS144{M)Ws&pnA>$5|n%(^uUwh;AOP3E>2I-!{mU;hA z-n~WMrGI0O&#AG|V>8q9Gt=B`Wd3qvYwO>n3Hf%JJ8Ex2$HglX)c?o^Br3sj{bW~me~2WK=gpx7($qgx_?;eN^B=*Z~I z^z^a$`RU2Yot@TOZ@qP5Vd4JMr^ne`62~P|(x=<5XeLoBDrOP+(ZxWr(Wqu+codEt zoTMlW@Su$EOl#PNAn?BJl;92p*SE5U;vCLx*WYiu{6 zXnb-4YU$G%Dq!y1Hs^cs_!5us-M)QmV{>zSe3CW5^saPX;}a8(64#b_(sy=l8VL3`gxcH_G25a=V7xVFtUxv54g;HWn*MhzLpPr&OTk0V%!(4B#rX$X)y4 zR7^Tgp-`00SL6QW;?CV*0(9QbySBC$d~`YY!a7fz8ucnC&OlCHSY zzQKoNNHSMIp+*8dH1ui!SNsiaFPYjOSp<6y5^z0G@4_;^ zZHwxRa(u8G0(QK)X5$CT*StvPUGZ!TH!{}Gj?~w#UAuhkJ)RIcc5H64GfCgf!GOKY zvjdG@oTaN1qhkzY5E{9R3b5xV6P6HdT9KvAVSX2Ra%>} zR@=oSKoT2JPYFUaXXKEHroI8^OPIsJGx69HT}_70RF)g6 zscO71I5583>iqEaH*eov{?p(8oqzqYj{-+0i)7+Gt(fMhG@^~57F-18O8=E%X0N|VEr2gdQ=qO@>uE;#v4c?@L60t-S7}>3r z)$jl0WnQXP8yN*5>1seS0tc+z`rqH39vOb}{DV(F_UPG@i-aDmCLqMDH(R%sS6;dB z_A75+zE|HO3U*SZ>OX#FtZdlN8?;`%@Xk{YpP!i+W!rJ$r{GYhX@0T;LOqagx7uf; zllAS^YgevuYvRK7TWzx0H`6+DS%rv@su`1tjX1qP6PB-7u;9#5+NG8ns;pm~P&Ca@ z2+EwJNZd1TSK?!FoJ|6^ofZv}i4-OUowWfA-7{J+w#LP20tC@4uVGUlypUZVf<*V& zEzl{U?4`v<3Mx*e$zOR63v@#6^Vr+FS+8%^>l7nE`@q?=b8~ZJVrmZpW@W*SKPB=!2Og$u2#wcSp7l-M17fo z%+Q_A(gLhs$#c0|+pH|`^aLfa1B`5q6#(9$H^rU%9t(^7N_s`Gt3`Tv=YZHZU~Ecz{WK zI(L*kIXSbwxv^E>T3A|o@cb;>3T^Oqpk{M(b8~KPo+r!MVHRSte}xl|hdT|}Vaeaj z>@?fZI^JfxTL-wg4TXHs2~;wXhY_i(ggw}x#_gz><;mo0QRKUnAgwP#M4EA6*u9cP z&wj;bF|9Z-#BWFFtQ*1u9e0^@*wm*IwJVh|oI*wAR4^KK<;;L93uXo(F?SP==X+A{yrhZ~zHmTnGrHRRJi4 zpgaarh7;ZabWkl3^uMVkOP*4*g2qWPIGUr52tFb!uteOH7Y&J2vYPdCS$u{WVJwCl zwFo+36cztzYD!YtMIe+aS=h>*jI;xRMTDGKodbzjLucRw2W3s?TY`imft@mpvj3CtX_hs{z>$UgfRWIC-A-LKf+8^BQU?yx8#0pA? zzyV0EAB@Za+W}9<(?D|AMuK}CnrGlel+-+o@8TNR4 zjEd$@qV{+l33KJ)TRTIpI;9MO0L7lC(93c|?pM+nB4otxf|Mf4NEnfF1#y}R!c!5d zt%#gGkFkI5l~N3GQN|%rq^xG)4runI4x5Ao3% zBP+iOg%XM^lyPY-3jdtk9r>o1Bw1GtK^6>1b(JQ_Tm-fk5q$wDNDm7rxG)~&6}3_) z`T;ies?r6t`T&*4NYJvv$WaX)j}nF-}&y#uU)^p+67%%`9mROqkTiWg7bh6{-DQM6^Dj4C7VWs;lL1DpkG&^rcr&9e=b~K6L+_*G(m5(@c*GFxc9~$|`7n8u5@vC6 zV(3CUMcTqs=uCB)1{E9%*SHp$O0*38Ovs9vr115=t&yeh%DKvPu9O7LvjwG2u^w!JnZnM$EHh7$HVq$-7 zeU*v)G4|Upxx~~ zwg+XaA8hgFjhpQ1!|Q%~dRLcMR_@$e8yXpgPQ=7(hVv5LzgH`XjMW8%1Qho^Wy_4jL8c;^c zvM8J%V*{R)gma}w6GE6}$f@2uVq40bqKwq}vW}_-8On4TG|P=h1!**3&D=3i(<&Z( zIT4^{4`~viGe<`nL8*y#`rwc>I+3OUS}0UwxG;gqLxB&z^w$QNgF{okuUvBi392NR zvsEytU_;!DR>WVp&xx>|asVa&0COuyKXO;tlYVYpd8;#&16tJbz%|`Bb!m&CF3`P< zRmRk9Vv1j80TCcv0zsMDzlC1KlG*?Ul(4ysc03Vgl(l2QkI=NyU8qsLL!S(=YzU~XFB?<5V5R)bgIBcE}Uz-J6L@al7Nq!sdI|4|TN@f<; zAqpgcf5sQ+)9ipK&4(|Imp*}2XgqVaFgLR}H?y%#SiUniJAIOcp*=lJ5-?NmoveWY z?kCbw(WVKkxj(qRw&{)C9=8kwDVm_efGf1Y900{&8p7=>G*<-fP84i91|v2SNOt}{ z0fExtNSouFE5B@PYD}|H!&ZnuWSSE6O-PA6?|hU>P+>(Z1X0UoY>r=4ky{UBqiZ2S z5~)x^g2rxoPg<#lj$Eu_8!(?gwr_}n5wr87zy}eg3=}PwBCJpFHi!3uTyn+6hnOM~ zLh`XLu%JFL(cl3hW@;%Q0D+820#-;2bcrWv<-CEYFl&8nEk>(7_~;aU0qelPk245! z?k5EZSppz6Y(};Pt&bl(757Z~?lzi#{tsVf$IxH>=tuf>q!_bCDs2!MDXR(f0VYKx zkwVl0O|uNiut92=0Ev^Aq?%HM!Kqi)wNE_y4s8Kr)1?9*Dkonme? z($-O%6qdeX8BDE&P`*q;qWq`xwSK&XgoI>knaWR!2Yu#woV}sB;BJDBCV~(G43Z!a zsTQp0zu1}t79vABr_m8IQl1Le54!N~v|_M;`X-e^Qsfzx5xLZaemcP{cO1&7rwYX>2qlAYN+<>AN%O1KmN?I z$qD8>p|Bvs^!s@iSB<9$r^lXpg8c@*{LOD)U0!G36E{mnGCCFtBS%Q9H9Otc-?{eD z53kNo^C&PTZIRHb=8AwxY_vPCyz}l4Uw!k9E7w+=?E}0r>nv2WMx`)-#*%3JB0xl< zWWW)tDK&KOg>oI{6)}Q<6J$v#60KhjT&!<8bxz+p<&;Srmg4mH@gf+41LAsoiGfRL zvC#|6aRUT*>V&;w?is8(V?;m=NZfTaMIDHs^2%fG(O z8k}2}0|NL~eyA*#8M{{nxNBP(zX$_(cx1p*&H7%WHGafZyhW;OQeX^l62u5B|H&!k zy)p<)?<0l0=0YM%u~hL2QWcBR=L9tgI{l4ng+XA`!W$5lciYV!`W@o|I%yUK*{^AF z8jq`MnIUwBh~Ahf?M7qEr351*1(-&%p6$lg-X@0A&u${e=I3}ce`|Z2b+7Eq zL0Hjf@yr0b^wRCb9+?YZ)-S^wrNb&}bQ0^J}`s1s?i8qG}&#wtN2E-f(u^n$W+Sy=L zc2vR)X#GIxyGT$rv@QnFAd0E<6Xdyg3LoL{+y>5pBx_C)>kVKFx8OXb1_VsNaG{VZ z-ekg3h*{wY8T4{y_mlQTs-^P{d(!SD-$IfuTMXHO>q&${lGsWrlnK=e$|mVvNuf0Q zOBR)V@2&Z1z<990D1?iOOq~YOW>^Io2$>VVLLvI*urVqRPyyHo*~m8(;2Ax$2!D{M zSm4yuhyd4^POyLsc_9a)(}*h(6*MZR*d&l5k%N8ixrHwU?rm9(7go_OfvEAIZ90O$ zEXdLcKuf-olUy=zGI^u1`2 zu?IU4*Vup$FGbVkU4^93`bs1Vuc|4Q1_d_%2p-N*pcep@Sxo{g9GnAR%1wnZ(zT>w zoSXpB=1U4WW^!mJF*nW;Q$#LX(kx62jPF2_AS)F8SslzlP1&CVOCdh|gT$&D1{g!F z@_%2k*xv`Vs_V&p~nUZ#vzwi=)N;#ZGNPkrdY`+N4<7NLrI5NQM@ z&pLtd5=tpy+o6E`a86!vTKbqHddNYpBv4tSqFD4qKGc#E#wjBv8UY2Y8gM|?sTB~S zNFF~Wj8m z-%CRccmt7LyNERts+E!DJg8h=a-`J z6Qc;{UR%{TC#Q;^!mvW2Bg%|sD#ZcQ2`>8SyDRHg-n*#OOB1s4j03^WH)E{~D zkxxGJ)WXyRPYxye2fPTtg)SfjsMEs(SS-tkx4-kkN`t+!pcKSP3XlROw5!!_t*u{t z_u7M}jt}*-XR(hqf_WLTH7Fon+~AY)WBIy}q*Y?R63h}}IMI$eMf zKeI(p1Bhao{<3`n>n-uYELNnO3E)DRr&ko+g2}5{JA`L3D8}LCM zWbg}la4etrGGidHR+TjrOiTuhm`d9cI4vf6q?U%ng;mI~befFpo1j+0pW~hxgS5$b!6apUJ+s=djUi3^oSm=jI49SjOXI z{Lm0oHk)mRs9Fnqj{3#`a)vwg#^!c|-DW4IC&wq){jpo!t-V7s4f zfLh}%whz?2n9+oxCQlFUFayZF0=5hU1uy!h6J}KatKHf>m4{dGX=A8BfJhKm#Ul+6 zAh}9Kfnu>4#VFyVGZs~R*2o9psWKI;NhxWnG6?5ixFFE@kTum12)30q0@{aK3gGlC z=uK?i7odV9NCfJQ!J;pcbZZ3%zy2A2@-3#21I1vdA`P&y^2Y%{LOCd5?CT{h-k|{$ z;YMWUu*NIkdKju5b?U@;;?YfZLk7(7R}dv=EL<}O89B&@rpI9yB_h6^%j24F?%Fvg zs5x2;nkf};;N*n?THyy;XoxZ?SWa-1RuH^RO#~Gl&N(YMOLuO0d%yyUe9&nA1x&aV zk1g6ng&!V+u)>;3^oKf#!W!UkCwys|5&>FNn%<4BPW>uaMJr_CRCGmJ@UBwqJ^+Ef za5EtA2b4)Ue94ltD|uLibutl!Whm~@+qYO>o0$8 z>A(EP<5Oe%-FEB<*rAv*5co4F0uzj>whYB$3gv_xUT7i_xd_OJUr0(tk+&M8Tt6gy zDPiSV0mC=g0x0B%el=Er@y#h7fcpH{AspcX5*NbqKOj&{0VlnI#3h!VbHkDFL59LV z!hs1`dTS_Dc5=kx-MxEj>o@MK@LJ*?c0(s8ZBqUs zFetORIwL7+n=#A?N9)tVu!3NXPy{O(@Sc(lH z1_yY3Yl0S63A2PlUqMK>%j=Rj@Roy)2PomzJA{kqo4`usm6&oC;qnzm_`;uqopO}j z-QQi?SX*6R<2g>^equ^U=HAEb*BO*RJyu){ zC;~vTh)fY!fqz_S?hlY=N8_7+3BwR9R%+Yy&u{Jna`KF%j(mDyXJxImn0= z0u;vMlL7*mj7<2`q*0NyQHw5y4$&akSpeFnLnEOGwEPBQ3X1}?Wl-xyA%QdHZ|2b< z2Kfhs)fpu);OaEDxnVTaHx18+-42~R zoBaD$tzC9>>uQR&HllE!+uh&ebzt(uWE{`+j*c=b*RG9@32%Ro-jQeQK+ifeckp!R zbI8*I+_*s^TRG5IeS5pr*~17SrPcAJ)CcuCD|=y%Sfth7U0qoxj)zRvo=r?nVFYXr z-s-fF1cIa~@f~?((UaFqjK)Xt1vX)6NrE-B#n7gd2 z%~+OaP*dzusMMm8o=r5S15} ztgo=K6M#xJOz3twEh{CIr!q!Pa=}UJP}(YN8P;Ck+#0bn8hPOU2Oc>85D`CZ_{ObU zG+Np>PX;%6sMgoe@ZJJq+=E7|0SUa7vc(Gyyk|%EP#gDsv_sqhbd(RC((FVD0f4cX zheETH!_9aK$zc>CkL(B4((dU-!jILZoPW7sHi|pYbTD07QHDl^H5rRVVBQGTQ8p;> zql|d$!>tdRqfp3{TgNc3NZxWG9%=KZLWvf_7AaB`(^TGtU2KvQxw6?) zT6Gk@s6}1j0V-y%6Co2^rrBr0PO)S0N-Me5sT5j~tpFo3bSE7o!vhzbD!w4geFNx_ ze39c3f+21Ur!+kF(uGU^^#7j!li&MI=JXJBRo3z{0%F@w>T}7pH|i0mVnA;Aqi{fR z0ZSu+?fd1ekv~#A6p2(MJ?sRJ3W$0k{AcCMsrrjxMDD*|0iS8+;b@{?u}J8XbMO}HSPB9aDyqE20_=ymx6h7N~s z6od>F5$OsK3O&TjSE(r;{>h=Si-<4aQZXKtOaNO1pBJ7*q3H>zC_&>xG)a*nstA%p zXd+K4)k%~_l?HkpBnd_NrCM?&1!|o(OE?rje1QjqB!+20Cy)yQDhN~Bq`+c)^>6L8 z-g@_HlWFatA*qn0{9qCk;nhWby}$6m2OqlclrUX2BLc}tsT ze_2o>3H)2ieOiV`N4ax?_i69$ZEkHlYC7ckE?-Nm%C$24+rpQ8)0vUXstUf#S_MeeYJeO3{w^PcLEFSyDqIN79Ng+(SXZ=RKqq%0z5k&$17Y-Ob<*O8 zo?(NeBvED4)jIbN!3c$@Ove)a)3!scftnUY+agdv(MXAq-F@`htwx<;E8{`N4D<#B%==8f zx0`6vATqQ9k4T+S4OGoc&!8p(3+CjQ!ebdRz41(jOK5O1;gP6HH zUX{a>y!6qnW_xpMi`n>*5oYn3GTWy=C&Vvndk_qvJm?tsaT}%GZ1|wyv&^E z`VZH*xj3J}&tALH1RBo_V1B;Uz5stffWOm3{l*QtFeVU3`rY7~2}5=u?J}e!AVBnd zN5-0%;)SuCAua_~Bm&3aFYfz21h~bKutALsEpvu1~yVmyK_*dWPg) z&upK%j4SJ0Z&(Y-MJCaP1qHybZd-4Jl>n6_ z2}bJ}6)&f;=T@%`kVKH)5kxMgq8B4p1Wq=pta0GRTP`z#&Z)2b}~( z#U%`Bn9)TpNU$~nu_oqk$DyyKpm!gXR^dX>qKQevyUmt;kyQzk4QzOd@mWG35RGr7Hi!(|IMEY4k}AXaQ~R%smi zVqJ6Z2rm?t^}-8gB>{cBCxt@FXOF4QfR10Yw(F*TxAH zYbtcMu_~(eYJd-yWG#Pd*?S%#^=|K~Dn&g$0{Pv!bz|;Di$FR9tGZh5|$k?{S zJMCFuNk*D(M;-go*+4U(LYegdL>uD=*tai zC{j_u604La z(W#TQ^eXh)YQU;73CoEd6b6*o3Dp2(Ex{i#RtXn85s&ObKxA$OFk)kI_}&razA(`_ zYbuE62+|F}AWMI&dW>^Ou+{SXFN^MdbPT}0Z=DFkUIKOM0c0q`6`v$X74?AO#FJ3; z3wlF`?vxS>iMV#w1tIs1t!J)1qea`{ z=@3{zft_|!?GFw`u-v(6(X|r>`P#O1l!u)27u>;S9(0g+$0mI>5|8)Np0%y4y&!O=4B5p@D zx3{?=KxdEL!Fc?i>^NTT?RD?(T8kioJ^q}3?(5~Q7QH1=Pn&y7j6G^DSmFtPZZNU; zC?Zl4C`9uFfN~?v)>Yab7UP{A+zen4Mx;#v)uZ-Nht0#{X=&CHjJ`5jC||0jl(ZDp zw1Bu?8%hjGy$ZCd3rgilHuZ%v4@R6Tsgy~~Ur3-he-5S~ ziU5QV(kiq7tX_s}1ahHMk_W{GD40tSB==q`gvcK(SP`D24!|D8^mM-Ec~@sUk~OlWEx$qTqNF9-)6m^TH-o zatd82mf=02Kfzy@2ymE{@;pVut35CmIzFc7KM@ky)$m8(g6=l zrgw#;@IXE%b_J9xx}YQ+#H7F|odQwr%7jv-oh1)OloLO3NaVsX#bE;1b9gX9ibAun z^dMd>L6urmu&GQf2LKi{pU4M?OHqn=#gDwYAd_S+6K`lGh!g7w@I#h@^>lU^t7Qb> zfJJh_uNCc$*bmywm57*>4vdR}0?rj+x?E92X}G-x5F|p@8Dda>nsrzGK0%Hc6u&Qn z-{;*fY_!ZRCbo}u7{Ln-tds~v*a?w~7(M3}?BLDyjc32}y$8;om>k3-v-1a(C>#|b z$;w*1Py+xCb?Fb*7JfCO$`ce{9)>7oGG7!|56U?|eg{xkDf$KkV8R@kV&t4UsZ}o2 zkCF;8ezTcdWj`_qQ&y8mVO6zBgP{Nf3f+~kgiSKnehX7u0CH49-5kX`0GFTy_F%Oz zAQq6o8A&J!klKehBu+l^<0-~VjxmY%pJagYk%{&I1AA9Fzb7%hVty zr~Ls~V+{gHk1#Y<$gnPUOPJ#-U8?Z>nPX3a9Vm+J6a%t8Rdu7(cQzoxQz}6#(2$1h z1p#>ZjR|>eBeF%9hG5bWS);T~WRf6GrE5gUmQMe@JIl)(b%-fMD6Tz3E1i{(jf0kMB7xO`%kU3InF}2mVJ}WT!7p8^I+8@I`sy6Cfe?(F zaC#T$@?cx@r~0?WaR7ot=Mi5I`UUNU>X9z>wGgYJBhN>Tb?j(GLwi3J{)o=Sr`#SLQHf~H=jSqPIvDw3f<0mQfW zOr*tT%Ve7m@)8&@tB$hRbCcl^nXKVLaJp#ip|ii+-s?0E+B@9i=NVDv@0m2$<&2K- z&^JqFd1kl2cZ6F2yB*dEwb;+c2__puFFW~Q$-a|!*Oz&5dGGFB-owLQpG4}Yi9JG@ zw{yjAZV_C+ejOdUss~KnE;ITwGcypsy~C0)`uTzJu~GVPSis;$C&$>`pGU-55ynjk zCM~%?V5DLU%K#v3(@69`tT-~AAc8BNNahh0%z=tLEblX~y!z8y9NxR&Ap_oo${{4i z@X;$@iIeq23|ri*C_0AD()^J9~y4&JbzG986V}%HdWfK4mN&xrULV`V+OkI!t8yT zCSx%(o1FK5Ld5|y`Tn=OP=2r13;-0 z=Vn@|E>5p3m#T$iX#pxVAxBK0HrrT-U#01Rcwx!}cN{aE6>pq}x@WCI0nU(-Zjl09 zs&KSGAfR5wZbwQ%Ko<@W9o*?!;h02z!6*|@qR`A&0OAV>AeBUDt)WMwVGx2!Phc(i zE7%RwSkUJT@Nla`h>W90E&51$g5=RaIMt0?6SPnrMDa86f0|BINwy;8(EW17i$oL_ ze95jK>f&&fx6PwKgdhlWgz7HO+5=T^+4vzA^Yl$lFD))jP9NLcY1~~|TU}jc(;phP zw;P>p!0GhS9}s%-#dcp@;>&P7NDG()11yB%aBf)X)d87l))xtRQ>$q`0xJ4udUUYCSehj zFaDs43>_Z&61ZCJ?D2&;cHG0F4H@u106-o^Z!qLAn^ckV#}PLa%dFr`jST+V-}?09 z=kIr30j4ybeHhqwW4R|#@TVfh%2RrrSni9!f)VOFSR8jE%(U^{UAgQBx!sOFpl&iq!zLZD6L zAddt=DID8a61q!VOve zrVYbmXd!^pSwlzg1PW(r#r_{XvuqS|42vXW5K!)5fz|c~q@)IeEmJx{P7JP%gj@ne zNy;^l+OD@ma4oRyVmJ-{5IEx{pUt1=4J4 zN)gYYB#KDoqH0|F@<_4Jt?Y_E(hVw{A~ZahhowkZTibpckM z(tYs0GX(T3&Y;^1c<@>ly@u4HQAOy7P{OV2`yV)U;>5z-YW+Hk1Z|qe>ai#)rbD>B zvbwy!ab{tjT_|Op+7(up7W2?8eyqQh2kp3}$Fq+;`^TonA2@U7;fEe%mgd_(eDQ9b zfUGD_#5UE0JVY%zIrXSFfJj1e=I)Iylq*@XcO)EeRFLKX_1)3ZkSwiU$BoHEzprese6qn#2ZNUt{>2G6(oM@PuAfxDbGUTR- zM1a5pIzt03mU}8HKweP-ztIyYq?<#$4%KuQg_sa-Qx%|(2d3vX*BpLx=n}n3M?qn; z!jHeP1el8DgDCP>;2Dauum{tm^X#;!#*?$Ww1!%Q0+7XPY_x^lHtS^CJ6ql6Hm|~9 z&bZU+Gbw03x&mb=T_ttQgq8X5@bu^&bkZ0E_`JUj5X$=<*q_vu&H zRxuZ7!jc(aZf|c740NVv=4NJRuUxru_3D+S<0o0B+uU|vAPi}GdWz}1ogEAwX&#P_ zkFjKkCP^b18XldWos&Vfso3AAOoJVV7zQx9A$(^}lcAyGx;6tWnCib3br z3o6rp!(WAu*es5KxU?~FqBc-z(W#aan<4@L3bgTu;?wUO^-%?-Zk8fd*eqPytoX#J zt^w&OlmNl~e9D7|lmkU|Kol`OaVM@qTclMOy%yq-2O|TQ>Bo@hW9){ZITHUobzS{{eT`L=0ywZj4Tf~88H4`88YmnZz4 zHqhq)8xVRnjU7*@AL1}whN;G+2amMqBNeM3t>2QQq?e-2U_|mm4x|}ia(BqBhv+sj z5yWGwB06tuZQttdvz-`s>u%jyzH|pqcV48BV~h zg=4t2zRr#_d54>$^`+yBGxKw}B#8}v`@ZxNN>)p8yd)F;ha35|Y`V}40LU;cTw9W& zb|G|x2tiGe0#E`Dgrg7wOn$9|yv-ynU+!idP>M5&C0U_j82*zZE*t|qLpP$K3?z_Q z2mw!;DAO^3AXW4A1ce{;Oizseo8SC(C-KxjZC(b`Gg=6M?D_;q$EBrSQBL%!6mo=I zut^=nz>M;e1jR(Z`p5(KeenJ>2VLgql9hU7LoKo6GP|3<_4Yr1_q&&_-}1dDeY}qu z?b|&9gfHp<$hOiycyDX#yD$Fa(Fe{AlbGgHxX~Nr@l|VW6>zdymf>p_HOioOV z^I*%M&@ovEaMDxt76TIPMmn_o$CPScDt3M3!z)FcVm0!_Uc-v zdmoR^8etHV>L19(uzHx|WD15rOkwuy&5jK{{J?#`^wftQI(K$*a_Yw2<(FT3_4dX# z`mz5~QBnl)=$@H$8Za=0szCYXtgMu>naCqG(h4X*IZ9D6^&Cc025(9nz*H#FKZ|hK zOufIa&Cb8nD#8aYRYYV}CKv2IY-#i~+#oPUibsy%gh(qPmJqd+l7cx>zEqKXB$}2C zf`r4ptQaADBT(aY?{H1Lb?@FXt4xkfO|*Bp4MOqzup?(C1gBVy z`Sl&O|2TRG&=P z+iS5R2%Y;D3$#EfZj&P;{=4p!b2Po17ZR~EghNDLWsDAB%#j%#Y1|YVQnM-uQLCQB zN7_gf6vIfUgIrXW2BR`#S6pbis=0U@^aUtgX56VlbpZLKf!1;GFwmM*jZ6aPgjKhVY44hNTvWEAb<3-2niVuo&Q` zwaJ|0Y|TSrWkdWPa;aTQj#uU&VcG!YZ3(22n*tG>z*H2V0<4VAfk`=Io&tm-_ETU9 z6TuY>r38`!K$@Fa1w|v&4Mr|S&QM?mg!4sZiH9N*fE!K%u__QS>Fz0?8h|yTqrM5m zwjnz+-k1#Ll)zM}xOIz|#kq~ z;(jn5ff-!`M}*He62h9`KJE%KU{RC-ZB2v{`}AR0+@V0VS&TNSr$Ea4eMCG}5RL@s z8>*6G}OG9{Z2Q!b444-FHZ7SJU`k~)J2H-p#8)X?^|MJTE%IX>q3iHem zt`SMVF|&`jjoHbG^ADWo?l$Sr739U7C<6IdxszE;19+n^>_`bLL$#1((4V+d>8`F- z1oEIJ;56sRLuFH*OTib0g@7AoV1t&*p>5MeXn>Zr#QUc~Kq@{Cf;CY3A)u!S#VD)% zN`f`XD>e))TbU+95pI6-gMWt)4UN{!wW_c`Kv(*X5Ke+c027*Jw`fBTVyLJ8pp(E7 z+$&v*9Kky8f}OCt8zv)J9ATMXdT7>Wo?V<4i%~JRdl<#E3!;Wsc#{A(ND$ED=0~sj399=g>?sr4WD_tSJJbT!yts z%`+RDNWkU8E4B{hE)3`hErp7-Nh&KK8gH%E=d+u>_ z;03~8(H& zoEFJ+w`rx6Wb{7NcQDesKQ%pm?$n7VA3p!s`4600n8&YCW@l_>B1bA;8&?w3Q2loZ)lv*U8BTbs|-D1p3@bhGKEFhuw7(s&;XFl zLICM4eJzs#K-L+jEyXJ-1VO1&70adnBo$H+jKBqL*yz2{Jzu9dJvEDeSl?J>_G)2v z*1pscw_}88i{OtXKimhvkL$=X00jc!pCJO)f^uQPd})R~Ux`C9Gh9G}6A9o@NX59l zs5%}J>>#90^3rsvVhKG>BuPcMqs0}5Qj;2mW7`W2kDmw+Z*K#@BY8U)iE9vqT*ePd zTvbIZzy}6G6|&gU9yFj*z%&wz6!}%QB`NJA!gUA$79|_*sEmr}Vq6SNR}N)RvbW1( zcUmMF|Cn#`eB0z|A(sz?x~reS!xZm!64N*$fSK_lDNi)|h4=9UD8~MtvVkdbe9!o6x{3 ze$gEF_L=5q0LT3S#wNY3J~j~LJ^;`6?lgE)?=G+S9UIvnoZ=3^aPMw+Wo2b^t3ES3 zGdVfExG=%%0(gNyoz+LR0oEGrPqB3nJN`F#GMVm)=|oI}MLf*(LlT>W4h^%P6@4NP zxfvla=Hz~WD+=;3q%XX{R%oR?1JuVd2@besk@Vrv!9j;sQ2p7&QK@FYzwN{mFLfZc zBy)5F+Dgo-1PBzRc{(P^ADQUOGR3lz8vCfg&_DuFc)7WV^AqFI0b7+hSQQg9U*YqQ zTg5^gu~9 zrXLm714t&hz$xz6?bKj{RH%yE(Zr;xSgepBgf`LMZP!}No%l)*LJY`o5)C8zX-&0naOXtae@;eXINhJiUtpgb5ia;ha!M;!r4k}Fyy9uCU~NP<2% z93L4yb^Lg3aFn>8mjOE++S-8~hUhFUZ!&IS!!R3Ym$!dopWNTa44A&6BB2W{ACChZ z8Z+-FpPHWLX#faREwav6FZ+km{N;$&=z>Dm7D=i`0fog{slcfg4xKVUj!hv<$-W6p zfp`y`wg!R%f^XS0--f`tRb`1kLNT$L6wniRP>ianjD?C)3We!F{;15+8P}FT5pKiG zT451LAJ(On2+$VmtqD6p$jIssIv;4ay?{#D>6U_ELbl>Se{db?;`pf_Q*0mdhBjfbPdI*d`tk&Squm0Mt zmGuknTs?p4xDw_NH=Vo9hIt1Ynj00^+)0511P^M&q`uP8F^(i+Ae`N5?cTn3dwG4G znRNUd3sH{E@(%dPiQ&Ov@B0HlDrt(qMsuK2wNbmX#Djw?jpoa5y!!ngzkKoC>#OxT zE6|Xt1?kxO062Q~#3Fmf|Kf)p|G=43qfGtM0AQK?v_(q76g3KudY&R(5W#?mjSji> zvDMjo{zpH2_Fukp@#>B3COhuAuGz5}8=x~0+t<^feD`3p)45&WdE@HM@BH}XGshP{ z@$^%l`uN9A%uV~sC7p}(BUQv-S}4}W#6x3u@44^4`0RIIz$*a865LPE@a(A%p8u2I z`)#J=N$U3WfA5u7Klhb?y1Ba1H#F4CvoqlY6~cqJ>kbZ@`+G03NbBZZM%thG)lW>b zc$dv@5Bq=g##{gE3tzo(?Y#qD4>dFf;R!_|(84?v9UipycVE8rZsYI21iX(u`PeZ2 zvdnW}2Ra6b3kzHoLLrq?A^08fDNWDyfhjIKq$I7BO?eVci#?(U`jYxqV{@wkKyX4J z^?+3(EU;1V*vt%*1@w-U!YcBCk|vC8=*z96W)UJ22fB4$*w=gF*!;}Y#M;g(lP-XE z+Qn80C_gMXURmGRX|?9YM`((&t(t6`)Mrrl^ka{F=&wJp9PC3`8>L z0j-84vS7Ccv~=#mreDX)!CIJCDE?DM3?51%9lXLoMuBGUBM=3bc`NlyV*nC?5f5N) z5peA=rApJ*+(FJ^B{YP6V^s%i>{w&oi21{y0T9R{RDjAG4MF6RQ|RJbEJaW0pG`=j zj$)tn7u?(;X=P=VkTq8pP*UtGPy*MXFB=u+%|Y7cTpXe>OoSC`Q*?GzPYk%<=ZdPF;IR*Np(6!i^4Vz z!){+zTx4l)X~Rx$VQC~wS&QQ*9f*XaCSdESBvAE&MM_v@SMjjc^?zas;l8=o3?RLejCD+MKnL?O1|z&v=2 zrLnzp>(;HYu?d#=yDdjkTDIz26Jry+zL%EevXG>la}-dn;@DtZT!Q1IWN7c7j%?Dhfmj%Rhj)^hzMy-!3(UnTk{>j!l%hR~;ha zqd~_#>LOv=`4ELHZ&pwO?1t)?C{%zN$YuJKl7vq0lLcPTr<#?8fgnK=j1IOq7jL;X zyDB6TkR&or0x|D^by*0AVP4yEI7KC8VR<4?pQ}BfF>%)pQ=4M7&Lnt;|uoR7z7oxHV%?!(jc>E@X;27`JU_B3S z4)^wy@2O>|!mYiXc58iWi|v56cNiD2WS01zX4*G3F|{x^H-F*;oBj?Dx+efYsKOe- zo#xK`%*_1!9Qy*%qv?p9*vzCd>lS@mh??<0A+`w;iUy%ciG{x0D<$GY65%##2(d6n zu$Fy5!y!tbqSYd1jmcJ^Ns63`={AtFhc=?pt1~#V_fStsk8K4ZRGFhT1KGyTCF2U5 zyPd5uF(2C=xMUOeB2x|u>@Up+H-*$zFme&r=>>A@+YVKhE#yky;vX&VLH+7S8xo_P zuWKlyv`SX^7Rw4qUOWRyxBKAP)1UqA-~8`?^*LVRz!QJuDLzD9iqbW)B&n~Tk=+}Y zuKeo9o+d_j7^i%UeTl0W4dI|y>&$eFDI?#`TpPT#TL1i4zV^>Qc%k0v_OrLf*aUp(NFWOJ@e_?x z%bFSGF!p`>#=WbzpS^J9+8_PyZ#{hOG>hBp8( z+3Bgr?mt6mWx_{fISsdBj>qVlQDQB?z+QC0g_hx0{f-RIv79Jtx5uEy|r~{^_(GZ|>0@Zd>kP;UwZ8k&T+5?kmx^mO~r%N$aX34`8q7Lmw zGL8Jp!>g>*_x&o|kt5P`FHR@RP+NDGnTGy@Zfm!(=_9{~d(FmHYkP~qLHa80?F`s9 z+`FQ?UH27WqoZRi`sYD_?M9v6l^J)}_Oy8p&HcF9lAlnX5Rxt%x(?dB%csjr0eC5J ztIMW9w|U7kuYO~Z7hNWEjGLS5Jd4codxCY1iuX?pjj;rYuAWy2udS`}syLRVHMo`S z-JAYkp~wg6N{)p?EFBwdwpZ6T zJ6)I}vWG5L^$-k<3=V4_MeLYGA0|D(T_R$A?m2Y1XU8s~qr)`@F^7FkUN*4L9zyU0 znfx>MN3$vo4auvvg_`sT=mW);AU`Tb6V^0zNIS62VS=1fSXZU}s#GW9E6o?j!nZcX zk1$g#Z93j4c4=+FSwbL?BZ>Y5qV@tv?;#)Xn3dDi)H6=ixG^Y*LS2lIe0m}wgA4+L z>_T`*qtOV824g*_A^f0_<0svz_+&s5%;uV3tsPnb=$#l6Of)yZInh_8KvPt7A`s)xppsjD#dI$2vn$QX0NT}$A^N59C`Q#kmlSv4K;5bZSs$H}F=CG7s)>Kqw z1rTZek(!gjBG*OIOaK-NEul7)Ce@P-G(OBpmNqRXVp1KkU(TFAvvYwFTZS;Gq#89q ziJ_=TRUXBnbNXQaj;Y^2FoMz3^jJ^guJbD@Y&0 z(HTM%&X7?1(t9^nH?~gCPeGQwI{fK(9e&+itN)L``G+?yUjrr1JggiNGmQ@PKmFvR z|M}nl0W-KvPX6GP*Z%VFzH<4-ZEgnG+ME$&)6>JXosBCuZ?l{1=@TdJTbjozlt=o3 zM53JY!t~+I&fZtQ`HjE-$7h$fcKU`!2F6Ai8OC446cCk0h4uJ^8olU^^{v1Cr{~_h z^zI-3{%3ySk@LCzNrz$^MjsKOmapVRr;MRpl?)7Ay|?-opZ~)1ue{0>=)l+{&4psK zVweB`KmbWZK~xrsdR&K25$RS5u{j`8SLea93v7WxX~$cF4LZ07TJz zrz`RfjMZR*sj<=Cds~gKJ@+r?&YokA_DkRV)}>o_`iCcw61tQ|%)y2c5V20c`Sd91 z%t8%~zIO55uYU7eC;s>k7RJXl0n30YRXI-_Bm43KxCl|mlAuK%i6%9udQ=ytW5h{P z+%Yi(*vllwMx)W%-F3*Q4vO(ek;NDv_VvxpOb-v*qmc`$LX?CRwDJ`aq{PHrhb*?R z9O$2!m>^UIkyQmRY-38BTVKOhmvAG;YH3z>6zGOfVuH;G4X8a70hM80npK_9Kt~u! zE}n|Nf>^G@D6~*aI}w1Tq*z4cTis92NI(O0I*FfSpq30XP(#3%_kdxFzKjVwH}t?m z5=ZL%JfPWerlioJ(uBCNgCQp93)9LRh@z&YBq%3I9v~R|HsG*pvz&;Hop}Zn=uF}g zgq%Ea^6aTI)Mo0J?Pm5nZJro8oSm+o0U$%ErKhD%+)i1>guxk#;V>}6J~n{BvDYwM z+8)+uq+9uQ9gnPl9sZ;>ZZkwvANZ71(qt-7Xm?HuEk+mgB#G&Df9y0w&Ek}pp2+hnUBfC$58s-W#P8MT$%LaCy4L|QB>O%*rE z%z%a_JitW`G+Ie@u`e_uG=>E9_|S&Wq#ic`*zFk+cXk{A;8Qz|jZU*cPsQzDbj73D zOnf-U6qeW5*XWJ)$UNWG;nCrJ7Tz58Oi`G~9?N36KNImh1K<`NZqC7k0hDh$g`d0!I_7 zTwU4>F)ppshNkwS3>t%F)IJtOqN7MoXmPdl3?$pEsxU~yL$^jvfXD@v)GCS>FjeD? zlNkFfPy^JO7N9hR?D;~3CM`|=*!n;bwqYQfequlLelCZSE?h7hgFz3X=a)VS4k+NE zRc65hdZRDW-DvsR9KX^i9Y?$drEQ8PM26xq3rPvGDhN|lm;GYYBqLeMxOyq=k>cU; z2t0)qX(|fHlkn+JXk~ATmkWP%%T?S5;Q(wJ!gpL656ZDO#0;jASH15Y= z7$AD1A_C@4P*IZ-E8^i>waKXV>dXY>u;)qVmto z0@gyqe*5C(E7z{$+jv3DFkB7P7LLv52_YN@^8$U$8FSB&&V!(xDTLl393CFe1wydu zb3=puv~p}2U#1ThfgdI(a8)#H6JzT#BBy-EdK@r%K^pDEo=~+G$wv~(+_hX3LdnIj zfK75ampgeCBo$BrJ+C2%-W?QzQcHqWtAygxr*8a>zNI(UMQOIbl(rhmVYMzah$!P* zf2mAOj=p|&%kWNTddPGKE{$T651&ps40K|YxezvkK>wiS5U&mnBXzt8Srt7(ReF?& zz>tGB>h4DJW_4;(s8VC$5M?VE{`U8$#zsH>%rEeC=>~UHjdV@d5-WKrS`2=9ZR6&> z?##?BUb}VcFaG{3 z7p~nJ7#V{f4NAk7HQqCXmET_5tT$TD4T7++6B0S?9T;5RZhr2I|M-vJdcL{eJ1{Xr zJrL>S(U49K{`4Ej0<%^b{lk6iiP7!+ zSd}988E&s{{rTtr;kz%t)-yB$vzmWqTJl>8>J-7F`EiPSR7W1(B;e7s!vo%-Hu9CP zo&3}PaC~~)2CE|cgW|Za(14)DZLd!OYU}-N{XKV9Hxe5r97_QG_Q;Nr#73gj@%usRx0_+iqnxXMag2U+|>~JU8(#l}L^2nB=C{60Z6ecMMlqD)t z0W4Hbfn>KJYO1AchnQp_CVrI{PRPSkYC0##Uh5;nOsO!f%X&q$3}##wesZ^K2f-yt zboubd9Wh|zYMncD!w6@a!?XlQPnZXAVN4gB+Yhp!L$FhTD5X*B2L-o$S%1JYHS=@F zcxIHQL-5Gjs>!iowsWQwLlDAHe0~uiD1;o6kR#LCa<6hAk-#Lu2@Zs}3xnGAKkiHz zy(k%3hDOB&5Q(%s@a?~(CGe@ZFi-dv)l_whE{$bC;W9`k2k6;>ji~=WRrlJZS#uWl zefl`v({nyPC+X26kOUSI2Z6vCV~io##_=i6O1!-CBCqlR@*(mOvR3jku~*hgVw}V{ zTG%E68G!`?fdEP9_?%~+InVUW^i226M>$cCMt|rq05rrM^7G_19bR!-&l?bQ)c@c{!d`XHmvS_`pho-do$GW+zX<$ zw6|~FRQrX_HfN%Dy+3-l+mB_pX3)YapwM0LaKKQVu-=MtL_fK0JbnAZxpN*S+`e-M zY3f@0ds<#__|$Xzb7wcrJL?IQ)?{t%{P_!)FTZ3Wzl}uoCG6431Rozf){*$cCw}OA zZ@z6u5bnh=2t8})Z+UO%{ryXqFX>8qGT9t~JNG&mK_mRsVTUY8+{`csxscea#^s}?po=|i(MH7YtonoLwa$til&E^F4yt|P%cv+clS zXe+z&Bfg|>`Q_+^-V4rtkcKwpYSzHqauuSqvFNy;|IZR;AlT^{anK=wGQBU=beRo(%I&lS$8{#JSSBCsrX1R=UH$)}~?jg?{+8Pv!J zfy!-7nt3Z<}EDDn?jVG!~un!Jb9l>(# zSLA?^_+{aUfAe>L|F^&NhfkS|R<`#(*!k7tP0urXm;1@%oB!+|{k^~XnIB~zAd+gX zE3PV@8A2Y*ITJtAKS>aiX?$C{>xhvAHVRF%&gB~h!D89`-_hYGKlTG3|L8}4@2g*D z)eB5Tf-E-_@~>?e(RuTocYo&7f2DOw#I~45OUgjmvJ?3F@tv*XC%7;Ln$B3Y+cb6l z-s8iEPoMwV=l=cI-+b3}l!lxmi>av7&Z=4(jRP$*0s{?)IW z-(s=a7HfIosFWpYOsNBsYmtQ-5>hxcE<3rCMAJm1W zm-(a{1rtcLAo{`4v;Xzq{Kgl){3pkDca2Zx^V1w~39s=>1TSQlhT>5{h{KlF~Z8b99Dw$TN4gFjnLf%kN6(?gXn{?S)I|9fA? zvPvh1al;MbPd*(3P?e;Il7$`Kx<-yQWClL~ztE*UF}o6UJj;DFJUpB|)O! zIsU=zd%ySPul(o_e$=Qt#u;$@m1p$}e-GM|`y|tLhM0%Qqb?ju8ytS=s*KPs0GP{* zvbfMxo7Unv;KbUCtVbjXGqTNP(i0AD;a9gt;^}NKxQwFI*f>DEMNSAq+(g8^lX|Ge zS91v~R~=acz}riXykVu9wP9&0E$^|Dc_q_HD>Matbr!e&xx1lzBAg{let{9zICscJgvNZ#cG<>?(=HqO(cxDrk>@Zy zUd@bX3S#X}hFQ(YZskia1p)~{e#r6#BvdF@TwnHt%14%coLCuFG8-@r{m+KgUReMQ zx|2g7mCOW|nh__Hh8a9y2x@Mr=a!?ag~V>faJ&a25tlbq9Rs4&h!|D|Zb83%6A56w zXEwIk5wsAA{WA|97&sH*d25IDChDb7`v46Tj(XyRQPd#HKfQa3F5Y?P2L11yJfSJs zf4FZ1?!W(}P`Z`J*2ZTIep zy(=%hY$3tD{rm0TV?ExZN2kwhU%7NiEG|*#Pcrrwdho%+mtTJE)1Ug(`&X~szH`e+ z7IauK$Fw=a*FLbwODBP$d3NrC)DyMf=GcS*ic`&Ws8^2`m54>S8{7F{HITl&D+2NbbCXN{}kVh^W7?>lqCEsM9 zhIYyfz5(C{vP)EX%1cPjzmIw_bU!2mJ!)gz_k2lRos4`2)(hp2zH&w*$votUWokib znjM~|o*5wdWUYM9#fh8nj77OINTJZh;!SE+=93QkTVM^{MaPBn#f<}T+{By`a?L>5 zB9Llr=j>s2FG*2?6?vwo4j&$xQDZ(fhva2%FPvUEvZF@&JBVPS|16dLB)Dc=NUz}y z3jl6Fk-s!TDC9;;@?ljGKi(-%n}o;mG*7h^;?!|dE5W8v(9%lg2s;>kEh29ZLp6MF z%RIf^H{N*Tl~+H!y;Hzvr2o;3>vQt4_58WVg1tKA`STY27%;eh|K^?Bq{M&KN(rF( z|K16(b;fhYvu7u5B1WQDKL~x)kU9FK4~!0n#Se3Iok?`WdS@}%Wfz=H*2!`o2;vf+k;`tmwQUHzHJ;l zq+-O;77_ovOmg7dP^9)5EGhfP-`ak;fZ~Vp=e&?CV z=2JG1gdt}nXW=finnr?yi_WQJL-mG{qJRCH|NdWo?l+%q9N*a9jmuGlyUfHnQys>F zxjdU(1f6u4G^9Ig-+1f2|LI@;U;piY^IyMoa>shRkx+6-BtwmhB)qxxr7!=<=YH#p zNA1M0sdnIW92FP>Y%(*;Sj zGmFz?H}7p^2t>HXGOa+^yng%c=YH!yT)+QdYkMc5CDo8dtdp?_c&jL*iHK0z-Hpwg zckkWaf1rDbih;>*jCn+DCIVv=_fv_=(gLXKTb|$j**Cs*_2$jfuf7s9%P}Hyr8P529|Vkh?pW#~Ulbnvq&^E~2_wYf3i8bi(z5lA%>uO}_%FtU%DW5U5zF z8XQDXG#VR*HQ*XW5!E+HoGSNF5IrL6PBjU*Y&1+P*RxPf%ZeWPKx!9!{TKY4{k(SkW9S$08XeWM#H|I;i( zk;)fTDb+to+LO9BauSMw+fB@O1gCeS4$es!HOA4PBh!_8jnT<4*hg^TPO5Gx6}@2# z=-8)|>?x%%ZC2Azx0#jtbnWz!pc#2Y+XiOC5SWFn2F|qzq`7mV)v=xuRNdMvUq|Y> zX7$1Ty$cuYx^of$gKj!lY6{!*P|NCBV{;k{V{^K=dFkhe8r&0mXZR)KVh+t1iamCw zu?rW^JGVU7Sp0?aR_9&P8L$WcnJZ_`oV6L~BN4w2hpj}lKQ{0XhTnR6i@}~gv+L!- z@4xrnYaja1M?d<;ox2yV-?-L;qo>cH@O)2{Z;0!>?Yz#Nzi{#5JMX+_Jqs4bW{%nM z{GP`_5zaog%qOs+C>1$UoafCc`ZavMXe|8{py$y4(n>NgK-21?H!4iEt28%3T}C~ePiQcvF>3Cvl1#PsGEdge zv?qSkd9(yg4>D@RcnUNMEgWM0xpy2$h$}Hv3?UG#=tmbg`kgYzT_X=3nfDq!fd;Tt zj&mQC&=I2glud1jo?!IaM-n$}udcT+%+M$6>`7()sW#*?Y?c`!7>Xeuvf;)z(uSuP za_KwmXg8crtEMTsjx~J}sgv08$?wUJ(y=Jn2PmlhCYP!! z$374x^|YXKtz-<=nUc+^%amRz@3?N`WR*VIvH`A0{d*E&>+9a^Q=G-}Lp>3D0ZoER zeqi}zw~06MyY7_nExfpuT4bG4DX9x&qA#9vNm0@b_EBKq6pkeW3xJB5y4{F}^fLD6 zj`}Zj-?q8fiQNn9>t3R1n*eSC3mhtIdwB4~Lq^XWPwehox_I%V4aQ~-*(|hs0*WVl zRDx>;deQ{b9I)o&IFocf)z$ra2lWqt6@L^+L z(d9oi$)~n!Fn9J&Zyev*=rOJ}(33ph#r030n3-m0HJFNvLynuY5`%_?v>1(?r9vrE zyPKk<^!<7Vh-TdaYD@^Xm04|LNCX`^CSuRhMcdoFwJ8o}gJ?+4Ufa zg8n!XUi!|%N4CyoEHY*Uu+Lad4Oeo>ltF1-v%9Va2*~_W+hrKwNWhg^^f>j&-hygs zU1L^^;Ft>Zh+y<#VDQ@QyIvCagRi~JDx}FYh7(^=UiPUJV5Cg|q_LHtR3MQKJma#^RYfEcoz7L{X6-!05Fed{c+6}ev zGdck4kJY>&;zpjbW*t1RUgz`;+)0q3jfm7zuY-vXyiC z2%}D1aV43|0F1yvFg0-ms_qmrA99=prnX!0QrO3j_5?*W3;Y0wSfwC zj2AYVmS6Z|2+6}(Pghl>MJZD=2dlgtzVU-?VdfAk^~!Pt zXR_iGb!eJJLMN{Xm?xXVL_(GVs5{AfI7fW?i63Y59}4RHe#Y?`4Hu%O3ewFAE?7#5vm#9+iX&>B zP4@wSwiIeZSUuY@uzK#bL0sQze312eDcI(Q7(rlg{IT7+8ZJC01aDxA4{ibuWAO;x zf?bU;<1l%~Y%!6td)TxT*bsiSlhw0r5;nRop!V>9{Rq3CpMbL`3~lB4{JEihB(a~1 z7cZSWb$XeEBQDoWdmwPz+r%tPc=$-IRE$y$!z+2|;&H2f-+t?@mtT49^qJF`J9zJ1 z5B|0Pb_cS5)D4q?>;`n<{@JsqUw-*zFQZiLbV7ES6l&~qxaOr^f`|Q{6cq-opRk?c0|Rq_aFwKTL?|h@6IKq!-Ky!D>rrBy;JOPlBH~AlVl; zH-&XZO6>hY)A{=u5meviDj$$N!#Dn82G@nlseLhw!l$eU>_xqV9i|Zb4kf$EOLqo+ zCdDr5Erf(S%QslXnmN=dizdGn_8B8bTj4~Z8!mw|gP1boGV=U|FxtTn>YTWSnySEL z>Fv`_k@QVU@{iTZOPbL+*tmv73E zD-~4(a+-jVsWhL+o(z7Y!ZBbO+pCII7TIUb~7N7U5;(>1!*KqP_iZj4YxDiyznG> zV#yi2_XNN+HXU6GiSVX)Y#$ywXaHw)D|gm1Q_padgR1o)3+)`=K6%P+63;Cc@VX>qspU2Z_{c|I zKYRA9cL8SQc%n-4cD!Z4w1?DI$Z7y%Sky00uZcbOqpT}uDPOsaDc)-Z#Ry4RUYU1OlbXQ?W`r_Mp@J*`9sN!%0mve%UeTbTwMV?FTNnW$`Gv+n zhWwhk7VL2H!S7yZ0Z*)!anQ~7i4*HPCn7B?ZsHRYDv!$M@%c0JsW8JT6p9^_1}ol% zXQ*WV3BP&ju*Y@;i|M2M7{@v%l{oU_9 z-`@4WyIqParQUJgJ$D4Qt}cP+^o)KL4wQ+xZEd}G?e?#I?%#cI@BYroQ~980&$eyv zUo<64K}HHQikqlHOeh07786nCAfWIlX5-+&lYjI1FZ}qYKJls7Umc;1uoWjG^-vJM zw(W&kfBNn3c?W$?g6gB!7oKwRLVv3W&hG8Jbm9Ev^XJUW_Bi|7Z@+i-_C3>Xjmy38 za1jTB!4^BWeCzGEy{6#ocJIHgDmTe<{M$NS9C;@78w=G+!$7{**SyyUJ8p{*ObSMT zm;S!;(q%i++`4o7-K!rwh5B02a=mejG+kwdy(WI$v=7(~Ki=-lrldbkhK~_D2>T5AXm~xYlro?9^Np4{i zON>+VB{BY2G-gG48!JG)D9z*C`}gesb9!&@r7M>|{OYUc&z#=Tk`4LEp&*UHkfHX= z+-j3~60}+^!gGsPTlYwSKdr0MQtG^U^Fg=UOl+J$^)b0$3oAU~e4Uo?UqbfC(w6WgETy-$`PaHYSpb0gXoGX*bTGLg^F9%|VpfdaT zMnPGec0Xa#boE9vAvJHmjJ-y=R96J=D;Lc{kNw9HxQvnCBdw5O%CpMNt(|g2TE5-1 zujwE#{UebZcf{T+c%72YURt*14h2r_T)1o0o4;uHd=W3^^v+9GNwQc;PzAJ&EYO!saY_ z%jbg!T76IZJtlbXz4yQRr+<3orI#*Uxh(E~>)m(n+`sG9!nO&4?)Kfg?_GW0TR|_J zm;03UBt)cdFZ?G`TN0tldK@9dqXTaaeZF?wJOY$#J+w~$@go*N%1lgG#ZxrYHP*94 z14AcI$miiA3i;FOdPW0vl`?5c%wpoJ$M|D^?#eXvc5ab=CP>Z$$BFw(hR(oecoZ+O zR%Zz$t1+Xt$(e~{snDrDz16d0EGbz0z*rLksxg~u=7cmbuYbI`u`>&=dp;9yWlY}| zf;Jb=C_wX5VDi;m3mY_9O|#VG@`gTYhX-d6u-L{^*%MR(6M%=B5rZ-cqv~!f*iaV zA*)WyiLQl-7y0B`1;1%jjyl#d$(o2%LDcCJV-uFy2PR*kFEvso&FNncc@ZjTZo)!F z#V=dSrN5OV4HX`Obkrk}I0#uh0+kZ`smIvJoP_GfN>9P8b(9+zAMz(mOrbLXx!q%% z$9L>Yp`yn(8#Ct5QWOz0w{Z*a88-;*A^t5Ko+dd8+dD3^7US=bR^iE8oF&|19K1a35 z(=i%I3ms8|w$f0tTuK_{55WSh3Ehl{#9NF6L%FQaXMm8JN5N)EHXuTNty36wBW3bu zu?}KWgur+XE0LaiojL${ArOnw6AexIA95O5oNA(&GJ%kPq51m+qybu!nsUbw3S92~+4J*4Hkg-c9_jUZ8WVy&>x&)~Z7NJEyG;|Ujscg4{^u0zx&>Mpa0!2ed6_hobU*M1c#DPdM(uB!$17fuibm} z*xFykV=?loM_J9$6K{+CsZalvU;g=@ef`y!PVabk)P}z9U%d6c$wgoN-kZEDXlPVM z3Dpu7)xl2lo|2u}-i!E@nM!2Ay&6>DpD0}B;N0pU^4^nUh;ChN+dlT}vp@FfU-}zA z_u*HrXfLg*`Tei_$-n;1-@bbLzCl6&X+2*MSQkN2*;s&;D6e0>_)q@s-~Q~6fBMX> z_c*T`WPba-5B|5${ltom?FZcQUrfwDyloCI>i`^21npPE;=8acUkB4ww*4KHrYH*GY$2Wy`%4 z0lCz`(_^sVuXa@?D3LSxqR=N`)!aeg_E=~S0P0Z))hAG|5t)7njnE0P7Tjbx+3S3V zsmKwQST%lVc<;q%AJ41{peohY-gPHO;BG&e4cP4bd(F)O1OB_%gG8V?mP)N4*f38uQ}ygEE2rO2d`J{M9G8<^c?YfdtnrEpf7i{dVc zB8WL*xa*9iq$*KruGd)*Ftye>CQ#Cjb>3@k&wfvUZG=s>vz>r$OFbLwgGsVSr8u;PobM5N;kL}>LwtoB8Ef1Vt ze$^(iSHAHV-~6Mm{PE?NFTei!M|OAj-h2P*z5RzC5x{A5$@Dhw77&!4KYNZ0_U~Io za6o2)g!>O4c+vHVy%UBD$+lNs?C%R7^xjW-2gu{CyX|6Rf~qci!()k;_(^^_ikI|e zYlx6UziqBLAL{gO5$Yh+AYY5?U#B#Qt1t0WeHtBAU$(J)2 z$_8;}FKTTCwSdH|5T%m_X3Upa(+Nj!MV7A!Qjq-67r7Oh3bv|Ks>3Rg#LBs_P~d~N zO7&TYowQgTMa=J1v~(R5fajNIRweT&E|Ej&=ue?(Pro{)03zM0XUh#OtI@{rBP_^l z=_S9KP_++aHwIozxm`yny`l_MN3vEbP7;Spyu`y|ozxY{-LI4mfQ>+#nVK)~NR!;v zFH?}R-DpJ?MPmyI+LY3!cLLe;mHyIz>$U(crzu?HR9Gvjg>0ES8dys5gB%4T-Gqc6 zg_4-$$}gf`q!`n~Mmdw9--utnWHyawd;5|e{;5;<_8*}fGB zH3AIo=1>#bnWm_YxZ$^B_uuiF^n=G~Ix-Sq4JPOB5+iIzJ0?5bXKlzE(~#TW%ofp7 zC)D|m?l_1yTJ|MZ!+4c+1N6x;x?teg&wNUT(5F}e7< z>k%U4x;-weR`9;Y6EA3r9@qzQZYGFpHhX>I?Oio$u|`&4m8}S6v;t@zb*lMTPxh4L zNEnU~QNaL~e`o;P$GM@MQ=W>*uZ%|AZag<{|NAfg$Nj^ns?Cg}Rk@AnC28*G>Cwr} zwV(gV&;0T){M>6-E)m9^{rxZg!B>9cw}1D#7fWp)=c?LCN}qBZv^sXUzWMuq@@Iel zmw)k3?1=K-lPv#)>Wdp*5U@lwagIu&}3cC`8Y=`Viv$N#H;`1fA9aLz=6 z!U%+JZvM=VeEja+pZo4V*Bmw^PflYH)Rc%2tb6(Ju{pRWGx3C;P?sY;#{YORNm)|j zpfVyYUa(>A)5Bl+>p%O?{?XsRbmmlQc=q(AGkdRI`UN@u#sB%ahiZyZ&oPB;5Og<2 zUDgl2bo%5!{Re;d7k>IP9vC%Dz4mNxWBc?QA6oxAzw+LDSHJ%D)wS&&RaJackUCAW zn!Wk0y7Ry;DMS&O4`Y~jQ33PdC?Bh7Z-{CJpq_T1ne;NHTnxLW7LBsgCx}dXp4w)ttka~ z`7{g3&Hs?m^F#;kK+6s1xuOrKY@@^20HxVODOj@W78wg?>PH6$`}=n_iDua%nGdY< zFoaE*-U~GK1ib9`v4HIGz>-2?y#ex_2KdEuO}iJMYfyWW*X{Oi-PzsQdgAVy?YUbA z+^~XqaVIVIwMnSpQ-s}y9QW=CHZ1DCd*|-Sz0+cGD+eAy)M~V?_VHU^GqdQ%)%Tx% z@WE>z`tX&PU%Gnrx>tRYl+pd2y*;n|`S`~_{*7;b^WAse(^z;cAY}Jmu*3a*^NXzY z^K9VuoqJvbc5r;BJp)A!2aoEtt*=>gY&%fVk%>k))mf>R!+f9Iw)vI)8*FN-$NA{s zNj@bzNT?}!rM}dLLOqp6LUYYI92y$EVLI(mTK1w8&&pEjO&V^TH^mtuc%ft2$eEVc z+aj_BqimE5cwt=X&(z5!GEB#Ej5#ug`VKierlTNBDy*ajjY(&b7b58czk=0if>KMRYNj*zIb8t8 zQR)M@p9M#CBCIYaW#B-5E=oVj*SFVVqjWEx*KDIu$E+03E`l26^n6N`!J+NOGM@s07=?q zVrd5OC|h4F*H~Pl11SDXds#A>MrAP_t+5=v0tUfyESnnplVxEZ*cMAUdgwKKr>Glx zVupUWe$tBnY;>d%g=iYdjhW6xbom4Z#WF0ZlUT6mAcs8g=y5WlkKq!N_mPo?Sv|b@ z-t%K8>;~ZJ-J`98&CQD!&cF8ZD+ZO%o;-Q!T(AGIF}Wi5?%gr8U{u}<#al>l{+!lD z>l|kCY3B&rb>!YjDJNO?wMJp8ZB!&0BSf|1f62kn8L3G*zZ^@hk=m$$v}CHJ8&p5) zDnopmYV%wF@k?*N`;JHOwgj;vgE(h{g{qt_)l!q%VMxl)EYf@Q^y#(R_ZpPSJCCX4 z9EOFV6t%tNusH(Z=zGq_JRnx>8QY?u1|dpXGR9}15H>=|dg9*Qm@hEROeXhmDRIiW zn-mu7K&*Dhuv3yO$9ODIszNK3!=5BkE#WlXrhV3i`~XEmUr;Fldu=MFjIvkpN?%wl zPK|;R=OlBB{<#<_2PSl77*?GXK&!5tm*GSBpFJ~E+f>vgpGS9A5B$y_{K+@oe8(oC zJnT4pAmTPa&kj#-t^b2x`h|b=E5CH<^sX&bEeJcmcmCop{`|R9r~moC`qv&&i!ut3 z5Q128RF$@OzWbiHgnj4bpZN*?FM$RzDarHxwd*(T+|FT3qI3+HRAX>}UpadG6QBI} zKmL`!b>+<7(?|B^Yu0#$Oq@7&{8OL!!Iv+d`_}s(FjdJF4{LIhW#1!gFIi)<2__Ax zo{1;yWtJji!C%aeStKsyRp!H=_@R&g=fC>5Up}*U%szn3KcbILY;66+CqMD|FaF2B zc<-9WuLY_LK_fo6#5&t+Pygyq{@Bm_=qHaC@#x!f-dYtXkG=8gfcNQU)GdS5a zS#o3a^L=vq|3j;Da02uRncq@apADbvU+dGZkcH=EiHY zGRAS;IwK-s?d$yYmf{)08dWM%rGqT>hjmz#nMvB0}lWrxbkEFl&@V{l*o9AA;4`*su}ELw@Yw;6cj-w|1WN z+OI54X%u8L;X9_Qps5me5+`W-1I_Jjoy?lqGu7y0Hg4Ln-j)=}-V4^B?>-lH3o=ig z-re7SVB)@3)hz!Tw{ID*1AEbCtF+h_HXUqEHtN6 zeWsn=6W{&LcW>Rf^V(}4*0q3dKlI_(QGM%{J*BM{J7Hz<{eAEG z&YBG8pE-5fzC*_TuiyB;i*CC1Fg@P?wQ2?eR`@(Q$I8_o91TNR|S zO*)fWP0UGEJ4Tz~9ZNxEc8f#zDZ6p5iIAftv&d~!JX^0YV2X-F^Sucsvk+%6&!blB zdsN;F5Zx&hvENZ!gnJX(k&)51_f@Mv4U4J6)eh(p4`?+Ik5wnO0YVuAq%E=ZD}NG& z{z(O#I>aq7QbW-Yw}p;eib4}fk4Q8ygeQH#o8r(Wld8vfIJo1_ zojLqg-P~Ss_(7a3gH_s+m6Xzaipx2z4#);WnI``Wpox3Q9r`j=Fbs=kP}Pxia)$$P zAXkZIz)YuY?#}|kWav7z_5sq(iuv|K>OmV)CdKVxJWcm|s5;H~1(Dj8)&OKgI{Qtv9)T24EQ->%QWO60 zYv26xpM70ypA3>rzv%^7789<@g_K8;VxnQTgt^$3VLoy!*(%Y44e}?p#ZnS~5M1#~ z#-6HhjeX7}G4~V=%p&F0@lwiA>L0YDe{ z_Hp{ihh8QB2L&I!5gk)wya|Z{T8`R28p#bb6pzo>JQyfh;m~!6jcztJgzdS~=G0td z7#S;TN^P`YZ*gRpsJzz%EmTZ?2<8M?t~25!k#JlCuDtLqj-xL|u`4|@NYV?=&I34! zrOl1<1f=}Su~6k294R^JZa7a5WsW7u3DtM_=yF_Q`Q{RHPO~j1ggWVSxAdaA`m<3S z^?lUkX0Cv?#55!#Yxqg0oVXuC=4(!CB-Dd`1|Pz5_IwI(tv~9G@RqVdf&@eUN1I1x z5(_GUBQY5kEX3K7Xpm6aXeRuWqSb*e?P!@-3@#!|Z8&eK3*UhgsPaQb=vV-i1q-UVY`&tLEiD*ysFh^<&J61{l-z>mTSO zcvrxVf&IOcR{a{D$ADmQ_vGHCD_7on>#aZk^FP0I`SR|r7uM{E@jXD$D|r9<2Q~;Y zwMvJ?G@fJk@9*yJTI@%T{Qi@J$J@s@FPuMbuBlbQw{PBVI<2{so?6US;7=buLnQ0n zwH3CRL&t6Yvwq+W#Fh>`ys>E}(T+Vc41m}qwCQabIVCk4x!K+{ipXiB>mjf?(srqm zjBzc)5Mv$0l)j4c#z&E>glgM#DyC|VxHF~9auk>=0I$4-h#8Y}mNVMTA99eCTumEu znW~ggL6*^FdhT?O;xjFqzxyJd%pq!?ZmG&spX5ZjFoM`MOw*1O4e1i#s}^zQd8iwR zLl|SK!l<1|=_y?y$J_!qAc!T5$NXdNx#-Kzd$q1x1s&6qRhrH5js|r$MMHYu(;O@Y z9#H{I+7w%pz~cz3S|^UOr+^hwc&O7vxS?}QCdnYbvZ#i=>uh=9ZpFFeOn%8UgUDm^ zOe>eW%$(#gY&5p6ih2|G{RgHSS`4&rb-zu&xOKGI`jb)0r)_$0Z~ueM^~>k&BX`c~ z{wLmY-V_=;6MGl>A?G#DJ0Wv+^FXZXk@Q*;H^Vk`L$G8gHN~V`S4DmCA$w`v!OO@R zi=gRXR4W{^xb0TD&W()ECi7ZatP4WRg<;wXN*}j_=`4BxSkP3%Q}f6%t>ih z4XYw-e(SC8ee3Oa%~4ro7Pw(CbANJp<;=-n`njL6C;8EVk*yS6U5Kuu(>vRL>*s#@ zD_{G@x31kVMLxAfK+ZHbwZ8eaZ++*+ojb3dJzd8#p=T1*U`A|@MX9JLd3YliTpvEU zeD2iiFJHF)xc7207_|n}I;Bso-LZmeHqwTD_?|kIXfoC3_8fU~^aQxfISLY9Qmf_Z zAZy=M=CF2|9Z5NS@&m7b?1z8gqm}_zeOUsXENWgq=7GuCQzxZJm!|lTyz;-i`^y*3 zfB2;MboYV%dy0A*|7ysH%{pcq?zAf|tIaKLw z_(noBBtwjj7?Q+OQ<=tx$()q(Oi?^(bJmq(%KDw62_@2@z`WX^WEm!3{-85Q6*DK8 z8b*YjNGhw-g^avKzy>Gl05*34r9D6Z06+jqL_t)MBLOH-a}Nrpm@DpCrmI8`2#9`l z0@U{fvJwj^VOqh?TdgFLB;*9X0hT1W4A+@v+2&$il4$EvMn&RPMd%P?7LXSvSeR1N z!|^w4V*X!7lRVoXdclgQ70PI+c4lr^NftSojv)aL>TCVEmOVXxW;0NFG*8bmPAh|1 zp2CRq&qv$Nsn((PYKW*A*RmZmR&-3YCd}gC=Cn4=!4T30ZzBdzkfDnjikg{qRUC(; z+~1sy_14cY_|jk*ukL)M(-=z7IEkB?rhv9|N<Z#3|A(w}Mv&3D1Ok)7kV z{WUIN=E>ZHVHy^0#<}ahryzbtwN)0gnq7Hz9zWjPybrbM z^%Q69pLQ>tzjWi~joWu_;(q7E?&Zr@?%ca~^VV%ca)#v15i;M$WJ$AwZr-|isxA7@ z$#!JiXKfvkYYz|X@27cs%XXgczkl)arM;6UU)kEa?L}gD?;>XZ!F^O(TA*vd2u&e7 zzJ0<(eu}4w$B(?tV9g6dZ4YD}@NLfy?%%5eV_CxP9@Cii#e2be1ZmB&4`|JLQuN`Y z>07SZ4cddJjiU`Lk8Z1N89E#c-Bk;)jE+wQB**F{Ya;mSo3GBNF493-rzyWxrR8P~ z_Be%ZR*ETR?ekN?Ii@5V&t@bl5YVN1NpzVdkV%?O|70 z^A&(7u}BK{hzUnXDqA;dBnqDbl_&{SX4$2Zz#?93sMSr6VrxwxG`7VX)H#{VVXR2S z7a(1Cx0Fd-EFsj5Q3rw=L;{U{y;oWw)WUAi{&TbaofBLU7J zIz;$7em2ByZ@=-AEWsKP&-d)IX3(Q?j~c&`dP<8?+JeF$OJlTJ1YI-!qG1t>f_nLl zmbo%%X-H4tp(HWD(#VJ+gBsDga!jTYXg=W}v@TC=-{Rj$_o;~g+O0dcJpS8%*!Fct zN46+Ad-jZt#@Z<0{yx&-mTq6VM@gsp)!25|=d>@2pu zsc(Rfwu`{rgc&uChn` zRN&?WKH(Phj7tL&t9g)^9Ww%b;$7u~5D9&yNs7&*L5KjZVv^ldhgbSta>@*Cw2lxm z`ZAcB1(`9WPz)!rSl&2_i|5C@uyLw+DNDtGvvIu|eSn6ZN;DXPu!{Qkh0JG9P4N6r z|K`vClVAOnm(HHi@)FSB{`sH%vu}R;5B~5EfA%v!ZU__IN+bQNU;pOahhA8tGf6IN zHN$J<4AIY?{?JF?_>mv{0XE+{k7_teaA#OL_Us2f^r|T?-?{pEWe5*pk&=#j(sAwP zt@o~7fAzxI)`^XXP;&VAP^5DC?3u0ohj~~>k{8n0o%i_3U%9M4nWkACRYb0;mH8U4h= zz*gqja}VfGoH&tmhafD*Ux$3XB=Y6U7fLgO zbCypYUw{3>ANt{seT+A4$*a* z+*xR;w;0C7iA-3h^g?;EZ^;RKV+dKq6s>Sa7MRk}FGe&&QCK8Idu$p+!~b(2M0Ohn zS*>e;cFVq-jVi$|%=Rv~T9J{=;%9Jyd?X}q=u<>K9u12~T#xnp!co!$2Vw0d`Rv&F za~CdOIW&FR*N*c9P0pwoXlkbIe^ggays8>ohy%>?;2EzRM}dxr zil*t1Ei(T;)eBufmd-&^M3NsZ2D6_1?1m1HSVy0UpG-livsPIsfEcAA9$^i6lT^kr zyev4*hmzXWfjojs-H8W@h#Wc*Fhib74Rdo>m762#qE#VeGA+duZZ&0LEwIA|_6?@i zk84Al?cEcejR=K{*z5t|IdgoNrC@wed!m$yn%jmlnje;Y`0{HZR@zL{V}1*KD}ug&#uY!w{G1$d3X{KGjk}>TLo;zlf;f46QDbJ z=bd-1UA<;V#EW`14mLeEKYyh^2gD+ zGiOtZMGkB0C;72?YZx+Eu;&Kn&Yt&@>pS=E7#~qbP5Xh@hNf1Uf%B++1@CPg=qR>D zo``dMV+SbM$qtSg+1s|RS4Erp|5p%-b&dFFl2pVYHOGQWRs5 zqj-RMrQI)G5c#%6oi@27;ktO$7tluO77yLkl73^2QJk1Z_@i`Y5__FanvX zF(gDqY%Y8FqPh%;5ao#Tm4AswqK({Qn>r??Eu=>8FvLFY{_4e(yH5`ujs$I8vGtiB{@4${@u6ltdPW~R8~cx+eDiy6j)zUsOoe2@$yPlR zUpw~kH(uX6zU85^=wG|8$;_H-mDP@CyIVV-`uN8__k}OsSKahNSHg)SG0DAy{cpYb z=4XH86ACVYCV2`}e)`8g^~M{oTYG?KU{T`4VMFVSXHLIz;f$x_N~Ge1?!!-D4>oQd2~x$H`g`OP$H7Q&XGG9bmN&aNw$ERxYz)G`El&?e3Z zVB33bnxq4ab&LfCvlRLI~T>$sXd$M6SEt>kKfK6O$7ZqJ^sBs#V~GNjIRg`HNP_^XOqw`$mcm4XfCt@&D;UBO z#3G}mWHj|8?PzxYK+nKM-$F2A7Y`?;1FAvO8I;Rq7xpD?3RW2V43adHK={m!2tfMC zf|=D=cP~lwMISf{LDHF%PJPm}WSWu@yWLb!X#%YG4O$4hQ@M&y2hv4ZCB{S&GHm0j z`!QLh(zjy)#~YAC{zOUPfpNIT&b!F)XB55snQfD6dd(}V@qFg_V^h!o9g3qap+zB7 z{g0rkRgO$yvWzJDil%vHCDw<`7VidSA3b6)Hr<&E=o@eQY7TWIY?0Q5L?L@G)u2;ZL?BMB%oEZJm#Y>{?Teohp5@QF< z!@M9fgGNL-Zk7BRV7Uk=e*o?>Qe7!wpXwd%5*)y-b_8Mm2zvlR$ z^<#S`?bzPWC<+s_Z||~v9HNuT*!)8`Uc+y9BYhUV4iO{*y!Pw)+L=eD2%F;a__5J# z^UAh!PBt^$zAt2WXhmV-H6mtoXlrLnWS}OXne2+LMM~+kc$_9gtGKiAUkPLN)n2kp zxIOtCt6$WcLiq*y7-a)h#h_M(5BM0dJOq>GX^)KBn+*ti1Qm8kOyO>&C>{cmUl+|rxNpllDQ(xjmSI^TOJ3~IJ4^RzXbPj3+_l=20Od@nDOw=;Ov9SRj zf~ERNM(7x6g`;aJvYd<{A_1)AlfeW5%FI#?s?@+&xRXjtW9YuUo>QAUgqUzLjHx)2 zeTIo;pM2d~1T(;?NDzp!$v+jS;gqR-X#)64AUm}< zrIa4#QwFEdz_@W(J!Brl#>9!U`ZyfQsZTXAbf*=Px!I;Y{pozL4CZMIagU{l5>z3A z@g*)~q|lfd7Rdmfg$Bm6fN_~@Oaao>j#S=)fRUU~>Dnw9=PYy+333A_)q_&u(bk5E za|U7R7mig$YV+meHQzLphth=(fOo8To0wOIH_ozt2>h{wrXWwNp0MBy?@&IpN0}p` zX9kPeAR%?3F}XP;Op}Wxg8}XSx!;>U_Re_6Uo+!9Vbn^s#b*!U>?nLrHRp6_!;>pd zC)@LDpqL$~&V%RKpUMnn(jeR(M|lQf8)I|9O}9=9Aq}tRPD2DajVN7IH(yvlXN%2^ z2Hc4&4GT-IU{+K;o)V)DQeM~vFtR3>3{K{HY;n2a)(9$<&;5e(|KBG;O|EFlrI3NZ zV=5GTVS4}!2kt$8ZY4P6k&zb~A&1i^5|z^AXT;9@Od{j!EGy<6!q2)ujiS>0jKZo0 z8P+UUwoe|UQ5rI(Y8gi!3MHYHcB^R(re|Vd3p#k64NVY$8OENv>ToVMhk%+_JJyM! zB4W5&mCRlfe5`7p56or%&wk|>FP%JbNc&?}?qk*YiEZ18G|VBzY=-MMZ@+W(h9_ai zAp$jivV%^ZKil2h{J{^uY7g=!&2fmo{Zr*=;0d|HQ9%}Pcrk%{z5 zr*_90wvFK%?o^+9bx3BW!QVW#xh>H0 z6#w08?|X5thi>_+Gux~lqhK-0BAs$zh(I)h*DNryEPXho>fjLImcx{F;B3}36L=cQJYpQ zpY=xDOl%QC?jLW7dKiwaOf@SDBf%!ZxgXDicc~m0?n?F+E*XxWAb}b6d|(>8dF&2v zU;+}NIbu1aj#cE}PrN0K=v09aB=&Ek6H?xMW*zXTXw@VeE%KK66FTt%pu%YTsr4?A z@FW_*hHj^euqM(S@gW)lnegc}EAvwjfEi36czxXhAt8xX01V^G#S15m3U433`(VGN zQ?~Hh-nO-iaVA3ns-z>3WLl%dTh?NZ^z3+HB3-l$E;q+tE=5o)s15hf*!nDdCq@G* z#eLa<4THUwz&4u7u%qF0iA>vvsI{tuSjo`KpKaH|Gy;)z5MTZv!MKEIthdt<)d<3w zV#C682)lM;1hW(y265-fcflMBMs`qXXinldkChaYK5)qO@HYJqvaF^iPoNK}6&ii?DNMDcXP-a&my^z|cBz_A0f*5@X-3&Ya>58}6{k#!{6fxzH zlAOKx|fN~4cVXI^DnV<~(xSepGOrDrqq z5~_kAHat`%WNJxMylBCBW|E;)RjZs-4Qp{A!b-<<@-eS*Dy3mxc~fc{FHcqk&B|o5 z1Zh)`GJ~n`Y$eN?0%ZUajNT)V@KtSHKO(Tcnd5Jkgf0P+YnWxud*un5GnSuJ2!>vT zz^1~F@@yx@vfaGcknw$dLtVTBBt|A3o;s1tSs}O_wUlE%17^NLwOI5bwFFs0bPo^1 zKh)b5hG5@DT!iCXBrL?pYny)~Ys4ioDrVAR4o%FAH0Nkme@S|@UrwkN;W{6jX(Q`` z)E2*=bHvl>#$n9-LqdMfzrX&jdkejHg3F_iWWfiP$9Qj zw4gX+i1CDqRO6tT%N(oN9nYrTyf)#DVD27J2sjy_9MMR;q zCTov5x;KVNF{82+fRvdb5+@SVRct9hA&L*z-HJQ+=C^Z)uiSMCTI@CK(z@(qDO5gs zA9=yQB=;NUg7{!odjnZju3V-8(BzkQ1#Pkr@Jh-2V9{`?;fs0n_MepUT~(W)=0v7& zI7*5a{|qU6t$Hd5vahHwTb`vm?xR!^$WkeY)n#C1Sx9pc%otBZxjq_XbEVG+mlPCD zG&yPsK1>4_)J5vkS^c@s{-mipM`p`t7y_6O{8nH_M5r=KHa4zazj^E4eeIcB@el7J zK37LiPoCbpqdeU zO70{cx}1r}7IBdcfH;ZWTqu%ZtYc^!aVd>!V?dfu4AHT|l}`ynlmcsBY|wm*`bNSa zV8=j0U?gnaYrF~?`hz~4j$kBV7-b%R_rm|rt-z)d6@)9CEYSifBKgT=O_gqpyaF;% zM{m&?t8murjbAK76x1?wHXsuSB+@gRwh{C8;)u&;5SHlWP!{>jpi^aV@3y3u|qZuI_FIs zJh}VREJebgU&tC7sM{)RUbwyv?(@&9OvCy}B^~q`!^`-F{sMnW#YEyS_gHR3`w-Xi zu&wP0R|TRCCE{+KnH@n~^yGZ>)g8G3967+pL5BZ=FcCMB7kz)oXbsI4yyJs%S`6wXwXH65P__wqUKE%1V;62F%KcP^`~Cfrd>PlG~It^_s;!0 z(DxkV$Y}bLM+cA2it*2#y?*_=pr4($*TUM`sWWHxt*E_mLtl%=8erV)1;d2orNZJ9 z?Xns8V)50o;Ej#_`}@~!-Y#G|dMZCQS(Z_}OP4NUk@~LzN7E*4Saf3d-1@oo`}dUQd!IbDRX|-t4+IED6hT>J+7e%Q zXFyluh)bHyoT@N&0E{K5XUWsH(Nz>yH5-6rKuMblUigT`l?0&i;v%H%bLH3QF5lPK zK_-wkDmTV5YRf)E^ZABJj%zI;0O~5BOw&D1nSuoxlwuD-sf%F+uTRx1qZr_NkR6ILs?e4ToC6rXAtT6|dsZH)oWfI-!Hp4Y`AQ3T%!fBKE%}6sGhR2o~kJF z@Sa0p851n`rUgZE*kfQtPVOed#9QtxEMj{0QwJ9xp38MByBf`B7f8)Igl+%;l43yX ztYd9c?^$J6)JTVlM=NcJfWktc?KZhW(ZnEWC70#}5%aPL#y-v|u4|pG6@StktsYvi z{iGhESQqPCLZte+MW}9)3g+~OqcvGa>ZrK}P10NdJYcXaCCXs!rTa zf1(+g<4#@%KJfMi%VO-$Q=c;@)&On`h6ShwORZlpQp*(Mo! zA-9!zwqDe8xO4B#cdmWS%XGi{)_31|@A{p)j}D(0|MRwb9fRt7)GSU0F(coI)_=`V zy9-%}+~Y`k9Yox{I2NrW7r~h)GF5q}!+bfHk^zQKwMu&gFu4Y37%9MTQZrkCUP=Xn zl$BdV*e_hebVE}pPA+p7EouyZF^Hur=HDJ5g4$kHJ6z`J=ReX18Y(CA>SD}lxw*(U zvCtVoyf}z#bHjo_!NY+??_tR~VF)BGKZ?Au_~xCv0+VMacIpoDWMgM#Qlk7}UtNTs zNEW*-(d3`o_wJiups~y50mr3UHv`8?>qPl;gYE*t5PR1WTC()iDZj`g#gLyzCt`3* z5HoS&8j8!5+!yh!cvvhNoXD7;3TRZ+acN_zksGQpxiXG`YEf41BGYhDphHr{I@&?> z^h5>n5vQvPlBzULQeC4c4pzs#8AT@GZHA$;hjkx83cqZ#BN* z%w~8OBSp-LpoTtIjy{?+f@`)O9PV1(E3@$=`(g~=4Sr1&F6Y+nK*I*6UO=m@l%a__ zO;4ZxQ8qZp5=J`ZR+$9;skH=89^HL>@4+KWy=}dp6|Nt9{75(YsklCCTwk+)1H$w{7D<{B*!YM> z*zjNC@Ss3%g3Q-E{I`pbslQN|Gp2uGwvvHAJ&hhd+`o5X*PDTkd$PQ<`~2CXsvzA? zaAYF{>=c7wZbVz)|4@NpGY@3K;Whp186c}?MueAo&JV{%B9JE z{AGR-VuX5XG0&1|>A*C)4}TFFlJJ$qP0t^0c=TfX85%M!6=~sUp+sfGNG_}=cu2_l zt2T`b_hN2@8YrCcWRAtmGiWl{NUF6`PZm~X7a7D_z?|W4;U_~UkXT!?A_68nxREd_ zqsl}@8I($Xi}b>h=o00Kt1Fi>##0q%+Sm)`P zfd^y%o&z+iIB3nVGI1%zgfWqdJk>acC+ckTDUM1JMXYUF6pyawX7Qzib~MYE>Ekx|{dCJhIUZXTQU?T8 zZf@B3xtDzu3`iQ0G$Qi+@Ys=f`Y1EWBPEE;ERL)L=0>uq)WM!sByCtm9`8wDMu26B zLhd%g398SuJ#a1Jmg-LAc1Odbe{bXeExUE8&)fR%I|sV-9%Aa#8p);Ml%nh5j*Z+ztPrORwBBS`mb%{&W2PQT-_>&|D{Ba^JM zoZa(J?!)?u{$$XtNSJ!s9N##J2RaD>gGpQ&Fcd4c3l4G#$z!qxq;h04rtaGfHW`3L z+D^-xgu4Mx9CX(;^8GrQdmt})t3^H>liZ9|`rHXgHoO}1Io2*Ktr;J5S$9;&wr|E# z#4@=2j_r2|hU`A>_(MsTLw%X6q(cNF9@i%WhW(ebdntV>7tvYd6rScem=TZwCVnEB z09Fzz#>^cJ!yzBU3~L5&N> zQ$M7=86vNOK_u23-6$oYg#$BlEMA2S8ViWFa!onx4G`_mq7!$SCZi1I$`GB5E;FMr zE|xaf7dSCs&i*(7$=P0A%LQUS33d4%*tG&_0`_$2SZWOj)wBNGt6&*8_OAOaYZk&N zqRj;@_#gYoMx;e`1vBj$j5!^~#_GrVgwqBR1a!~MMW+26*Z@s`Kkrd$Roxo+cy%XL|+j+U9TwS;Q2ZAf3OXG z1DP;2Gt|)+uV`m6ZHWe;+6i&6X0`@ z>@=yz_Q0gBXeES2leHjWk<-`XqbCP<9@)2McW3*`m6u+6*zNE)qIF;FH%oAi^w8@;8F~6~2x(#>c7Lv)B6&WBp zp)J%DR5kHwLiZ&(I+NB^Cs`*|VdzFW*ef4esS%Hn~xsdJ-BjUTEF*d84H11{x*9EBGr4Jz1e|qNB89fSyLSJKzoTnJzk-myhhszPgpIAR( z#E@F)hPq5C@S4$LG~to(F6#Z2Vyv z7Xu7LjqV^yhAUjqEe%^FP4}hYTfF<8ieD|wA^7L%GU`gZ0)iiKR|i6;HiidtuC(q% zd1@CTcm4TRZ#S}`5>Z)1%KMBIWs5FE7aK%8sKpGZ%RFp&(Y(WsYCI#6f@?79d0?*RoPft!C}O* z2d`#L#*C-m{$gU-tN?Ih4FIm{{%|29{6mwWoBSeWV2&f`YOsQh zL^)o)bosO;U&hOO7P7E9^hM=JCix^*jmGsGH;aMCAh{*Kajy`b*4DoH<~#r0fB!#t z@JtAh)-dF{8-29xoN6DWwe>qTO+?@L_hB<>#B2@Hk^Sa~reY?{5u>P`ki}bsupE=GS^OY;wQ|F3>HV6WT9FFA9pEc+7;Q*l!Zq^ z5wyF*K|+=oZa>L2{Pa$h8nn4JGLo%mXcP=flowB%@!(~CW;HfL;nMpc} z1zUxjI=OrN=7Yx=s~*)DE+|+V-}rq2HNzJFvTx(~UaMq%r^S-SYfW0! zTy-1ZYVCynUM=f|vRERKVueL@5wtMEvW6MoK4XbxnSMJHmQr=Ap2vTL4Li{|HPO>l z(WHeXn;R%Gf_vfog>x4!dUca`gL%KbRbMK$>I5HovMwP-g@QMl&k>A$qfbkB=y7ed zsYP02W}$@V0^>@)@9pz8=}EStOQ4a}b@i@}juGXDV@J`6Zzc!d)Pe3lHl5|*06ks; zsMMGp{u4$HGF_##s%_8*J4ql*ju~JmORdS|%Onybxk;RVxjIs0byjZWXDEZ{1S3Tg zEyPDx_Sak?HjYm3CKhHN3aQy0p1fJ}(f*ya-JOc3R*Z0H2S<+|9v(b&Q)aK`9p&^I zIMZlN#3ko@_a9#S-~*HLJ#yFd0w$_%+U@Z+Z?!)Vp^2f9W}0sMb>7=Qd;Tnf1>zRz z8pykL{q1WvZrQ9;T@*auz59T|_+h(rdZVy)eJ13P7vLV`+q0APv`_zzo^wKS?Lg4JKrIv)>!{n?tf02at05p?O37kGw?gIG(%PY^hqBP!91E zKu|kl0J=8n0{bfB0ZuSCd6fohs#8W-1Hn)_m0ndIxy*2Xo-7lJYu3b;RNg9N3JY5L zNh*<|H@$geS+BG9`3LC*b(M~MWrt$|T-^piwor$>EVNP(`y$;9w}*0BK#-8)mi!V% zjdNqbl(L+lEpQ?rA_{$+&e?=rVH@CXDG>joRAfiPRO<-=p0#zZ2OR~q650Z9bj)L0d>arX`5?MGIP_0r?G7_nB4879YMm-OQ?nU}QW%QG4ug^C>XeZ0;0{=T}8 zRjrhVj7A+yA$)f1q0Ip_RBc7D-TPcU>!?bi%(~>YHQV``@54bbJX82)cUu<(9Z?OFIZN zL=8Mk9cXA-elcIL6fA}zZ2^kLDm@978e1qxZ#gc8154F&QDN#2bOr-Jt}x-uT}53E zIo(qxw`o0Dwm;Oy!eUf&=B9$+qi6UHmK$cCBs5?glLT##gmhaDGLFy6aO>4p~3_=8jjHJ(WH+97n5Ddu|djF zdU7qo3H<_Yta*x;0Wr0h8Eam1phuqnfrXH97K>~Lol*{x2<6m{aw4VLfDZhis4`BN%pBS3k!=W48ROEGf}l0#|$XeBBg3`)@qGYr{B7M2QTv%drsq?39N6} zZNQq1$8UV-_zZ9>mJo*;5Tp3*Ti7CJ4EIqq*&&x^f#Q7q*m>$|C2sk#@FPuAL zN7QKH#8dH6(JAqKe&HSMczAkI`?E)xrUZRmR1>} zxBeP>?IBM5*X$o&56JiLAY zJUC~cz1O_gUVA)gkaikxPJxp9MNWNZ`N}|?WUgG*jn&Jb(uIRh(F(LX*G7HGlV#{L zD?um5`z31*MmQAybbt5J`pW~TBeVXChB9Ae0&oUZ(OW#ecj22Q_I-ZtlF|3+q&`PqHqk)~- zV?e=$AXAaEJ((*PLfY;Dy1R;$d4RvM6A4gA@1s0Qrj`dmb`5%EA)UFd(u(q>iOuJu z?w&9eXa{f$KjsV??e7lzW(wXOcZBBN*jL;CWA3fGkQWPp;4hvE2^2ZAxP(6|WlG4)qV^o|oDHa2hFe`HW!Q2#9X zwN>AyB5#N^{l^6uuJc+FfE>yNH5k_aJ=x!*zWVMQKt7)j;ZKr8e`9}>p@wZ_5UuEo zNxqUAhUIAf5u;R zaMPdCTsR-1=@F5WB5t&>{9M_Q+=CNgqT?}HB zRAmD(iiS6%^jzjlGCS=@3IJ1S2PW(YR<1f6SxGWiQtrO`&4SA^4>hMX7b$xlFOLTI ze6avWh0b{wp_c|=5>kM^0_`vR>_Lx#me`E2vmka5qos*ZX$uW5qr2Qwe`6U9;+$Bi z`~rg0iT!k2ooGQEwXtb8c#|n6JRicNh(~N3ZdidXBLFUlz_7dso92@?78vn_I62l& z9Bpj<-gmzLfBwq9_`@H(=OC?(?Q_tol5|=m`|aV1P)zj9K3<{Ya}vP(Ncnhg*ZZt?w7?!%+0nw&{5-29>Z`&22(~99?lC5OB$9X z*EqQr{csU{!k)7{#3rL%x_f!X0fiSYtiAVPP-_sA>5w@>SzCYl?BJstHw}KD-Ll{~ z`WIe??x;Y^$f#0iUibpVdgYsU?mn~-A}d)spAGj#&Pe0*)~2^5IL$!Q+jNRLYq;4( zhLfqKx#8435sY=!Co%ZT;5kWI5`WXkr=l~?jwJool-QLEE zh5`QdwQaRWBP?hxD$J%v;OTYPDEe1N(is;j3MPTkjeNZ(+EHgPZ z+@_L@@gILKZZIM*mOt3ra{}zn*=_dfWDYP;2z%1UuOsLqDH%4g1=#%N(Smpyok?;ud;@)4e%~A# zoWzvX&d3=kF@o^=zzjP>2D?F;@z+PyC0otU$2)}>BQesRq zGYMtWP;6QTOyb~8RUSM!0l(x{n_gFF$(b1Hw@2FwcUb%6ydx}&Kr)pjMdT$_7c@IW zR5B+s8rDYNTWgNi7jH`aAZE z!zCg`*pLy8wKOu;wbb(&@JXt|V*^kpzp-n0;|=!IHtsL^gAZP8Muu-Z59Oh@;ikp@ z2m3|>PHu6=dtTVY)w=bv?pd+txB!l2tCt4k0b4=1u}x5?$y{?|*N)6BC~|n5kXnv} z)WUzfjHfe3y#%yTQ(GW=M2_D@xGGuc1(X5`!IXi>BO`9PP+BRavLh)0 z(~GMYFZ}#ZeZ@Ki@=so;nz8<5yyL_G)u?V?MeLgZ+-itTetPq)wTu6Py=N0 z96w+?RHr(`5_yK0i7cqA$rJjMI&rGxEJ0Z?`tOfzh?SiMhLHst6@@~3gr?Rt`B@6b zTLsfvSCm{0=rLlz9>p#h3a~0QgoSRAkAKWmOA!i1{FQI+LZ1L>O$eyz&l0OmxnY#{ zoFgM1`Io^A7%S>`3O`Sz+8aPVN4IDK%`y((JxVty(I7taR?)H7oJ&4zBo*oCs*5kJ z(L*@I69gD7jF)6neIbQ}GLMAiSTR1N7Qqdl(Kz@6Gz{QmWAEtLuYU7i|6l+7*FSR- z(Mjt=oVk*AxCt|)djAWi#Rai9NFs+RK{=^7Hf5Gk3k>p>br$)*HdV0h)Fhdh?T*5q!o19Y+r42$~eR?#1X)UlxH_?OGqb5xN z2@FH>ES>`8&7;q!YvDzj3yNsaM6zJ8Vc!9_qt_I1@E2Q=VbZcOon^V8sQCNSZj*!t(Hmtw#oJ zxK%;+&N-_!w%Dge$_TA*#2E5#H8$^B?qYwfx&Zvyk@HWD!Zr_E*09GC^IKV9A{>cS zle58Ez!(_mH{7)lsA_x9l(SMhmd)~N`z;YJ)Weck1(SPE_i4wXpi??1&%}-|9SiEv zJ=TE{FsmK8{h}sz>o@aFK8VinLzRN67k)X_wRQ&|n zYe)4w!I1sM^S%4GZ@mBB_a8rc^u}B7SSxOwKQ<_aek(L1borXi3Y8Uf&wi$B!RsciEu*cRGF~NFt=JtueCh zjlYDl|IBLKri`9%Hj8B3%^P^U4FF5H-a6Gw+<0Vf4LfB=qOOXw3F}d;TMI|SR`!s! z_i*`kjso_K`jOX((YP!-bbFU3vO+_tB$==Ce99{N(17yq>%!Q;Kew z3J6$X`qchxq379aMb({pWQgjq6!&AJ1K(G+J*mV(#SxE=8u70ENh>*KvL2NPBchIF z41)J`dYb`>liX!J=vweK-3{uoBujN{%3yKZK;SL267Op`^Y)FC5 z^XWWwhXj`8ryzA|6|NRZEiL)DFhG6EorB=0Yk#LM#DLGV5=msZtLm&oGZjDi4!;R~ zuA*xy6UqQh7X6RXp8FxYq`Cq#2&kZp1F4J2pK1m`C4&!n-eX)yhdJoqH%K~KNCGk` zlQB^Wo@<$9E*{Udl(>A4iu05UJHPfcGkw29V3MT^;V8g21C>w$u z(aFac5NJ@RPnpOwqT@Rerwk8sa%86g7$Um#X|qw%)3b#|=cZTA9B;$GU~3dSvX}`H zLL5bji-b$?EtR}yibiKq<5yUkYDXSgtPrn0BQB*H2C;;yCa?BJajSS7q*xqM+ke6; zMJs%qFDB4e*5FL50&NbM7u379T5(oASpI z*h(nP6xf`Es~Xh!iKHHl8iz%ma4{<+VSuu;aD5eSvVGKI=|H8|MQQLfq2t}QhhKj4 zwg2hw{C7L2wloB^noTR2M#2VgI4j?eRp1s#ethFIXUVTwSK6boO@NuG_#v6tRMW<# z9JHbFDJi4kV+f9L?#NA>(yi?phIlxvFO{;lfzv*ItCq|onjJd1+1w@K=7gp`vLV<*+C2OCBoEDs6k$KG9Mq^$CIQ24E zK^+i%HK9gj1$ZjMSc!;$Ge9tIS{>}bt;B1pU4g~LkcU^-@KgyXoghJs8PceV5TE{M zyEh#^%%K$A0BEzWKlXlYjB(*}>Z8X``rYeMnqTDBVnyAuo;M+RNv{M~#)K`iI_; z?4<)s3jn7)-kbL9#Yv$VxnQT49A`~P%2yO6=U2=+A|cDH#28v}O$-tf^2ZY<*W@$| z;zFa_PP%EDun<{^G69UCXa;lo(#ONN=y}ht z2J@|JSO15<{WtG=wL(Gigupb^q7#)>7mspVt>~Xdf<2^uhKbEKFv&6Yv36YBhBToV zjq^Q@2Y&ndwR5LW-`_vP^XNp_gdd~~FUQtDx^d^npM3U(YnMPG&a5bdD#Ms+Z0FtM zZDi0I$hBKf_WtCf8wayQ0pCQRqL38QT32}N=+ecrm(QP}oxzlE)EpDTVx80^A(_r! z5=EIE3NfAVjeiZQ-x42J#;RGC~8JaF1nFfe-ePbZPQ+Dk(2pI4`HH5J76rIpJQwg~><@Y{6mMb<@|J5Mir;bF`jVJc7^aKnREW zsd(|Sa2JrFr!31~N!7+GTJ23cO`kcrfyp;@=B1U+X+Ltx=K zxrUNnJUe)B>*M!-^qreGKRtcs+}5U(G0jpR+c@d%MX3o)qvkqF#-}WPmZB#2F5#?0 zjY$7P0|IOF7e6+56h=k3aG2O~adVZ`R&1yrmFKu{O^chy4-TK~KYn!oR_o?BMQ`3a z{$#x9u^04Y7USuY4e{sUQ?&XpD4Y|UWr}#z{Zsih7*@;bML?r7!2F98x3(VYT5qaM zTL*hj?%chlr36usZHw<=1&=>@VlR}pHSg&&Ql>tkHkyIyT{ty}sp?FkT{}Dn4)#vZ zGiT3wgUa6Cu2*($pEJiBw}wv$60OKNkl=_AEW4yetB{~tbhc=y3Q{Q(unY2oIf zl{H0R$AWc4`cMYL^*E_5ar+4OE$cdEL7G-5QBfyt2{7hF7d4|yDaLPOh|X_Yie^1b zdm@Q6=||P^%ZiRp)b}*E`%D&KSGs1n$e)6-SpI@VMX{XDWge)$vc&0b1UQ4rJ1w0F zc_3M=W+H{EzIK!^o=xy7jnf_-%WCZ3`YOCE9RQ9)TeEIKdlLb<(3(N*eN_ol`Caap zgr>icaU#9w(tlw@ByvSVnUlT+LSwRkP`jUOfPlJ~z))pYrmrfMu{P$j;RMDlUtD@+v`F;;GoY`g9yq3;7L3r+2kMl-UN(;qeBP&x7?^TM=mzn^1F~VgA;01 zqc(IQ7MtD~Sji8l^pUoNC6vw$Q(!N%(5Hyz+!xaT28e-DpaK>b+7pj`QbYuy#s5*e zi$aE?NmYhWJe4WMhCC+Sbm;H7bKc`?aiQUrCy$Ho5>Z2mhQbEl2$OW{o5?p7JMm#AXw}$-$i$!E z>*q)Zav`tlI2#;5M?yf;kuOwAnNJ)f*#tU%d0;G01023_*Ri-V^>76}`2H4SQNc87 zC8WbbZS0GOU#y+@^>2Oa|M;7Sp9LPdh=|fgXwgDK&M=a9S3N>S}uKGzT;711v(6IUpfPoN-BJZ*6DLg5YF9AoPWskDgDkuT0e)VJYEwfXyx!P8xpn>O^=ntJ8MW0eb`XnLa-1W(LI)|uOacHt42yl_Hq$Vg z;-67Y9%M318ko_KnGYEkk8h>`RzXQMs}GrVr7N~}BA58$@ZiC{o1c96qsI^LY@gh` ze(egOS#YnH+Y!{x3#wgh0g;hxdjrUhd}N?~lh}hdKSn%#ClaWt86#*ZeUhU4ae#?T z;|dO4OpVj_PJ=K`3b2w@S7et-7Zomz~Y0JU=*ns6cZw2$grFUu@g4$+!iA0H;(Km?YTjFHOPDb zsR?z6Yz9`EiP8bEBLCvW^K<0f(Q*oqCQH12_3E|D7cW14{0MyN>R=Cljy>%AV8mTQv9P1hq!TBHXg|7B>1Z(>kt6F=kpAH_F-xyAWeP}I~6L(38Now-SO6Ioz70CS{)i8$2=S{%xfB3$@&{fgcZ;!wvXsI6!!LZ`J_h#n#%cWRw)WX|}hjI}G~AQC=R)YTlb!e^K&UuH$J zxYSp^5Ue;JzDpTxhIm}NS%MjxDI=gIlaDL(NA;M&Cgzt$JFUJ9I1P zScYeZckbP{9o8Q{C5IG{}19>EOjf6xn58@&%(D5kM zWqC$AT_wvX#ltwH1ec9ups^WED~=2zE-=H8VEc^J%hC$)A)TBRO6yY;^7J}J9bmsJ z^eedGT}mP)tJNFPDYTAUAuVU zwX0X&yLm^pu>%G5lyZc7ZH-x92n&WsSbfzp}0s2p&f*?hY|ap`01e-BimlY zIx1>j-n1lO_^x!>_AlaB_v3z6YWAE|&gZ7H#o7jEnl?x9Z z+~0lTT{|ZUcK68>r~Qxwm|L4Wtj6h~Fk8|y_>pZ+phG5j3@}cg-Zt8EKbhK7a0 zq|LtIH1N$mSHi?zi(PHpJg+-Xbh~a4Zn`UL|(#-o0Jxdc1mI$5yU2i^1A^ zvh||le>OJn-@7kZzi|G%3&Z)4I#B5KH(s}(X!r4xyZ0LGSI~&1nF%&4Fknws4+|4o zouqrSxd}FiUQb{@!{*j0mdvQCsWujoN%L(+N~;HRR7$6e>=t!oU8ZrWB zJN@{OCc7hGMXTylm!|IZgzv8+dY1h$f{DT_32_!$!!jmQw&3{=R{yhwrToeyYYO{- z(=WK=5m*FW@W&I1XTLzIZ{;daO_GHk8oult@ftT_;$##ZiRtORNw++}plVvR6XKo1 zYO28L2nOpH>vIxQGdfXM!SFS$8&&(l3v6 z2~6AS88Da1Gzt0qgf4@Fd{m%Sd{MfJ(7*<>Y48<|ofvC;fyK0{fZaUnK#7LZ=U`m+ z)1!qnkdcP0vE-OYW^ojcX?f*y%OYb9XZ#60ITFUxGM=iRrJ<*7)-fc&UkLH}u=@}0 zk&D{RPd!+>Ut`s22FaktNez2xLGY$#!FOOMzbY}Z>^__9@a3hcfeUr+-^oywK+`wJ#CyQsRdI9V(JR=%XBR}wd zsyd<`VMHhx#4z+Ki~n>ex7pg{o9PRg44gCzS{tq<(6G^t6!sLJR#SAsv~+z`OVOH- zT08cYFTA5q+_>8GCzZMR@bN$T)qnNrgD2~!PHJQ_$3!zfkRBa6SLNrw`sKg-xBkXY zfB6e%OrxjaXD<$)5}G%#LV0}NfsxLqBv~Y+{K{h0B@e`MFtQ<8;wWagPj?eYR=;Cl zV1dymxn(s;YSBIBD6E)f%u_P)ch_yMDlGYtQA=`OO3T6o`R`!L{VXBMjUe{w1pg$i*gYcDUJ-TulK-um6| z{K+1pB{IWgD=U@-1_N{_ZrpqHt>62PUwre86X(xusmf!h13G6=9yW@&qW`tsgBQQ| z-5-4Cy$@&g%Tkp>mmr8Y>w9ti^7*%~U$ufO=Kb`wD5r#x%8BBvynXEdqFkh^ZdeO0u4o&up0=jW3 zXs6%vT+22-Nm=@F-BGLcRR-2>*0n97_6vAv#Wd0=8*U)Xl8`cNXH3}_I)}b)${1z2 znc;vv80$q7pa1gZrsH*<-#@%??kq}g-1zkF#_{V{uh>R4po`lz#Y=ZFnIkDJvKM1j%XuVMkQKkgX!2kgq+w=iH4*=!u^{8 zK2Ozz#Yzy;XE7>GY^2sGk7*FQX4e_y$JZ=s7r{FN|Ka_6&kpvjjnUb2ypORup39n8 zJ%iod#}noqvrhlQ`BO?ieDuK3{;k_LK`^XtM9<+rd>J|%o8w3yv7ZU%e24u6a;VQ6@;)83;i!{TzYu_kv^rFCKiYWLhHRg z2hU!6{f(>Fu34%7!3RG!#DD41{#$RoY0Ms5cW*ztd*`loK2H9-cmLj{iVYcundU;gaN%7%yLYdx#tzaZ@>8lyfCxvj=VCm zpNq~X>cecD(#j;#%*a|DkhOz|o0Nw;4Ig%cksJVtV94yESQ< z+=L#vCn8b>prHl=OD^PAB%8|AT9lKy@l!0zePwdy5mIIip0WSB0b1VXH$0fuF)|h) z`D$TTWVP>jBp(oQFt1O3B%XFCL&AC(-O>m8Itzp%a4q-A!Zv^(|dPTM=N^=^r zqGl>5$|AXp6+(X{=Pwa;ksL;R@xcZ0OGDj~wOnN)u@UybDUE8U*uq-;;dO=ur1$gc zGvIMiU^pI5??gwZ5VPSQ+1t51f3=mms=^KjGvZ7y;CmBD41V;68T81nHS&94ZH zHy}z%mdVGBOW6|MM&k|1xQZuC+1NPXs}*oqr&MG3dG>bgSDP6T-kG$0?ybGIJDlFe zjSj&z56dr2rGD#uKP?RT5xKJRYKBDAB3nB9 ziSiMV*ZZlidr!AdZF{%(>#tpVYN{!-^v9CC)YFk3C!I8GzH<+%=Nu|q9`|?Zl(&kU zJ9DOr%1gd@ao_peM@K4rDoC`68=jGP$lWsQ6qc&9<6;ia<8>(Wh3l0pvZju zqdhG^WcUl<(z~3yt^;&jr)%jehrEP|cjj_TpxhA<>=fx|wX(<)#xK37BJ>jKhB9se zYR5nKGaZj%z~y8~nsNphr?U9qKViv-BgY7il}%4OAd{tRWfpu6vazi22PlA1KGko~ zOsb61CY)S~4>O?D)eZp}dWVg6oK?9$wh1H!1obtw$n{JJFDs2(?wG=_HtG+dTfF-74_$6umfO z{onoJAOFFR-ZKL-%Zc;;k=GBZTDSX)Kl7FU`M>{r@4R;T<-y+br~8c@5@SPZW_zAN zHcTu68J#K`XWVYUND(vhAbq;2C8|uRnZZapthHqcDOquvPTDD9`D7vKtZw?WV#yMB z6Rj+;x((PO`jBNLn1>@mq>DLfqE`7+l|4#ZyFCfad9vE0!W9?)v8$nj{4~ zk(P+S<+codONBp^D(3*`%k7I}V%D#H=?j;4wm-V($iK~$lv2pO66IPyAr|=e-}#fT ze(~ME^z%P=@$@D%jh@K_b}*9)1yIS;=P%6_{OjNOy*rPdwybL&N9KZ65o3^RYd`VU zo3CBDXh>oyKDDtNe59OO%ax>}Tta;HFOvI&(Fc74Sz-@|j-ZSva#wCfm6L%?VC!iR z(#i5!fN@!DDYYcwz$H%hd2n#(j2(xvdm&mguHN3-s7@~n&}7VRfZX~<^T3T9IF(;$ zOgx3gqfznrxcE^(#7D-+Sh+ZelKzJDNX|hNA4jg-xfbu)>-`tz<-D-^DI&!VydA#7hTZilGHkqs*}aBFU5{>RRed;k}xeBd?kb6^r_qMNY1#2vkx;&`1PM@*YK( zhUe`!IC1iY-kJIS{YMWU@=~YIo<*mUh}0>GO|y1J>$KTgamVAHza-ZG=%bGzH@kwu z-M!t$4_{b_dy?+F((BH>J7RRFXKrwVO}`gjn99tE(%Uf`^Uj|?_0HR0@al8T@A>l= ztvq;e|GwD+!8&?1eF70H{V{j%{)1K~2r+CqpvcXwo40S;RjjXa@!}=uEDE}P`;N%s zjo06>$9Ln}`up$ySRX*w0`=DY8y(cou~&Fv_-79u-uvvc8^-u_RIa>ydHdEafq`(} zYl!uRED9or&1YN9%rr9k%w_{2!gMdBD4m&KF)eMiTihx(#qA+8DbnJ3Ix2|cld53?k^ZIklt2EI4AXO{ zOB#7rO}eAnOgPu%iTfU~k96o+-4!)`RhfCJ$y`8cAf)Ec|H5mfv;eGy!WR3JG3yPR zXa2bXFaKS%ofVRU)cGwX+6kvF(F_}w2X)AbjHI^k^}_oY3|!lGj<#`CsBau_{TPR< zCe{m8c2JNooG1y=a2TIbtD}vw`()%<@JwqUVB9FG%SjG{q`OIp<(|M9ylRmDeE>}$ z@lzR0B+p7u=C0=_O(5HM5>C7dCz?s>I%xA0kxM?$W`?Jsf7io{Y*jiyTqK$rn5oyk zN4TTEa0U&P{>nWWF2$q2M=XQBt@ zfV@qeFzP%qW80Cx4sQ_Y6LxYnz$t*Sz@~2QqSr{s-k<GQ+ecTET!;z1K}=Ct`@b0B)21?x!|84hDxU*A4` zhEJ}$GGj!Ds$N7!q8ygHTBsX)=}|H-b+{M>qkcj`WicHTmQeoT^Ya3Ong<>&QK_e> z?miS1bhSufECDKAx3Xzo%Hk)J47UtZp1x-Z5Cf19!nD=X>QA*@N)!y_X<0m^-k3lm zIljVN%&UqHrV2RzrkU?Sw;+s$xX0nR4OmzL&u}CJ@PMJGSa}p*FoY$DgU7=)FM@ga z; zE+Vcsntpcw>975pe|KqT=V!n4&RL-->4hW%Yc?lv!rLAX?MOjXSmd(eCa&%P61jA9wzu$!tbk^lxoR z@lCxo&#sA^dygLOh#$|MJ9+Xn`=8dciS))@214A|YDrjrlWKr#rpYi|-O92e z5$Ca9g${O7@pdTjJkQ9~CHSjyEhSIY&TJ5m!gk#vE{39qZ4*U_{zw#7mAk{+6> zc2cT&jH8$PGpgp~7&`}85uPG3X7R}T0{eQYN#K8&>^%rcz1C?~i0BeBBcLx-?cyGZ!ag9lV2#D~?82XDEz`}D@ApOU%& z;Nr!LPW}P&{Kbnx4_k+INKF0bXpg-#8mQ6#Z`EJ&fN~;4m{q+o|mfG$bpC= z_o&y?D>%ka$yXBhF*PGcr&#Bp8>-Hvv1YmhVwVy=rtqWO_h^ETC?WVv`( zM7ZdY@1FLDdD*4*w9Mc@3|N;wB_q%m_KPFCeVD?Q9EpO zRh_dtFMYJnCepdR1Z`4I9BY|ZbHsXP{#P(ljvRj(Mn*ZUg1)qIu=XVXT9Pj~<%XGeXOWSvA=FwFPU8zDk;L4}FkSe}V0F2g6`u1gn41I^Nm; zk?&GlhW5mfwuikkc3?tOOf^{L$6NxK_$}Vl2A~HiwXOuhwnIj!DY*LF9mrXEOPJc9 zwAi53o|dgEIl5Ep+I2yCV~oQ`SHR-53AR@thv$WriUMlTIBhL}7P0difsmG)DC}G| z3RE!$8*(T~@e)wc_%r}fKM?Q(eKt1)MCQQn7P^=3fAq2CTbh{28D65#(^@Zfm~O6K zyyO(138M#aq(+Pb$(iPWu5wqX7%un=DgGmZln3cNA@5}26XhVu5OoHhHmoH+wW2&? z7oj9%tciL|LOHVe<-xA$tF)(1gvko|H+4c z^5YL0Kjdb6@XCkCWn0*>7k~a2f9~r)@uj1KrzptY2OH9I%bl8MijAeudJ=3&PkJLh zNO*$K02OLa66~zwoH5Zl;j(uts^X+eyL`tY#_*^muq8(yWV z?4eb0M;db#-eIo)NbtP8lJqBh3wJUzmEWhKf<+Epg@u$A}```V+y*B#$EtYx3Smsbp|K`IilfU}}7_qUiRJU$m&4dH0&CNJJ|1^OC&qy~Yi7O>#=?M^u zk>;QaHu?(}i*jZs*WA^Chk)azN`J;e9ch~9FgT)RpTI_)adv^d!Ga*MrZ{-Zk-H~r zArLUJ(C||&7*)s5h+=dV8_O@cAdAx2!1wa;KTEfI(`L=>(@bh_|JkW6N5P!m_2Q@} zkJ@Z->Xays#|A2zTc5BjvQSS}ZG)u>8VHXP*vLHj)#Um_Li9D-RZUIKWtFdroO}x5 z++l%ZBa%a_GywN#!^qNei%8A0p&At%R`@%TR26Ngzw_yy3d7rywt3Pz8^_vbOO`zF zNKwSNvw6jVZm_T=*lnS91!s!va0uiJ(?(25Fd~(rh|I6CJKd0HO1wPb;Lgo)*P6Ej z2Y@#A)||us%-q&F+1@^9wo<(NWbf(v(K9>ZSaN6PZJc;k%O@H?(fUI38*6KNSDH&o z7lI?h4m9Gxi(S{GZ6&=0xH=LJ?9u1h+j}awCoFctCqF-N!g)Zziq&-(q(CkBR=1r< z1isC{e5M{9^^=hHk1WjJ@6zn)ZD)q+8(1179KU?|(#zY%&o`VMgf{ya2yY~F%=8QL z&TMak+3U#?RU7|^PTcQ3VoqF;j()2ElAvj~(@qzC^5Fgh{Taj%u?12)=gx{Byldj2 z2k2>nz>2?fCO^D>lz>|s(M+#Vs*1p(^FRd$);!clYg@dr$b3`g(@}NmrYKv0kzGv= zy3Jn_5T`PY4vIXLvpSGA{VsM_tyL`hLw8DVn0(@>(fc$enXnQ$#14~!s#qohn|+et zlkHEV)SN_+P@REmFpQF_CwLJriSuTkLPelexK_Nd9Xy2;oH~kvu!R#A{Z3x68k2(w zcvO7Fve;M%pG#k(r^J$& zFyNbj7o83#pBGpZQE5N(6@@}3@(S6&l@#RoDJ3EHtnV)8M)JUCit}KE9P~sy8RL{T zhFD1(O=qP%<*W~>kf&;|8c%4H&5t*)SueQ=SZTr_@WnjXi9#ow^p>|}VQD%{{)Q$U z4D0Q6i3KG5hmrYCY+J^NbOZe?VXNF~Z^4Tc6lXq(yU$9NbC(W>)zC3wfv-x**-5s< z|L}zuQ^pYQrsXm{43dEeW^9@g6RNW+ZB3u2Xhilh5@5l`^JfPZl0JO=NS8e;%0BJn zIy88sy<4bBTGeO4$uOyEp)o?!&QP9(EVl?9YCVu7KqflTWK}OI$zFymaFi9qJ|pvj z+Ny;#><53P@^XdAbE#T~K6#JXw#^(1Ka!+8i{0l{M(uUF${fVVoqtg<6CkM)&ISBb zlF&?kWen`qW6M8Gy3B3* z@ADV=Q=}G_qSD1ruMsWnfZsguonFAw}Mj z2goR8d{2OTn2@^y=yMEVPOjC%u>z0gVJL=@0Vv7;1+EQ`U;Wa%UwZ3}JKz1G{#b>V z&+rc(_yQy2Bme%pKlDDM|Lhm+!hZMag&ik>Adthn_jvaQAAbB>fAB}Y|GgjXYT`}x z=TS&@F~n@7HvaJN{PxM8{qmQuoIghd!How0mJaIZZm1qw=UFs%w@6av8mg>Au^MP2 zJLL$^a9%fK<(&s~HkEA4=>umM{be74p)S#4A%EqxLeGfICAiCMOv-TkyoZ3X=E%76Q1^x+l0nb;M?|}e9^on?3qLG zy-7KGI!GFhPBP_C7LU?oLOtJN%NLy_iP0QS9U0=>cYe?EPG^8yup3F!1gp~-L-J9F z?Y`3q5#>MA5_GEH`hI7O3QWC<_~AV}IGo>aUSI1es1;Zn#uo)EV>(@Jn=+W-Z`HNsWiQfWYgNhnoeJN46RR`o}(2$lkx6I)muB{(9 z?o(@S*xt_LPV*$ee4|@FM>@i2mYGJmlAbR-d$-M^;siaMZy`EUCcA}Do;JnOPCkWpwgP6s_R%!6jQa+ z{)basO7`fmTLy z^5oHO0c2Q+&)(VPL@QB^0*H}>u1<}orusGUmB{)V58MCw1v4x?iuMNEVwcl&9TQ|! z#mrmTv0%~k)xaC{cNs=FN4VDpXQGh0-Ij_nl$)!$Ad{IYe zmAe=X1qn(@KFQTW{P&gp?p)9_D3DC7@2r z*j&FX@{iY99->1je95<<9QU($Wy!0LLZ0@8_WZTvzf5fhe1t(>7 zyVwZ14|&-Wf4|QdbxG|B=E7zYh`!bA2J3~qoB>xAV?Z|{r$aDOVrLfoNram6m_&_G zm=X>cCFZ}(jUCL867AAt84%O|@v37O3w;SsdjEiRJ^vzmVP+JRuvsWaC>x`N|Ng^! zPj;WxYwF~I6CLZby}fy^O_)B;4dAC7?8)&7#|6z!1-*X#D%ap}09(hLZ@?GuS9%p} zM*wRm!Xfx$qmkj`Z38|>*6(!}*D6k9#Y38veWT8ZCb5-uDwr+S4i{F&=zNWckd2da zbd_)uw?>5-l(dK_9R)IRjK+bOB1$4a8(8mhXc*&2j^zzxHUgCKl?-H(vXTBg4Ry+_ zrAcFfSJ+R_S0E@+vRa&(hB`3OGa#~r{FClSm23f9MmOk|%bR*;EHP|~lLi?Sj?L`= z1g~o2v$p=pXSaUv;}1I5maL{g&rk;62v1n+z{GdodJAF}8>jVO5>os^z3AiBzbJ7% zgx777HSaj=tw8oPl?}ehyNIoInognP8I>8Ky((AEdIL}P_a8id^3Js@d7gzqPa>Ev z%BQPUAkU}ujU0aJ96gN$s}An4uA0@keDU(*&u&_M#}m_u@*_m}TqybC;@O=yu3jb@ z?sHOyq(xLo|G1T{Vz**0H=|VCC5(h=G%<`oOTm$UCAW`S0WP$cp{9s(mxIJ&!buQy zVpl@@{Glgt!J&^ckKBG@PqkPm=JnC7NDi%%a-@DTjlhUXFw#6wcK-E6<`R@eaF=6~ zLrr9)_#Ufh88$N9DeN4 z-8VbBVWCS8LO4jvO`}#(KR^7V_dovl=G`~0{oxzet~rQFC*S);KfQVLqt9;Z#dVMa z@1#0`u1EmLd@wr}`sLwQ-+ue&e)8QNr}-wXuna3P168zrPTj&>xk?(6%7Ux7;}56p zUz!)9ofeY4a6Fn)|6b`J0Acc>sD=9|3QllSRc(9TY$|2`fq9gFhWxox_bG*DW?8c$ zf*BTK!&sBOmY|7W(BKF~My==)ZZro=6cgGchlP&vMbzBZcrMN%dQs=;>N7&A_GnA6 zCs=WWOX26y3*O!e5a&MI?JXn&BTv zidPDRC@{~^f6~cIEOwOnQt64_W1%dk?UlNY_b(imjY@z7to7uD!TW~8+n8TVn(bRd z0{$E{p+i!i>)_~_b0iHrMb4&yc4vd81}AeGoqC9J01KwKEW2;Q2r+ary|rRVlkQH7}1_#cDj?DW4DS-}V}?y(dpB!oiV_#HrI~)=$_-ZPd)iF?uhaIQK@l zZ-OOUCId)LJG&|Lx6>ERJ7>=q2snoOlrXkUYu?Ul&DG)aCv&*|1-o)}f{Y|;g?CP# zJmF=AFM6Nmg{@OhclU4KxpV!s*DqbXWE+VcdRFF$EgnC5VmzJ`F*;8^y4FiDVUDEE z&gKb|7NmaW>=|mgdGppu0s5J3B06*0nEu&2cW&FgOCI_~`XRgaf;@V3=E*r__xHRJ z<)zNU&W^ZW4`FXqOF@T()n4R{jSoNicyI6Fx%20&_-`cq`0;xWIzwoV|92qDas3%9 z`yDFsa(Ay?+ow)CPTXw1sGq%?F7P^mm%A^G1&Eg@h3Xjq-JxB)x^HB%#&=akDO3&* zT0SI~x|RWQhM87Da5}FBFKxQaIn?0wuXI5Lx~G_;PqCe^ceZRTG9CAzZYtj-+U9dE>WAX8Cso9@4K0YtoV z@5ZBZ*Q!AnOQGX?!AirPP1H~IVJ8kUt3^<> z(vC4+Z39J7<3<=9;_^NzOVgv~Q7NObYp9kS0m)YB@x%xi(vsW4MWjYnA4@k0~HT(EgS6OKEYH)tnnOR3{!tJT8IW;B>63q4%yB=w~w_TjK(WG)R!3G$DC zk^;|d^K|Tv(3DA=npLhxYQx zpZw->^eAOX$N-jW#Hq7pOQyOnYM7F86tnu73 znS2*>!cyJHfMtiCLRWERhN-Cz@anv-{mvi!@r^t8YnD>-e1$PvR#eMrm#eomz4iLF zr6!N06A8hTXF$4P(n%X7aD8|G;O5`_8?`dtRJr=8E&rpd1=kGyd|$ zDI3c2KhK%yzVYMx57nBCFaj1aC5imtF#}rKqpb+>B$1GJ=2n6&<}Z&mL!DYHJ+uJC zutr^+_?%2Ppkp%;kdPYRwYm-sQ~?|K<;jAb4(MYVsD%0JU(C7*f381~#e1mJLojQ^ z3^9yL=QDaV5T-g;L&nW=EHZjj!siMb&}f2r$*y{&(giULL;WIdWV;K(itbn*>U@pQ zK-ub(mWROQd3UnDDxT6$pV;{MpZ@Ca{_*#|{YT&PN~d_!>P(?2Sk={rS&U*=(cPy9 z_kQ%@AHVmpO|FouB5bcFRh7@nCZ+{Bu@J^+RJ?p}_1yNK`^L|{p@&&bj_zpa&&^Us zdTdtEzmeC-X@TlHKjXvcO*eB<0CvU|#8%o{`T+T2GUknPr?s!6tT3#=jrq$~a8G(n z7a2oX8A4#{6=ZMjvBdw`fq3BXrFgT?wBLD(S+GJI&4yS}@G30G5DNbzB0(>k7=0AF`qLt|0NpyJX#1>&caIlVr99vtv?Z{udbn!w36v4>^ zR7d>KbjER=o&aU`sVv(`GrmF!IUVs|k|oi57w#~g?@(I6R!>LZQnX^5e%0-&6obLYKkH}SwDBA>T0sPtP~>t;cT`1F}=Z}_q; z&mIiI)iKZ#Z&{&r@;FmrO5Uye=38&;02pBxYI9TXfAHhiuD^En?3oWg`p_yJV12)M z2*Yu`IH!LSjb%R+P4xF4K76{jjRp?z&aK-P5t&Jl$7+6V)BwMk1YI7z0#;#X|M^{l^%0Ht*E_~s?5oAk`RCLuclv7m%4O%T^Dj1!55 zuwF#P!B`+HzO2z?q@yHP=y_$3+`k$A*V#Bxonrq`bkcT$UIv}H5H2Vd3;Jus5n~-C zo^l}OQt=w6TvlJnA#xjMbK$RV|6!P{Bgw_r2vU;Tl23sVW3k#O z4?j`@FqxKIzE^&dUTbJl$*>uYRmpHGx(cdX|EFxhL-`b;9h@pO?s?I3G*)<+DsMMK z#UjlLPM&avf6&Hj%t@!2ipOYz{O0S6+!bS7}d zIL>#dRJh@X&F761lu}DQE&bpfTF0U%uE-4!d3~V(AOr#a7gQ~PLgy)GgC>)P!kN6fPhuU=Oal4 z9)lptia=jwS~5pP^=wr#e3^U}FuAdhUkNgdaR;=AnXP(1A%W-1v&F<*GYIg5l!d|P z+chW=5r~1&Rqn7Uw)M>qZ{7KgZ+%<1tgT!08TXZor6AbH` zVp}s`ZBq|taSE4BoUxefM&1fXcf<@0U5$AT{8jp3^EGP_@!U4kxOpovTPO)Z*Dy%pC5Iq&)@$?|Ln87y zPoDVlo3H<^zy4QVyL>Ui8VG6TEfF;Wu@IZseA1d;Lcui3hK;g~Cgdn>ZU6B3&D(dg zG15XH2#GlhCKv!T=iM{JWK|x2fDxMNs9-y_`{_&3(PHk?qn`MMw5Pr^fy`;uTtq`z z2WmOT?y?*ihU-=8a^!_I|KD_^EeeI+yQG?aRqMTw; zE2O9yMs@cD{N6?$q?*ot=BE12oazV%( z8nB^flX1*EkI`qMmXU1@Tb#)&cT|Z^L~hRJQLSfybd=$lA0R0r5v2XuJ2|8a?j^4> zv>}*uo<3U!{uh;MnsoJm%|SUuRAXs%m_x(3v}47V9T|CS``kr6M+bx2?%!&NhFuQ~ z8d$Dx^ML6W8@^CNqyPXw07*naR13%tPe!1{spw7^n4q;;{DXIEjSif!x!%iGx$i?V za$elYEiWMUs>w6wcJwhaV6UYk9nGi}Q{wFB{JG#h?Z;xL6`y+eP>_3Iv6YroUQ{Yr z4}ldQ!G26rs^J%i?_9WW_x9aK4<8AmS>#x70|O#^0iDp<5WY!tZ{RVGZkhh=+qb^| z_|esCS2@F5w{Bg$c!6pVe&Nz31CbBfC(u!$UX<7z{^KXN@7^(%f9bMAJ}=(*?B?C? ze)r1dJFma-rY-^Ey+gnpzOjGfBQ_z3`)}O5-r@YTN%br$^xdFi~5==l3xbY=s2jmTV zxku+Nn0T>+Gl7O;6HnGm$PaQVUE+#b6$!6C`OvDlU*t@i61V{Oo&8IU#CzVK{Zsl!)l`H$htxjt{BHqFTEJhfX1-Dp5|g?tC+;W zuq40wmdE4CEU#QCVQOzBHe2r}Sr-bCmIdWJPLF)jXhr??t^+z`m)BiMl^VL{+a2yACxwPfrYRGsQh_TwhpMRgo-8E}%hx9<7Az_s z5x(-D{`0(TA|YJ8P^;g9JP#Pg^XEKRqwWC_0COlS7T|(xU=OT?h00YNGSM)FhZb#c z#4g1JhVF$2l})BuNSX4lg`z31F$M!Bx$&{Q<|WFZVb;9zXdP*M=FBt5to*YLpj~<` zpEP&3u6^QEG#ilBoY}!sb>}YAjCdDT`>5s(C}!|E{RClp+x+&h2amWD(f!1wW~Pb+ z8`AodI!ap!OMa^$xs%ptO)W@Dij}PoE|kjp-H;cIo`RQ2mnV%d@X4H}!;yVpdx{RA zwEIf^qgWZq*D1vyibvyC5_F|^&W>U}2IpYwlHF&TBjUlOyOU0#N3iw-2_TL~^St<5 zipNnOLxCX$Y)o>g748e>#=aD*0)?%x4L;m&Z0;Vu_$R;qo8SBJ)Af^0Kv!%58);(( zKp#Ed**^94ul=M=m%4q`mr?&91*ugmV-SThA$^5vbb`A~dbAniFab%heCm_S*jT@Bh&+|K-2<)wizKU~|x8h1%jwoPDpaKRGyb zV$AP+=TBbtUYkOfj3*%aEY8N>vbwfj6w_^RRD0z5nqi372F>eic=l;|` z_?O>w8VJv!x@5t!sU8V&@W#%<0v;l37@Ma7M@x{qNYecV|K+ovcD~64j+2`@09H&3OfB->eYW=W zS-Z5zaA6ap^ag~BzVX*B9&=&H8KYZRNhoubF!Z8bHtn zNbTn4x-r->ph5sLYWA(gg&Ega?Z_?q6Bo8;NI`p64?Vl|iK@nz`iz*`o)n0HRIX_T zp?CY93yw7)cHGXQ8r=Z;_vtLX+*9Lj;uX=J>2|j^ve_5wnUEdH9T}t+3LuZ3LMwXI zIO!p*Yr;;QDH=r*+iRfS@R#@XwrP#iF?Qc7ULzKcnKY>Ha032;S*#(-yOUcuq?$N= zlmimhtMA00+4RnPqT);S5b9to633hAhzqArKi#+R-p<-R%0j2<0<#V>n|Zdj=~3%# z9&evLWB$U{;WOvXdBvtCa-+2-b&he}hUH^ke^5Wcf>m~B=*@m@D`!6Xp~EO;?DIaPO<)LuU-H6lTUv1-g|fN-@kJ8 zs-WCyo?3Z*LZg9p09dYn`_A2uKl=F0IcveT0kPru!Q)5U54X?TEV6U{>b2|kt9zvHKc^Kf1de2HN8r#s9fl5 zh%QInBR{ed^3Y%m3O{&^X9ik>o^u5lDAB!Gt1b<|N*0e1f$K!*Cl{f;YstY7puuhZ zeKLS%9M&hkcJ~6-$Y;*ez2;x;KkoP23FV#%5`4UU&flM<1WEsx0OZ9NxO=WNiGE{8G22G?LreZYt; zhZhs~twG##z99_x-Ow7uh zZE3iBzdcX9G4G(O5{0#Cnh%z5eQ)D2;Q|vl>m6r_21NQ6d0tOZv~VMf3=$oEQ3o+5 zrf?KD3{y4<3q##YNFRjOE2TOh*~3_x5TeI%kHKV@M5c=JHiDp$kF?m9Gokf27lrwY zoSapkPoX36NzYz=7+JdNmP5&(C=jGoo<-#5(<~hANk6EW6^fyjG~Y2p2)>3PU&CTI z=^7ne#>{@C`~XcYh*E@CwI^q>ZH$y!r7yW8tk*Ale0cN^fBiRqa-Xw)ewixjlF;tP1d}4=cCJ05{)xB#8zW{qz{#KmSt?KD zn>_yR4}bUv-}}LFgQ1%zNizFLw=h4qHs}lQynFuK&T&1oYIoE4XT3Zqaw7o-F!B>F z{aJMMbpJ;me)QAtzBL|@07H~8Pn@{_=*h4A+HZV(>vpI3ku3tMH~9gcYNT<@o@K@$ z-Y1<9EO?3)+O#+ck(Q2pmhZV7;YuMhe@jENkTrzjiz+K2BR-OAf5or;bKPWKpj{6C zlRGsfl&BjPNx@HkU&ULjT%JJmDGLG?(9z*$d6?K2s$(4?K8ucm7(>fXMJqAHbg?84 z)VUnrqF;UN<;Cr-zy25g%*_Y)fAin{2Pc{FHPjz#Q>?G923bU>q6J`5>dt*0H0)Z5 zrKFS>&tAWH?yvs&U;64hZ#!}&|5D!nB(eY}yJ2TU=C*_&^`*XIw-0k4nG~QTFk{@Z z4YM99GjPBUz}yow^;9-%Uce^9vXs4?Bhv`(qPl{~48B%e{JrLIv9NQ4bSX9JDh{tADtAj*|J&vspC=ySI~=|u{S{-mR;j4;SEyc^t=b; z1TyaZ)Xa(M(bqGtJ*@*04W2Lm1^SwlWYJDH(;}^nH-scwQKFU2&ukRd+H`QLp?@(n z;HNgNF3cyijnzxI>|V10&nr0h9d2`k;uD*vG)n!n@Taz;!6`%b`+MFV0Dc1P)gOiM zol9xuNfA*WZv|hX`I_iywzPVv8Y#4vnp=?#=T`CW@9%2Ejd<6%r->wwXk8AY=GNiV zMbP2cx7_~}IXKhv_(@t^Gn?)_{fk$wC?n=I_HV_G5Lsr2?U)p>J-~!JVQO^gg`90C z7H8kOd6S`t+N~NA@mrPDyCp2L-`cb~i0Zu6>*A#=Kl<^He*FFiHVK@x+((3NZ2$Br zF|ro_=fCj_pM3hshaZ0Y$;Y2)G#)+Lunwo=M+x`Ei=7LXE?l{C_wHS<3(&{F5lS#^ z(!`Oa76JjK6Kfa2De}?Yo>?065$y4F!80>Jizq!qqIFfHStXlt!O66X%4e{@=%~Pr3{h=+9w)enYT`2Ib_UAMJuXtncn&b(jhP$3IFnKa5 zMwOpnbTg)$52ypv#FZ|i5)h-f6NxqmY`$5`5AY48JCEhEw+~s=4Y39QxEteC0QM@3NBQRkh&n^}( z8lXxCg6so3I93Q&LMn&Lpf6MmNs?km{#9Z?e9F0k-YwSr48QXzFDgrE0jP`wExDXna* ztID=5BBvWugMSe|DIqU&t_Jm2B^??|jr9ISGv4c6X1uz{K_0}7l~`&X++C*Vw*i5%1#hf1tFoC|pdwJ@=VwYuiJ$e#wX)V5BK#mAkflV0tn~aP25Nwa%&wS-eJEu?I-#6GcEq@S(C>O^! z>=OO8Z~fLUeB)<->8HPX^z3Oa+*6XbnPhGA$DiH(2mk!nZr*?7lsGgb$J8>O>`;Vs z;+;3&WH**FE9J35Oi{OHBpLZEWBE>B_Q~xZeDk+{$3Z~vTsq4>MO6h{>n9#PefAH3 z?Kl3_x4!M{DjTdI8VRS#iX$j#9>4#Y&CFTI!R!=JBuiLzH16n-^$aF{)8wUN;0Fq{QDN8yW`M8z@0uMJYUl{X${ld=Q{%e25 zF$UlK-QTyA*}`*-$rE_hR3%sy19-Sv{afXFg>sJ0ON#o_gxGk^8Z{^>vSGhcC_ zQzNhg5XdRN$WEHOg@H+l0fc4@)o3@bl)$sle|4&4!mch)D?f57soJ}guSR7=3QJy? z4`y;mwP4HOiYqB+DX#MnMJ+9iGwHgqZmj+4#fw%CL{hDIK5IoG&|n$oTAHxuNrcA9;RN5ajTtnP7ew3r(ob2nJUz(lwl}T`;)Q zR`^Dz3i!xJDBS34V+#c{p_vctF&mGi`d(C;Va*6-zM)wVt)6-{v5hGWG^Tt@nwk`e z_slrzj*K>9zZks%U@wTU0LPd#6h@iZ4-air7>8D>2&P5mVd&t8wJlv1hu8JWOcTxC z0PB2tOLT}U!gMN{VCYPZIshup3}xwAJhyjNp3R+e$r7FRtrpC;zjkTDsEnoQ!`SJ& zb^N46CINzwo>QA1#}m^6CQNJqhCq40S9WaQ?vr9-`g3x>6^nc8z3Um%=D-Eh+uLVN zu;?^wpYcYoa}M^Q5C^mF-tG+~XLok=L#*Jr@!1UrgkHIHS@V(b3z7Fk#3|0JA?(jZ z%*m6619cdi{eBJ}}QD_m~M1bG_!4FJQcn>3qTG4MO_#W7I@7R{`+G}rIy?Xuq4?pKWLZ;GdTrN}X!$fkW;CwU^1dW}0ZM04Cyye8uI_Bf#0JO>Z#|S)3X+sA8TsUcZhRms| z%1JI@&NvS1T43nmyNYhVT}|PM;swpTU~ilq3z_^d%9P^>)zCJ`vX%%Lh9joOT2>IO ze_nukJkl822Y^;rNP7SA)63`1>u4|>P#B0=8m{x!{ri9Tz3>0#zy0m+d(GrVGu`#R zKpti@iV*}%h~gSwzjo;te&(kPlL>uC2e3`-l09W7wb8w5cLXRcsYC~m)Yg9|GGmrI z3sHqogyB)kD)*GbZ55Qqa!kA5-~P@Ik8g>t{4DgP>twRI_37Qm|I6S1zc>HKzxVYo zziS2=op{uqfB4zm|MegIB}d7Fbgx#vLuHP48+u+G$CD6Iy%Wn69kkn z4S(sP_MwEXk)0VtF}4=Xm)F%G?Q1d7MUz2H%~L+x3=G1-(6#jvxG!G6{NMgpf7Q)5 zfAS#jk_rr&OSiVP4XM4hIrSGo?W|e?w9}bKl7LW)Hkl2+deUC zBj8NIvIQ_Im0yTOLqrC5U`N@sR{+XfwtzEc!VhC~H~SRx(z&MFMdOz8h->dv10+OEUCW9|XyMq>sp?!?Pcq)3qz zMTw-XAsTF1vaG0LJ09Y4T&cV{e@R~RkSC`q4;d=us@P6habiby$tss653(dnq{QJ~ z-U$~#W9kMPXbky$*ZG0+>HhtG=j>rkd+oK?9*@ESkgNt`B!sDTshQE#Cw2}fcRi-Sg=|65+4xrql ztVO47BEd`o2R~UpHKV9|K7Mdx zO;c^neFM1G$#qA$`;YeaDf?r`WVGV^>HdTJWa5N=XBn@}du>6}W}2e_6T1C}4~&kX zuwB2+x1X}*cg8`E?(W_-x@|kJ0dRp1VC6`g2O(!-QxaAXz5V^~lbtn0+K3D1cBq;L z*EB+PTBqmPGi7AB-4Q(s$9CLjJI}4*sbh5X==<;Axc&RTfA!k6S6_YY>ecHu4BWZ1 zYuN7IU9Sb$goQR4%Cm*{>VpR#y#Il2lNMeduO;~AE->}@@x6Nw?BTj{`HJ4CQ+m{5 z4HgufcU>2|d3D3q*3yGktJ#Gt+1*{e6qBeF+Y%sMMz1e;oMw`QIW=P(_yeWXn`be?lQ{~3QRjko7^XB@^5 z)E226N~*+xS$YxK396`q@TT(A>%vg0dPOvMEiN_Tx_a{QKw^gyM@=*1)tYIgBN~g` zoccUzLY;)_eOH#gHmE*c1Oueh3h9jh8kYn?l*yqgT&;|#yz+Sqm{tpagG5q*2jR#m z!Bn*qaQXf9EnuGVbp|{F`}OG>7p;hJPu4<}N>5ABVixPV9~l8F5!?N6LMxv<(B-74?B>ycsu@ z?=-X(q-LvxuDa$+Tz`8cWYA^~foNrW)jnh9&cUcoBLbW^U%P(g!i8~h zCpv7;U%hr^%i$G|?7FW-shr|4Hif)6vG(Zj`TzY}zpcOf^&kDx*)v<;{@(Zg)xZ6B zfAseI&yJrW#{A`+E{G}zcR(w(KpZ=6aq5S+ZohWg`NM_(?ZN`wIwP0#Z zG-(mnrlL{GB@Cu7L{Vp#>|;r$+y`YOhdiPu@1kO8NSlbRK6%Op$sCEOW~D_VQ8^dU z4ig!h_$eyns0P^;@JPY225nLNigAewf22ldpYlXhR(1H6)boU8zAzJQ(OkCZ3X9#k zZ(Y9lSAX&6c`EI&FMs~ZScpynrXK$P-nn>aj$cdrq5cJIIu&(`Lwd%GAvi9mI7GCqpq2D^Y_3nG` zzw`a~P{!#u6lWiU{k7Hy8OX? z^ZKi=$s+3ct_@K493iTx47zV^Ju%+l1XzQBE>i%~V9g7{D2!dphx^{!b^qdpOINO3 zId<$L#|6?+i^LG`RHLNS@`cXxkl@aF0^^Xc%t46Z55%@Hg2lQu49!MRZO(S6EQZw%hO-$+@;1Cr<%|eWv^ygU392> zmhY0%WvyHcOO>8q;fN`>T$V15yu1m5XD#DrHo%ZKrzKy+5MoP7k(zIlo{>7X=tAYe1&n)0Wpvkz=Jtjqo6$F*wwg^ z*f3p+p*w&Ju$K^;!dhG7yeWG{7=?haGPG9-*SkJudegj)n;61d^{*Q$jX9OUxUazu z#YgNJc{}8%R}nADK1#-v^LsO~R;x#55y**hTmd?+hv$$hZB$E~fTv8d^|`%e8p#tS zp;F2=Go4}{86&HE9369V{mFyQ0!;!sH)PGg7&bB5AUG@)iX(kdt=CC9>qMAxZH0fq_Wgh^D1X^Hh4@C{{5?y;4yAV(s+I zseoTVg_bHN4H5!k)B~7Nl;EWgD;z+otUabq1^TLAVfe|A_#8jOor@-gT@opvK$nx1 z)keF%h36*(G6Dr?gbYrov{*nBL8S)b{i(FhQ_2da9z)KOA_WeJ&0f7C zG7#60qM(PUBgZx@UHeym@U7qd)^~N@G@J$oj~P%@_!1{Z5-D3hzH!q+Y%R@6=hwb} z^R@*N$2k>ZjV7jo9b(HGs5LZLSdy9#$#Hv*Q?A6xz&6OTK%XB~GhGA17U>D8#MAdd zE<~^@hygP1DW%R-)dryqX}wX%Q8C4hh0z>Lkw`m`$%Z`WCui$*T;IAjye+xGLnuNxia*+rRRYKXGyU%)j_I zzx($4x4dwunX{~w6lu`N8EH|lP!`zA&3XUNx7OA^|Ebr1=EuJD#ZP|h!Wj-lLYK7^ z(xsns<$TLefJ?bA_9Z}(Rr1LeilHnkh5e_moImrWPriA7zoFKtk)3sr@p*B0d1vd| z#T|ZO_D8O1p8-<4N67~>(PClrnef4^aJ;?PNCnNPRP0XiIx{Oh9bJFq;K{+A`**Xl zRw(U z$*0V)qv+^$#cC1u#dEy}!18f#gK~PM(MWc9eeF%&HVrc^F>O%wy%LL~mo+2;6o61g zVKkDd%(<=}MbxCvKK16D*z-z^x(^vefmYsIP@*@1^{$5Qb!ZLBys+)HcC@wi8&1Ro z-&7AtPk+FaV57y5WW9;IDQEun{=xpehx>H^aCq|B`Uxv}4E{fyNd>ihlfSh_(iWoV z0H7L5jW%x|XsuTw=?FFAQgk_2+T`av`@MU+x88ej|L&86&BKG!sJMCO=I*VV#BgTQ z3fQv`pB=e#`yk_sLG;x8HrwOViyO zXx9h3b!XQ&y`IqfH*Pw*hnN5G@u8jL0P1ph``Fg$ZF}{mYw`T>(L);p^bqdezt2I} zSRHwB-}0gh7cS@t?B2Qkd%ySZ==Hgs^SVx&d%erMckbP}clR_UB)=1#9|{!V)3Eh* zi&l1YMMShsZGO= z=5ekzJIr!0iF$@KP3@@m>%cpO$@HC7g2tJ^rESVa{h+TtbkS<%srGJZOli~<2FF*$ zU3CGcGLlx_sZXA|lpgg9PyQA^q!)hGYUj>y%#bZt3CEN1h!gcc4r@dsh^wc9>&fFK z^9YVKff@*tQbm87W0RTAZXj+L0jN$!OVFoDlNl;)O()ex>!86! zC(gN|EYTCBnqShCAmXba-PCH3iEHQ&Slnq{4AII~i5BTaT!s6ar+5_!f}a540(s^2 zkaa3T^1(My%jjBoD8>Ax882xPpac@vs(Z77JS60~&4gGDaB1*RZ;uWcuga4^bOXfL zCBINrsLRw74KI-FpDokZsGANdH-dgEQpHoHXfiBh(Li>)u`JaP9Jy+js9fNnrocW9FZy&t&G`=$|wq>X=TiHn(g~ z%V1x-76E38pqqchDWQ7~|JvJ6n??nF$uS+{bTD!e)0Ltm_1Z|GdE_{%s9bU7fB28^ zDYX(PyQz4I6PHS83ZW#)Ud9B`IDRI(b_g6nq+0ex{tK!wbWJGn>Y(Jeg)DH<23;ga zQmODejeWUR@(jGVibVzH_+=Z0g`ov|Lsgm)%c*z8Uu2A|AVC3jHDSd+7KWvXhk!PA z^me;VXcYgoghUW+^djOSdW4q@hl9w6BAq3JDgr%zZcyuIzwr~FeEqs{u%#)A{>XRr zomo?88H@B8nu%OeyrfS!$kTZLAVJhb^)R=(^Az$#NW{;czV*f{KlRnG{ICD;*INN{ z)H_fmR`mWNIiBXrBPYx}m%7d^9tIAW~ zhP+0&I`42ziI72JOjqyj?f=0y|JVZbvzure;m5;tBd2|kls2uzFqq}4ZgXXZj}c`e zlX(OXS*wABX-En$2J+BWGU3S~-)5)fhTGX6vS!X#2kGGL=KVNZ13V@YOXTb1{fozp-2)gO8N z+U4K={crw-0m2Zfl z6)G(PGBC&_Q6>d9eNyj1YRGeG4kS(W7+ za_Y1{dAfV&1})YJ(V=;M`0SXqNT%119y6zK;`T0)R@?@U*Vgat?cINHWBcqj!E^tO z+}m&S?z`_0@1=`ZUVH6z^Ahj8^ZmEq`QFBvO^v%&oXftuds|O{^vDTql)(W)Rsc~K z*2;NMHuY(n2ytEuuzTjTKEO4{Dc!!cYv~b&EYRQB-fj_rV&k75J~6pqengRYfx$i^ z8aP{CZhg<&CmqkA3g`}*0kg!7%z4ABeZC~ld(uPhy*$l%lB-U)0V5RD6=TPbO4D6q zG5JZSi4hVit{U3uhm+{eJRx>E@RV~L`sz}klXguL^_BRzWnuKc-1t*+z7s~~qq;Zsf2zBD6?7!l zDl{d6>|j()mQQvg(IhIk2qn)b&_arHBY!?8;wR1g0J#vIg_xxC7#$Ok7zsom7cjkabAnG_U~jjrmD-fl@bsTnZ?75{yzPYmI<- zMFD^Gv7ECe#ABLbXh?@pdSIi{X9}w4*zZqJ#7&anJI+c~HsQ6yg)b>4#j+^PvWYhQjhLg{&B(d1;Uj)DECJBtja4CuTI`6*esiCa%M*4>Ct&lNqz52m3cE;A;>j8zF0CNf4eBRaTB zAtJ^Te%G{nz_=Q&(C9xs_~Pe2^XGo*8%D!|B^gP|=06aK&3&wLP6N9%31U$=gq0&q zgRfE^R{9Q!h{2mU$0q_Xm0M^bo97NS|D~V#$>08W|KZyo-aN5!CcXxo8%Rp+F4v3M z?B!1a`9PA8E0Pc)5G^Y_&!3)r@$4tR`juBNUDQxO6EIp$I(f-Qe(Vch`1ODBTi^TW z_VJA~v_b5!A<2?3`eL?|3ganT76i9YJahbq6BlVSoa2vV#68II==sy{y!-C^H*d}R zr(%8VXQjQ;@ZmZ_fg-+0nqwCyJW9o7!?6}RAutlS#G__Q61>7zd=|e9E_JFR26!d& zVaP*s$9{55rw|7y==Bv9NVTb6XSAf8fD-t$5JOH_5}^wWMG^evZ$)9HISJT;q1i(k zoV@Gv$1C-NcL2_hgn|<}do0bS1K7`OocZJ%AG?0#tDpPSAAINS?|%P%BZYS#>^IFg z4KqHGXIza(b$08FcQ$|WjaNVQ=Bsa9yK?c&1~1%b8500z((@BjmF5tq7^@m1w&)%F z7*TLNcGqX3+*W9fLYT7LIlXbtG31ah0vLfv$Ip`}XIy$2Yi7$Hlng1z^BluN$V5i~ zbPFMgQnAccPE)Du>gL(4?dsq$heg_G`@}hyUN~Me-p<%D&|~_z04+Axz`Ish`gut{ z*0Z^B2Azv==7D%7-8`Wep}-E^nxjmb=|{%Tp$=QyO#GvV53KG4g90$#j(WUw=?Wp+ z9CoR?m)W+$@j{Cd~ z;IZ|iC#FaBl+5s?KL$DLpg^QNeyY~k$Q@VMVxDB!uG|f)$}B^=`O!yQ`HL4X{m>76 z=KJq{-@yO=-W?kZUVZgd`*(Nm+;tELcZKG=L8mfW&ZxD=SO-jh#Ni(_(bbtNf1T+{fI}19K>k91YqDKPuJZ zd9#=5rPiC}7=x2dBuuNTD&$m185jj<>h-|Td2VhRtk!wZ-_nuMWuoQ>jCD%Z=y*1} zi86IX;_PN43MqRvAbT}&idM=RmPxnRkEuCIUAo0Mv&&EhL!U&J>;P4&Me;o>e~bb5 z0^r6Fry|+LFZYWx7fk*?(=){|K5m_4v5mA`HW|E_+(1}qS-C!ll4z}7g z6Z)ztN)9!VRSYM*>ISva0P(gBNFu4Gu+EAGG5h1i6NdHWW+CD*z^`#S+z1 zGK{n(^@v4++6VWN^)tz(T0%0O6T0dKRDw&KhA0DBOA!vqmhkS&1CD_C_FMKMg8qZg zl9KJf;`nz{bPJe;m87J6ix0#AX3cOi$ip5(6X`odljdcTYW-qNm(0;Q#G+)dr(1zQ z6mJZixqv0^MB{G8|0jhn$fidIsNzK#(@P-A7zd~w#Z+qEEkv=SbmTwDo(i;&qM^e}B^11;=P^LHQYTaReM z&ZMDLJG^$H@kT`*09;X{b35k@PIL^Y8p}W$E0_hU`U2?FfU)#;iV*|Fqop`UPaPo| z4PWlUf_~f)Kx45WoJVSG`E?huu^(T4lo{D+*c)Uu=d!>pZ5gDhP)5VE7Ii^lZ!Q1= z;d_#o8Z5?c0#%Ax>{|<_OEgUS$9oR0>Xe}uE^36A z$}iRSW2$%rRe`z~_X_m+Q%jD%`o+(C$IXBI?%TbnvtzC*r_lyx{?whlhwr@q;TJ#s z$!u}@xx|OUH9z1b#_|nPs5w9ahtSkwup+6jY))@Du9X214uVsR4Gehss~{Q3X#g(x z08Gls46L9SQ+6_l@|)TwD#ThG=kc4No+1no7*lkMJW?Mj$37M{>NGji;|8CzH02D4sg%=cnHHMoshLd667s+P5DbWJqui<6P`3FaLFNn)vBH|>s_U< zD%Vn|?@TVg3QsaxA}B$vfbrd@E#z}g5HtLj0p-*-3bQBy;F0{&d0BDcr)#KE*a#i! zt6fqntfC^h$I&BFGzGhMg17o`f1k66x+Xir;ghGu!AaMGerS>q6&f` z*~d5D`~IW-dzn&xQl`LOIHo*kd> z%u%#CV%vyb1u+25QXssp>-@!I-pr-^Ha6BB@tKI7 zq8O_x9~;uUvoS+O-Q8 zFW$X-*O4E_?LPk2C(how<+THfpi;2M@A|JKIaxnx7NxcSjgi(7c>cK67WH2Vs~}aj zD@}oHnndDqRgk`^o#3=qSjwuhXT?U3i$OuPL{upzN)MKy;a-cc=F)N>Ui6Hn%rh|K zAQ^y}E{{~>uPA$@=&A63Olvj=5D9HJLFBa2P_nR0!Zd}xVXa?VkE!+*X^{h8V$mzt zZuP)!rA89P#CvtI7DamfTpOr?fh-{>^Y}%N5NIVgs)8v^ z>8y&9Vi8RgC90`_Xvw~fh_WOvuO#oj=*(uRASr0<&`USsWD=p#cx_f~+)X4Q-dWPV~Da6h&{6m*Dio( zC$05YJ!-v84A1ya20se2t&3w6@?k9!^Uap$MV#F)S5sK!jJ#nr@}%{FxoOiUPc|4y z_VfOu2aQx7w@R{;WVA_k+^n6nf4^hcY{s%Y&p?N#=CFDRMgx+P=|Zr2Y`Jj<%@xFv zTTDYmIB9fp@MB18HY-FA9RWwiRVx3{7Fk^jXH)-biWAR(YZ*+#?ISQ1sNEA{XMA%s zTfM}nnvi`Mj=_~1o?uMLz)}DTM;|6s_gw`T?CJd=i>4x{tOqG%t+&U*wDoxk5j|3^ zN|s|`VF@4jLJ>T=i#AEHI0Z8GWU{cks_0T7V+!LH@v5}qWGLxpP=eD}cwr$on*ROWU42dh_z7|LVW^)h~SNlgFH+lI56WqY@A=sDw4h%_Qh;x`YqvJfTnp z*e`7SWK%w4a2G$TnwL&5e4|RDgdM>vqWzHD)5rhWPyE<7|M)xq*Khnw(WXwp9Wy6q zY^RZRs^uCX3B>=3D{!YYgdIM5?b6O){pDZ0e(C)4$B$mhedrtRRXU#H&;QwP{K2=r z{agRx_Z_fjt)|RLawX4rgbMaWYasRC5X#}>uYLI|fBUce+OPl9fBvmM`Sy`#-ey`( zl2*nTR$4r5Tk(W01puTH&`F+EM6E}vjK+tcOi85%k!)2O+tJ!>J51tdRm>?!N{h9Y z#H1VTBgxV-Jy5E^QfF_mhF)?~`DBbUx> zUOj(KzG~YPOI#;_G~ngcefXy8kj)S`noy#J(rzY?$7l%oDx@TWJdsAk4cyUkp$JKh zNKd-NsL9k`Zl1cZdFqXeJBM$*!6?kYnCis>Ss!diB|hqv_4c?3vM@&eN(pSdEtD_H z5{(eA_(_|JMQfeN3SoJkEbdo%{d$-YpOm~sVb1MXY5HUe;*x!CoONGVrV zCRY9D#bgp{x%wJ&o@wF>v{0SnXk^p`YNI19oUp$8(VVKh8ZOb9LmICEV0lPou;RMf zH5kwl&O|=EM#!){MO17t3xOp}9`PIfe%*1LEdHTR2mAYgQXPhxF+Y9A=9-iCao+mq zhL@4OcGWz<1qq}?OFd<-hP9Teuw@zY=z%{sio&jdI*nYylej^DtJ}HbZMXG8Z@bPM zR4`I99l#eq`E>n=4XjEZ_1UowIWYKdzf#8 zh?b*z_*e(Snpe&8%@5!I?zjKYD*w|Pml|b0cEqxuhkFk+KR6Eai>Jo>@7>w8OTepg z&+TmM=jhC@cWm(s-Z^i5_INz!!kYe|7+f+XU;vF8u)thgYGCR^u1DVZ_UN&B0!+EN z{^~2QzPfF-dtFFnNRv7Wn|ZKyfM(KfM0UK9%FTE?9Ks77wBzZ_^XCTUmr{NIgZKXA zPyXb>rHkixcDV9p5^mnQ1)bLpkhX=uOonw;=e7yJe4zjUKmbWZK~%S2x%!HChgqqw zXDG@GhHvlgn#XwU_17<6zD)Ao5Nt(|NdadiRZou|b$H=1py?w)*2=wybhKSW|K|Bip&?e~!`A0a{soszzwRvPNRkH|eIXgx^(wsH#iG zjWw)iE{#CnGUStv5hzhH#0+DW!gr9zt6{|i;7`LsLmYTShgU9P6*sgnG-OG)Gi!Nj zAVQ}Vw$e$jWeC6Gmy4ZC6;9DH)0ubJ;K|upJPi_AsO+*ADGwV3% zlW~YbnN-~oU91U7ik>x)r7hY|KGBfUQ#pq!N!?Ubg~>lzrumAwL|Qt(0q#3ogJ>bT zSXYq?#JR8Y*khr{)D;={fm~h7O^S7*4p-w^h9tD8rLOGAZl-K&(sZa>!;|f#Jjyh_ zrs>4#xJ;G0j5LAKqCplA^7w1VQ7rk4{ez06goA17Cv_5y9dxefO~WHQ2??6}T*GQZ z@+pKI3olz7q^`=m0}%A)y}eI&1Hp=5Q4_B855 zlm`SRFUP~V7PY(3_%T7|YOPziu-PCVnpP!LRn0G0bRx>#r3@oys9S>>+i}Vo5K9b7 zm#hj+9O-a^fQ8tXPRJw+>r$dolY+&3OhdaUxFqT*2E@VVsUlel-tQu>spSGmjNOXB zC1!zb^r})BI%^-3Q928dl;DbqtMcOoNl+FMCz94r@>ilIqf=5VS)M6P7x{3-nH^^<@7SATi;-o1bEJHO|| z(hfh1HF^4|B1&I|q#v$9iC{kCCZ5v7rGC;*iGw;Pc*zI zhggEg?npm5M)tF3=TEO+JnbmMhP3!o%WpSFp1t+@Rm%(rVUQ&k1k@T}TJ~edf9k7W z{@9zZscov?tkN)S+G=)-XuZkl^3JxlddysK=Dr?s2CCyVlEJDOX3(%sq$6i>|Zxmsve$tD>Tr@_*qJfb~ zl&dSULl?dCyP{XBvZS^yG!{+LOS-_Vk_WDb=zNJv@&+#oLg6Hx2Q_(RHzPg7apI@O z0;QpH1+Z&kQkN15i5s4Karg`FXwxUMNzKi&Ewi=hSt$y1N^m_D@!Q8P7YwN6GW(gKd(`VtiCSo3NeTG8^&$ncwQUjdXY0koh-O7N+eQM*FH7zwC zUCkd(9K)j{(DEJWXzLi?;~Q)19LMc*=hrvSAQvEU@TkH;P{vAt^S|cgpA+8ar9C37 z6Kk88GP}|U)$^l<+c`1oo7<;%E^M4WtGh4~wJxY-+scA|c)geR9uo&yq5H|f!^3vf zKD0MZSN!Dh7khVi-}&CRZrynQKfBU14P;WJC7_{3e`m-nZ?4!%(8$dV! z4_KSM`QkL7Vdxd?YCcj zhF~u#U}*UN(yw~~@JF});~#zV>iq{wZS&0I{oA*kBzyb*o~;1-V744+GG-$N zt*PxA3qo4jzM8H&j|KB}IjfGuVFiV;wa!tcanGnC_rmtWbuVJ5Z`a&Q3(JhyBrHJm zbnHdoYO#&JsND!*YT!}_Ml9*S`lKp_AOqE4i$Cf;*5e0tznVPNi(qE?RzxoKK+LK# zjil?J>L^g6jwxxSLh0JCrOug;ZYHVYJkz(~bI{Ei=9uHc88xN>zGnQIH zA*juujPCfd=^#t-!U3pyO#TVWEFxCY<=jla29uuq*~F=}F&@u+^1x$I?t3`h)0_YQ*Pu67)Nn%RViT%CLYbcqPr+ zqF5+kf>A=H57VPLf7T|BUI$kK)jX?5H#{U-U|j`wzU0yf$ctX0=g9EU=^Ad2c1h8o#FEsf*@Htm?;jg{_S_PmQ7&-Kh%G zel8CTq>Vk*VH9o1t6=#kPLRZ3CFXfPll%u9#7h#J@6L|INauoeK!=8K7+Z-Ui45A_1jN%O>6|%6YEH#Azi@yfM^3@6wI}Y#Y zO@*A#>T6oOWi#0nxbl#E5Gh$WMn2#z^baI7R7k1w+(blN_AuY+WJODiB}9f4rOw5V zji-kS#%tIT z1a#?UYWof!U)$OGYrpb~zwl?j#x2gGN;3?k&d`RV)CE3!^0~KO`@8@3-~6Bd{y+TJ zzxRhm;w;XMi-d|pqrG(qRXep+8@_q+*q6TWLx1Bx|CP_YasBAw!4JLp>M#BDH-6(^ z|Bg9w!?oD*&LK3N+FX16($$Z@`5IpX65_|Y+O@s8{^$Pe*T3+&dlomtp-^)Db2pA# zIA#ydPPk}7SMpbZ|=W$#ett`Z|?MY$OW=blXp-A0VJ%&I9 z5Sj^%+yZ;(UsS#RlmM_FBq=Z_XTD@KBej|EW@L81e3s|mx9U4YH=nVSDBma5cg9jFJHR+>Z{jf zzkT<%MvMhO6Wb=Kj66+tHJH&-D<_QdOY!8?V`IrJ#$ce6ioRj*6l3~1<=#D&i!$y7 zvXVn%p%##u8Q}VJ%B%7_hkDTj| zxyJAeGe3EJ|6i=Lg#Mqb2jdlo+hEjFsJKVne;Febeyl^o5iQW4T z-hS`hgU6Rnuk(r>`C=Mj&lorpVQ$2p27~JwUnM1=dHN3UXy`zZS$MU-|MME54**`_1AyqOe!Wm{KE%<8fVw*Jd$@H1m4y^d^O>a^rM%%30 zU+29I6j{~P6sMj_A)DMlM+QU1uhLa5LNx6y0{n+Qtt7ZiS$l8?y>uBwA?|QJV?SkC zUC?9%GDhsd($rKf`YbJONz% z{-(}`n}l$U_ykGQk2w;}-tEK2U8yy1tkbbz$qnjva_Hj+_T|d;ScM9N>V&C_p(sJ{ zdoxPUj&$-Pa?0cfzmq;FKrY6lct8~Dr0Qz|CQHd>A4x|=;HHQwh91NZ-F(hK45-kx6?h3#_8k$QT1q*t5=BBXA6i!7E4DKKBi&L=l<&yxKHPC6h~iqtFOYu^ z-}ctlM|UhfZ04C=fq@1GW`Hcx&PQXK)wAW zIVcC0WD%N`pDnpIY5tDn9mQ`sb$=Ttl1o*jPjl`DVuZ~x6#ul+OobZ_3@^Ev^; zcav2SPw~gtOrqk1ziMjp`4b~}pZ&xefBBbw@#{bG!)uDIal!Dc7+Kg-=2JE+eRy{G z#ZSI<@xS@o|Ics!)^GmyzkTP%M;5eZ?4)^4X&tA*dp>^t%;~qTU;Fxxe(@V$`jKnr z&+3Ge$GOw%fAyDs;U~WQ#SeFHJFTxCptljNt(`YZe(C%x=g)0go0^U7QWmzz{P~~! zI+~K4#3Pp4Duhw;*5WlfG^0OLRLMr-lz`y8l>0NUzc%F%C$nxWQ6veV()8+;iX&)r zt7QPgbW;t}dR7I{6G;hSDt*s!n-*mnfSM$}M##QOF0{f4&48ewxESQ476Y+7qmF5K zJkY_;ij*Q~oq37fBBK!nB33P0k0AuyH)=)_jNU?qU%;XyI)|7^ACiTvL5Cyo`de5< zMJx^2fu8-3)VS{|U9I_0UA}7EBu~=r{xtndLMpCdsAS60Q=A4Wdy?Z0 z5VKk)dcx;9`eJ0tZedSg;fY|Wk_*8jJE-dz8N^W+4D;#Po?5dy{LMGs6#T(QAKc!( z&4Gw><5`#7I9=UD!6V%8cCE_-x48@>TWW=JKW4>?vwXN`eAyXE1W2dXVXdb5 z(zL(%0X6U8iGxSRk52KTx!)z+ zw|oUqxLmq6bH0vgW`~Dzkhj)0&gck$i~zPl-rU8O_WhPk z_KEuMoZVzjGv(^a^l;!+CLC)s5jbK}jB62r6zl{^obwu#tQ>~G!L z^}?`?wPc}ZWe*Z5c=^k@oimzdqHvryFViZf1N%81JvMC5bW@uY_UIAs3APn^XZx%k z!|4~>TJ{5n|HyJ`&CtG;JW9@BrE_)8U%aRa-MDdsi8l{GQnz+@l?K_KKYz~az#J0$ z-Um0pF(98@A0OVi?<}tu7cXCW^J8zlcjHE{Nt^{gij(6{)Ju)4PePrj%qT9$HLr=j zD>KTO;&T9QK#{)(svuQcYGr3_qbH^Vn#wTdZ#RX0fZsxwClehKQclrifsu@Y{Tj`~i)GM}pdbQ?wxq_a$qG{`Rt z3RFhb&LAh5Ek!J=eJ*ylsMn;?f)XsmasLw6+qL;`Yzv-v-eX5|3Kk8);0z$DAc*Zm z3TbClC^|9C@<#pi&c?DRp~N!LkOU@l5Q6QY=h#eUVuY1#OqTDGPf3-rJTB0{8wi31 zI`Bn@WPzT!0jm&k%Aqou%ppM`V#7PRipYOhs9WKm1m(cYTvGWjczKM7OzAKuG(_TK zC5GVWF_KOk?jKN79RSiwiQu9>jc#PnpFLqW82H9l^iP2(XYQBmX=!K$j$mUq-bK*d ze4dcSQH3^0N5(^FIpcRE7hRD&Qc42DvAC&Bi{R2*xJy|XYS-%@8xv`nK}N;U#G2XO z?Ne);2Tz~gahCAmGly}h;zl54Ga<;uH>7d<_O2|+Auo`*_P!VcFEg;AKogcNnuLIJ zgoKbq>@m%1r$pX)l`M+uC$xqrMX-paoHa(|5w}E!e*V)R|H&W!>gV73*!KD(2Q*RmU5pu1bDZK@ z5O%y-?u1eriRJ7-q1m1ib&7R3bw8xTBPB?_0||xOxlkSfQ_Rl}U%!0r@BHOo{laHI z_3OX!oB#fgzIn$)uc)jJCppt7&!G`N3vFKrb>#tn-+K+zer+@q_AG?0_xcxG% zl^>)PS)kFTl8w+XC`8>)+m0MOdhPu7U;m|F_^Gdc=@0+pyWf82-COtWn%!oQ?P zqmE94TKKLk1u7HOV#IEakL12O+Y`7V6BtkQeN&$H4G9n((SwS(?t^tIf zfP}al9?H{&j6iiuf<5~zcSW^0QkKdwgE*24=900S?qi`M1qpzLz6Sh&9TO>G(w|UF zFRB+6LVP@fGFN=0TLOStfD4&h@Y2s7qGyW9peUss&R5B*dyzMkiCy9f?1P4gR4&qq z>aiko^8=Z95XPvw$F0)$NyIKxx0NE#XxhxHxKh`eMt(U@(}21~ zGD5X4n^{)AvON$jK~-VZBTN0)oQc0}$k)+8+iX78mKx`SHNU;hxgpNee?ZyzWc|wZhm<5 z_16v`uHU(Hn{(D`b+1Be>%xggj~>(?IDC?e_rjphac7D;)W&eE2Hyq*+N?`rFi5|~ zVJ@72?G1y}81kZkJ30z-28w8MzjVtiayFXXZU;PKvC4| zkr%h_+_`$~sy)QsC;a_)KhW%(3ed~)qU8^7@7~>F{zq54eAQMXFV+qtcJ*a$eO4BFKgjc*hcA_hRN60ZSh-t?TJ-5js9rYtJ`wsn)RotYw$65d;W_LSr$$g zGeq)am|$OsT7*P(rL0R3?C9}W!Op9qP*!({&PA>90w)N~VaqD0e2$nu6Q zDU>#qm@H%W$|qb^(O?i(N>l!5P<<98o}xScg$KJ-ZYF+`qGjyhl5f&LxfJosCMGD^ zjhEyGa>9nOI5C9LdFp1X#vp`9FfXCWQXX&C(WKh2*jRpKjm#sBdG}q@87(az3$E5J zK~0l05gwNn4J+(P1Q{zOmt{mMXo|5BySau`VJkl4u&0nLJVTsEnihblLK$2Djkr}g zb=zf^Ppk2oPjoU`)6$MvbBLRsfN4hqEp;>(fGR-i_SPKOoUyaFcwQVf+8p9W&(U3t zB^-BHhbqOx7jDLlT!cOB0gJ`tU8$hiXA03`$LUB53L3QFrqpgnwQ=qaoq=N_8#wrcj zsEV<6e%OfHvCzXv#he8STU_)O0!wH?w~Vhekm057guA5x69)HPCq6kOb$RRVYe@_P z82s7XIJtA?+^g5Fe)6q1e)!X$eDnIX?TvLWP&wk&4F(*VPKx=N!5YkgsC){|Er%&7 z`_ZiLY@PnXr#`X!;2}e6bc$JH&qT4pT$7EH$A9#5pKbL!NI+LdD$0x-06|`uM5Mp@ zil;QYl7LcS9zNbYaqQ20@rOVE=}&y~yYKw_Km6vmzWbdUAMIMBs{J5eV?W!DCqH-g z`qe8x^w!5d`-!)%U%9-24EytpWh|fqK1CRk1nRG#m8V_#D1svg5BV$~zjE~MEeOLVvOCuWMt%#rteliQIEB#I? zi>4)xk!Zj89>V%cFOz$j`ku!a2*?uXsmGuVgCfc$GP@D>3V(?oiVG=Gorp>4A+NZE zgQUC}{z1IpUEp<39HNuTgaOsu*K{}+Q1=CoCd*THT_lrJ;WReekL;Naa z6-s;Cfl}&1fJ@3F0t&u#uST|UupmbLB9joh3rE~x2A^<9K&CWM0OB(TV+_afviG<#38IC|=Z6&6-9&97}mqaA}h;glMlG>y!y?!a$ zs=8cfYq-zvoO9BPV;*nq-sZMKX+-(;S6{z&t^J_yzVkMl`{t{!TF4>XfN|}{827Af zOWC=?CX&}$v>}Mw*1FP72Wxue)_DJQTpUIgrS;&n7cJjEaf)>EU}r$&*b4&-hHY&Q zKfS$m_M9W=3B0rl!szME zbza(o2loXkdM;agi#M&VKQ$ihLq)?0A91;WJyO4byDv25N)yuk$wKJb_TpRA3_vN9X)3R2lr@Bym<22Q1n zxKymVIhG@MDgxT-;dtM#u{o7icc}58?8ATxfh%kiK)sjN42tXpy{Q&wG{?{^ZWMPS zDM8D6lob!nCCU2&2JsG>&jRPI;VovOmYS zp%_&QydM(*nIpre zqOpLREQrw#dZq(#XU-8*y_q@L^%=NpIAgO6RM^E}#5*Fgm1?O9g7Q=ktu4^WMJI{s z4!z4hP60PiiIeylnTsn%Fux?p;=qHne~`V&o0zGdKeI^!obQ zv)h+1o`2=?#jBStUOazxYolptrr(a+neb*9nt+K;pJW0+!Xb8cYebVA=pkREneh`x zKl%Er|J~pDTgzEu(DS+pjCfFo1P|J)HKhTRY>U%2+BiMR?iRs^GD!=mEb-z;)KXOA zOh3P{vG(;ZeCEeK`zf!WzSUH<(JqS(jyfoPbK}(3>CG)ItueJIXj6^i33zTrlmLdo zngOlqVup#GCZTtb#hFfdMcQg+8xGc-5VTZn#5^}hR&A9bI!7caXE@a0j9lEz^K=sM z3>lyW$BM5J?(&;5kLHQ5F#QN%*kigRC2%d_C9TT(Do9|TLdrC0+ziO&CzbIm-4sZ4 zOvD&|nP&H!l8U0rCcG}?J-@T$Cb$K|8Qb|s64(fptCW)rml&3O+$&c^+_35A?Aaxj$ zMzaS|+!q>(I7zNi>1IjluIH*kw*`)07wkc65kg0YqmRY}gA zz}QIvS_1rA>LWbMXZNm`+@5aR@up_g8d{AB7|_K*>1Rn2hqPICt5&Q$+BAN}zqD%= zXvx!*RB_Uj=ajruJ#entL|txT$z= z+5%Ru;qk*4`@L*<&54`xJ~%k^3eTH&?_0s-B+#ecq-*Y?vqv94xpn8@=?k=&;djEf ze%`G+ckQTSW4F$pef0dud+)q==GMJ4r`x=QBP+DhTZ_KZ*xr+<_&Te~I)JCo34CMo zjBQg2;NUP@-Idz5e(CX3Yjq_)kd*#|_ zIE8Eu&XWjZM0?NVt)(;n0=jxi$*7ALm^OE2V2cArla1Y(fp3&@lzZxk0#Ob01e871 zQ30y~)e)b5Q^nLymKiXoUE`Ug!izO&WUP3P<4dtb0^l2EuGri1o8pey8mLoSm*F~c zsXh+T{1%`V(-DnfkE*Gf?dA=p>JhPdjQv%or=Fs}*l{Lm$Wm@#*@%!!oxfT3z-0^~ zDb=k;>49Y6I&OoNapa)yc9SHytSD;8TE#cS$(h5Hjxj;{pCQeW8FGf1tcM&~8R6j` z>my_+onLn)=^8F3bq@&hDJ%w1=xz!u1HbfkmYhrWa10d5AosA4yMzWNfE8UF_X)Zz z8r$gfJYUkR^1E|SD1}d!|MaH&nv+$m2Qu|Ig%;|hUF<-S^u>wZ>Q?!}H#kEW4SoqP zns8b+G}4^GkdqrL#R-}lme`}u5any&Ko10)G{uRKkOW`IK$uoN5R-Gz1-1M^2zDrx zbj38J2@Oj>0#$ORM@{!_&ptEBa6fSP^TB~hOhXWc0S?~(zzpNYDaSIL8G~h{!l{G@ zxoREO_JkseaeRc8j21t6e+v|PzBTjttWG#EdpwGR7)o3RfHafq5VDdmigHH@Oul3& zH@aoSv?o!N#8j;La6zf}qAtQGk?JaWZH;C_Nb;M^i)0eam0JDfZOZnQ`Zxw0Lg2sql7iY7Tolj)z9~Pn@`L zX6@q6Hf~6z!c0Kb+mhho)X`u3*+297&wbibS9?F3jl#}h{{ zo!ugt|D>FzsxmTqT_li3)!qjVjI=6sOPo_y6$-R1W2ls-sKiD91cSAnLuUEViQTyL zanr<6Dq7|wy4%={7P zBnkt0-T_Qb{y88-a_ch$US`2BmwsCUk_6Qfi$8UU6a{ zwB^?g3MCy4b(bV13gMJfF{9X(NjK$=x)iW{1IwVGhatpIx1}mFCcAYlzaWp)z?#VA zD>fo9Q4`#>y^~>qgLCyEA@mm?FEdAX|C3*R$<)@IwmdQdsy5QjM}L0mxCHQN%R9EC zc{n4W#;A>tl3FB0bl1Ysur&gySm!AMCU)AM2^_BJ$KXPxgo!vVO<|un!HYFUW8nY( z-hE`YZhvFVLUH>?|M;VCX$cJM@0`7G_Uu`9yb@^ao7Jh_jMVeUvH?oL9|ukR@i`sN zkceZ)RFq_^WKdpphg@p5kK2*}$cfEkxsO|(aLRVZlWV6<4REWuunkT^(6Udq&EUMH zQl<+Ca%|w5RX&?s;f7hBKaqPzqD~MUOxEL3oVr_iG_5l-)xc1~XrT`2lH0+<2Or&d z_gjDXdpACKd+p@0D_1TX)W3c6CV$N;cWD`2n_zjwaBJ-NPkL)kyuWb%yn(R?^BUmZ zVaFNe{qLN+eCf*8=`*5EJTfgX*3vZW3xjx1A2=M;X|D$sz&x{h=ICZCza77gG$PP7 zdH22d?EU3rA2bI_SC2VzLqklq7U&w4rw;eqSG@nC^;V-vPEGaR-_x7wm7x>;>BrBW z?j7vj+dtSgd_v3`15Wv&@uz#<7EB9u9y$bR2j^i*Mxx2*U=q&eDA@@hX)4fZErNJ#9wqx6=+?SPE%D^ljxPe_uHXi zCWs21Jh5rqUIFMw**UefwaL$Y{_LQZmS!wfi!E@R$`p7SLaFqQ9OoL2$BlPSkQM!D&USb*gLt{el|ms&XQlA$6uG8=_|C;!ncK6D{SLnou>#@X0aB z)sC@8F;N~+OBg-A7QzeVD7|;NYe(@ zh@teQs+=Sm7-g@7vTuH|nm;YokWkD>YSO+;DNCtkDOd5Mr`-=ZMZWEmb}qVz#Np;ir{1+HL()Z>u8ge?uF(EQiJE0dmJgKQ{_=CT!qQUoM(e`(dU zuL}u73G``AI71tE#iCYPON|z^6IP-rEy)JLknYQaI6xR9Zq`AQZhG&}P55%_slCn1 z9mV|k(4HR4S56tPuzazzNe0yOdjC`zcrh%vdJ*NkCC;g<<55$1c!X4MD|WRdQy$ZC zIu1arM4i8BETXu_I4l`r7SD=Jwg9V^5tldu}S5f85O$=%ogF4n=zDTwjI1yzQH~O7mOF6YPb`^(2Q>D|H2mo07L_2$?>2`-T zGFPNaC6LcqgG2bh1|v+VQovnFWFX6)evFNk0Z6$=iw1a}hMTDwld-fgRgHoP(+GFs za_WW&oTV`0hE^gBru&(Jg?^ECjDgZmN&OeF+{eQ-L?c$=Q3+(zwGmtWo>Htl#`*^q ztV$uty3ci5U{3)IJt2{zvYV(#8I;mXJZ0iJNtj{xTm2!nX@I7ctJ$u%5vPIhlQrso zbVbk)SeXQ%E@d+cv#8O0FSNy(yD2U?`<;M<7K;%UH~!Ew)Dl42KG5NcyCHm_QXmlv z=Dz(GRAmdsLY?3e4PE7OjR7Clmqf6-CQA}?PpHe!LNNjgT45N8ApfOq$T&f0W$lt- z(i_6C$KE8iv1tJX2c2C}g4zXH1B*48bBQToy=We#aoHnB9@?{Hp@l>1(UB1w_nLj> zbLs~$pUiFmVm5`0s2&!^Li~fk$Q}ODv>2J66JCL4 zd9=D^@r#(w7UkJFnr48=U4cU>aK0Zsa$LQaZ}5g4z_Y2V(pzc__P1Hcf)IwgwMngr zw>GB16{L1Tj5aPzoiE91Te#aAA0wWoSnk~Z=zHJ$;}72Zp5+fa7cW^u?+stx1K=!F zy&yz3nse;9ZMNQHU{HVdIj>uR!-F?IcF&PI5B5Yz#ruEd>h*JH&+~R$q%v-lo`F3H zDT6od*zGKk4CZArIx{J5O|TKAV}{KguWva%Vtc;*UcSdi zcJFv0*wt6ABhfrS`zW4zTY^^eC{8T0x9iXLKT^~A#D1>|bwsa?9Vbue!C3oCjj|r~ zsv0UzK@dyUX^Kr~)&gjD6q+)04D~}JYrU^69p*(hnF5r3&eY(+J{GD_X^$F{W(Fyr zK^+`lLN$_7_Ec zKwz$s#59>N(hFEx!`7fz(vhNeD*8xH)vM9X*uhB-*_O&D5rQ_OY26f93$(aqfQ&Lr z9Ke1msc2fl`zg=z8#^KN*cXIi7%P;QN}`&-9)u5sr8yKb=69f$YJn$ZxuqTz5({cq zMD*=HKjUTiKrUD*l~On~F+4OasFPk9Qyh0o_JJC%0xazCoZ+mLgegBgue=jKNiXr_ zz=vi)z$Iz7;JIWQVY*A(CwECf&+Usom2w z5`XLZ)la{6g&W>b)Vw@Zr-EivXcL8|3a^~o`Q)3g*NZY@C1W6}${tcFU=#I%sc15> z!u8kw{LlUJFZ`^vtKqCk9~*?;7pq>rffQSISZl}<{ZvBml?-cl_1w8@moEv$5!v=N zl?r_Sp78DHiwp+l*#$JAsuh7L^yMa+ z-7JF)03}jFDrY)48X`e4WuH%=6H96qrr;nUU92Z3QBn1mM1lV@VbRe@OOt6J2NG^_ zIR|#?4^ku!NNIu?-4aO)ajB&i=flr%g;Bg{&uQpAr$iwZ?~d183bmb!u-O&EoE zISf^c=MS9V`m0>i&`K~O79yi{WGAE>;9@IA7h5@ZtB_aK5{_Z24DbfJ=DgaG zzE!Ak96gZ}`}d zPmCJ_m^O@Pxt&np#d(_evqVNqM96h)8n10yg?a3R4&4YN_g;RsckkA%_iuglzD;db ztP}eEJG)*&yuP_%5q}O+r+#f(W3zYc!5t?fJ$q(;#PS|v+TPms(XHFY&{5C(H$!s$ zmFpMJUzPx4-k)U+k2OCy)CXw1A8HL>$7bk6)uZ#cl&=G|6$S-R_;&J=rp>iU;rVmR zjQF0Gwm1c0%}T7}$8blA8z+uA6v6a-=kFXnda%E@apLIDeB-PC-*5b8&Em;ZckbNP zaPOTwVQr~O43+i~7N?|w@ZpCafO+-G6-~TDJc*uN(0Nmogk=Z9iDS+$svISla;YPx zGorh-XIVf5_a*`_H6vXvgi@f|`Vjn1Z4Nq>FD^&jjDqXpJ%yb%cncUeQz@N2yR&)q z>doCdh8%T8n6#N-s!>sroattg7WWz@>`F_KGP28X*HpN%nGFYKpVIg$TTZ`iw@Lx& z1YU!Z+NE^0Y-3TXHd9R|u(~rENmnURfU2>ACSq!C0knwgY(%efYEK@mHyoYmp6`*c z3c+ZhZk*aP4*#MLZbokK=o6#3x->(g^+l>z8IUp7 zbO<`{0+@~#i7KP#&xIU0d8ji`TL(a0g25sqg8J|1qg({;4MB|EH54%vLy=WliD@9N zD(l#@2$=Z!jXj@{mC(78@?5bfZuv|im6n@xboS<>b!&jOdGdC4qY4RI~ z@|AXByi_$v^gnkXI#!q$O6HsBBQ<{nBEN}Y*sWAbk{D?4Do>F?nM^D03yI1iR_Xnw zcf)8=>e=!%X(m;<24^_jvp`IP5C`1sJ>0Xi*YUlx$LrXJcbGd);r>&`%lpVLUAjPN zjApb9#`=@YD@AfFaaU(Ko42$wa^2KzW{y%40rQcTHt}6Q7_Qh%FYDU-L}tv#o%ExA z9->nC=%kn&6e)E~Wcr1l{Q6ga^h+uo6h=R4hSUl*L7UMyi)wjfb8SwxU4)s~#MSdqp|NCP6%MN4Fb8<(=q7ZoSb$@cVLHzX>R3-CC}$k{9(*fM@*p$yEB9OjY5?_! zvq8FwL+>h)(3eYBqG3KsNs3MvSc!H?EOPWCaPvuE!)nzxI?KXVj;nlJqfs{?kd+-R z%WAX^+UkN9*BEYfJUpW&l5IQWg)2L23D}mQPdKnmV5L5>FwDJ-TzDAT!m`S;{EiW$ zZ7hsZdS=?e(OWs{2=9|QsbJ|tL{8g&8@8Ob znjtnM*NPLgTy?qSCZqSJej3-%DG|X>wzhgbU&&x{t-CO$)?%D^?tE!G2 zYjMuu6NwG~TjZ|qL;0+CcO)nZP-e`lZAqt2X$85hPUn2|QX4$J7fG@Ht)jqPBR@C0%!nsc>}=uI?iTf-3v(SS11u{Gd9-Go!RaQ@lC z4Z$V^Iuv2+%(<;IoyzlQe{19P^(*H;xao|}V~>tKzxm!fpLpZ?kAC4Z*7n$hVe`Tr zTRQIIjyy3}=~AuZ>!+|lyYwub81(5+y!9vFd3)6d;HN)*Zp#8;n5iJEG~UsY&%O2B zwrQH#sjizB&U+PrIU*ej>jGAbjPzm+jn#a?llRRvhSzKW{kD04<%zqEQx=t32W5>D zZ(wKV{F|@6vUmQW^MwQoqMJDdHE7ycX9S^?&#g5-dT4uyPI79bCZvTBiJgmLrFH2p z7ef#)d(p%cyYOman%DQVqnWU3as-b`juLl^h~;0k^-rHXRxe;RZ{$QI@{G|Oe|>x zyQftV7G4rxL)0Rd2%BpKo%qF;u7WTBLd}G(ZVPt>ec^td<888~9Yv3ApWDi2GSt3B zu|HE?(o96k7oI>)qqGn{bd)>}hK_W}RnVaJh1P|%g&I(TS@Olq_vpS1#ihRe3R|eJ z&`Mv+PyvL$G-ljJQj>3y*W{}hf=vAz(avFaXb)4&#M2P-&WxbZ7b7s+P6FEAGGA!_ zP{!GsY{$d3``#ivb0 zaNRG~YJdzXu>OCl?z6qFBuVf)0T3h%Nf7j9x@)FeGqXD@Ju5x>{+~fF7$RKxI)%~~j%5n76OH)Re(Fy`Vk24`T- zBwHQnsBEYYCm1Q?!aBS~BSsf_;5dr#viKwgS@2S3kxI}cP8%ybCNIgNSO9gYTw&58 z0*@nWDTZ$GK7%*beI&DQs>+bdWpISAlh35$`C76YWU+Xh7ZfhKhYRgM z5);Kv_ewbOQ0q`ZY2+Psrh~d87#Y{vc@G2$jW^Ec=9y2U8%c4|pZ{~=@Ajyecw4H+%r-#ZPQVo4GRQQ<<0 zo-F|HKRG1y&B^)O6X$+94(Riz3k%*q-8eWuH_{U=inoH_u?pp$4j7N$Y5lqNoj!Nt z@X&e~GvgN1dpYj;#Tj+pym@r@?mZ_d$u|t^gw!Mve|yVdnWHoA9N)FDUQ5Z#C$?~p zTz@?k)9N}26y(qwCu(SU!D!u_dZ2YvHutaZw~R#j?WDeTlsV@|Tk1S~dH(&wyQ;hk zb(aK z|MYi%{MWzy)lwYIvsGxu^6YuAtk;$aq*WDMe+o9pm-*~oyRmophi@O+0H_RJnVC--3d&%;@P7L1T4u-Xs8`$n4j__t0EGE3nP1;xu`^lvpec_T_b4z`++ou&Iyfh)+FPtt>VnhF!e* z7*e2{DHpax?`&IkC%rkJ)@Uhw)icdbkcSNZrA|w$>_a~#w}P0AJ^)jo!WC?_O?vW_ zrb=B&?;@~NQ^30{00_H*zkrDpS4gP_gBvily(RCb5RtGB{!zn9Qd}QD%|8w~&WS7< zWL-8KOb&-`<&`ufU!OF+QVe`_rrr$sFdNu#vB(82U_&FAzKhe~MT(s|$X%dAi6ow; z^-c0IsY%pD^29Br(ibF+j%Fjn(91P9&B(x5ts4KOh(O#7xU{3Js$n-;iXk6He)}eY zuz+d`7I;VU$FGWY}~ugCyoGhhF5K=!_`z#ySwkt-#VH1#=+s; z2low6sG@6@8xGF{P|aq=J9b1kv*L?YIC`gFohfQo@oK9ZwahArRyA6^uL@+;3~kIQ zWhgi!WOnCfI?w{cT?t+M216A8#x~GqT^$z>ES-*W(#t5O^C?Hdv8IV7UcZIU78yl3 zqGiX*lmZ$;Oo8(mY^w1=Oy)Ab%80S59F(w) z>sjJqY;pw5(Ah96JzF8>+*E_Xj-E#vlxKdINMq@uGcgPdR+wO$R=pHG#dsfzjh+5o zvW3V`U<#@ZLaBy&fv6TkJ_w5{4wAwcL4L@9ri!7V%^Kw4Om~X}y4mAcb=M4fY6wtV-7MR=#9PYL}Qe5OD-g{!JXgCjj5^BZJdoGDkL1 zC%)fMXIidKD%@!B z8gN5rBVf!2-DmPXQ@B)-F!|_l$i-gy0b;UpBYA9+XZ$Otl+02yZIln(R?JRjNSi+9 zTCtQQoUs}j`ih}giOPm2clEJL);GGt3q*3qqr`($^hi0)Nm(W;SlCv|u004$^+RB> zua42)I}xxWMAS2^t$r--huEU6ZQU{vXgT<`*i;KfVAxvtMm1HPup_fk=!Uld=T>GU z!dz3~Vmp!Q3NLhv;i3v3`yp-IPZtVM_L>BK)rnS6cihcnf@<%p@zQicGsL0W2wF?P zCwGK=0Pyl+T^lDf!srbAP8+{saN0sm!Y5x2t^b(pFM5?4=)x%g8|kmF4YqnxCBwrS9Qjb|KzU=4fMPNMR9s91X?ccP_heBHS)a(fK zVknY>&HHah6%?Y7E4nkRxo~yghm9pif8s zzkhdT36OE}gB$zS4So8+JvHLa+1XsubH^pJ2ql&y?ZZXG^uA_F1U-jN_u##=5YV|+fI1=E&;KR*Z`?$Jye0%%O zR_C8yTrg>#f~_cgWoLT$Va=Hji^GP05L4>(-1sWh{TWJCykd}{yO|=Np=3sW3$JHA z^@I)&Qz~dst%8+Zg{D%`HNRO#D2Wxx`U6+zW(MH1#hjXIBi3qmfPXVwWDL-_pCM2k z$*+2c*V}1%^@Fv*s~Zy|j2D=y*tbdpDrt2T{EVb|Oubcms-!^cT2#9Lm|r|rZoPX> z&+yvr$KCf9nSJnb>`zyxn^D1z71fL-T)2&5Lifp&!nmRapVoj*nm+NF_|Z&vB4)&< zdZy=!c3{;P{^9FGRXxR;8is@BgG+LP8koryIYaIu^`w$_a%|87w8(1%T4t4tF}G-x zEqzj(V2~jm{E?hs@RQSoHli(f)C>;4H36kwzwR2U(X_!96Y-J&VGtPQ!8&5pGL%*% zxMH(}g1|!9Ees)uMr_Q}JXb+)cn@HulbK*PjcP&e;f)#LBgbGZ`(kK-35i@rvq4ld zFlxwbB?m@bO9zsx`--2xr8Mb>KkOMD8QfP)4mRvz%yoO~{_&j$_wFi?DqLk)-OMJ; z1RUMC;Xq;|1X`m}g#L!fSQEAF5K_Zx7IcBfpsH_|gJ^87hNT{9Q{g(lW_8sRSy0Q> zR9Pe6hO5kiU|1$Y_r&bhPYumHXNlA$tdJ3V08(E9m(`=&g>$3trLoeaeu@fhtxKvY ziQr2ZbT*j>epVNy}2C%X<*)AxKhc*dE8pHj%W7tm`n+Wj>Wa%mHg%f?`NW+VA(R?cD8Z#ZB zQLR>J;SV(63}JqkB$G|CTXyiZOG}@?FaZwUe&SxTiKtVa7a7nnJNi$jAtP}pA`wAG zwpa_nq)2f555}Pa$8cV(lBNj_RPmE@VwMVJh>m)_fs|MMj=xBzo^XNO+(&P5Eiix) z9Ej`-cm*vE<}pm|krF$MkCd32>VF!5tY**?ZSzq(U{PzYu4q+9&fE+xG!)pB;u`FS zbvV{_$4BOuH4WRs7jtz9hp|*Owh;|dVmajlFLbf)s*n}BKo>68HS+Ykl%b3?5>@Km zC1)Ryn5WSO_Ee@-5y?{56$feF=$`}Ky{yUCD5*(TL-J&`AiusEgpChMZF)*^QE9GR zM3oCUh$4q-f<<@oJVJfrLL`)B^awv9iJPbhRNVwN-vOCqurG}2c?-nv+`aSDFF*hK z>(9LD|KW!ps>WyDKm~3^H=#R6Dp#a7rg2!V2xXGmbrnN?5eWQr`cW+PhEsEVGE8A2 zDc{?`aT}ev{dr@+AO8n!l%$WqHXpAv?2)G^&t3>DK=w=y)_Wi6QIbL>IKK0 z7bFxT?2jT0b-g`(`Rh-AdGYKQuOX`^uMczW8XB!H^7fmZ-TnQ;J9qB%Dl1Mu%-deB zVmjfv&A!y*2GKXCXI^ys?&Al(#ofDizx)1=PPwtFg)$kdqqNJiD>ocv{QC7Q zdA_#S;^oPSj5R~f>9>f8wr?M03FZ>G_%;=M`0)AU^_ioeWW9buXXjX}V_-SQuIUBG zQ5d}Ux-O=8;8gAw<#+|>_AkFZ!;hh@lUFb4%G89Jq=VbH@7})q{euT5KBuS9^Z4JK zygJ<5ee~#|Q$}BX{d&rafAjkO{RiKF`@oW+tyizk$`-PGuqOZh?c?p8<5#a<8ns?U zn5)g}J%9V=%DsDk{?p%m`|#n(t51WWoUJe?ibE^h!-MsnR^XK*p&>maswZ|q~ zK77c(KfgFU^cMA&x)~0H{MD%~&=Gud2kDq=mACRTUOw+e zajo{z!O}Z#zHSybC~xpp2{2Dy^%{hqWJ<6yJR^$?zq)EToZSF;8OnHwy%?(OT{RZU$7)o5)sXMJU_7EpvT@!USiBw{vR+FDm+%XW)xtLxmOR)K`Vh_ymWG|H(W zBA*`il%BZ{G!9f8gDyg8;!>xn2oFocp@ahTMs)=w$0#BV&LBhqkOIPY*yp(lic(<+ z+EaK_-%u}P>rNrSyNJwU+MA|@067Q2r0*NPK0gx6l*8wg~ z)s9-bL`yoAO#+0ffRaSS87qC_T&lcos08=ecVg-N_uGKuX;47b<0+U`nx)F(M?0TW!bm;}f*zQZO1RV6G!)@-B5%(@xB#!T=?Zz(%7V2D(|`+P&%6~K>%qJPm=@m3O-Gx`x3)HOhQ|74<3ORkM-{B3S<$$OVkA@ zMC(I?*^I!FtlSMvv2iyE2Xu^Hdc$oqn_Y$#-9=)sWYyAEm{9dAXq1=XET*fNWLmf- zcBAzP4B_gC@GPkU>0tya!BWJ?4ttanEv)mQpP@>DmtkUry2fL!)5s?_z$8Yg?}CrB z1~qy_xMZ$N5e5AYH@A%?3UV3ttLiWqsa463>wK9iY)etayVbM_a^Fokx1!(Xq6fU%C;Fp}43Zr%tXwGS?}WZE%KR>ND$xdb>F z4ycwis|84`7%!?z)>pPIlB3>uSgOc+CJO_d5uK4_s-HNvLYJH>7K?+s>93aAC$?rp zYLd@y}9{_j(#D1gxhEk?RaJa)HaE1%qr35v}q<=xmtYAGtq z0fSB%o|u~twvGE3!{$x=?dP9fy?n{FyMKIq@7_JGxdGYh7w;?$)imHCrMIhb7?{Q? zx20+@jBybk!&xDSHGXSQ{%H{@>^whZN0 zbuxwJ>1~#1i!e6~8B(xq6{Nw^k6$>_-kW>I*4YULWsGJ;8}+8eKc`I;z}*`6-Z9|V z8|PNj(&6Ea`wzTN<-SpMt)aJ!5zvO?d{9_%eT6Ik=H*Lk?u}vdZ*M(%^hj?3Ls#B! z5k!mUmBI)6H;wLlo3d#QrR&*?R{(uLg1;8zFa>YT(ksv?vJ_d)4W@~so3GiD751I5 znL&Jf`TC8KS_{gJvLAV;%ieX|TD8yHoQ(f!8Pc2m1MfSNFV5QBJ2?36n@2L>*C$V` z0@28xoq1n@9u+a2mVa^Y?%hBA@Q0nPA9(-x|M9PXd-dXl*Twws-M3zX{^YkOKR-4l z@{%um|IU$xzVMjd&|l-wzrT39QuzlJq;uNzN&!2hS%^q71@>CYKwI!Fui}0Epxp;B<=FP>u zd&k~B)bM}rB47)4zfjH;jm%1`ugsU{%iO7vaaCejUiq{!>j$aCI!pnM3q9vI(gDhFc)EjOqv=CV!wuR=?Xkw@`ime%)N?O0&_=Gu%^@zm^8?2*uv32h=* z+-R#*KAf>Ejcz)vNHItj@=6#VxdYy#Fzn%T{xP`f3dp9K86v1$))29oq$v1}7vDbB z0PD^JRR`eGPiiUs=s!#YF~Ea&LRRpk+^{cBi=ICP$sihfpz~#X{Jh- z;|m<7VU_}l4Qjl~#o#22==vL4!|TEYdYT6iNG}?fzvi0bY*22J#8pxcn)Q(EvEeEU z2Y%@gP{U2b!1^h6@e$VFlW(}It)5UkrQ%Tak23+eya;Sy3xhv;(T;c4?7Ea7qzvw= zGUn31H+E*!=lJ;Os@I0!xM7UV=|7IeG(JJxsy64Esn8fh`psS#P#E1;JR0%RXV619 zd3~yakG<$-qPWsvEBT5_^|W!O!BY!X(s8qw#3R`#Q~JrPkpQldWh4%A44D9ux+Y}p z&%i{Op(ku1qla`c;R09aB$;IH%Fzm819%N0VOUr~Z8R-WW{qlo=4g3TIJPkf3x&LI z6li5AdkFgq&R-O-47$cy2?n~vQ@STL@ZmR}2PO*fQ{@#z*Wld=_e-k<3)&DX)3K~z z!z}EQi_#3%0Ts$7lPnV|rpDXF6B&F*qk=}u1uXVqF}F5#N@RS-p0M2~8p#m&ke`rC z>x`Kt7A8qC{3Hy}MK27agFL|j-}v<_JW(TrLbp?zQy5)LY25^8r~rqhnr}A)9>!6d zUcd-A^=gVivnaw0T7xT-V8ABvJg{X`V;-AX?iaS0=HA&hh2D|?LNQTrhEXI2_Ida5OCg(9h zhkR|}9FtPRb18DH0~t+ewk(gyJ4{XKwjD*)BFpRBL;ZdN=d4wr>Kr-mAVx_KPXB*GOy4yJegMow5TO`>Y0iyS70^MQ+K3b z1R5;w@DM#Oe$lRX(HZ3IZKdkfEV?cNcZttWCh4uLXvqj%X!Vf~Nm3Rr_hDJ_WDky( zJS!`N(SwvRymB$71F2DfX9;9~p(>hdzkAh`%ag}m3zLWD1c*QX`OjV#WeD4wcdy*L zYcWp8!EiCg%fL`fu|Rfil!w0-Z<#V_LY_v&Q$w!5F(k`zv^v~%;!uNyH^%t^a<%Cz zCr?(>RKr17w07mfi2hnO1DykLXix7bJw#Wxz-g0jt-9g3vvVB_U5cGUqv%%g*k`fz z{{4k<{lgnKu6#lN99v@&U%Z+NLtbokeE0sNN8cabxMh_;o^_M;jQ95r40;(mN1YRV zadu8a)~uTkw^GP%0V=I{ezl~>9^b5;fpK|r_AlOZ{pFc8u|Tn8kQ*(^F|r*}@>}zB zWvhkn)esHJum%e?tfD=gy5FJIWa@!-M3hxhOD$iMyO;s5>L{;$_BPwpKb3pw(F;Ddw5 zPk;Mx_Esn9=H9*o^qxL@Za|*l(d(<{&+c00bno7GkFLHtz2l_OH>V~n9LUK?KfgUa zIXt{c)!rp%gYwIlE$TV_{Milx1OMI)`Kj=z9N;a^F*PW`; zsbV#tBGH`XFVobl&6?Dd)keoChAmctpKjFuS)pESJ7;ES)(zcmRJEAV2uuc~!l^#6 zncn+~k#@C@E2!!#OYQA1XBBnzlz@}%&|)AWRisq28!L8-ZiIcI z(>J?(h+XjsG%ZP{vC`8)c^0Zj1QJ_|neNha{7pr^G05--3#(74<@JVt+)DsG{inC3 znBjHYiUf273^5kon(wqx*uvGrrTJV^Ye`n>*;rpq#>U{6O@?(bFw~3d zI}a?rWzed^WP(O$1b$DlfFtW$M4FNT6cWYxOcugKX6a-B{+Fimry#8IyMzph(4iu{ z8e2fjluv^*@#$d93o%KDSDBwO#0`_B6Giw5i0>eMM~Ias$EyIraT({SkNkon7>fU3 zt|xKe#735%np*sYVtBoj*|U&A>~~=M87^Y2yM(_a8p?rQwaW-Q5DPoAR(Ccc%ioF1 zOwKn0rchTYkjvAE>&dbyu8~$KQywG(CW!h%43qN0HOC3C$&hs|)o41+%wHA2*jsY= z4vuK9|M{&gbgXl4Q0g6|{D!x$-)NSEN~1yOfI^;*?UhcpTS!KP#o-h*)Y3|-=`Oyi zj^-l0iBlqkb`EdGTFE^zU}Xj-iHP$|p&R z3uIupc{S0NKF;XoBM4709ZEG^1D_k7^5HYWi{(8AI2-%>}Sj`$up@l zsR7~d=;ARx6FchusR?;4bDD}gCMCoD=)G6xR;P2AQ8n;?eeWPfYy4bVtYSy+$;+3o zPhYk+bx3xSl1{Bx8Q-``%VgAP*#i0Y<>wC<)ZB^A<{(T#%sb<)Z+-aa`{TP06a<61 z9JlN3f2Efz^ZHwhR(j{X?tybT?U8jd5Wl%zbxy2z6ntt#-#fPU_j$VdAI%lCQQ(~% zJosdt&UIM~mRh*W-Z1ld&*4ta7~Qw`*Xu;BRh|K|Ek~T*KDu%D_RTwYj_tlxTEBd? z_aKLO%;J-ELHqj_=-#Xk;5g3RtKKiR`{mjXfB5d^&71lF&hArXF)G~)oYFBJJ$>@) z+k+c-?%aKN|Nh~@|N7&PKc2mPtN2sX-~I4MC%o#k&fHa z-kqJB9=UsLZ2sWqpMO3%c||i8=jnv#8X-iqFv%yw!~ znVX$6#Hvp^W)#qAi;Ktu4f7Ng#y>@%pT0%Zf@Eu@Utg4DeFhi>HtdGSl6RidKEd)) zG2sI9pUp^a=@hhEpe+GaPtsQ>gmNnB7>e42gkarGQ!xq9sK-lWEK$_^uN$YUzC#^D zOvdyGQVr-Ya*a^d8do`(R-N@!Mhcu}vm);(G2?cAZf@@Fn==Fd zmh~&rsvQGAiPg%#?BBKGXwZM#$pWf16Jj29I`tRq?=fG!25r`C)L1RPMpe#kSJ3kX!OIt?yuctowDO%_doaEiE=O{=wIg72~DjOV{ zgKz*!&|l;w9b~_UZR)B;J>JGHL(76TRsd&YGAaL60qjUo6E^nYX~PKoQ$9)%E@Wg> zy!9D*qz>_AK(=4vf~OpG``_`{LKd(iKryQF?$>4V!hNiy`yn3LEo&Qxxj<8DmM|tB zV-!0m6~iH$4X15%i!F}*l|Q8(+gkj3OtQJGvQ%_cPQ+X#^goIoO~52rwOCAXSQ&`8 zsS`~(yf|rV%GI#Ehj;HRN?_RHDn|{A?$#J&*BqMRkj2#Wv$J}ptl_vX)(xF}l z>1t$_p{9|kCj3Obk$|8IpGP;@X2gO_lhaj%)jzEM{furVVCV^41x-X*o#8DoY0}(C zF~K33&*o>~=bkL67a#K(goK0M%?c_Z)N4yL0MS(w@mh@~6B-p!`;A3q8>`YaSuytZ z#RBVX)2BCf4jNYM-1k|XGOJ2L@TE71^90P@I1P~*W`)B;g<+J-Kr`Gih?rdQBX3Ol zE(s^Ya7&A^U2t(plK(!;^4oSSe?Pc$%wfCjT~V)|J%!`@hYxx4E2qX(c>FCSQ0*)E zR%y3;RDU>+rB?}>yMW=eEz-QCuq^rTyxZp2=T?Y6 zozG+5Vc+5$zm>D_sPP1@*Hg)S`!M}yKUdS?zLWf`}z|KU~)5)+-$41 zty8~ZYTJv}>WN(2JGy=M_|Cok8%H`AIMS!TPB&jZnN_fr*ZTsrZpZou5ASPHl_kYS z-cqZKwA%mGiPbO8-0vMYA3uAMz*{G9T0x^Yr+d2n@(}}v*RR=DY&O6GA~~*wRlL1S zfZAlbHvm{pbMxknyT`Y^D3?;qJ?tJIEqkF9c>_P&sL!;&ccX*fuPM#safYT@gFk%t zE#>_D>o2?dm%G<3{5KO}t&%yAH>amg_;~&H4Ts&kpKsi{b)r1cfFVNWGXV1bk%S?i=`_HC(jy-XTD zC$Zy-eCRll0NZBocN0RVfvRe!>NqKm!6jx2OiMl^QOT1`1XZY^rAozA^&q9cZoB$Ijkjd z@$@#K+WK0)4O=VfKX|Eq0~?LW6R`ANofwz(n}(VAsk|hLpLmTCcayvyD1_oaNW>xT zzWkXOfCWsa?(=8$&j0*h{`vgVS2g}mzdpWbYn08w9er?U&(7YQaGWi-6M!!PHT!J4 z3!I2?_`Gs_26<3Z(ZZcov(l?yES4idCEM7ZioWrlhA9M02PV@WsaL~Kj1ho#@h4(D zyTxyM=u@^9H91Z=Vih&;Vn$gDD)&uz6eTUk8U>q>+_L^6Coz!0T&!RrqhP;M$y7O4 z9$hL^c*nQ!#ttK3QiGyY4=0lVxj$4dcIB(i6mgsL2@*Uz|o zhra+TB!2prXrBF6@NmFpR!&dcG8MN&z91hUU8k_+K1c_lFO?;`pfTjBPrF34NLJI3 zWe|}rVDzy_4nUYi%+KfpA-u0oXkz8}FjV6uGk`WGliAPoBT7X7x!x*anW^@|hqhoY zs^rL2fifkWr=#gyvM=``&jLaS&k%Pv%#I0o2;s;AAi2wIc^1c$Jc@ZdxpU4Dw~56V zBH!9Ga$Y)Qj0k92E6+x3kgD)c9U{IF2!qq4>jP-^6AM)z1bs*dd(L_XK3XB-^aw44 zG|iYf-eRiPI_RV^yaSP)pxyt zbym@g-pYLxvWKa8U>LOhUg|IfjsZ^Gbg)OKX|DWBzk&Csu_Og92aC2cVLh~6ryPrp z%#&GelUFDJy^G-kDA*y5C3F4Z5fFbvELy2Oas{jm05E`%;iupYJHylhoj_n43PYw$ z&$3FEVMKHA_Rvk=Y}{Sf;M}m5iCI}mg=2%_hpWi>#uM(qGFpSk-0d6yZrlw{w_|Xw zaXuSVw&sMJ#oX9>Bha5dpMCrgp& z?v=-C*YV{?MVvE}z(==^!2SHoFK2Jx7>af>gz$aWoxR1dos z)mnq?h?CE6PhLFz?XkCd@%C?CKVr>!|JI2y=BphCWKb2Kmcl#l|MlrfJA-FEDq4

2PI zKhe=IztXx_Zm|Q?CprL@D=D|6)hjrm*)O+)W*BEP{%|pDt{_dKkD5qX~_SWd$E7ZB31OcXW$xv;24a=j4caH8D zn6Sdi8YhSTS<+)U6$T{diW<~^`{u2+L@dO5DD|2wmmBs%+`V)6$#0M4s!Wk>Ri)KY z{hC|&_~DcHNPv9$?8${47kL3!_xE;TxVTWq^?D%h6Eh0QgJley935e%g=n>XrFjY< zqZxzBxMf?96*$?<& zOtWyHi#Ehlvm~rEmF208@KO5|%vWzbiiycb%i{2*d}+7TD_D|}3c*2RXvg9u)4rsU z=}J4*?jfu$=M`1l<4NI21|P4JcO(X`l9S_3HC z5~>7{ZxRie!8hQ5#YdY{1%R2DFVgB}LOX{243 zU|Q92>z|m!5bX38@3ZPwGX2rh9x`}t;XDth_IjtjX-6? zuoc^a0`?6J(76tx4W>!{luAH=rV^G&WzY0hCX}29Ec`TxXq?TdyNoVQW2X6Oi zu=}9npD>edL!>xSFb&s;RRID~2MVdz9zOxmT9H{&U0j3a*S|P(FvFUK3oWx;>moph z6QAv2eg7WkB*#lUs;_gk)dRIUQ64e;OosYRaanCYNC8WI5@P|>#K+U*EuCK~4b?uQ z*OG#$!)>lD`)YgVb%G7fT)7Hven{RDuKEXAm%J=P5@Sh{A}9Ns1W# zO?>wQ5KjRl8NF&NYzSbQ zBg*h8A~u7YO0jgFWbe{Im`3(eO?Ioo4bXbyuVpHBroMiCsxr5V-l`Bzcetg_m(R~L zJd7F2$>q5eI(e;Vic7wT>r7`2*Q|Z}ORtBilXqFe54ZneS*?8T03SV{b4&RxvuKNi zlUBB`Ia;56YiL_K z7w^x{6e}e+fq8hnbfBqqt5%$6H>tK~e7BNr2;b3|dDuuC-_L>MUdORBw@z_Z1%;lH!GPcs_i5^6Z(FHUH^9 z|7WA`r>Cb|SB~%BzkBQG==rnf9?b&kA6iXy^7ic&a}M4`CCEA5yiXb~+uCqq^^);` zs~7LA=XvxeEo?DXCJ2lv0({?=?s zCbGNhJ;b0b^Py+8wz&r{{JneU*a|C4Q|OKPOe4_l{hg0jdP~98>pSoC{HDLdoGdUK z-yDW2d#bG1X0LNsLozkHOfPcc%2eQdWA(u(+~dFUr>#xE?(3eaE< z>C!{b4ff`NHG7?sM&^D}PrUb?&yp~%{big!e)5N=g6UzzC}MdB$vDDqVsIzrhH)^U z1Y{;s_Ll(e
kQy#4-Vpfgm$JG7rXy^-FR+jMY!_mzfj~?9fnr^2Ju;CCptD%>) zD6#ueWz2QWuc(+dTj80NxxZ^WRw8Q0Oo+4lJQj%~F0!wCqi=EUW{{^8_yB3<2WYSJ1 z$1UxY(dkTBya}PLo>Fv?LcrY8h?o&xRW)=+4TCF| znHgAOstwqV+YAWc7+iZrDqACD+;uN(EPWoMQAo9QoR$s(U!;o@l+WMhCVW&IHbTLs zbW7wV@f**~q0X$-**?|WX?{p8Ef3+aVIbL}l<4?R=ZR8IZ6K0ZeE6p!&kO^g1vevd zLN1{#nzs!xj8#TC1+^GVVz?)X5C6ZVH;y`~6iJBR9W`)^;N*MQ^IAsw$n;SEfGZ}k zxnV8CcvKZ@u;p)=Ik!_$jXWD;ia}7Bj^DPkZ-l-2MFVklPlG^vi47f62HMP8AgH;; z+Qthn)h7GGKuyvlQ{(n%dlo;}p8A{TAve@37K_N_am+3bVOe5?DZ5X5#pYD@Ng)wj z_(7-Hu?P9D!ivVUeyn_o^Z*OYy&SQk;42X{B^~ZVyzm1Jyz$AvmIUz;fneNDva#Pl zGl@bMaCjtgU4Wq~Q=$M+AX(>@$VE-Hm;2wxKwE0=ove;sR(BeS!y~s&t!N{b736Fl zPPJ}32>H3ZzqVi*(Mx4RHx5%?^zFn#d?GDg2sQx-SOQmW9jJ7(W~n6msOXGN&t za6J&aMNZ!uEPn^LMIE-Bn%Xt2>xEbXW973mA>L1C(N3;TpEMFS+PAs@WfbeTwADMu z)9oBeGZ|FZH9glAtRPx-nzc!K z7p~rg{O;nz%ahY@PF^`KNT0L4pLYpV~_~qlzzde2O{Dm%!;RYhe zduxA~-2V054lR1uv9eCpaUc+@SGX-Ms7A_?8EK6gL#U9+5`kmM)pnLr7MRF@s|_;w zhQLcYft8v;R3WVpm&f9zse}$Pr1JhT)R|G7W5&ueZUJJwqtbwDFhvqjlx(eMvH03-m^_f}&cOUF|`Sf+$YpE^xu?xjOgSsC>c49KdZF@KS zsc5O6s-bjCzP_)b%1iRMwA9bkJcJ{RI;g)cOYf-{yRNCD(6V~+a5A%8NRNq~J|=5n zn()#L+Om2Yf%G$UCihBND?7Ot$?gi<&$5iTt7UBFH$Om7cpyLVzLg5MXB)_~Dkn&OyM<1I5(0x{N`6MuG zXaxYDZ3wd(cf*kAAO`p1HQS0{be3Js^N1SvgJf7Y1cg*d$FR1HbJ&!_$UoSv@6G04 zt~iSL^yKyXbDO2Us-7(Wx!qv_hqVv}Pt>+eZ*|Blo3pt}janufPS|mj(5};wv0ihU zQI7z%tb|(y;H&9Tby;%xXV4fTr5=IIR@$&e?wDL`QkZ)}B|OM-hYr*H%KO4Hl~h>g z3t&j%X@VycQ%9JQ+a#3$+)|dXl_#U30OD7jsIfr{Txzb@Nm5Q0B>o0CQWXpV`N^ee>IVp|N3;4_2ATz_qMMY{2N;(kUSkn6(oN(%48cUg z#2|~=)QZp&kEH?^IK)coKRHJea7x@~G%zI%i5TVN48zN)u-J|8x%wH~QDxP`rqIDc zgob&WGE<$CX)+*Y0FVw07x@-u#mJI&~zxx7$8bVi%2DA5uXx+Yk1Z}i#iXlL`Bg8UD|p%^Bzvvogw zh9p6<$0{d}u=?i5N*;|&uCVSiUx3mjM(u^t=zeN=T z>p-}X+ksY4AWLg;kpZ!}Ddj}jhy?q_J{Jb0s=5*k+pC_4=@9zT(9&E8@f!1jd{RjV zD3AG#hPZb|cDX{Gi#jkHrW(z_RikplTdZ|kVU1LdbQX{)PWa>!kGYs(B}OD2hHAe` zRuKSLSX|1BDXV-|mGc;&>Uf9eYDTNTUID-5kWMwfbtY0?cvaJv&%I#QSdXl&1Bb|< zuI}m^%NmM8E4liaE!^MXx<4etm4H8yB~3d3o?n@7U5l z-afiTP`iV)hDXP@oo&iMkkCmmS}vEV+zCM@7Hn_8-MxGFc>lAhAhW2+C@kA8ct1U)Ss;LW+R{ejI=3FaQl+?4i9#oK7ID;G7-)PCH%dXSE{rsy1k$?Zg4;Cf?Qy1U9b!63+x0{`vzthDc4tAfP zc#oL_pJYTcHKuL!Fy;K0ucv2lSyQ7T+TD#wTz0&|G*Xzzx;AwQbA`WJho&mX)6h^| z^)J)4i|E#QuecVd*rUv& zMoI3p9<|U$!t@II{n)X5`mGt_q+s&1B`lkaujmc+^otX+B_4PNpeF!AH*`J~jW`Q& zHTpOW)k(x~ypw%9oPke9TQR3;1g^>`MJy#$8gbt8tO$Cy^D!a)wFj-}qyk*qy4|L4 zx0df=(0c|PP1~Uh9w3B#ekfx!AL>zlEKS9^Ad(B{e1Is1ZbvkCr731OoQH5sHVjbp zahVoO7bsUn1(xngyS_s?e1!~(MOA(S48KZAr5sji$pFZs%lx__E6q&&ri}!XaNtT8 zY$-Aq_rtx^aoy7hGsl2}3o3~Id=|M$wNOiub_UB(EX7iaE3E3v@dE~HcjEseK{!J{ zEVTk>;;_J^>zuk_`&n8@fpu9lFeuv-Wq6o z_|CR%L=Xn9XgM$8BJ7R-sAWHO<}s_%)n(_CYYkLb1ZhR_iwF15A3d!6jZ6v@NW~t5 zLRe>gWG`VUFRJi{*I16+jN{3cNt?{rPVngw_?^F=Ywbus235u-cKMlGI><)jZYwus zWvc0a^EVka1SFYUM`z`AoNe-QicywV+ISE*=;ABRpiQTejvUyQG-`2OlOJ0Ppt3Et zbb~CwFa)uriSD3`Vq(z4Vkt-gFvi3NX=uZdWZ_scCTO8PX!#)6O<@czOVC<}!tifI zCVrD<5G`X3W}r4{!!@^5D!i!0$dD<^CWXpoC>FOu;%@jP+*Hs7XMg>P^cZVWEqM90 zfta6BXa=GhJYE`!-`R$OcB z5{1@!T~-Bghqo~=kv&Cznl^qMIAy>frmCb;hw1U9SG6__L~mFD2tl6-2%EcM;=dPj z3Qv-XMs3PZN+H@*tSm1z#njy3aAX}^gR|#SgDL%#I37uEy_&MYlE%X0JPHKjQg<%o zxy&WE)z3V<*W9{ZU)5xiffddW`qZqBk8}A8adcMqw+v7ue1xYS5y>!o_({TC@d5&) zf+C00n2o=7u@yTVZT=shYtXg2Is7e}H+N;f>E7Oh2M@me_FIeDz3$5J_<^YbCuY_s z4?Dz`_+)^gxU{GkQ&P%pxgNp%Lcwyw=-`aeE?@*DTv={(lz*tKg6 z-?X`{uU?^R=f=lt*KT^Dz|EWZq7Du{RyoQR3(0iY^fB1Mv97}n{&yP{tg^R^!+Ecj z%_b64ZnB4F6Bj|^=`RAN|#|{D0;dyZHo)G(h9nAmp&wu^? zyFY&a%_FlrUdv^1l}T0g4^-M@?XnGiL?F9fgarU}Do0IEdEMCQt5=q#JbC(z%YQH` z(#qRj4S023zW?sqAOG?fi}yc${rJ-_zsiO`{r%t3qz%Cw)5Du~R_s50`D*LT!9*3} z3;nx`58K=C_O98cy?glSy3Ud_KdTR4p1gK&P-9oRPMRp4u`dR=l>o*xJxZmn0fPFH zQ)^9SRcWH4YKsO`=bpy>X8qlDFBZ%K%vx7`r3Ox=o4O4jPdBz8_B#!Mgb>&4PjlnG zoxULG>b=Ri9D|1KnU_^KCR5#9+qKm4D$+VxGH#Va136Q0o6lYo1Qm&VF{QFk0=Z1A zxhpEogy$c>gc&hCGwpG0D{q+dewQgwUzv^Cu#w`{1yvhJS0$z{+?2*aQt3;K{OpV% z#N+<=6DcRoP7T{RE~a6e)H~Ws6&3FIh;yM4sBTWTX>`^3c`Z}vz&uftV%5kST%|67 z^z+j-Dj_P9fK8AKIR@IbhGs(xNU6If;U&RI>T#tQ$;a0PIkJ1;E`+t<)Ur&-@1QP5 z>P5sgCX%XeIFhSC1Q%EBagE#r`9U(?`zFT%lfDXZG3z=}A*}if(ZU*I{a;?V9E2(TiuV zPR}mX6vjxZCElL4;_b5Ut_g5P&~->*S9w_^s;z>GqouJrv9~YN&gByZbtGH5+i3AX8&x@n=xY8Cr5&cw( zVrvj3>>?RRVWYgcS7R)c6~=N~DU*t#t)1c`A7e#8W6SUiY%vjEQC)eTA0mSqZ_IO7i*?lB(y^5=d<1Xbm@k zjCd~e2~8$}h9vz~0gpmPM}VgehH`0FMj;&J)Pj)WD)BZM99@V}OZ8<{$YLrbrX&fA z8R-L%LKy+Eu;%cNJdq4lN&3zw^aZ;l|Db_lI9NWINsjbc$`JZl>WcQjU-sU{8V6VC zLmX~}aN);8hS48dA?Y53-;$0fLK3p@iS)E&P1vV2%!9~mxIT2<1&0l2P9ZSfZ55T84#dmfyz-y>a=329+@uL)FP9MW{R(NyeQ zN-|=X2t&-LyoN09iV58i%BHfPaUB{CcT+A;XO`u1_?#}OqSGFG)tM1cRjy+xrQlI$ z)|(`1x=W7la380?i%g2}PZRN48K@L8j44%d6#%d_hlyZQT*Fdi02S*?h(R`1C{YRX zg+^3(r-z-xamC2apMLoJKYaH`#D0DJD+kv>E;kMyrtK1u9ljJhV=~kt7h-$YQE$b@ zzzqEJPC9~C0S3e^b{?wb%hJhxKGoJIKGgNS-QG%}jj<}BHo}_zw)1-Rn6{{0E0_dt zmZL7f`10sBS!-Z+)^SX^8%M|$L1l;bPJQm(C|Ix7nhRkU=SksOH zOL&as8{|59`TV^_?JZ#XeBJily?yTSvzIR(|N6^szdaRhQM*06My;Kbp?7rW=(Z&R z7$WYi>&H&@DVP1teo($3H=3903{ASW2JH6jJNNJ0G2%|u9j9UX;{CR9^l~Djh77h| zzGElC>(kc_^YboKc2=*|UehbruIm%+IoRi$hmWkufz3n&H~qVBzqLl+9^Bua{HjGZ z%zx|F;f*i*x=WC2@m;gWdON!gr9HRb`s=$HYS{9g1O6=EK>rS8{fdttS~6AF)CT11 zvqSpKvLAhcAOH6Ahj(Sgk3ap*8UjlIyaVC+tCu^^Umf1Of#dhzw`d51GmAOX>`IwhNQtF${uKyjU1&Fy9kQ%LLRff^&LxTYQ@ zEyJ%QqLBFs-_+HC0+|+TOg}KsS&9Y|e-bI}y|FQ}3UL;zobucq9##LmVK`Z%OuDzI z>{=Wn(xQ6bgfm$j4ul;#3~*Sz36THp778Jz-s$6 z13_paII3!86_7Hb>k>>PY{pN96G&q)lwb#CybRryjzs=SzMdD^aRr5n!A~$XU%`p* zQ6o6@`Y=*d^p?WXV35HV>KmJdZqR1BD{7Jz9vllI-QbSbk%V9HMRVf~d=?HG#;}ah zoZU>+wPKHba_WUR9V{xd9#MteysW)Q77^~&0Wjc#fFlB|eRP=EVus0IG$bFcmZG>hDT*{pKXY6CP=g!dxWS?unN2@UL&lNl-#?4@2>_8r2Ie!R zag1YhymT|E6GejRhybTd4tpVA%1Pm9soIL5_jDDw-!z$~ikM;n^(NEOWVZ|P2p!*P zWQuFiS!Wu~hk~mXKVNF9Y6z>RT3|p6wO*s2Q5S4jO=EdJA|{Gvk`@81#8331GG(yV zCT3GcT1(y%#(Fu!=+wC(p43q*s=uY2A(A5^3BaX3X5cBZ0Miiai;V%BDlz!7El6O~ zWkF4OO(wA=Spp5kQ1w}4!tDUQJ*-ps`-wBd;!f z(*)KGwrE_-O(q}<+VvLWAoBiaf2~t6@*ENYODHWvjgxFJt_vMSb~jd+yl$MScd0@& zr5>rEYs{~FSW=4F=nj_TlNwcaIVwP61O=Rj#x&XHe3@o((JC92ki>|L`>Bg00RmME zQTQ4kbBC+0t(&1N&AW0?U54|dnUYP|n20*zW&l40CtMmRQxktgoCrK5ia?)4+km@` zlL{_`PV{Urk@(vdh5zvVAFTxa+fP5e;7aan-#j?9G}M?73|R6X!j%`;>5jr|;v|?d ziw7CaU6R)vm30DDZ~mQyF10VhH4_N$UO#B<%Wj8#vmb#|uExWC4~jnnh39dTpw;P!1t z-CsBE{`}dimoJ_>yvH*6y@P8G?zI2m`STY){q)yYr>|JAA@)a)9&oFzdN)avl|KjR7?OsRa<*QT6b)tUly6J)A+sBggsTz@XD*56DB z^RW=fil(1_d0ch5we|evi99xwU{cE_?F;X-<2ZQBTLtrzGb!~}NXz~@EP3lI>ZU0y zW8q3T1PdjswD9!f*9D*FM01 zSADWFF35yW0gs@n;hhgC62S{WRe1}aK5Jh_>Ss{ZMIeN*P*2isw&2uv#Ru}Y}L zuEaF7cu>=YOUlSeka?8q@;BozrrK%nRP6yvs+9m8I3y%H9>SfJ6X{`6ht@aD{rb}= zc^b^2UEt#4vHyUkT%^MJE?g^HR#GtXGvH~?#?U=vZbE~M7G`oeLKfzP^;|-K-K+gU z^*kaC$Ah-$B~L5@@-SMBTlQ`6z|1BavNH45ty_P$?MT2I_g=qF3)|Z-U%dMD$zvl` z2x^4lv;B*!8}P-j0e9bAH55ictp8E>sz*#E!mT`x8S0~2@Y-z2h*c>ow-RcjoJ+3r z6@wZ!$w5O)Xg6s~)+BNn<5>Y*cSORQW(ds+XJZB{Mcj@)ATCkDmo&ng#952DqEpla z3jqfG%n3pnmO)$ULE)#YaetZHK*}X3HhC65C8fqC5`iW58HC+ziUQ@9uo12xqUJin z4}970=-q4ss$#0^kPdcuT}hzvw}^%S$`k{~$nP;F^p&poQNu{Rad)fL;23QbE$>hu zLOD5d^*lAm2A7}U@iXvaCEb@QPK$stY99QI4HsL8w8IP*tE7gl6cdAe$tyQiMgZmN ztR4o-i>RX$Kt75(bBaP5?LYDo`VOLkRX&^hC{MzmLP0W7@jA~Wl|WFdr2-9Rlc#uY z=gy3Kj@l$G+7-QoT_`1B$(~&1TDV&eseGdom`l52iuP^&X~ZWA)~d@Yt6{=LIH-|v z5CL7WQHs`p%AlSqU~(-xjSG2iSye$N3#sQ8#r2ZKA27H5d z;h!oxON`zqcQG>AtNEb;3;J1_rvBqQi-fwi#bqi=y(5%jnDRcX6=n7Z+I!& zp}VxtxSOJd$K1r~Lret>NGz$-PCpdpnr=q5lO^$vlOS=Hfu#Mh!j)3&&|YeZ`%Q5j zBul@S3Y+>Dwlpp81=k-PcWn-EoSocL9RvVRipxc99$t;wk*~cKn0MT0UUk{pgBXuc z#o{!=aU7@1j#5Fw%ykOZgvWWzjNtA@Rba~U5c*Fkt1_LEKOt2VDR_X`xnZ5ww;Uf^ zaQ@q`zsR!tww*c*{?472Zns{=yI;Iy>x)w=?FOiV7`e0AFv>1@#`|x{gb!9*`b|)y z_))O}-E)5b!BOume)qvS{qL_`y1lKvwPJ7Ee$vKy~(1hJLU;)JFV!(;ajVBKD{<&p|DwVM3G)% zV05o@JiYKM8|kc<+G(uJ`@GKHo(a*XkPqIuc5-n-=}s#`;_Ew$_+^>hrXO_99B0G} zzW93O{PpV}fBfE!JA^_iR$mYrIqnf3^O7`^qjOV%oW{++W3Eh68HK2m}kgR>_RbRlTRx&ATdua5DRX@o1~Qu$YUE(x?rVTzqgk#1 zXh4_0;?e~DY&XE#rfprMFQ4CE{h*H2kua$MOwX+I=0a1an7{QLGq>9eD+&!nmaGJX-go%_@w^qNQ8pdfFrndNh5RuC&t^7!*@E zjSDVOjF!`MEi?XQ0LS(DytHq{d4IKtElQWyp3ra#VJ@SLD>0 zUetH-{?*>$&F3#q|MvK?Hvls_g&wVA?>^i+sNZkGYYrvY_Zkv$*RHA=&3`p3tN-o| zg0gGEt$#(PpVoU)Ko=K zmZg)gfV&HATzD99Vq&Q^jm>ks5COlVFgis*x-$WbtWer7K0Oj($X@z)?~_dBbBNXc-2Tg706tO%WVyl8@J%0~qa4 z^#?9gE)o>kLpe>GF*BttL5H`YypjqVekN_OaT_X9mSw_=?=vWgO*0rJ}Q%!ahTnQe)7>x8Hs@|g%5KO zr3!7?um`$3y)kMSo;IaPsEb#OdS~P8(+uUp~fkTmB+vV<(z~sUtXCdXD4-$E+KG<>)3j zdL@bMgpJm^m03YodB9hb;=VEw7Q{w}p5~Jf>we6MC>g*?gRArvM^YwgNn8?G1VuNH zy3yyv$(DhGwn6Otj%dP%cYZ;yp(=L_6Kd>@n;31$s&$p5^Jb>q4-WGX&?xB;bXit3 z+*G}${(KK6ky?Tp^$bs_&C?<})-f=G=aZ&Qw-~A9DR*4RjyFizekvR8-#tFMee1sU z5jEh4eV~KMam>RqI)P&7ilPmh#CakeIB({N&9Nj&7?Hac!nc(IY5r zTr?xlej^{XLlrEZr?Rt*!5B4u`M>%7sRxxMfd|zMcFqUdhRY{qx2MCTtO>~dI(XMn(?CNQ(}V3`3ViZ&w%Cq zOSf2NikP3s^R%t`#x_Cbr1m>7gTLDmj78#$bwY3H5s` zLOCARhxJl7H-3~pBwf-tL=9cB7ZL;?uO4dg3M!?v!{RV7j8FS8nskwB0LMzpMmO_? zNkkf4zw26v|30v4Dgf)6vt-i%UroB2yL2pqQMl~s{YPE)V~Z5`4J24R*}flZ3eAM3 zFv*?4V6BfbTqlg#uzG9z+Ob3djFND0Bf$`jAY`~BHbY(h3H7EpY?g!acA^UqH0Fy- zE3`*f73crRoAR{0U?Q1h8I=kl&TUO{qm%cl^B$_K5HXTeJ|I9fqg^d_m_FqHMlQ2LcLl7x<+5Ez@7rx*?YN(Skqo4}zug)%z=!c4+3N*oz}Pb^&M)bapF zxm5|V33(QhIRejYE_rgl;vxyIw#6;59fr5`D8EGab-OXncwBm$By0res%JCA$!7UIAbE}|VQb)_+U7>x1$4K1P8Igc0 z7ScErF3>2L0|%~swx`n@bQ&ga24Hs%d#S~zE6o=Au`%3)bg`PDE|%uG)^1=1Y#B!k z2#pIr=^_EL?0$;6{KzMzO{GSC;axT+c7TT8m9~VNbd_NDkPTMA9?F}A-cyQG)y+Cc z?&%DQ2<9F7aKWti;Je9ogP+zybn=aAZyW=j4)^cCvW^)HlmaqiFc%hV@ShsOi(t$Q zJ40#lmcELuLKGR7{nSg|08l-f8Y@tX2aIJRr&$gNRmd1IU!2%>p>F3oKe&7U!QH#J z_Vy^r88}>#JGXCbZ5{A|zV7X{ILDH$e2m8HsUaosDlO#-C0y!Cz6E2@*XvYVXpzkr ztN3S?pP%OS8r#aH$|c14c1v))QP>7t8$(+NQ5W*KF;3ZlX+X zm5SF1oT>IY&mJdcSByq6Lw=QG|M}m0Eoz@N&*lT(U%a)d#d3FV$bI?l>_XF~{=~L( zL{Cp%amMf8yRUrtXv-YyNi02xqoX@k5s?)^9O8kK=ZGIibG~`|8dEyBfBoxU-@dzW z+K*|0H%?SNKMQha`)t?ybnTb(N@6FU652rRmM(#-j-ROkY;$yN4JRR1^+mXT^4EUd z1m=le+xHS5O0j#GmeDftKyhXZELh^M8?(o-y#R*)b)vj&n2*g8nHx>CecEo5tGl<4 zkC1rv(p1S?$C7#}!R_PQHRV^fEYZ>bVXF$H^6NG(!^UZfwI{oD@KYI1pAAk7a zfBCQflh=j*<)@!dPfu*wuq)buOlR*esNEZmosdNB^f_aNS3mA_;8i8u-s=u@{q&ww zzlt8&y#>sY1p2*d?Xh~n8?5Sdu_e~Z*M4WJXV%wy6-BN`{=^hFC|D6g_FL=83@g+S zS3oXn)cl?SlgKBu9r`)=bN=gPpcc`ogAfJdk^pT}mKS;>IuH81$_-w;J1&R1JgOoK zqMEw=tW;UsQ-wHHq+G6BP)+V57uS1QBtQ;bPE-9pQHXh2#QP814sT^7Lgbbm*Gm)A z8LD%2!@HbwcZ_Ig%3FO!ALBbz1Fece39s;Bv7!-Q@o3GDVww-F49)nr+Nv6>>!_@H z!zWJHb1e~+LV7Nxzo1j&5~)h58YQ^7T^CQquj*;^Kgp{o7NUSXO-=1OX=^MaH_v4t zBGqWb2?D9AY!MrPam|Ct1{QqyHV{jgOP!2;)S|D%*$OOuOE?m2yqid^-~k4*#8XJ+ zQ^tgyn${$YyOCs3bDI(2P+Yp~*=25)nWUJa%h!R;5}~H)-!a8P+hB%v+{E-)Eb|UL zy)q%`x+$AdQe2nP-?VA5Hj)Udn&+~pYCnAR`j1`+YH{bCd*899*_x(yU~P<^NJb;x zoi`r0x{2R7m0Uor;JT3;JuZClJ>rCYX;x5aFg{F& zA^wXGdwR(vM?vw>qw6~Nqh>$Qfma;y(Kt;X!NYE08Q{|L!n@cgZMhglrX2E86##!Y zf=lVi*r$3_v?pnrjagKpUf89jFQ!wz;7Ug*`Ua?m82FMF&`*r^JB>)X94#cd+ro*Z z#2}vR^f|yob6ALYd_mT27fUmhu*`d7B4I5~1`kJ}s=!8p1tT%!ijSM><@$3PvU0qt z4hYtVRF_ymuVt$dnegEO%_@VS2@h(MC<-O%XJw6F@o256g>%8c7&7o~MoDb2TlX>3 z-DLROif9X%Yv1(9`AVL71WQoNU^HJFO*3qk99;eEtRzhmeDtf9*fE&Q+cV=sq-d3F zSW&qNDJ)(m3;oOk>sncYXyTRxzRPGVcxF02!{Q~1R6H3_EZ0pHH5{i-VEk1|Wh_=!0x<(r2S}&*6x3K%e2P66dku=~^2VjN*HT@a_ zej&R!)Ctfjs){%G{rPO~{k|T-_IoueDapCS8|Er>4F-;e(m{$~r?qhNiRsvn|1w#b zvK}YvD11p%k*m^!q>3qb^YGf975{f{AEEW^#S2d3k*%Ev`}`DbR4dW<9noV+kCf>n zrR4@m>g|tHWx$u_KVPxlL~~$?vejP{Q}s85A{*4uOE z0O?@2xJH?h!J2qX^T17o5RldV^2;y3J^tn5=*Y1@W(>Fmw{P9Fd-#{fzgQ;l^3|7{ zw{Ft<`_~^VM7jUq0g?aq)88Kd`tx_+eg9wo$A7+ieEVPi?cd3C=g#f#zrFwD$#2Y$ znK2>F|ErfLZ|CSzYry21c>vu;S-i9JPM4*%o;mtkCUID))iSd>%IaYY%yf0?Wil7+ zo`3v!v9tBA_hK^~n}>Ktiod#%sj2drz?gaCQ5jg4ze<73&h}lW3``9nJ5)TBsqSeN z*I?oL=SEKLD?KS{{@M!s`5lJ|6E1CHpBbm=jH$ns0jB8-77_P)3fWvcnA!|r-*)P* zJ%k?svd!_dWQezt#DulMquvLhK776=bD3bTV$Oiu0;jG}0n{a8k#^3Q>N$W-3X^sh zug~AxN5XN#_At@@sjM!7sbQp_T0!Cpo??k5+%1`Z#rXEQBmcVJi$Ala2kZp<2mO zWXb)wrlbK23q-LVrm!0~05P6aX-k&f3mTr(4$?^lx1zKd{gvFn5AVbX}6Ug#V&0u zJQ4>r-&E3fG&1aU*G;I0P^GXrpLiJbB`gmXOqrf(i!Q-Rwg_g&5nE<6_w0SrMIBG8 zv_epg6@?8y-9|xJzwnPz3r=CH-U3@_Bnk&L#mz7+yd?)f8>PxuD_7^!h=BVkBr7DQ zfn}1#^uNeY*&NNpOW89&@+F*{d}FblNylZ0yU7;C+JB{X@eq59zx5+@LuGJ8cyhMb zk|cnSFgUvro|2QyXTV~FFKQ_9|6}S-do)Y3!@gHlW@YXB($(8c4`;|35u!wj5JX8f zWLOd`7_tm6FKieF>=*t{{s6K8ztTzrT85-#g4Rf+D3KCnQZt;Hp6;%$eap(s%BrmL z?|wlR{~RWZsX2HErKPlmXbKr_ zgmyE9KQR;X_#1cSx15j;gVY%JxyZrcxTGsp<{U!tD?r=;0~-&X?^2F%b2-FM9zC`h z;KG@+@85i{w>?_%eP+iybe$s27fC^_UB2wX6zCkOGuL zPRfQXxR`v9Xrl9j=8`M~^)%$v^=sFga(MUF>^%z1BYRa4^jcmWh}A9m z{0qMRGcO+SZIv%N_`7-Irl#L(ewwWEJp-xS%D)ZwC zw}61R`KGNxuU~tAcA6soS^b8VO&Lry`dD6PwoIzAg=(Uj`HZfnZHzk`n~Ho&Lk@xZ z5uN2d6f*^T*?*-(9aj}Bof)Fa4Ey0=>k^-Kj4C}_M@41Zd;7*K+RskZ=}lK2r;rWd zN&??XnUF!Roa|$>N^^bP6+YI0UZgaTf_6|wH7lq&mN=O!^I zwXxRGXbPm)%%-X^MJKW+fN6Ptr`(GImQJT>7{x?s0wpZj@Ue-Rs!OmWz9(%U8$wGn zrFS>gA~xMs+sjZ~3=8hCN4{8`uvH%d)5z9$S@=sDvXl_UD2+Uj(m2bRhThYZK#5q4 zrxMJfA4&DBhD4OWDN6%B4Z)zS_8qjq;H2C^9$WDo*U_vJf&w4fJ#-4nmmNsVVx`m>&z9yZ=k`AwPvw@X@+-Y=@h76R;4Fxjyqc`4{VuIPAkIM7;-tNOk zkCjRGbnn6a&p*Gr|8gHf^8^+?wuG<6{px8J$riH_6^9u>;}&=B+|k9wPl}D^P^f)R zT>Mp}WOakY<dkk_*A;GUC#Y!G(CRftM@=P2}b) z9)3!BBVn1e)FpFJlSt}H$)sQMMPslHBOVqZK~tGQI#LCrW~Hl9xZpG&^e8IKqo_6}7mGzY-*cpy&h`=YocXNi3*jfZ_;zAnsNv9JtAv!pCJUmZ7GUYJ*skJnr&! zN@*B!krEIaS!gwwWa|W#5(F#8ZA%T*-maDNwYp%()`Uo``sd4R1Q9e$4r{~JN;zEi z6`sbAu(Sl_TTa?V03&f+bq)`3yv$$@SKC9`p9ZSuHvNetUPf28N4@$oZ#7ourzE(} zSB0;yf+rVJY3iA*SZ+mRC%E6}txN|a1>G9y34il*wb$jHh(Quh75PyW<)9J@VHlsA zSnuO0`pFScKZ7W)Yx%kJ-#s|^qfbA-`|yeJUI!QMdQ&W~GxS>I6Q8_) z`_KN|H*a0NYgL{YiaiO z_HaTHKJ=}dSNY0D&6twGd+nrE@`moY-S$RYcgE0#3%(R)&Y{6y*~MA!{aI8wW8Uac zH4n5<%$;$Xrw&XbIYgvCr~Xrl4aDT&RU+yjf6IX^5V9m+NB`c#`wt%*f`5(yCGYIn zotxKh7~J1EbDrjC$l@Q4woM6_uUx^ltkb$5J~wOOZ53?@)Zx2)?TQ+1Z7jBN$LgT> z?|kslM<4A8(mB|Bap}sn4?nnL`sDG`r&hUr^0kjS_6{fA-?K~5*+&<>oA@W6e{LZZ zSuiOF^r~2lxdu}%%%x(&VC&oU;#IEml(VA)FNr?!ikWb>8A5ScY=wpeO$ns73gHxm z3TVMuboRe$Jdyp0tKO)@^tga}^jwS99ag+0q8fT8=ECsiw6l4w$~&tQvePM`a?5$K zFJIe?d**P*iZb&t9e0wcfk&w0sfaMUAj&{JQ740XBZSmO)$IV#!#tz(+xaBf5%{Te zhb+6rj1YYx2iA`LV+M28VRm}pR}mOVS{;II3){SQMnX4?QwT<6E+wWpeCrA%8XrNczFI-#naIAt zR%i!laBdQzhy_(9^&qf3>Ax!?n;^iDcQiwtw0{=@)f%x9&wDi&21gxl`lA)_j8_jP#ERD06t!vk= z?;RX`_QmI3d8kft(%E(IHPms>N||R)zv_Ap9V;EO%~`hE&zT9hh&VDT;E`+u7hyxn zw3=Dvw9q)l*}yM@7)g%#m5e3A6~O2%A<7UcsgU+v`!>%It#UB34g-o)fh%UjFCeTk zM^c}qsFI}TqQeKG{3_}4D~bh@2^3#B$hR2OB`oRUs|=vW!8Cp4vL}$?eF{wZU*IMj z-mU}DzU4|@COFY=u_gBxQUJwCIad$H$ zhaW{T(*ZRX+~RT*Q`nFhS#gzj7>Z;IYaRN@-PqkEo}W=}VM_~$Nmf#K4GzT{Ys0@A zFzInV1R2!{c9(u6^}R5lTnY&)C3*iI*3&q~Jv9VggEQGNposxNi?%&CM=)Hu>oIx! zPEJ-Xu{jx?#t@EPldi{{79!&TNk2ISLrP+|oBcU?Ir;gMJfVk5l$LF;$I{0uT!Sqh z!{U1sKj7KZ3KLx(dzfdUq``)aBc>gF^5W%h{LA0@-oN`pCmxvWG$P#EgWCL~ofAjD z`0a1pxpCv_#S1zBONApq$VbS9+0$kMWq7VXS48!~kPSAmRCk+-x&uD@|mtkPK{5Zj>|3vxwB6 zn)(^`2F+v`^~MJ7T99C?E1&h=y?a)}@a`=Ew+@~shkfK#{_t~ijVq26^>;m!E>5P5 z!XIrJH_e9lR$6RnBbI9LgHsQAvTxo($VS0z?|3_xm(-qi#*kAEEqRaos%s9rVc#^U zuin^d<~Ta<7t7u?{?fwyUC!JYosGQb z3s){*zInsRo;D9yQ*_E?jfrkY_=*_o#!Q=Zy4Kb)k%J!%?6HS)E6(|(wg1w5FslE@X#8e>sPOOA%GpgHaJKk*S`MsuT$E?hxeiV_@fW?bEuaQ67Kr->j$r1e|i7D zy%1h~tt(@0L*B?mc435@My;%wEPW?}s+3ChqyskZFBp6(QI^+)|_uCzna+b5~7J13WRI#yKy5yn?%O5pPN z;UNeXMOh5zQgOvFh{{0qyfh|u!T^Clu6*@J?-k+Wd+!^0>zFs4Qtd;-zNZcdcez^y z4>7?0;B*U{h{WD}YBK35mmw9VNOXD{HIBaNNl}}9oT+P6DX5&ORq3jjCxWC?`C}z2 zV0#4Qe?DN=^AIR;Nms7oDHZ|*Mc#NAia|hL8Z!DNV71}+e~DLq_Z$5DR1>-{CUD^? zeDDbQ>OK;JL%w)`KaLxWueESn{4|uqXuyN-m?*5-*@j4%A|^~=z(YX3ls7&Di~~0t zRh3kD#)urDV3wO9l)M($f6YXvrUX11U6M#N*%_Y+N(QhAo9s%Dl}fCweND_Z~j*p1{#nd!iSb88VV+O%gMZ^gEgg zU#twez){9a4V9FTgb`59Vst$eF^R*0qgb^jPRdX#JpDjbW(~wL!b0m`RA!Q}sD$(< zo80x_i(O9?h><^$H-H=B0f)1A>Zy8A@In}%oE=gHY_v%5%4)hu(d?nLVb6R*5<7rd z%aU=_5Mg9yYAY^QRD`81vhqEM3W}A%*&x8U5j_oC^{W7nj)7KLg~eaw!d1$?)B6ew zW4FQtF6;%BK0zX1rYLjBRgDTL^8kdb(n`GKO2TSa$B^bE1~}! z-7w)Rx?__HJ^rIw8mweL0YNslH21!aRg*T-mfwEjr3{IUrK(zu^)wk3dq7s$;GNO~ zwhJjlQy~I~`}dje8C@Cu$f8IIk2yDsKX=_mdZOdG+$Cfsa%prl9SF6PGFve0IsM0U zvl!B)js1+FD_;AV?9reF6zUSB$f^B?a3m{^P*=b~7(spXV(;GbJ-+S?l^dTM>E51n zn%?t$ljt$)n#L-XWra)R!7$Kq<`OHWnf_F=+S*RQ^_ghRj@ z7#f0f(1du(K&2@c^#Gpxz$bJ$N_aRym{NbH7YV!5xR1@HS~1K z9kf!&#Ci0}1&8@~3$UyA#I}BaUIb<}$(vbTf>Rd4W46{vaz=2rY4wibl%3JC{(kyh zoO_ykm!fE*UT? zhfkjGcCgRcGxFNs<@4tV#rwBx$yyaWunb52U6SvwUf=!Ej~_gK>^;yw^T{_Z+V^~t z@9^r&FTa$dm#bb_q!{O2-Z z(|BY@488if;!&~aw+ek(clJ4LpwLu0HAxi()sE>eRLsWVyseDJ(7Ubt@oZG4>6{;( zK4nDS4iK{uwvV5-&PrjHqc8XQ_pe`{Iep&DPX`*boRNKCFpD2woJ4w+bJ(w4KXv2O z3UHUXS*ax$T&cHpy-bhKdhGgkK3J!SDYT%C{|oO_03G5w5{=U&E6#N>IuFa+9?V#5 zVY7vKRUIl3LI4d|byp7{vQwa`Y~B-$l7WUQh*9OLo<(jdka(aw+(|f9QY9>R{M#RZnliZ164d;}a_X`%Dj=okMe2h#=2mMw`rimNnXxzy zzPenZ80|5(qB&|POk`ohLqEgh5OBLJR_JPq+zg-KCSb#I_CB{mAw`TgILh>paQ=pc zZWFNvi4pjI_~^-h_@f^IvodD9QWQ^GXX9}%~^X~n7P6QJoX|<=~;q2rK=X4cR zM(AF4IN7-~nX@Ck=Y#P^JVaX#_1QYz5P)`lw>h9BuXuRed7Rq6pKk$|#m z7#eXmg)ZtfyQPix2&>X*?@-{?1w{afXgIr4q~H+-P$6?4_;l(Pp+#>nOp#AbK4o1A z>Ud6MnTiFXx+>3KDQXlKvZxTc^ds#s6L_o-I~DwqLXSD=i}Mv?51Y@VFVUo2f@f|M zJ*g*KvXWx_M(WVc^tQ`IAt?y4$PRuFu@=MWsF(yvryhhV zTLV5!s?aBE#!ne&P%>o7%t{lRK`s3iJ)d~)CN)KDJ)tl|{NqUrDnBFZNrj$=!# z?Dg4{wPT${RWy?QOhTkfb<9TM2Gvw!JP+J3ACT}RQ@5^_6Vw99JPnL565u4O7%1P> zIJwigS1r1?BE@dlN(`|y(*bZQH2JxUHICAQj-j6_p$ABH*vg5N;Le5KNZs;YZYzAl z{szmST7a=!k4~zT%gTAp4j(k+JAR=h7o8@OQ!6uT?B8&E4Je`zaKsLYr2qgx07*na zR7nyxdXu}cF!>ad3+Yhuw2EP*q?}a4Cw2wZ8z|jErjY4)YsmBeRKAzh-Ia~2 zrBDkuGE0FHL3D6{_?a{3HF*?E(#~1$>WIHy&il&#;8wUiw?u~9essc+^&E2l_W8?~ zB)xp~8iirR;hajZRWYpw*LO;d*7~32{X<>{;FK-D``!-b7$wpeMRt@C&T`+Z{g@3R z32rQ3XUUcW`%SgBC!DTfH@KJY8n|!D#rQudIybiiVGhq6ozrYSe)>c=#>>UH&30Sc ztl&M;M)q|vIM|OKfB)d{=z|YG;12%G*FScI=O6sx_a8oa2E*Z-{inP851zg_zw^){ z7E}FFp`=a674GaMz|f6bw|4jUzxd*dFYi6Dcf~spnszYg{p<-jtrxg-`O4epC;0Rd z;9UR5>%%NVa-NcT47s@ba@VTjjv3??c$b;AM{S0&4+i)fM~zdw{N<^71p@zHLD-nW zCJz;cDtEm>p1*fydkKQ|a8N1CKdsAxS_25>{`zF!*(B}ytCD~x8=QJN)6D{>92i7x z<9`LE(@0yS(0eN!O29`l+Fzzvvs9V43PTUksZC5~`{2x$2{EOE&a7oZg3;w7X5T&O z;9z6j=u|j2HCi zdPK?ztg16M_@>FLan(#^RB==pDVJc(mF0ZFNsMV0czvy?i(JznGbw@D(OH#DcYrlX zDrNc7;R3XnWYk5y{g+X3&~NJQUb4~-XYsZP4zgOfeAobjt`00=Y?T(sRH${@!#ZI! zI(10!W$DCtz!6!niXf203v<5vP#ZGCg2#SKJTc+zuHUGZBr3zn2+`w$h`CZ1tR+wW zNjNrD$t)7cpz>bmSEzhDl&kU8)mCE9JR5(3B@)! zV0nPj0E6?$3!4&!XPx1;47DT6I}R0pIP#Tp`DZz)|#Ue zEI7OoBFyr&0f(+HDU*Jxvv{eP)v1zj;Z#(|j1ndQ$@*k;3+KWEDki3WiI?%1H$|`K z;=)+bsMr>iJi|Ojf(HUwaKl<()jNy%YK`Ion`^OxmsAoW92FVA>k?6U78dMq@Kx$D zqQcWd*Ctu;n;@u5vj_`}{M2}EhlkNXoKv&-h?GO=JbeAY8awHD8i^@hvAI|V7wGyK zm`U0*AZZwB6#d`4c>6Z^5`FrG;z-JcNT73Qr5l)GT_?y?SeeM0*KVzis z_ATB7GL3)dCPxv@W<=0vd?f=-#t02RAmWvh$4pTF>Y>}jm1?m&%*K1*@}S9ucppm) z%U~MLX=H##rXWg_hzoe=&*P+8Rc8#W2zbKld&KbICO%`Y`}s=s!^p%eGhcm!Nt%SB zSLRAE$irWFeE11vQ5p8wMUPY5N;{c91_3UATDtyn-N3B`)h`0co8`l`8I}%{XV<2- z9kS_uu3@OyNt1dKe_GtUSC#UB`R$w$ZL(n!N4^-EwWLR-#ocd}Kvj9(2j8UhPajLmTc+hI_-GxoL?rZ+j+?$0wOKyWXWV~EJX`QxcuIm2|Z%ZP+aci%jJv3u*~ z{>5GnO+a4hn`WS#XWDj$b8^mj+~tdxY)RNNm9lrh=K0CwHRnJd?InD3Wd6XrM0vgZ zRC_aWmvLIgu{M(-CyezO{-?eAK=s?VqU2953L&j4Bb0l7au{lt`o_zYhB9`psShM! z4+A?zqh@XJbiPh^(AG6E<3Dd^*gDbl4qiFk%scaK@xH9?JraB;N?S!$r&qVVZh#+I zWurKOvg9qk5hfUiy-!Y)T}>&AWu9OUJ2Mec5+A+VX{iv8o0(2z#f5=#It>(zPqAvL ziWb{H>;zE13el;Nr zouigVrOb4PZJrv)ASN)iPOoXeQ}Y6I1HvaoSK3qc8oBVi**!ONBRcJ2ca;gtZAYsG z_)-sw)vAPMD(aAiF|>oXL}_PA9>HqZLR$0ch4+V0!PpQg!*^YV4UN*s@bN%!=$fxZx`0?2f$=!EL!M>}#&9TOvR~jU zK-`3Ho`nu(KQT&QxlyCR*bHNQq>E6^vsh1awd78pRwY75_91=alW%xwimgIkTgWeI z3)IES>Q-(&9u#M7mAO%Kn)XI1u4YN=Xdb<}aN+E;XU`sc=gNbJPwqTnavcV7+G0oZ z+qQ*m8&qInCbZDa8b{)dWN|aO-;)J!R#ncs%v9a# zWtHmm0<@;9yIY~7OVp?>&4ewLtVTd7uHWH{>BmFY1`gy(lGajYsTWMA(@&aIK4~f{ zX|*qzFnEKVZfsP;8-f{{aN9U|NpV*el_!1zi&|XH71x6&o)ykUuKaD1`hh9OZNU68 zxQJCyBz0|2FQG;VP?;>KhLNP)a2L-?&S`U1JcB zsoO*Pk>S3UL%mUVI5WMz=4W_`ddL^&N-Zn|`I7=|6=4?Nw=?E{?p(3=3n^5U-*S@f9uwjOPzhk z^gD_oEtPvJU=N;B%20JvWP#!Iaa#iTlc0b&_dBfj;1VU-qTeP4HRUF#O0%@-Ajr_f zDixMwunwq9nJA2f-Wrx@#6?NF3rln+P2lC-JP<3|fFl%`+qhcfV)+=$FuJ%~5&%Z= zr-H={@C~s^I-D0c%;06%2;~BdDrHL2O;S$bs04Q#bQflOn8y(=ZHy|S_SKd4M}!hr zFlQc~pVOtZG$P!pmA3{A4Gg!m!y)IrAeFyZeHDlj+{FDhPl!L-2&;v~28A1Uv@)cg zK(vQX{%qDB&5Xk3R%KsC^dE2imtj2?E2bQXZ%m3Ase;;2pE{sH$#Jy8Gm>+WVR`iA zF&8M$`5ou-@&_Nhf8)jtPBS-$(=hc@igQ&}bNvm}zB({&uewILMfZe#*E_=+p{r3H zaA~D4Bh<{Df5K-Js~R^N#U}KF42$>rTx)f_4Ao?WbES=fbFyaG)q0gO)!J&LmJPKC zg1tC89d3VmGRMM9t|f1~B3pz7P7BiElWZd=D#y|?ofl8ch&$A0?}ax37-HXl<*hh3 zZ@uS_Lioo{^0D^ADp{_u%=In-gR;c5hNYZSYI`Ht3vW^*i0aj;pDmfcM2iV-yCvID}XW5%#+17*7oQLzuggyYi$7iTigdZ67>i} zmN$>%+96iAv3C1A;&@Cp7D15oO?&0#l^!f-K33(Znn4emF`V)Clre8{j%Jjm)yael zdxtByou*8G#A>dc*JqB5G)$$GV55zy26*Y$Bwd}5D(8ZhM)ST#HPb?q0!4~S=7Lja zI$o6!tDbs1lw$f%?sOl#hoD!@Gy$W#sp-QYJ2psz^AC$eVXLmF_g53>jCusffc?uo2HWhfTU>UDV24*FtIEjXzL|Q_lq5cT9uT7YN6}L+d;=7jI z!iI(75%4u@W+Jwt7Ea7`dTN$9wNw1BPk724(Beu2&j#iC2~USws`a8!GT+k_2#b~B zXudqBqXkk_kOB%^=BSoAI^o4AXRea)-Et<_Uv4IX&0 z$h~`a)xS-bdP{lJlco(D%k;8wHMO}JwCOWujgv}LC3*}kCTL#%Zsg1= zCIg%E67iKcmSv_T<;^OIB2IU7M)kN#sG`#-TFr)x%3#xmPo^~LDXB1;Oc$BS{Fa?v zX~PE&MQD?arC50#mx0fP0>eNHz8G=qk{--v^~px%nlZrx*7qh$hp(u{&EysH_sa%l za|tGQ{Di*jjgADH6-^`6e5I12T!}de22Xao(y^Nefgtx#42%*aNcPc`A6Z~GpE?TY z=qz=_=@MbIGsNa@*oG#UGNzvJWY{pD$gl_=2q=;)4auQ+klT};D|eOP>IQVx zS*9U$W8^=@EQ~pny$>?FCgFr4lWCdVH8G~QIwl(1=e|-1Bul_Z&E5Otq>G9$wD6 zhRULLbhx$q_RWiXPyhJt_kQzV|Nbxisc-#_zw(!V={w)LxKne8Klo!8f77>|_3?xn zmisYbx;mwk;bMjaSrQ`Y$Fc-sb>Uez3w(ols+!0km)_)E4@);mx{((T3rj$w@-fxG zgFtVTKXE}oiG%W8y!*-5@VSPyh(J*g1+9nx)QC8$|rHs%EDe`2tzZuG7?Odr|tzA$m}kFW0)BfBjd3wF}G0SY2?D>R80)D3wyBO z2Xz4ac%v!?agD2S7h3v~p;h_R0Wb;h=1q?EjEowC-hH|M{9ylC+a{|zV?Hg+x0|wv z5L3irBzk=>c31udwn+e@w59GRx}%c$;YNJrMitLM^y&CX+?!PGUf7fg2k@-3Ch~1 z#(i4keDs=|Z3vvNY^1|7mxH|*uf2$7#+8zz1A^y=XFt+~SY1=isj5KQ011GU)2Z{+ zFwG~gsLj~ORdG}&IQQC-IZqxwaIm#`=Cy}fYh^2Z;Vp8ME(d9JGQSqreA+<#_Z z+F%)b0VyN$|2eXGu7?&ZIhlu`w{P9nX(58y9R&0?bo_|WQAh1}HsR+`PHygNTRVB- z%wt3$)E0Nrj0|{P7M9@1O-t$@J+Xj)Pb;~7z%kBWWJKp4J(fx~*_X=0W?E05SjYS3 zyq6x^ys`Vzo(<~L>oEUdEz|AmS9sYcKD*1kR+w#zr%vqAhhC{ppzXUB;_z&b-r5Iz zpZ-7k;6v-ae*Ec=b;pzu{F?$Y7jgH#iHGN2A11H&ZG@i10M~C^(~Pq17ITC9v=#H!?+S(`=EoWHnOZGJKX$LpHM_iN?o$5Z96}l^W$X- zgli=7jUB-f=BwEpyuEF)T$4!p?hX=*cT1xcN+?~!iE2{$rGb9noEGsk)^tL7FuD=( zrw76^XY6af^y8?D0!|)o9c~?+J>wKtTamXMAw_s-nl$LObuzQ_{1ZnxM*qY+8XxCA z|DH|M2f$(zEa*>O%--cgToGwnL#l}7#+`DcMqjxh)BXs@;ZkNfK>BL!fJ&97sTY)+ zNu1!RyT5An6uvS+zPSN3v0zeVI3plY5tB##Y`mB76ahl73!P96xTGrA@t-E8uc_)w z><$djAi}Dy08~J$zl0=3g)8o7+@u=;1)&mj$la~rC0*Zf=bU2UBhYlyCb*hRlH9gI zJC#`&hYZz{qi{Q(U@)CivAIM~B zzX31wW3bTDds6|CTr6~t4|t&*tmu} zQdD7PG*wXywgHP4{k~S;F)e0R<@NUM#WUNtE}d7rBwQspak`Q%>{Fo+vlBL^fup$g zCB7o+tK3U(@tr>UO8g1`3S(u{L+a`Vg(aBwVteQ5?%u!n-QW9r|M=Ix|JmI)5_&&g z+rofHf(X%!J>pJoH}~yL?<6{Uy#MOg{?)(v4?p2Ln$|LL8}7pFxCIP;89 z>_j~KQ9AJ^WPQg`WcX{P=wU>sw4jY;Tw=r-Ub|XK7H3!C;FSDGFE!3mZNv3=8QJZlIHE0Kg2B!+BVHhGjk@#sv8i zmo)3nH+vrxnc-qcQ1naCf!FC|VycR&%PI>M2;7frP@xXu_`jGS%Ze$-(4yhNe~2rt z{)XkyO&e_tlvn6uZArVHSi~j;FWGvHnaym#aNas}z=gJ3_|U*(8?ZF>t{AAc7&?=C zDS0V$X&liwNw}b06qqy{=Ch#*jotoSQks@3W0OZq;uw}(V>xeFCvDj3+WL?k>pdDC zdinCn7JvNg<#(6b)IQRV&(51O?nqi#n)skLp1U#fHB3X~hQ!(+Mjr1Wi zIi;-EjWFFS8#yekiEaq9)88FAp$&`hJ zBy8RfXRq}IhHNV0Lsd2d2^JJYRO?|t(1PaF?Q1ri<{?A^cj z&~@#~HHYCahST0zkgbH47jMzLwL|JD?Wq|2jR6>AR}LJlW0S*?#oo@%Go8+Vw19+b zY0%Apnrt_IZkdhsK-^77=2$^vWKFM;@pyZH0b*ls=b9)w-TK?p`9IF#DeK-^n`Gqv z0zQe%1veFM$(d91DMWA5DW$dnnp03Zn_|n+EJkYp1oG6$# zuO+t=S&z;h0FB2dpM3K3-}#Og9q8MzFw20gdOCUX@iX@D?71Ee#!Of~dH&qQO|LUJ ze?ApmIB%0+WksetPsG7VyDw?r%Eb$&!gMX@cxrJSr)nTvu}4C&B2Ee`DC-RS@0_t~ zz6HQ){EB7eOAfL&MpRFIjJ%?m5g%vzRF$~RFluQ;3R+OYrfL|EXpi;twa7w z<-L90g0bx#3fKLFX=W(UKo$c|oO;_KP4dKcmu+iHES zwje{-q@qL#@TgAekLf??227ca=&A_AGfl))X*f|zT-b6~JFdjY%{cdvW|Jj;NAuO{ zi~p4eVm658S!AgPqyj_PWo4Y#lK5T^;s$XrbFJT!Mrr63y~7N60(>#M4BeK+v~$<&EUmdOPWn@e;OcOk6{=m?;9 z-sXPh)akQFCl5~V>~X6Ut{;B->3{se4>=?^Zr-$|MAKtp>XHe7%U7HTu{KKSN2;Wz z@~kX+RDf&1lmKhj6i9g~X8g0XlwrmX21aO{5Or0|2{pqoB*}0S2#1^OKW2DB$*j&V zb4Zi1xLv}gyzrEdQ7c)P1UF7Ft6za*E9?;}SKO#aaLe#R5JK6RDOD;VPm*H#p7do~ z_X|rv1m+lq7>QOq0zeb}cr2r^%8}Mtks?=Z$bRQ>C1r?vmfSsuI;^ z!!+F*k78D7qo{P8amI2Cq?RcqP=s!!q=Im)s;GjRZU+vs_zL}!I3Y?}NL7yuRRh5*j^1&|@u+9ha&252I*`k&zldUaU?b#dFRmt+={<{y!a@T=R3ap2_r5lPLu7Cawc|UCBOO6so8*NJ znoX8$WLmKzip7Mhs`9m|)ERwTeq%+X&5_kKHyG0Wp%KAC!bJt zD`28|<6v09{3msAak#;@+6}N|-ITxzGVBe&tQZ(Mp zIj-uc7y%r^qRI|UKWk|wYo`J=#d6Rs1=4J2CQL2V4Vc2WEI$V(rKd7o`i6-QUu8NC z)+eo6q+=jozu{V2bh_Jc)oVogTRUeRyV637ifmP7f|A%nwMls}IF=6PIZXsR`pkaR zBo4mUt>~3!q`GwNrfB2kGVJo@Yaf5~@wKbh5l+LcSFn#T7| zgNqk0K6&y)*OUG`pX2E0;`s|7e{jbMK3?MHtzPGLPJMa*5o6mq_3q*Yi$3l(UiP>`k6)k&z^a9x~Nt0FArYX=tRjcZ`q)H z<=VBYC%1NYUU`MCmx^ue?Gc^A-|h%tFzIBKIhH4fT8WQ;{)1OOa3UC1He^lY*)u^n zc(r*U#j1j))EqZj%e*u3X?Z4>%l?qgU)C`8z)q%W5}FB4l2s!G(2Kh&&4nK-UM}qD z8qq(?rERgt0Ih2%%^2I5VXG3x_0J}wbK7TgC24mqrTX~xa2g}H~pj%Doeq_HUg9-D*)IgY$6v#QjDzK01O%(b>igAU=~G# zlRAnEa_AJC3KxnfYq{VkPlnb)D1D^E?{a(-e1kM5={Qv^-lNiOIvx}$2n@EQ2#ZRy zikk7mm&gMVwIL|61s;RNx1?dt47(j|3+Qk9e`BEC_c!iOH&I!JF&SM=B$G#viJSf* zZO=!nFT`4uxyd=c^|$tsvHGwZlcZ*|l}rmW+2FT-_Gf?o+ut#y!IEY@ zj50ISvEzAk?x#tpk>8j5B9x9Q?$nO?4ZkC8Uz2;;1PRjNq)>d4hFHm3q7iT1PF`-5 znIc`I6tL?_K+%(Q(3K|zW>h$dkdSeL7UDrtmds;H9>hJnDK}DbSW-S7S&+{Z_I^Xo za-%Xhq;K9)VCj-g3``AdEZ7o^z4RRJm80gmhe<&ttK5c6;UL830;^pDt)r;PgIUZL6m4GlKN%Bn)dR3O|qVbI$^YC#| zvP#YnN?C41WOnkjJ89Odvt@1W%dg{!{m#+^0@o)Z)_22)?fA?Q~^P@Wyhp(YuHN(9KSqxQZ z7=Xu-*i*N`4x2pym>QodHdSwo6XSTywItXoy7YhqV7Yp_I4H(hU#K&0z^Ns;;VK|v zCDqv0kWcBcGK?+mx}VN#5(f{auVXU9bP`4(0dTM~B+S7^X!z*4o^EL)H%w0?7=S=Y zP;80Bu3*@5JG>*-O5NdZ-Yyzy?#=~%?eG}zV4fNwamPB~7m!*~H=us{!g2zQyfHl0 zLf%b7(0-jcV<*+cD_1TXMYIo3IF6JVg~c~c^I{FAyqSacodYwHv1iNhwb0f0w4FL4 zxR{$pY$TkFnrdKlnt@@Zrgoq`%CB_V*RTwg_JY)sRu(G^lM$6KRg8qyZ2077v~l6g z8AHF8$iIKATnfCCQ?`_ZGvpMCxre_d{?3TXoAmU|i6H&}k+goQ)ngj+AvNQU=eJ%94lus**I z|GO^_EbRBjw|YlSyu90b{G{RJQzuOY7(w4U;XtIRGQ2u>9o^C116yn_kdB@kA6UT)&;9%N>($6E8hvMB$l22uPHV)^U%d3Ezw_;DSL`o% zWL*F*bThWMV4Fkb>#d<;FUqUWKmX#&rAt&=^lb3s>(h^M=!3&u#-T$?W0x+R-9ERy z`|Q~lUw+AwE}lDo$#g;E)=hjo+p(Z&Pp;Itoh{mOlTR^>nai8K71 zLDuP=1*Ts``ws6Kx48KJc^#AoXGTI0r`W ztHwi2{|$zOQb80eSjP0!UbUTm9l8R#A6vqWFjyR zqmwbQ)2D7k0qG(Y&+Q}+zH!-w4a`KAtzU&q!lrW)rqm-@ZjqcWhaoUecHDpiQMaa# zm1mtf;KVmmriK}^m@3m#hq%3Z?YfnTo@H}8u&Q{O#hz7x%(?nerGSPF@+!6SHJ7~v z?99$3Q;lk+raL?P7D9S$w~*qN^c&Zflv7G-{U86C7g))sl6Lasb@S!>BCnBff(g3{^5$L;XK-s z3222wWS|*#3NJATVZ(k93DVaHz3iiA+hgpol$nYS1t1}VY$}C#jq19_n1DNKN!qcm zQDfGYpV?j_`6jvLp?0>7B#bf?8^MRf=%oNm#z@sIzJ0~0h>>HUdU9pi-b(a${_%Hz z<9q-5uw}E1WUg`jy{8AFMs2`ngrflCH*dTl^2MM3#Xo!Q(uH9#3Y^a+I0B3PD#GpU zgZ;zrfA)ow>Q0?K9}D>wh-WTz#Kd)rOD-m!W}rh|cBaiIQ`)F8{(~5>jAryHbfc$~ zk#M0OftPduNA%$to`{=VA=28|SKoSx)5q#>z{pTSw zR2n-0yX|l8H--|x3J8Dwl99@T+WYL;!#9V&@jJh#5&7T#hyV6#w{Hen_0lDojE|-D zFeytt=!W8=IT>5cF6-hNj;ZF09T~<6T=GTyIMV`*Vb>_a{iq`>NsZm8RZ^6aRXT#~ zQ_RG!s(b>4EY^U|oNDblRFfQ;VnaG`BbdNq2T?Z=2mvsVtIT^IHaC2VN{7G}@3Lmy zF6`p8{bwDC_`#h&_w8@lRpd3N)=UejUgPO*pR&^V!`nAGK0a+@C#TKj*ofvrK)helUFNg~V3)XpN1j%U)}& z+qdZuMzdUMkFl;%Jwks%2}?&)tZVM(CaEzL2N(h|e0L|N1euS~gX+s;T*u%!TCTxR zd~L3Jy1luxZv3=WD6bBj*>dLA&6~Gx-BSN+1@vt-A8QmjX*1A51@V&1Y067Zxt}U4OG=DY>}li>G$gitvKLfYY%T&TwPJ7I z-M|09@GHh_fwLpRsM*aMH@Uw!)G_g*04ME;!ihCKXU|=*dE?2GM_6^h7lImMdx)L0 zNfJw{oL?vbT2wg(uw_paQAYF$;GE##m`W9|5q zK^;cZVfj464)kNp01eB{fZZty^6uf?7cV`+?0Wg^iBd;=Om1%-GDhbV*;LS*o^M{) z&3W-`H;-AtK6T0}fA4;lp;?Z1KfiW9tewWs(NgKlb>4z=4PNZ-ThVmxocAifHFt4% z^z`Naxf|Cm+dE;|l1&TdZSf}m$e>&qCFJ|kFL7FNLyVo%UO#axc~opGA=sgkUzpy*YoPP~5A+M^>IQ?}ojSnlOts9R~Leo?`X z*I0IO{!<&p#=8QULe#rd9_5tet5bK41wD>gWLvNm1*+;6-T-+id96giD=P7^I(Q zLlm*3kqV+J^_&=4-kLJGt0pn0#=5OfE)f(n8?^CIiJFK8XgrsQOyvS9l}3w8qj? z`U)%qQm&`@4>BULL#>&&1`n}9`a@$b^_V&&qG1irch~?A&FDfz7HMKO#>>PYU5U^` zDiVx3Dt_wk|L7;5{oWsbUj-_gz*JDxh?;^E?a$kL@#0$_fB4t`(qHgqLKA%%R>?!z z9m3_w;v@c+!HiS@f?Ng*XEXsqglq5psXAAk7!-~aKcb6$Bfxk3q|sq5tF#u70p z97BQ%K6$ji``P`+_a8mG?U0D7q(L`A1&z0n&^=$`(bMNYx%c4R){cSj^pNH!zMr|q zKt`7Imn^-ADXj%wKqUm|iW~mjdR`eizrkisuJK2cw={v(nyyC)Qd?Ox_|)tmP%KDf z6#Sd-cL?_1`zOEgfB*B}Iyhy8uHyoi`PFC*&N6&js!}if!Ut=54O8FizOGRE+dgfT zz(4!#e|zi3wg2_M|F5r}J0qVIm=X^ECEMHz8nY1)cSCCIDx4XGpvp~A^#PTdZ26=a#6u z0K7ZlO!q6!hqQ9NmJff)H<<%nU34%$tHK$!MxbaSCm@=L*3C-il7^$2Ur`tV)vrsb zM>ERFiIGw-H=GT_sb(F<(xzNE)WzVTtX7Yc5iX`6MrN>Uw&eiANwCB&8a2;gsbCnP z9c5Fc6qeyKkOessD8sPSzC$=ygt(}04qtH2RsFX<_~2_Fea-sySjAIg?)-5rv_>g) zRiSf{(gFpvUx#Ob+Kez--$$63MwHC~uC9os@Fn{!yOJN!Lb;TQ8?IAJP8_py?z}f= z7)f>gpXb$*eQR9uX^Pcw}1kh;D2@4aQ?-M*wXqpoOkWolPAwK^mrsrd2sRK`Q5$44n(ql z_|2;~`!+K3BBZ`~?F!KtkG;bD1d48|s~L=JE3=YebJRE=Sq_;6OMtp&b%GkrnetM9 zwcE+J$w~@r-vB->EwzS01stXlXTPa3JY~L5_l${+<~IT8rx(__$rLK0d12H(sT;w} zD9dFUn~n?q^3&QLtVop`fP>I^6_;hrCby}1Ao9$14|``x984o|4E=AHZL(u0st@qM>%|5)eCY63$CJiTKWqL=sYc| z0S_90Asf0?TX7%46+ZH29LHdyrdf=r|LW(-*IwJ=DT2Oyi}OzeRF>mYXipyJCnNmn zy|6NLDkZ7>&h6aLt7KJ;eg_2Zkfx$d;IhS3He{ftAAA!NTa&>#skl@-k2N2Y>Yad#bwU)*T$y!8x&IouDl`vxP`i!%ntT0SERW>9aDDlkaw3 zZa;haTnt~|U#p4K3z?OFRb_pR`pFelZ_uD55!qgMm5orAu- zm%88zq{g6WK}%*8bXWT*W~Y*2y+if)ts7U?Ym(y;b<`B98Tf$1W8d>TMQvmnL58tm zcHA`}i*bsJ`R(mL_`whV_y6PnYC;zhI>{E(r)L9G28I6OpZ$fu_{+a^?VQ#{ajG9b zI>vbfMx+#&E>=t?InR?D1f2$zmViyJq<5LRZRqrbM(E-tRpF{nMvx6u0wpPI6*7Fk z-8$`Dz0dC5`}t2kTI75uPN+`X{25wI>{HW8%9Qem4)d;HI>4` z3S~F*6=I2y3W)i6m;xS}ENPTl%`d@{S9KU}U<1-^Yup1Y^~Cvx-ctrd059{|{;l8s z*MI+?{`#JkKN@w~&i)5$W-~XC%zey=8f57^RFkW4jdd+QhCsz_?^qk~kACxef9hM` z`fI=P%MR%aIPzIheLh7lHH}0_6&FElbDO7W;d8a&XV?H=H-MI;8YW7+K4t%s5c*-h zVvt?+IF=jZa&>;HmLffxngme@ApOMU5<7VXF7k@XhSQA(`k{Y{NEgy2w7?P{!wx`I zO7X{R$2~cw#7I8BD^>Mdh6U^BcLabARAM;L)tn!G&GG zu;Fe9lWD_P#3K^aFdr`ktv_VhT3+2!L$H+y(JIMV@#FR&k5`r9R)xzq(2wPWwv5h% zfa5++9aTd&W}-#M9B$y4QogjXj5Na?UShlQG^F56xDmk35Fh!2$-aG!fRL|@PZ|$7 zI%WDw&1LK*w92wck=r+Kedp(X&Y6<1@f8dnP8qMJtp?JxAn(fng}hR3s+SYbO5=)K z85s{yM?FSW5vbQ8;`%R*c_v6jffBRs@nB{r>)Z2+S}V3r_l_;cs95X^qrv61)x_|#3bNa?NzVWSZe(UNL@69#UW9;m$ z6Gfjqe(0@OxZ%NLMXP%8yaPq+Twq88LJIISX6_vc_jU!__R41TNPFRcaq;QwHLT2~ zd@E;p0W&3L@mS{-Uq=#}GBt(Y2>8ce23 zIH~m76OS&+xtB zF(?d;aK}^%@>g9tbiy8r`DtlBa2GougHSa|^~n$j_lk2PXt9s2^_Z;aH8@;Ku`py+ zz88EYfQg1VR=}aRHYM+<8y_ir>x04G_dRs8}%%f^I_7p|98Xm9#tacBA z3~VggV!>0dI|+r%jXm@`rhxU-8LO_zs|w|Wc^>8CGB4DV%Vw9=; z3D@MMGA=C@mM5uqV(6g!SUijZvzb9V+AK{tJPft?+aT;`gTEV4TU(9<`&Ym7yWXp^ z*O75vs`NRs2ifmQrY_S`L1bJ5z=Bn~8LCj4QrKZrOxs~C?WPT;_6M{a7 z)Ilx~Db2>{dRQ$*jKe&3TaB_JQc-`w1_J2{8SXt>jqurn$A9M^efQz+-qtw_Wb=li znqrHGdZTHNsc1NepSW@GoZNx>ar@mA6R6^av!8AN`4?MPoCc2*(Pp(p} zi0(y;M$>XkE z#x6YlO&|#OCH6z9#>_Z7Mm2gurdMVb9-URC#?uTEPBL+mXT#?Jp%N*0qq$H|6;xhp z+gyBk>Mw_imh&z-oyTLValV3vzjhxYMfa@%-o_UV-(GZb%*j)VjFaY{?(V+cmwkid zI3G%qK!PFctGoD#onj_B${N$9mkBBprXc6AyB$y*mWt#PNJ~zDlnQ#)%2xeM{rxR| zu=ju6ynZ8PRzD9@*_gu$YjZUUt=XBjtoEJ$vLaTh6VXfJqf_e!K}&S1uet@v#nMU0 zS;}sS0$A5a0t%EurAB#eIzFN1gGP%ZoQWc@_A7VjuTIqF8a8T`moBpHDEm(6sC14( zv)Zsl$9Qe8zL6)997Y|q(joV^toXTpLzX;w%JqH5^>4NPX=|bm?mxI^J^wk2X1BH- z$^YQNL*0^J_%nazYaf5C^@I=3JbTB;_cn*u)9h}i))SgkuCuxIPVB*5{x1bKVQ%K3 z#UZ2-W0c*2I!^bo@qnO~4(TCs%8d;t(TJA_aA<{{0qWcit-#hcioJ$x=d`6;pMU<@ zo%i1-rn8&Oqj2oeIz08p?1W|c+5xFi{?LyxjmSr=_TNN2UIvV|gOGEQ9x zW~NJgs%2zvk7l9Pj%?VvNUvMoy=oq&PJgwFEXovTnuGvJG`>w96}JZzIDLbu*p;EC z&v|2nl|c$H-K2udH|di*k?U=y`Wl*E0M@C(XyLV=X)4kgifOSrEMV#VQRoCW`4EPA z9nwZ0t3Ih7E$u4JW%AN+MGK~h#ZeQ4b7i)bzp|&r3?5Q1VJA#LPac;Hla;Y0aUf}^ zJahwOVH^ZPMAu{v%9IzWYMw+`EOsLYLz4O#}u1V&CG$SVV#fA zmbG`S^|#{E+d<6bLE;PzMw>Ni=8o9#^W9x$E?F&G2CADu{7ihr#D{ANaXU>$9cth- zO9^Qy5qeJF{tIr_CAU*d(wD&tsz8?zcFiuSClun-%rtsbR#As_u&kx4ygqqyPrjIo zXl;R4NN&Gtn4s8-#VXKAm?Sx9Sw(zLl`$$5TpS1rDQY0gj9M-RtEr!9NBT*hQ#@L8RY{FQ z)mG&zx|1b*!O4zEA@SD!(c8cK-GBCPe(rwx zY^;`2*~1iX@xuEW{@iETtTq?n_kZxC-}slm{XhQAzkc>)TkuEaFe+&=vc6cvk|+%% z*hn+nlB4PcGAbXHpcXL3%A={qkik>fikP}=GH9WZ?-Z57%W$6xsZW#^H9sjf0p=EN z_kAZmDw`OrNa*+2gi-(iKmbWZK~!H#2}(bRND+6F8pe%2XsyOTfMl7$LX*ts;b?F} zHp0h=WznH5d?@FcPj-*NftU)&(C_5&03$pV4J^s1s)D-`izY;)&cPMpYMU;4EK<;@ zq35oA6a^$3M^&Usa(FaKa`tkIy3cLv3iAQ&_vPqKCqO^lst%*U7h1Yu1EGI91SIuXtYj}0aTPN3BU-s7e^)1WSt);3-B(`!i4#t=Rg9qoB z8om%M=jGX*G>XNA7z>*@`!g3}{F4u8$b#bXn(Fy*ZLCU8JiQSs1M}c2FbqYSHi9Ba z-O3rr8ZX10YJyHPB5`Y=6T1ePn3YSK85c^8QGslU99q5Uj9j{dhg&wb_0};<>TO(g zilrLt`ST}t@7{F^&<7uWq$Na>)BNc1V~5;x+vgl@&FJfoKKl6mJ9qe7wgGqpE^kj; zn^L!J>S+rv8&J`)32iTRbX4mddGF1 zJOA{_Q=L+5fAZ0XmL9!ue1n;U!&46X2If7!RxvS$&%V5S>B_ZBSFYnx@6_7#(`V1` zzj$%x^vPQ{Zeqp@gLl0GB5Ulm#|&8>n4Ng#4Y|x^sss7RlzGDzov?ilJ-SZRAG1|O zvqEqVlaiZEWo)24tg3@~JnZQXl&85#RfA*aGL|v85beg-B$SrX<&aEO>qj++Lp?y2 zR}Z4m$$qkpsYqmpT0-a7ayCct%y{W))+eR?3UA&WlyBLU;vT1qZ(DKe3G`U+LHjYN zvDjMO*F;Mi}G0+tkR$|OI&L8C2v;7R3ucZ z${zJJE-R6Ix@-AFWR#{`umkkOe!JRPFyBo$uR;<&i=AGLC~ zr|CHqFga8{E_NlIInQWHR({k{;dm-=RUZMW+I?5YBOm>0PbjKlj|?+7T+8?lckReb zpwX4BbE%W$!%o>H4VA8A&g8UxOE32wNZdB5+62h z0yAxW;T)ARO;go>^Y6(PPt$(EIeun*4DGT2_k{;m6NPYTR?2C>dJf*TG(Q*v8I27A z`IqX_BfXT>)i>)Ai~S`zk$tC1T&od6-DC8J__!?)E2FB9?VYn#C_WiY%ryW~0y?iq zrhR%EYYW$y++dVq;l^|947(*>;n5SOZm3NhlnMZ6K;u{AYlt8ISPUE3$+*-#k!rOG z3oCA8ooeEAU5KQ>(Fu)Ce@hth36UYCjl@DbRb)S~F!#xbBre_N18Jz_mI7;>0h-qG z36@Nyd~ff{xhn&dmLpMQdKT!8L3z=oCRuy#5#drRmC5cFtwmXAX=8tq@@O>X zZpXO~mIS_;Aut@r26Dsp!j>{Z;fu|t{~oFx1Wn>WlJ&?qKmo?>RDeTYn(eLs?~i}< z>%aBABO_Ak%Rx68A=yT|L)=MhhbP_~T5Ejm`eifYI?`IihIlpRPT05Wu2`5%nyd#YK-E$q9c9ELh`C>=+?J`~jE}L8v64g?bTKz{1||f9 zcH%q)wc%~WHP_B)JZ9?-3T z?5kWD3~6N`lNWF*6c2L4P&D0c`oXc$@X2ae@?xfR)Orci(p&7oue4IU7pQEXHc|L? z_oc%vW#^zcVIVGOtYmv<3|vvx3EQ_;%05p>laOEXxXd<7<_R^gd| zVRobX%O(Rd zcDQ*a)1X7j;%mOz{i@&;s}-Le9h#j?xNLIG;8=~XP;J+;F!kebM{ts>BL@5FofDSc zco7%aiUA}bV`gH^MDtw3e7+k-0{l={uxA}85rrX=z z$ZHM!LvQ`w+iml7;~h1$~VNT&}SI9ceNeFEFn8wQazuJ7Jnw8hqt@{?yzNiND}Ur5efIM+I# zC$CuGd$&(I6ezh54)}fY!n=yuq(efza@oFKe|PUW(O}n90LAN&?!vNvprw1d`_hZQ zu3Woe?f)}#Q*YktJv43d`i(VNH?LpUmv}a6(T8Ei`WlI1*ZbSIHU!IN=0)cjL>5sC zW+!j?r1MrbF#If}|MKS8mREtzPc!^66<$^bXwxtR+Nxi*Pd4lkXqK$3*4$=4zH|dB zyoyoXMBSIIT(bt&rQ`}2F&m1E3JOm-084)WL3rrJBMHu*%S$I zN=q1cGL^yc)F`FxrS8bI_3MCdRK0hV}M4V&z}NeP0D zgOz$%N~ma{G@|B$lpCdt#8680bD!qQgod=3@X`UxKEA|usWMP%EV_Si;c6#;gv;r3Ig>^yChiwfXYaBc*YfBQ&Rr-ri^H-vXV)$ zgmW8<@mugZeF#OT$S$&&6$o$9Up}5>>Jzd;BaHa5|F7nvi{oFXzUG zN`3XM!$O&j0gU~7&RXWHZwV_x8-a_O+X{R*#mF5U?{cDP(G9t+Q}-V~dGPr0*KS># z{>uoKG9$>R^i50VroXU=R?6l7E?va1|ApLrWqP=jZnA!|kR+8gTwtgiq64G%5(+`1 zu)ZTliLO+criCmt6L=iaFQP*T+=|=SiWEZBxZYm*Yrpj`z3X@D!es&UDG^3ig(f{3 zjNs_-z#H7Z`Sp)~Ggp)_2bQ@a~tt^}GM}>+iqU+TMak^HajAVzQO~qZmNqjk#2tk9=VYpKdGbOay4E zB^JIkB_IfC2{Bm{Awfd?`b>v;w4p6d;j3T{<}gGd0YX~J?)*(5oV((I4f|1wToF}? zanQ)tK&*r-dR(^9*5z)vu0}NDEkzld+a32zz5I^iLKH!V&LgFBr^&hYxs@nt%&du%9@(xNZ2K zi_(-bbS*1r@2`_ZdU8-cmT|Vk z@`Q4O-dXuGD|4L4L{RgoPCwdnQii^e(S4g`_1<5+bRy8x_in%U{+;)orDN5nB{z86 zed&b)&-DQ=UApLKoksoZLSzUR&RsIYV^p3iZjGoOf(^tjYxM0omS_4=w{PFR^ZuRl z=PbWzoY5cuSqAYKiD?P$(S8}~4a7Pdj_3i#@v~fv!>z)17?I;y+0Ns~5I^z?Gx91Y zcW&RY6ZhraFS+&UM-Rs`rt=r5(uqXQrgDB!Z9%p8+dJ>w*7q|oadPW(VU>pMtxhgF zJb3&aLHgmPVf=H$s;y&f{1 zJ%7&0^>z(NkgZz(gK~Xdl|ENP=tRgsJkd+_K?DEUb&JeqD|77THJ8N2Y= zjXJXI(7uwBBZj<(t{j0--kDY7aTcg>5PR4hBYF(iwgsS{&xn4yqnE0+&{Xu*M%z-5KH8WU*2eyZJIaa1NF;o!m@tY= zuDR=BREa=#d@St%#AT_6I;g7qkw8tjw&TOZYnu!uv3G~iLw909o2Wd zfb-0%z7wFDbupWc(sO{K$}ujELN4t_3l$8}c_Y$7nO3Ha6S}lQHew7lkqMvdBZCxu z3SA26u?HE}=(1F!oWihBDGqq-rMuXl1Oaq~7y9nvtLL~FnuV^oe!~{$R<{tz-DH1c z5X#UueIFIK^Sp(R#30Lt;D>$)FyTAw(9Y+WrKn_n6StU)>4r?yN$%7VV;QARw{y`} zCJv{$7K^kN?%_7u8FC?8nu&sBGJwQT9J5mVJP+EXmkpEUQmZdXF*uSXydjT+u*_{T zM{e?3nR~If|LNyuFEotS_MwQF1vi;BbS68{p8yw2sUCX;-Nh^t>m_$+%3?mhXQdu|SVaAI%nMgoz^ezL%!CljON^}X+Y^vSn>__sT4$}#iHk_j+?jq73{ zIsB(OPJi#We*F)>`P&y?J=G4AoXTMP?73IJ_Sv_7<;!3CcmMHEe(?6sbnz)@^+>+7 zqlE~l1xUIZ|KbNf{^qZL{q@tw7Sdytc$0Cxu8`&&(;QiLl?NP_=HXlg%6o5n)}v8@ z%aOuL+%eeeVR)29M5UC$UjU$$IN=7UYeZ*$X*}>HhHdomGKt?V2Qc6@Ty~p39#p9d z_B;f=Y=Nro?W0Pr`|Ig*R&@Q&-CF{g2M@Sp&f6M98={{yL&eO7ee1$^QmBV6x`xXj z#YuR_v0AM2u$ZOV)PH@@s?+?KQ_2(HL z##7e3FHLofqBx~KNLX-gM)R#7n|#n#j1TRd-fnB^Wosg}L{!Emv^i(Xq==2Z`>hw; zyH}K_Z&6(}4HdLRNZ_FG=eO@cQ2ml1mtO5W=a^Z#@=zL-stRJlCm4Q~cZ8Dx_+XMl zus`X;hk|t7)J++^a6uFyX1#KG>(-rHdKyn0KOw6C*JK&-B0kmk<~ddmBh<1_+xb6x zjAv#!9trR+ngG+GlaHOfwRQK7dH1u?D2?glF5YP)c*w-`1^2}y=nX>iMrcWk0-AJL zWGXZ=oi#r!+L_8?c#vcIJw}O(dhj*6BL?jmJEzRt$iyY8Gkm?oT8py?@-7kOXOGl2K z_~^rrUhLj7=0L~#`B{*80YXi~5EH0df9xcoM?`D6^O2*6S+`*TtQi?l!<3~_R83ex$dZ5gHTII1%DSnL0t&=a zWX81HKoe9a?)GS&9znz{E{Vd|V4)#Onck$%4;O9LOy%C#Amy-JWIv&NB}PwO9^VUx`xIs;cHw zC34S4@+n4&75c=(aaLhhhNpfp^fIZXBhlnC(6gKo(bOnysaD4&B%T6V6;{efL<*?1 z4)8QVW}Kv!A18wzjHKwWZ&qYW zHDV!?G6>wHD`|>~%@IeQxXdvKcQNb=!jgMZtq_;BN&p5N`!_tVq}szFP5$VaJTDnr z5b}0hsoLm~U8nDEsgun46Hx*Ql0nZQO#7Bp$!?D+gY4fs-9kC%b(=%;9_%X5)Pre6 z)wE-gIM4fLf(d50%-(Xb^Dq6N5RxPptcNy`eQLxkNW2e9C`a{@57w7*>n=)5ze?&3=$?5{4TYk(NqQv$ zkUfDK9hvi>9I;H6$o42}8tuy%p#yaYnY^A%sRRYjNvI3S@+_rw_5bUU#i80}N}a1% zakB_1%IL}K<;6?d>zt`Yon=S%5DaL5B_Dyv5;YMQ!w6PE26~}oDDTThhDF|sJZJ*a zlI0#OJo@PJ74FBB*pda4(E0^UUVpc?w(5Brvys9(|B`M0fB7kk97`J-m6gAt2Z2!% zr+xvHYaY#2=)Qgb^7nt|pa1?hPai(WOWWnHqLk`)nPgz$-mic0tz*vg`L{2= z^W(Seojc?V8&!uc5RgQ)H{SdB(%bKSaQ@`iY$M_`1#Q8c0OKx^5-J)i25CJ2C0=9` zr^^G$DngSf1Ma;lB#kVBNDjt2F4IekTw~fqgd!>0}&Zs$oU@StD zacJAKGD2?<2Gve9`CACtrBu>>vHX?;bqxYMmc{&%i=AnzI>oiWA#%x?GxN&@HdmSKvnchr9f9;fFY_a? zjh`y88kHd%8${4bO5K(%4YgcueV4vkk9Z_|y+^AK^$s^69f*x}8}z1>m{SWPPD_V` z#LGf`Ku@1Otz-Mn?YlRwU$gZ?ZMSU`>-U^Lcjm;&<65H-J$cd^SetDVvZm?tO0Q zuHF5upk=t~a&yBK!qzcACCm(O*DjU%(T)H+B>YlYpE-5LnE=LRh(Y0f^x%;hX=eC5 zeyHiI9r9x1;O5ptqmp;Drc4g8k^kPUd#qGgy6?cgi>J=q+PZ(`%C+}C{#42{r%&xW zaKNzu1Q;dz_a50iOj!8Rw4fRrh#3EkD?CzV-jWqJ z6G(4Geq$X(Epx%8H|&0Fr?CQXlpchXFJA=6_>)ZInT`gJD|Zn|2!W`t9hyNT;O6J+ zOUTkBWdExc@gOhsWW_9f=pcl)5fqU(MW;Tk+ByC`cDh`ql26H=l`B4B9RY34E)XB% zMhdiIHVs;7;!Qjh`nF~FnNF&PSlPg~aXDkDk(uOxDY^es+R&+=8{J7RQ|dF$M`8%4 zYM9F*$Ebs)0>#ss&J*p7$OtT{!XSNnOr@&YFw>JZqDy!{yDkS@z=v>f7MBn18WYRQ zE&|xs00$q)0wTmO`gmULl3Z$}3_RZ?Jn~EB1mY+C`>hwD`De8d23_-wqHUqvH-T|R zWYs^0(>V^2wR|nK|Gk)c+E5B}i2_{vSK5Kz+pJ-JGlvX64tnp(m%E-4ULnF#dm(nX z%(=<#uYBEgS;CO`m+u}pWiN%)b07G5;8`ilTm)r}Z@70`dt?toLrv{Mz=o=9a^pE% zt)#`kiuQvkx#O~}M%l6RRCEt;!to*klP=Tg6@nC0D0Qc_p13)5(Ub@lo+D$ZbSv*5 zkS`H1m|hLL^R_SdJ;5lO84L&*5_}T_-UiAbiPfHRoaqq@iYocoqaKNQ1OI zr3_enf~pKGm;Dy{OQV;9&4_l+z-@LOz&NHkEfj$=!M|Kv$xPaL*Q6mPFvYUHzq! zB+lBF!n%SGK;(K6;Kc%yRhZr9YYDw(p$334b?Y2wgTTqP!k07vy?Oht1!p6pg^U%* z75H+D1q^uCY+#x%^#)~HC69^WdXf$u!(e%1&mSG6)VN>nd($Vm4TwauQh%k;n;MLZ z>au<}d)9BC=byiL_J91Be|Y`c^>;2`cYMZ(4haic{lX|z+IsZhZ+`mDZ+-QP$2QCZ4iSW|ugI z(3K2>7TE%epMCU+VU26{erhia(Rgn(AwK`ch2Q%Jzkd4I5vYw+ethlLpMCdlzWaAS zzk2(=5pX2|#JEm_Pu6bTd*{-%pS<(I*T3}H|LPmR_W9S(bwc6TdaP!(*d_aVtwcaS zdH2IV`Rl*AXOqp!Qy~J80JZk)iSdL7+or6Hij!Ta`c*mx1~EkAnA(yo?^dZ^Sxk{f z8j&f!gV3c~WV%3VYveBU>Wki_Rn6nPQx_esq)n*Dt~tLt?^;GrOB`t<0RBe^n9F~> z1ScAx8yfu#r#%TXQu>a11+Ziz3*}@?c=gUtaesMYv47n)tIBd@D^1M_5n6uo$;Tdx zB6MaSJ9_MuGiUTOvn*@F(yayXyE+!B0xx_eRWsTLU%cmMRVnBb{Xa2N#di2`96EGF zH`3lc`*SI%RTC$?$WWX`WeF3^j?J7p)lVFjro?TkDE3H+za+do@smiT9$o^20?%Vm zSq8%PM$RpvN{tQOS@rtMC9u;IT-3j7?V*_N$(lv}*}kY+6pmSSB30OMVf`okeHU&kyU)~*lDz}1;o&YV4a&Ikau z1d*0Rm8@=iJqo8_&AJ5#Yt>U}k?~^^ z){ep5aSFSA)I|Ryo1NMTx>mGtjDv>{x-iZEmBU9kv)fzu^!VSod-v6I7k1l**tTrX zpFg;9>%{3-&!4||_4;iFD6 zpe716Wmqdk<3(7nF=@_;P|V_heUJ8P_S1AmT>(os0$+`BGjUo|B2r}KN=`m44YGLM z{%oVrdYnb2tIYn!z7{|O%X40DF@g+XN)P2P4TyRp?bncBU97)Oav&>K)th|^e;bqq zBzwvSM5p%%s$1Ah2?{l4RnNSxQPCwXGPQWA`$r$WIYOGjY{Dakm<=YQ?coTHF23IqYoLtMoR-f*D z)=L&pOQ5b>Ych1`ZZwKgbX%pNSmg#r2cj09q8)k?|3_OD|MYeB`qphY5{f(JToZK9H?)io7L11x!7Iw(~a z*pd~FT^+-H11qkZ%vI(dgPdSSsV;`F{3w+38XbX4 z8-0T?21*Xzhh=%zH5lO-xRW`fM^+7W?ZN!cGn*@0^o0wKm4jp)9s)L|rnAK~PgOH+ zn6(_-qva?%v3O|5*K;-X$tCw{n|c>hLKt)C9wAcVMKjr2)F!#8>N`>S5CJ$Bggl{^A^RDmvYFvB#Uq;cbKiE-v`l9X@(*lUuarU3X? z5oH96{r&$D%{){d_!OKeEX_ zcC;8KznX483VlWW<-&>|KGJvTJNR<@~|hqB%4*K888b??OHA*oip532a=c#2)3N%fO?uVo#J zU1$naSYsy{mCF;F_q){oi!qEiB4?N_?wtfZsCY>XN*eS(ka4mkC4%L~&o3Xlkb%?4 z8hpx>7Wk6qyZ8L!qe~xMx+V}QL(-Wvz6wCX%{X3r<+3%vNso3H)T-}~mL|LgxQ;^3#f7!t>{qaBX=bKd#zGNol#7Oha*G`La2-_8G{`CAgDY3x`S8P!FI~NR-%)0wgk84(Gf(M@Z@&4~ zg|o*u4+=M?Sy2fv7hwCv+SS`z-}~FQzw@KF-~05+od*x~d@fS;ZM%%)hd1B4@XBv| z^^0Ho!spJNIJ!TFu}GMBQ1tg-?w&l~_1>pf|M)Mzd;PxNhdt=elQj!)$FU@DzW%z1 zN*(p_)f@lMfBEabe*2xl>yKq9bhZIq2=3oloxEyYsyt-QBwX-~7SveE#D3 zLp4BF^?$nYFCDWM+YYWhc>2PLCp_JhZ%9#DDcD)tQ|+GJC((>#5%Hqio{s$Fm{0Z~ zo$CN(AUzp-86aq!|5MDS9FNbgURL#BgLGS1#WFktMH-DUyMKxu-54tHO60WHzU_7# zARm?mWZE8KE))$7!(Q~c7=CiPLs$hxk{mPhY}J(eDT3bTX;V`?P4ZIK>ue|J#v%ly zpWL~lh?|>!VE;j@L|=RDHQUk)9>nobsX;lq0Z|_+D1xb36dXj=g~FYKF!QG_>b9;o zb9~cShq*)fSE(X773H_a%~8el&*@9X{^YI^m4rLvT*qBKcj=5u<0h8*kP&|%iMB49 z{8f1*lm8N!S+Akfv2 zY2EI=4FSF4V~wxb`O@edgPR054;&T~FZ+Qa9+=15>x4k)_SG?s>ePu-wiz`azfi1^ z23^40rV^Pvb7$+WaTRP4ARX`tBY7bMnM#LU9(@;S(qAIC|~Yr~7tx-QIe5 z;PCyu8ynX@{uH(7Iq=D+ufFn%r43sTtyweVVb0nqV?36 zswrG-ot@49l}!X-K66pJtZ!x7Q@AH5_pvXvlD3G&e6kn%F`3E4`IX*cGK~Y@l?re& z8lP3kUYS!5MA=lJ4}xkF}-S@=3ozbTnHGRE6`j%>G@>SP2iT;c#7*v1}-f-HbV(d zgsO`tx6O%oSI3KXS6j3w03Pr|6L*JXXu%3Vc`D#s&n-gdN*FTYiNOS*XCraS3Vh|Y zZL6dmQ3RiBNjV4~F9zX2Psj`JgEzgoNId@A-YCzf1Sf`7;52&gOZ=aZA@YMv=}HF4 zfRa7Om%PJX(2oR#LeZ&XrG5*~^F)I#8;oM0!A(w_QU+NhrV>^+=PMyD;o#^drea}{ zN5n3-Q9hS459cL_m4PrB(%FzZDd&3OBw#4>q=rFZFBgWc5gN&$DkoJ%h-3tXDH*(c z6xii)pux%37ut9nUfYTsLxrOEef9%8wAtn)4dV~P4ThTVQ8QSr3nnS3iR>)d9>UKR zaxGlT2s$D=L4h7J|3=v1L41b6o|CUj)M9polN5Z?(ejtrm%_FZ8I3HjV=T!^NX$$A zf#Z=kOJ$glD^QcP>s21V&_tJKL19@5EI3QuHwN8DD7?^3o5aC}taBpO!&P$lLv*?%3(B>@TSfNQo3<3`NhBzL85d0}_<-R^_r-qz^(+=GRXJYfL=4q^$>|Tw@vymrj0g$7H%mfW9R5HM)(`=pe~- zUK-*X)Zw89*F#NWZcZapS~tDt+RfW;EW=SG- z7>1IHhmt-H0_7FZNffC=MKn}S{Sp3zOVK|>`w7SZ>brKb-?@y(q-y^BRXlPxRh@im8Jvz z_Sb*)&%gJ>@4xeY4FHO-gDD!ynkrGSe)-0&4=-JP^Xyp$0BML|iaZN@o?g0n>&D%! zrk4l=W(pR~!qvnQPJM=Q;LLsBw7mgJ@ zee`?3{^c+I-fw;5a~ICqxhNcKyUp4C`Nx<3{h$8%U;pjTZa&zyTdU+d6O7RS;XH8w z(4Bi)2=9J;`O>wUfA9~#@%k&LHt-TyY)0(j*O8lL+cns_BL^_;-e-{mW)?Y`_j>)_ z!>f1hzwzwFjT?9W_%FZr=Rf*8a}joS?~BNa>oC4BKPn2r`tAqMo_zo3?;qVfc;wK* z*G`|%UzV7YInB(ce9{uHN|QrsnW-eKGvo~(*N7TzR z?!EiS({U(=+2^NDozfR>;;0HlC>#qaMx50fqH&_5x6b*3H{Cr&M{$gK7O9-7fh`nO z8A}D5jQ%IGK2<1@({zPq0qOD9N}!|QPmi}m9*a9g8YvxD34ikYt&*7%lkP@my^c%n zjQD!guSY85rh}VOGjb|u@^5zsOo@`7Wf)nAMc9U9nn-PVfCn~3-8^*Y@ncc;qY>yX zYhlgiSGPUhIG~$f`t}gDAjP2A-o5qcKYp?-uJdhL4okJzSHHipf`c0leQ-?5V}+Fw z5|PCddeg+^H*el#IqeR%wq`lO$iLqGdi`y@GXwSDAJ}^Um9lX6=GJfB+q!Y{HdzbD zY*%Wnh0O-o1Dj%j7tUWGM`UY?0F@r^-MfD6<}2sULjBRlAD=vP&U)u9XURW#s-xlT zx$^>adkPt6c>CwSFk#3tg~uO%OtUfy6|t#D-1EsA|MheVIiej_N(I@q(To^9_tUD-xq7 z%@W1{7&{D^d-p+hU%$BREwc83B z*Z!QaBd{gKi{cAtCVi9JDKFUnTULm zk`fr=8^#eXyL%td@?j~fJ87~{$VaC955q=I9UFvW>r%<5Ft+Gt@(d`2E8-%xwo>xw z22Et(+i$}0HP>MwsCB zk+ztDKupWilVN$L(Yns4Y${=s_6#uERX}pXp3-wS-0m=10f?+H;h~Z+HPjxRLwrgL zj!(wme*k@H%j`R zyOaT%yz_h@d*I;|h%u~{ps^wAimgyg7Wjtzkz#$9EVD836`dGwWEl%ADc}X_v z<`(#cYOQ!4iN&O671)tL&zEXD3m%1b9cQJO0p7o(JeM<7d*Uw(&U zUQCv>yx^Jy6??1DKE878H@^Die9M>t5Q%0mtKx6my2V6l@ve->Tjg4@17nP2_(1fi zBOv&7EjVOcF%htV8R20bX-%Ruv`9YHdsopK4U9$0H{>i32M`X(7dbK#WqFp*`ueRq z58nR8`=XUvw@cz^jLel3o$0b_=((Ko63FjTRCCN{zU6 z?e?cP{__1xm;bvz{3pNm`Onz;QFqi&-uw7p|NH-B0{tV^gdN#Ozc~bkn9%{jq#MEi zbnE_q{PuT+K7aIk-@I`8I7-20K+MubQxg;qNhPi$k3`g8x#xtWaIdW!Sh({*GvLvG z`N5C>%MaeZ{ph(vk)<4~imx)I{B)f>vo!?mx%c$>4}SLUXD(hmvAMbLP@^+?l9zYE z=R$@^XrwM*YzMC!^{!#hW%*v(j-Bi&D~-$r+Z2Mbx~~lPzEnZzqD-)edd0vw3Kysr-bS;?_@z8#^$9Pueas=lP7y0G?1N|ME_W~+mIjR zlSp(86%kMoG}s&sJ&tMSh5ForPfm7>ij(N^JYALYQcGNXB%7XRy_|c&71tJpLX!{3oOmW-lX` zCQ(?}r|4%!c^n-N^z~FbtIp7(Q^A1_ev2{aqI#~805hN%i4AtCdQgRJa*oY{w(j1(e&d>?LXA_WPj4PLq%YsPSaA~eEogu8)Q0_g2)kwB zkDuz`zjF1eNx8-oj+657^H1DFHilTR*fJ~(>x_=)Q`?>ygS2h;6NB6#xb z-s2k^N00B`yMOD^4YJ*Oxb=6xc=y#ar<*3QOULp3>o;$oKE+LOO56JmJRzdT&rGp< zU|EKPLxrl_j~|c>o7&j7!A8t)$3bck+nVIU#Z?vsiL#cmAeJ7PX4+JR=7Hw4;mPS;6|+hT^tvaIFsUOe2({Z8o!d zpePhFdeehoPhmn<-Q;4Mf*ka8{8HVaj?~@g7@KvI#-8liqxHqkvE{p>mJ_Uq1!h3L zG)5BPcr?l~^V&(dj$9Rq5m4hK?ur6L=9thA9Rn4W5l)~ZF6b3uH$yJ2px4EtiD|fH zrYTjt2U^n$F{j6u#QbTkD8oyAm*}uus9F)BqA7;;HqjToM1>^L7*Fsk@CDX2{eAd( zb$3L|32=^Zw6A@T_{vBMD2~!f^PnQrVZytZnCf5RNmDkf#Df^tregpfiUzNFnbtom z--~CcPikmOouEydtuFs4oXE86X7H{E)~tdcUi#76h(lkhM>%%sOFcY)-QkIO)QtjG(boYKRbv457IYj}S7?#iKVpJ8+>16Rdfm)1X^eRu0i!?(K zSZqwG>UT7mbO-$ORD>N35$!LG#Ltt&EL)q1J2FDNP!ba{w7Q3SA!-2>;`x(jxJ%BS zMUiN4#Zkq2o)63gJvh)uUC?`mfgZeA(<$`9$CpGiR%w=EKJz1yDCgPr+jra|Hkqzo zkAJUcVh_WDE)BDhg&b)nUBSb&j^@_vUI~VSXpktzfBA{NCEG$47z4NRx$scZFa?{O z0^vVFCX3Pa+NB#eKe=*Uw87RUDU-Yjf-1zqMr{DkSy8^rY8-MiK*Dg56?ce?Jw|?;Rcem9Eb)Z!A#)LvW^7dc~ zL|v=v0`7eD;TxmV5{JN)&%2d!#YWH&}io{187Rd4>4t@?|MGK+lsu3cN(Prviy zpMC2)Ke+wKY#p0CHhxjrO;|7GHef0i9xnLTc3r>w;733G`4?Y5e|Tf|ri?qHEs-Qe z8-pWG6YT;nX1vuP$xT(~YLirW(YC0`_e(Q^Z7DdR^snMJM7ghi?32ZV;u}ImMqH~~ zSX`(Z^+~~i1JauSUGP60dB)`r+eLBXfa*p7W*&)s9uIEp(|fmP&jV$@WYG@S>~Bk1 z;|~^orf(o#B60|g>BW~25bx2em_hTf38fmZm~Rd7tsu#oV>n68_T?U<`wf^ z^b@NN&YwMRpL;11nmS_F<0r$##_kP9HGP1l18L%@5=a2tQ2`+nN0Xa{gN=0XQP~__ zM7pdJ7Fnr%3i>=63#lPuYl_Y!Rw+ra6yk*2-J5HogY2C55mO@7oC{fL0~kf#?-HN-_nnE5EX7e+LfRjPH}}A3Q8J6s>2~tusbhza z3nm=~Je_(^y}E8l;r+GGfBp+6pK8LrN7Khoozzx2c1#QADL^Mr?b&*;t-1`DK=JwS|$?ViX$C1<6vDmMUb-rKV&D73|+P|uh?IuF@>8APYIero=u~u!j6lw zk)ys{d8{RXt-4-2rw#~IG2kSZ?2w<2kW1E{c_Z(*LY(8h{*2XUQYjv}I%z{DZxd{0 zkYQZp6slg+9gc%JWJSQ_i9TTanj#r_-t|pItz8>!PoyypJmh0VwH_&;dNcIuFOpas z47yWF!CLUs&B#N)LXFAFwic?sm}MNmZ?hvcWfu503;uochna8`8j@lnLGiNd3;E7)4dgrTX*?bc+Ev=!dhrnd*}O)S>2;G^W7OoAOvk>2p0LmI4hGL)1t+ zPR8bjUbH60G`9Suj}VU4%+12qurQ8c#c1Ljs4hq#(}{UjH3g&&6aRiGf+QzLS_{#r zbS_7F#J*UkH!hcqtK8Xe4Np{iQsEDjo8B+DyA66x>NMW_Q3|LEA8f_sC7;J-&7Qo? zy$ouwp(!I0nG9^08MHaU$}HGD0!V@&!5DsdA{m8TMmYgR=zs;W;umJ{C$*blVIg@% z&?2XRmC~1_hzTE~UL-O}a}EozWYMKDHKWO3j{rP^v$KfFl*N90^{NGc2af1eLvY0n zOi;O0eQM^XDw{*t;7-B{q5|{c$iYK*+xb!0w7_1>KjjfBQSjf{QKDUFMocoSw|U@r zN*;(%h(R9_g}PKRNmQ^&k(hfL^2w_DSS|=HX`7?|;)72v-MP0Vc#}fHgt`I-bksPy z^UVtv9G{fn`|GujIwaUVoF{H5EYxbqoiitof9ADw?_Is7?FM$NjEJKn{g%zSo~G=Z zjXN@q@=3z0(5_yLy=!|NMV#A-r`cWZ{ZFr5xqW~0$caDss~`TGKmOLW`ww62Y5cd* z>oCvvNMUdQ+{|tQefxut#qsO=4*tJ?{_XF+{flj$OZQ&p7DpwU>~WSzhf20>{CD@B zD|hez#rJ>wxz{fqdgJ{5nVB(mp9_(0RrI=MS7SV$j)>fO`}TY9-`%=%%_OTv)a#om z%jkI2N@@!}n0ql+pLugw?|pjt{ZB4kICa9lpuM3rf~fXd#Pw3QJxt64Ald4_sHz=9 zM;Li0I{rg1!2krZA&z2(D=hjK8Nx4vUAe@OHc{7%(m+C#(J&oE_k_7s=&SfqUz~Vt` zK16&~{N$M&Q)a7N1j19-lt0))fnkc@oKP<5<5+bg

w;-rXxfLWS*r?Od+qhtE?zuoU!Z;a z*duKBA3ZWDNQ+7nLQg-TTa&l1{q%@#y4s3FZb~uN zA*NPDI-a0yXP#8bQg{YHm3eEI8gs(1=d<;QLcfHwBJU!0YN{woTK~7An^^-nQd1zZ z`0AVr;S|I`FRFJTZRH~Kn3e^{F^P3LKyB<%W<|>~lKmz&+hZSr+K%;z&tKLG+=MwS zfIZ06D&fqtqVLkyKUk6N#r{smSuG!8MDUyb)#1;Ej1D}$|FBgVI^b>9YSR@3r0mE~ z2#_dcOkL@NP zk}Q9YL~V*OLLf#oc|hAr+Rfw&5YNVyQ|+ z%5mXg1#v}OMLMrTsak9OBPnkej6(^`n5ouCpBUN7sqag#g zI3Qy^Z;@CK?J9k^@M;0qe7|xn`j5R8p)ncY`OM9Tq=ew*uB3eVa&<)zF&<^OL}$ILp6w)>5Q%n9UY7@=N+bbugg@NFx3=cL|8e?>kVm@f^z#eVODig zMldo=GRbcK5=n9`Wwuc<=|1jPa_Gg-Ar%nwBG#7lNheOb8%h!nD2a8IdVXE^JAp{b zQ_Z0P_%`I0Jmdm&hh#Mw$d-VxhF3`038u1nz+pv;)p6 zeHkJlMkH9QG$^TVgzJTfICc>*q}b-=f7+`fC}=;m?OaH@+G=57r zf{L7OS6s-(&1`P$`TU!&nfKy&Br*8PA|pQ-A}Lsth|A+s4ZR?SZhN`8aAlic5&B!ft_Gk=n&;CbGAOGl` z4_@p&_?;jB^pWu-JG+D{(nn5RC2jxnmu^!{KB`r?@Y#drB^G$_h+*|V6jQz;K+d`KA_z4$M{<&&tm2KUaxHHc1Y_~OhS{X&W!5qB z20l6lYCuBLU5^w93WbTti&)J z<9!GmM8rNRLg}(jt@-3nUDP6M^nC_BgT)oez(BnOZc>%k?ee3h;P#DKFk@>y@7bbv zZMRjc4h7>@>aL!3y|Oq;O7Mzid%^=-jFtPTRStUNUr~Zx;i>*r+OYknhdr<3m+4MW zETSSFsxmLWxf|);KI_l*h{v+!D2200Iu28gyV?iDP+#(>({+03Zy2JFw6O9ETtM>FUYvkC86V6&A zb2|w+UyRt%r^_GjS`N0&qW^<c8 zx9v2y+chC*34HOWJ$uCln6duf(Qukc9;R*U^B+BSXz_u~&7B7i3RoXMDUKhL&D`tX z@R<{E4KiBfD#Eofnl_b*UTV2Yg=2K-Q|q`*WwBFjpj=G%{Aiu#sD&lP?otvfeS>CD z73x)%7URzl#zqY2HP@vnB9E;nPFAH-z#PlA-t>JBq?W3QHPpoKtfmPtSTY^36&uz% z-ont8k7}pZk&TP6-0(ot^F7y)Z5EdEyw!+Jme_yKMoaql6$@=5Wn|jVmIApQpldou z4L+KyV-HqwKUMlKy+y$2^dA*f#di41#~sYreJQVh(ZW+4y2#iUIwlJtZO>l^a(wH@kC+TEZ_v&5H=E4HAzy zrKFw2mwN|PIfY54)R|?KF?Ta6ODpvTB<5s2F!WMYI1xg!(agul-!82c=E^icfGqGN zk3`Pg>-dX9AOWSC(QKZjQa+;r4Fq_eu|?T}c!e>L*cVoCkq!4CyCsv0*xeB;1of3= zmWco@4P7L=p4Ede>=t+-v`LJB&&#+^MSdc`^x*S&G3>f0-MF1(3A0z}yp-Ycgj^ts zV;^4&;;x7L5zS;+gi&Rn?3utQk0z>#hK3?-kFAk<@1f(#J0!<6Ajh6_;7$~D45EhY zww|j;4}oAss35(h*noj5x>}n3GoH9XMB5eHRY;S;0NS47EneS>;=R_Pr}|10`bXIP z#*9j+4>gczZ^xWQK2yTou0%3yCxSe{@aHDu)#zHwo} zAe+2$qwt1aDLvCja)%MQVp`UnjXnEbzi{F5)hoO1N(l2yk|sM8>JC8?2h0kid&tcS zhLxV6w7-6ePy)~&-r-krMd=C6!v z2JA^H3kX&JOGIvn7{}hj&+E#UubvcKHk$XfUsXglMJ%0SNcR%Qpm0`#z|N!dzYVNXC&M2H!DbRbB&*!FU%*5s{@Lsc9GXlNXoC31&{G}N2C#Qbvc zLVtN}-05g9-dk7k$)m>|lq=lOO}YK(fkLMzxJm!m0$?7a=hOS*? z;nL{_huWj)KZ#CQA_$yklWWDDwuI=_B#O4$G!rLHj{B1``IL7U5C8C$JknAr38ZVn z9QL}eFnTFp6nX1@@c_ulbJxE*17jrGY(Az(F=S_=h zYd>t;@6-iRBT`HtxqJ5>{w*WcTEMYMyIKzBPMPf_UanC?K?+~9W|gqP&WB6=jY0;+u#1~_x|zz{U^@6^2w#k>V1objm^LQ`fGRZ-9Nl_ z@64;O{QQHD8R*nOPG^709@KOKb%Y4pMNZ;~J*Aun_ zV7l+0Uy1r^I!syomDjfc(Rz1_Quu0E8LD#k*qpUYBg;XfY0Hyr>P_~DV?c*0Cgn4g zZJnNAuDleyQC4qA4VP)6=Yf4X9@ti{+jff52GDn^9hgl+_YvT};CG`Lk zM4(0t2_!n^4oRev)zXq*AOiQa89L9M4Sv^-zmJ~&;=OmxW;QiU!f_?Wp{z`LaOLK$ znc3f9-;p`(om*EvGsZM%Vn>r9+!R%3iHn&z*07<3fBb zkmO@L(Xz6m$QPE2SvZ#_C2^J zUl00Jw0b_@t4B+#8s1z*tF{p~jHctt|FaiwTs*HI=+d=o)jER>5V(W2zW(URlaDT6 ze)i&PW6=w_h)J?$wrF|oT3?MjjXDSa^Bq$o&cAxvzUgQ}bovJw^bxmfWU$Q~L;R9C z{Qw?6(#1_d>ySOcdK!|O4HsjTIPoUZ$fVfGHFsXJ*_!Y3W?tNTxNQMk*+!P|3^1-@ z5nkr@`uTGw?Fe3yH1$&WQVwV3ka|o7%t1(}YK2!$96NXVlt?`W7D9$r)P}Cb5>Ia1 zBCu`pVltF{gXjyx%SA3>p&0>BcGiCMvtMl8zi(t3C6j&;fys^NK}z|`qGY3Xp%J8l z>8DTcZh2!suOx-8PUa~(mkUyzxtNI)k{@!S(;NDFzW(u*>z8ldI)D14LLH?UW=b00 zw<6gMVDofj5`iVIz)Sw@iOB1Xv!{OjD_^|$%B!S#Y9lmd?VHwYocXoVm$sGnzuwiO97jZnOkwS*&|`N zD)UK&Y|+>^3d54H`!`MIj4jEfMxT~z6xbrF?;%)HA7ANXE&>uqetjnOIBWIW8p3g* z>7}d4T$dJpjYx&3Ov9M4U`Ma!0UJI}5aHqM{t+Wddn|D&amjH-g$wv#st*yb%&28{ zdMg<=F`fOoW-Je|%vX2*>C>+W>qYtZx9(|FRN2mXWLkZ< zJ38Qj)*xDh?~0DldHkHg2ZMN`aqY6FwjV|B-7ELStOqv?FjyA6OT**N9g~Y}+rNGM zgwuufLhmzBarF4jPd>f;legb_^NrWH?mxO~YSGi3``2&Tdzi4EeDu-&L$rKod+UDs z(QOZO>~#(reru0Ceio0rcHO(*=BL(&kfd#JdPQSa5*|9BS6^`!YPuI=v1>A0 zy0MJnyI#>w;}6dI08~jN*2V1Stu$0RZ3a(yooBW;teQx7;|j90ruB;U<;;-zG*)X6 zW3e<9k5~RX27)idyS3CC`;`$hnks7ma!2}vK@uA6w|?)yF710KGAQGE`*X_034}*> z>C=yfE1qPSP z>Go8IX%0n3F z;TQW`>}Al>d9>%j^RLl{8kAdpy>Oqcl8fhXsnHr?Ey;49d1yoqqlKK(OsfQ`vJ^LF z)gy+)HUUTwi+!bVwV+STix_I&COQwQ4G__Nw9HR(9j@bj^;vj;dMF+#%mlnO^EBoAtQg&c6rL`ZsTz;e1bFw&mIBQ>V_HI8~Wf{;b{7R*k}h0%ymC zJ+@`g90*|DRDJ!~Tnx`=A$52iLi&nh38H=qRYC*Hr)Q*z!)Ojg78u_NWIiLc;jjRt zSRw^2)!(FX4?zT!RMwyek9DbHLSD@f_5Nsjp`_v-K`E7)ESjRIo+jlbRT_|p?X|E- zszIZa4!QFd>|t+`sk%&ORZ%FC3oY^tJBtUau*n<)oQo{VT?AIdV%2Z<}+i6_Vm- zGMmh9;Y+;PeYzoAv<$IH?8u?bvu952{@FYEgn+3Jl%`1pxE%_2o|sXfY_?o?NCgwz zNG!eoE3X~hy)i<$CX`T5Mjbf`KGk7_n@v`Sps;im+eBBUaHsHEuFt0Ts~gqGlg)j5 z%mX=e;NZh2kDUajQ70!_uui)S`5(YUqHre-kcb-u_mUqjZRlwV0nfT=d>w4jk?N7V|VsRKP}ddNx(~0*$f=Xf>30m+mR0;t%${mIFqzUCl3mQ~%DEx0QdyBUMN!76&r?^`MT(RGo9XzLl_>(kAm}o4sxj|23|TPh zGlf-6zkjb**5SwdJG;n>3Q6+gXY4dqB(16d?`1%dvtTXrEaadH4+x=nw7%HlZ-W~e zvg{nEWb6YoJR2@XN?e$tZxv6s4f0gP2ti=}Y{LtZ@th1a;6?H_|xp6K`C2X6n-t5`s2Dwd>#+h50J(Ps?I` zAChEj9!&TD+;$g6=poIP0+z}|Co}-ASF};T=8(?XqA!!o{+xGKs{tDJr3gi`bUS}Zn}7n^rwbGV z?_`s`&g462NcYh0y=Tv#efUg2Q!9;-W9K=-QMSq9;Qj;rotnsAW+nxZlM2ZPE7Aa2 z(c#>SA>T;U(#l< zAqfwJ&v)Luc+px$Crjy(iU6;Zodk?|{n8cNv^?D0ILsRj$>~VlaP!t3SGB@QS(pIi za^rN({rhZ2(dOJk%R*X4QKG38iqDX?w%!ydqd-$!0+c{~cUk;FFcCe_3kc}J?P$O7 zpbR6VzJipHJRQX+dZ^b-fHl$|%#q3bz9dP~_T$?)clvbAG%BaPfdt1YdG`x^7_!UY zCk7uoG_RjIb<(Dg=D{mKFw*1t-uc$I|N8Q+yHJiKBbP*d!w#CG!mi!ABaDqX^i|8};1YD7`GIMv5G0Ayh2dgh zg9u-Gu|r`PJzEVgQRN(kdPHmwIbS zMe<7&9_v0u41$+@sKb3UU?f=9Ljr2<6gSLq?pbfvh_2O+o=L7luAdp&R1zBeM+a9+ zW}`Go7LyK7t&Ep=mqO|(n*Y8_V3G${QtbC~NfMD$f(AO;Ux7)KedqLs6zOb9*6jky3g+ z#4-jLcGTZiBiE+!5%R7d+}zaIbp7m6d(7x8vtqZoEq1ATA;4`KjP^k21e{{`J$uRK z=&@sm4>fW^KJB=kjhT>h$l!_g%L`p^6ueH1sN4ivey&e{c^))|# zwEfJYRJ)6Q_{nAaDrgk|GDB0U@DJ($qpxW`@#nNoz{J2d(5WpyjM8aCvRVXsVbw<@ zm#oZWG_U=M=`D5Z`z8tmWQkNgje(}NJ3xB;Mf#wL`VA?J`Dp@3xs=nY%VMtzVhNZz zHSshQ=4JvmErG^`UY1j^@^eaX9g#o_#eq|34A;mn-HSE*YgxXn)k2lR_`FdyKy`>; zgxfG*z5r!3`pzfJsm32u=|loFL|7@DtKi5EQSWpuQ?v%NZ$wPf9zc9xcW~${Li%705p&hus(kV|ig206`U0QV&?J{Aa@9obM@$K(v{j+i zXj+PM6`l8e(xKGRC(95w1rn&dV7|i-NIrAbb#LG;YcM2@xm9e(LkRmOw>M&%mL4Np zMd~5UlLt9{paE810ouY0g?ZNhMnycEPnB7&Jd{x7s6*k>U}N+T3a=7&5?ir^P;Vk& zzS(2QDx+ih9%q)Z?hkF4A$D&`20nd(2-B3GeT(^r^;0K~U%hqv+14ZHlJM!y0JnhY z;Ku%QufDo@@Sv^;n*}vpD0fhi4JIknQl4g(<5bqyrBCwH{!2cUd!jBC>9S^I)O8a* zsLNhrY9y_=cahvv^6L4>xX1aP1XI}`WixhMt3d)V@705gSVY?8^qUU>?&6c>>Z zEzG&_O5gBQv=!g+Zwe8D(;UB;O)Vk_>LBtk2;qTJmM-1&{sVg zTcEeS0#S?T>Gs33r;Z&xd~oaLU7;A&5I+C6P^fHNyK~1Bj`OFE<1B4VMu#6=y>Wy1 zIF6wx=DkTwFV;>SIU?L-dMf1IJG39J5PRLPlo6*qS50`rm3g*!pOMS(SOs6c%xGau z7LOKil@d3@O2}6d=#i2Q{tQ4S-rZ_h`?NFgB~OAFfksyCu4ji296xpJNE4;)5K+M) zw)|5%<2R<1dRJPX zhhHq^`9*O^4#CbdEtB8;@>l-bfBO5MeeL`l39kA& z&wQ?aHagY~cJa)Kw_d;aqxV1Berif|a}xMs;?TGONEK^rU5=`u(T=Rx2yMI9h=sTb zOd>NA%79WaU8voWplI{-;ukQ>CN`D}IJ+SwdEOCl-6iQ^eu0s&jB0FDp`O0L4d6x3^V~IVQc>r%s$O z@5J=(Ckh@ma`TJ9riuEJEC`_%koc>DqQW>{L@ZDBc87bs{ZJ3HT*SRc54L$MPE-<8 z?l?U+->AOA2FuDmF${XxA$#q>Wi~&|Mgp2js&rXk3~Gx^619x zx}=%@ix;*7cAUspzw)Ky$FDjF+-^eZKReDaq8qpFYSrMh-tpmTOoC)+@8gU-ZSMbSR|)sZ&xP?tpT-LLv~0hnaf<&tG#021mH@IuzcVC9=_p|;@w^cJ!I^4qifxfZ zP0Hz?t^5vw3z0(u2f}?4oPP_EVHz59gE^jlqb`@67l}+hCK+O4E}~>nZ9ovVq97BS zz&95rfI*=5#M4hmqd%${0Y-u=_mUqwI(r~;0TcVq^PsHE!(t>rpFVs7=6|pk$}lq2 z_1qiD{KZTIQHih0CyI)<;ThPOD4Ir0vPWyRnujd7G(@7PdT+zB{de!*yK?z5pK#ciROAZbRqFp$|c`it5^8G>&0ms z*Xb#F!Uv|D5&94tC)rIBPHQ3CwRiWYH?RNkpZ>Yox$2*0?4-5?Q+>7bd~^Ta-~8HF z-@I^MM`#+IBBqo@^r97z(2e?m>oI%hF+or)lnX+`Ear3t;6J&SGcE-w(}Ww94-oER zcuhfR)teG#Q?ijmrL4=Id+T4BW6YBtKx{;Icn`_+XQ~WJ737uTM(`fQP(b>M^Z+kU z=03B6AmnvxIk&a9A@9YKacEG%Y(KL8_3WwBA6>gONd{CTfhf}1bI);LpI-j@=ROl{ zJtT(p2ip&9DGm5MipY84PT$flZB(P2;ek%_a4s55mug{;(pAYzH85An=Qi4U<-NZ_ zya;XX(`SztTd2e;UDIqXeMDuDbw$Y3YddGKw7lhfl671}SqWe4+O(m2n|M%uWi4AO zWi({ND`WIb7GjNx_z_9e1hUQcA(e+(bQbS=o<(Sf={TGZ&S+1lFL z-hQ&{s4g=)T8+)bJ9$)Ur_hfJTl6E>nD8^S{K^;K`udkXM;y(@mN-?X8y0AW)%i1L zHV#F3lFakq37SWa3GIqav57qi4NvJ+FDsihUll9>IA!wU$Is zOqv8zlo$y4=4l#2;lNxP5`v6@ld>!D{a;)i>NL>m$xC&8N>I4LSx)~MF2Q?dleVS+ z@J0GiJQ}x7!Bg(hSnV>#>?Qd{LZ*hnX!38oETal9i818yJADr)W$=fHmH1_vBA6|e zCO}fF(Ybq-CLEleh{F^ZLO7sH{IFx8Ob6dGvtoVysB7X9XpVzXfpEZeK?}EP+yrtF zNJH_`>2+3`tbR2bKmaNcbHja!BP1rkb@TFbcf%{t3>IupNIPGgj#TO@8d-22Ze!L0 z1TqnZ)V><~&2r?Rw?w88C8tWT+uS&IB#;T-E{F5l(hPz83`f;OkxIw9opxuEQ%u4? zj(xN*f#}^=oNVmV`(xuz{oeLSa+g2hNqS*it~ zu&`vMyyz2SR{jbx(%@q=FUC(y(BHyeW3fFf>_|M2R17Cv81aU%M{H6s+Vc@RX%u70 zQ@N?#C-pidQ@?*HW~3l#C_F+{PH-U4ZAhY9*8buTyK&in&=xuatl~-pdiHbn z)ihDEC?k1gzS#06wS^05e1}_>ibna$(;?5WjOj#W7?WHS1`;UuE|y0@?h^3iGWJ09 zQv?B7L=}~ZcgXUr+R0sBX3Sy5#UhR+73<3mc0G5}I}_EfbH!fax9>2^qeqTHXgpD$ z4lwp3v-7Y?hzez;+@H#cT>~+EYR^C0x0l8G@D3|vY_MV+JBnUx`t-;Z^5TC!E{dc| z*GDTcSxcYnm-0MLI)9L(sYid${zKpU+n@gXKlxLWmOL?fY+qzCnJT|g(H(Q{*8lnM zee-|#SO45_EH0C90_lg4p`TCkPGlah?f%xczxzM^FaJh!K`mTKqRz>n1z?rZ`BTTf z@Y%OCW|UDX!U?U+_-{26gH}T%YF-hF48w1VQ?v8D%mxx=B15VsjKg6%ONgvdv7B)% zjm%w*6kJT|MI_xAa?J}f0x%GO+)06-Of?E{miDnb;KN1I0QYE=F;99v39|m0|Jg{K z!=226G>`B!cbZ@Aix+)bc#|-2{sE099VM0leCG8FfAy2Ur3hxLS`L&lk@@a^{Pg*| zAAPI?L}!Tv*}0kG_wV1m_h6(@Sx!7d6((!}WKO&)5p@xm6jlNri6bGFD(u5K$`e7F z#c?kqHkKslVSoPjSE8gDXOm`L^f2u&H`G%t^wJ*EtTi(_gjC*Go>brtqF78J)9tt7 zJ5-5h$cRz^hL~{nYGM&O&P4=$&3W>|pgp^7y4a{`${nbsc7sxzc3b?{^+00=B7h9S ze%3ic7`T81Wz}V7M?O1GP98b@um0e7fA2THVIF@2@_O0V){Y+7`;9Ms?gu}9{mNhe zF#LGUfyG-l6teT|;QsZmfBAE7oqx67%env-=o#vX!-u~5`Op5&rgWbZRK&Bdx5Vt@MdX6;$lTA{hHV?Jvk006m#+Scz>eyzwR}tkqSOy~sybiGA z7EFkZ?%aO$Y&w5Zv ziUVjtTvC>25el%ZQ4inDh#<=adf zO`D-o5uun@MBE{{x^G+nq$tyOH~n`dTL_!t<*WY~uf5zmx#BfuhH5Ba9p+5cuxft4|Y*B78weMLzSza#p107>FXd#^0)nFf`rLnog=7&c3~@ zg^?W9cxk9siXK)gPXig3^t5=yDOI6J9&qKy0%s&G0V{PlZaATEAx{osfeg*Sq4hCw z7KK+audka8wQFOiJ(3>W!|bu6#||9aNE-DYYt~}Zc1F8x=YRazafd6*{@#Zl{Pbr( zy?6hfU35HeFQ8*a5B3{>S0C{a+a1VRtftx)61Hul}?H}y<%0#0hPd9MvT zKDm1B-XpW%9_@a#%^hjKb&}6M`x;@y0Rji^-GBJEZ~tult9w85+0S6|lS@~vQLocq z9VGS~i_IC$MCNQ{+FY!XBZ*CxkUw8+sDva-nq*t;>7T(v=F-fb{!~~4IbDoTOQocp zSIBm2)iNp($+HeSGL%%uw#VA=iFypbfMS9L83Y^jg&JGy$5c=n`jr-_ib=h0=!%69 zh^T9Z%z9;!%@cbYR;$>;oHWUK8hEt@hZ3`Kp6;^2k{DW?J!3uf-yb{>VawTG5XBSO zF!kp)>q2kJJ)mP{&43R!OJ6Q?rD)mp|R}+759slCjf{rN)+R2D8NY z08dPsR_a`!{gMDDug8t$@&z7yahv7wUSREd1qXs*EbO71Ww?<6Y#*eJ^l> z)$@J=k_v&0Nq~R|5Sv?Sn%;+EEJ0HDg3qR)WvQU#QpZ8zc)g~oL>e>|c2@)2cc2F{ zNWqv-OlBe8$mK3^dYjP&r$oM%|H?o3(!S7GYshMH;DnVSp{gthz$gwpm+!()^TD)N z3WmV+7W9b1-5^QPi`k+U8S`%V8A6dUWRqX|kIqy!!tc_*FoH1+ga*NpGDQIeBcw$G zB}QR}#5U^7;o0}BTN7iTP*7oIuxH!^djYT#Vx-Bo0vlHEkT#+tWvA8c>HzaNn{%d) zn$ut&Dh3`VFR3!zP*8||t(>KM>QehHDVFcC3{uDu5cy&ez>q7KiQ1*~$3>O7_ulQ= zW3b}>xBl`w_xNT05jqOfA(X<1M~Qs;r{Dd-@BZW8e*N@`tT-)Y2_545H&m2C$-9e|2+k)CGAeJ}yrf?+Y^k4$)@uPK~0VOwT z7gOmbx?`Vhh?SPS2uXWU2dBg(ipP>C$#{;*`EgU))$@f{I>7y~lvtwZgaV`3{>27id&5Hm}p&1*N z+0+3+z`V)smO_)WABQroC3Xi*VRJz2fU)-CSHJMt-~93yOsBDg5O8f2>GW-GZtOq* z%4v$=)mTC_Hv(^K);!wyArI;+1SxoG387V}aq>GbfL29<)`cI#+s6<7P>v z#+^;*#^9LB0IVF3UHVFlaau;WGASda{$89uaqNwAXPRm;?>sA|?~F0$;_xhWcxIiM zC*zSdfN{_K@<>BV*}8}}aF9+HQv{N51j{_xyBkqfo>H?1H8|ufX>hu6CE?#c=#h-` zfMkTL&4(+>Rh^GP4Qjq1cYrxg)H+f1Y`rMldJdK-DMc-ees%eT$hq%qEl=ckM3iq; zNFyOB^HA`XEXge5@Hm3{sf4G|3PJJ)VhF7)`5Ti4510{z3<+>@B200w?{a}ZDlj>w zu4FDvFAx#y2XSv&%0vCld-j_+%K8zbo7x1@tp{x;VY)m~+8d_Gpvos~PoC-@e+Zkm zx?{asa92uFT%ultU+wucf#9-{<=J7ZBpL(4?>5%d<{(wOufSGUz4)KurYg)Nlc4#7 zUWhe@7R%x&+>${oC$N&Jw6O;%Lqrvyr{m7*6~Q38CFII(RX)FtS6vy|WUdB}GjnNs zyR!m_%N}~dFa1*jcZf8aSnT$TM?!q32HW`Gev$9|;^X%}c;8~*U-^|c%o@^u5c>*A zoej)BP98r&g|-c?)2)cZjFFm#LTU{GtwntS*Y4cbsPM|z;FDXoKDl!B`rUgb2|AI> zaEJwdYH$aUZ9LtlouKkKeB_9%_Wk^uAN}OjS6|sYeB`wYdpKcd{p!~v24Y_P$6_-W z#xffrCD&T3V@AoV{xDi1x5XTj-#n-kNR@MIvI~i^rjcFhO~<2;K{IyCrPNfGORb>k zFv~>RoT#(jvhK+v)bi2zY3qmve4aGjPppvmE7Ma1Nj{S5ejEFbej_H7T`}-TBlh8g zR-3fC-55N4!L3ZwW`ttLoEbw{tv%IoUW1ljZ`Y#=L)|BY*)hRbr+omO@NfNeAz zf{V6jUT2QZy2vSuV?Jsz8YdX|xzMLO4*$(p7z9w9YSt)x6-Y@Ei7c2$6<(}0Znl4s z@w8Qpa4Fao9L)y(vOz@`GpT36W}{(i{H0!#FJ2etG+HZ! zPZWSfkPliXXGY+bZDbEUs~TD9JU2OaW3XlOMx{1ZI_l2|Cbh5OWS|$t4+}#Zv_61( z*FP80Q9q8gJ2`bsjcSsbRK%1pCUFs9TmzY4I?Pnsi4gHr!N~0-lc@Utn7Xs(x|S@l z?*zaJ;5=NCcdT2thOTN=cgs>J>?eoA{vdwxNAOp3=*1BVg%u&&l0u=b?pAkI-MaI@ znFIlX1VH-tUwL@tje~Rc&YijDT)8rLX6~in1*L{HBK#~GyFJc(xT`G+--ZELrZ`gg zG3(hF~C!Cm8M}5u0cQ5lAtS=EuOW>fJe63p`1O>ve3{No(+kQ zUU|VG|5A_q?OV^qb}xI)&IK&0OCH^s&d5q&!L6kFJQL3_Kk6}MoELDY;;S|r^ifnw zQ(({4aQu`QJcuUiJlL#C$(cGgR<4njDK>}SQW|2v1QUS zW8OYF+dPM}CQ+Cp?hU>k1z6YnYX8NHCr`ckoGsEZAcF$nJGb`OVRP9KGJC@U2Tnv+=CXBtS$|GFm zM@O=UDUsxN67G`qP+A{(0#jt*l@$+}Q-@~^O^B7HyW!J`sV~iYdQHnlgEgo7`_BP4 zuy1JJgLGsXf41GHp^b-+9{=H={?Wkyo%im3YGM7}Uae#uCsQK5dF_R@Pj>RL=*Xmh z`)`gN+W&;9v^4d=^5NI7o<4iw9V!-M6!x`KlYxKhgL{dd240CIB7rW=Wr>t3SL ztDSu`^R^Q>_~Ogm-96)e-aqi(o%g={>Z`|3o&!W4xPfS-QG1ugxEMPxyrj$QY#Ks$ znMAhJmt5{uevGN^QUJT2()}(ijTNV)B*?08X2KgI03pQ-A1tc1@>=AO1L16 z4Z`7Tjh!#;k6x7yLnKs}8$Oz0Y2Qk$MY=^Qbt&6#OXn&LHmuGR$_)Kf>AIqgd7@r? zZHa(R>~zD~w3|=Z$0d)8oOBy0aQzMh=jhvHd!+;vK z5g?rDeKJ--lVQK+4jbd0F{fc8W~#Nxe1IfVNm;h4xhf>|QZv;^b5V7V)rgbq#@K8y z7TH58%Eqj~GhCQ09ltUn{NBwQ4_^F`V5TrB2PW)WkN01E{^d8n`}xPCbL79Zt$vOl zH6%x2@ey*uzwU*Tmv%LZny#hZF@s1B*3y1rOdXY}VD!T`Om+!HbRZ^aJJxsHJChPW zMUo1FJT9r+bz6mSkUjQEoyLL@DgaT$3QCI;8DIXlJ0Ay zlW|fh)^tUZ{|$`RC#g-6AtbAW09}B28UyOruYLN#J*^0z z**%9_It%6A5}N1$yghz7#piwo;AR|OG7a#ekyBTf*CcW*h;LYpxN(t<1k00%i_eI!EXn4<5_Audaw)b3LNR4!)W)=)(zkaSe+ zrxF8<%QIMOj<~^Kb=Q#fOp;$)sOb!t3DBDsK{sAF-h-6<4IS_%FKD7SfhLKjzMy0X z7?LM}lODnna_r?)m(ZfaG2#qPG|yiZzGHxdu;jp2ZuKgY3g?}CqS1N}{Ij=h{1`BT zgQutNXEu(oXa36r+>xpX*4CE_*872Yw;6W);?r_A?cvq>#K zGyoLTe4?UhycFD1rLiX1wOpBNPQ_7-o3%CQ(?~yFG)l#=vhNvC&kXYO0QjvAj+DYBM9VX3jOoc)Ew(O%p>3jMOlz)3>Z} zH3@4_L?7flI*^&YOl;5B!d~0Wn$Ft7lYV8+Qq01J!e055pGz}~La+1a5_FD|^;oIkB%T8RilrChgCp!8_XyU8w`)3s>jaWNpZ zGBCaBV>Mep4{$a2F9b03KKw)6f{o7z|GQ#t?HF%geR~TdEB(V(V;XXgS8V{s4z27s_;nLQ36$ z?MBw3xC)yE3>f$~5X(DdIfOx#)x`fMW7(&WbbI^HQ^8t|@gTX1q*PQFNX0TURA$P- z%Rl5K8&uDi5|+ByW3(ol=*!ee8jhxAo^tdbNs-<6nD0nq32UlW_<%%QR=b#tn1vh> z10hi$!TA?AbE|4_puUb~-tC_NDC@WK}&lqw6;e#45qO zPC2$7YkI1NPzK+?#nd64@dgLZ2o@Tmw!@>KdA%t}Xpp;aNH|uK(l{&-X{^tNP-lo9 z*3tp_xrhU{M%?h9oLt!5`^_&u`#=Bh|FA0LlI*f4urtc7t-t%j=Z~L1zkB7(r&b`bo!Oy|KYsBQFSvE#(ui_wAzIQL+rD|@%FjRkfRpHMTi2l`H$>+{CrX`2 zOyH1jtf-3J9~{}zaN+hikStWTiaTTohQA)x< z3V0GDv;_?B7CTy} zHSxLP2JBl_W?HFL!7 z&|5^u{sc`pDe8;JT-3%(%}u&a1}P@4DMo{vj7B6`y==J?D$qi&wmk97=x-A*#xbBm z2GJgcArMCyC70bBbBw?gXJ5H;nI$XGMDvy>nD~ACtTzfjvdPZ}?|-ni#@FmBSaUWr zg^+EKjwD&c#ob-a@plg%8dJOdo-H>nv57!UuQnwl3@0 zGsJ4HF^--wwlU>2QT-yNsdI^PG(>1>u5B-?M#)m!Bijg0jxjl@V|8P|N^X+#wCQ*h zO_xo|Y3hUl2!Vs+JmDnS!8r)42Jfqw>1$}8ay9bE65>^Ndjw#_)9iEaOgewTV;NSL^cbBP6V|!x77pQFHQuqz zpjQAJwP*4ftC{rBW?C7bsO?iRG@0l(A4#Q<7*o}MrZOg9#M$t_iEhIE*>Y5H zrLy=o6$;pJNPAMC`QS2r`1!po;O{;to@)z<>Eel2^H-wfuG9^%gl1rR{m}9iLyjAbz zp4&x5*9V|r3K|NhAdCZZ-~t~FF)=q|C3L8e-qrd?VKm5G2DgUyOd_TyQ`jIR;vpU6~v_tRTd;k8UCl4MzH2jZp0~JGe zBUL^IB)l%rUJbk(D9kO5YA8b*+)zx*;EVe!P!3<=2*WSUh(%3B?X?M;s2nne6l}#r zEIDwMR1;nFcb3p%E4juGi|rW#L;KCIerdzWZ=db&Sah{mSBT&NlYM*p%kRGX;+ya8 z{_Zzc@q(9#0O*z+%r4pM@so_- zQJ7puk&^-!myy&JK=crW6%5pR#GLd|XeVPs(ul1aLG33-d`8Pmz8hWRfLyEb8xe(P zZVfpyR2!>~mF2;M6j^~?uGzJl?~shdq9GE}z!11}0nr-%8i7$YSZQA_ASfp~MwyD` z2fPgO!uiiW{`lN~_%Fb$oI^lv1?2|aybt)`=;g&ryT(t{Hj3aCq9XdjS@;5iH zUiOIeP%*@nVZI7cxv8R0Qr5S{I9x0S0kQ5sWgp=W|H28wrF?sB! z$9+r$S+>v(Pa9c9k9=c}?qH~jLuga48;8yO6W){sbD^Bdy95brwBT{w3x4K1ni+&p zgoGp%?tfBcPl)v1Q3KFVz=E~pqlQeXWMIWD+SieTpwR3L1A3t;t`I9aRS%ezzW&bX zD;r~q9c{|$#ZOxoeYG`HD2Nm=G%u)2b)j`Hds@cjO7$45!&}mf{Gd zRkH}cnvl={lvOI3z@WQArs&t&pvB{hE4D-4+eO^rfhQq_i9G-@YhN6;wRQK-UHxiu z+y8MNFy~bgJs%s0nL%~pT^}uN0+zje{?#{Ls-13Hw0dgT-pfc{9=}4y-sP)j&RzKN z$A=bOUD~;#pG6w*nA%G)7Q$b=xF=gH2s{-w+Pov<>lZJK?w#!InL-s_iH4R<%Ce!8 zJ=r|}zkZ#Tc5Mr8b(Zz?LcEbq(+M)LNXa#X_CPh=laRtHThaG#6iq^(pRx@c2FE5 zgjQDEt{0$h)6m$KH$Qug!;pY}mSF0?eXDwQvOA+9O6w~Sz^Qc1VmKDF}Nl(~8W#{Vb2vJ0!%D~z;i z0*NQfsrKgNmpM+#;z$V7ryS`d-iP7|$~S|&v#FuB@ZSCc7z1{yBH8YtW)$`lYC(z~v{1)HlN z+I~Ai>wK6`Jd@^EhSnBwBD750`o?`~TPnAu-LWz$>&)I%^3G@?^O zQjaHT4H@x938*yGI27TCQ)yzT|CS1}i^Tr_JCcI~Z^O@&9;*=Eethrt&pvqno5#GY?0+#^g;nQbde}A8=81e>DN*F7Zka+qB;*$1gwq`10g@FF_ET)g)4thdrUee(R^ z_LVCt)BfSXv;D(77ifV@+li=<{miZF*Bd2vy9Y{<7ypQ@`uHuhA}9{6LtX|L~CP9i6%p+7VxKN8<4>qu_R6D;MTNcn8Zib*h|yh6quR}?;6K(R^ZWUYp}%@oAR~jj zX=zU4Ox?6*XVox-4bC8inBK}zgeM`I6keG?Sn_opSRX>6giV!gI(!uN9DKl)ACKu0 zlwj4;RJ~mnQZ4c#I*Eo)rYy^U=v~BdE#CN?2wKTXfII|^oHi=Xa5t_zg}BVj0ub>rf?^#OiCx3sX=+3TzkQnPzyvcUPXs9~}*$fFXf_jg>_@Ly<$# z&;TwnK0JEu;pyA07gf(*qO`-A>N5+)>;hD6dFuhJJ14dGfQ5UcT-zBdi_zf<73q2u zUG-X~loUcaU*xP*85b3Dcjv!dEKcKOvMhQN%z`j=t+uTyGdbp4<(alllw%B;m<&l1 zh(@u0A9B=K2 zU1r-G@cTn|&kb99X(Y~!&FO9~n;4d{INbUjRb|wCmYw)VcX8gkIlV7EcEC%V$>k^B zGM6Q<(W?e(80T&gK5rr5{gYHE#!WRViF@K7d(l_R+_o?7UAnlv$2Gioq3d&iHNhf9 zD@Q|T&ze$q(JuZ62O{}>Iu>h#%6In0->oyx_V-ODG5Q9w2hCr8{rx|F{?+q?SLZLB zzkKcV*^5VypFOu+N0*90+`Q2nf{!#w=J^;3dn>qPV&CbvL8@mF)4mJ# zFW5Lwrq&|gzG3%2Pw?&DMn^gp$5t%YS$lc=*3Hj8`AEM(qW82M9vl_!34!VZ6?Bkh zEiY&A&RB@K_;Eza804uQ#dWe?M(1{}W&m*WoKjwHL@GCtj$`PIPjA&K0HWi%js+5B zZKMgXc`_6!QwsbdI;0d%$;v5d1yrKCZB_e}WcJ;ei=L?1LABSYrB`ZPl}J}IezrVx zIMPQubF@QiJ$>Uez=+1&SmYR9ICG&9`LnkBX->6v((8geMBoe?j;XJygI{+~_i0s2 z^|zMWK0M7gQc?7-yed&e@LmMW7EQ3W9n0k-SK;gx3(fwo0i4D&XR89J1QNj`6)BCi z#$r6PC;)mug}?VD;f$b;^*?rAccaGz+vi_zo2A#J!x`*QtyEFe){xbio)!&Lh?czm z<|l>v!6Ub|+1TVl%~|Lbyxu>u{AG0Lmm@73Rssa zF}V_b^{;GICvvaOyI6MTJUgf`Pz%lAs=R-hl^EGh6B){8{EQEfW0a6h0qt0jaDW#7 zjmBh!T1{9c%gTs?aQ)%mm@vd1(R=w#|5coE+BAP&GJ1SfMJn^#7B@9`@H*REZE$)UVHJ~%v}M}tmBuWZZk zqNj|nU$s(2J*b)YwD+uW#3p>&hp&x|FCBqUpc@s@I?`E+nZfSRAx2-=G$AG%&C#{Q z0_sS9Q>90O8>WYEc`=nRrEa+J;8zXWyRi4qe)Fq;|9AgjxSO?iH#{Jap^D%I7+Zh) zU;nuOzZ_j@s$VrTq4wt}PYAYmzWLr7;3rN7%aYx+_(cOJf+h<9BKvhgZe$ORU>MyT z^T-J2Y#_TRi%%m{u{uS9{thiV;wWHQo3bM|S%yNl=4`Q?U4_gsqMKAoWPf2@=wfiP zlo@9QXrmE;2G^<+Xjxn)$>h=G){xC*pcXYjt!@^fBvH2|(d@9v(83R;)M2l?!_k@o zL>%wmyW>p^Pwqcz<*2OX80fW+J5Tptkom7ZdY{-$e0_27GH*5ZEahEx`{A6k*aQ72 zh~J0`Og0vuxCWaP3LV@*R4r6Tp4~Hoc3`IKBoz<7DOFk%m8gevY*?(RnPL>b251X8 zoEg`7b$ESm?{9wd>#N>#r(iM)x?XO*R``Y|6yR`~SGI4ktubA0X_lX$FCwGb7pW~k zKNaRjqI`5aiNZM4QuRcqIGGjsfQ&{$2?&o*6mrN5sn3BNYm2fm)o7JVD>N7r9X1!d zC8+S|XdG&_?qT@uXOgG-4!xH@q zgA4n00M6N9&>{+ z24AqGub&^mE=wBt19FBSOia>L8ZI_7!~`5KHa_(v#1tv;?RiQL^#rzeZ2pr#dLvb~ z0v-uijBCb_*+FfMYLeKcOD_(O9zA*f=Wo7!*u?#}yIb2&_75LEe(Zf-WU>F^fI=QU ze@URNywll!`wUlNJB#o1F;UIc+TJ*0#BblyLR>RhxhJ1 zdi2OZ@r|ojO^K1n16s=U^(CJ4HI-y^M#sf zZg%A>6}+?9CC#MTt!bs)PjHPUm1_oirf69P-qfn4TuJLeW%oMp(E(7rj8`sQUWe1734N^RT@2?{wglHT=Vf&MVu6@ zbPtkc7y%q`V<1(|3eA6b7M#5MxbyTp*)cg?6yh3F^9zu2{;TanRcON69SH-^g-k1P zlGhF2^f!hXddi6&f0cBuNS?yiY&a18#R7%mr}a=HdHLZfGFa)seRA(!z}AuT(5Q;6 zrY8{K*{}ypUBCJU2{a;*F?d{!913rT_GC~aI%-N~)0LyZ9y^WwP7Q=LfmS&af2EPp zF-&u=Ivot#l_JGm?0KNdCobY;Ko&DO*o%d2gqyNDf|>bLS^b+EtZMZe6U#b^wu6lVpMKLlf8oDu7^SLNbu<) zK{x?`_N~GB?XP}u{nFmUL(7%ytPH6;V_+G=84s1e{^5uFPo7-8cgNrxFiCtgJBXM{ zy{JEZ`Sl9}ZWs2Fk^u}PWd8Ngv~OOU`iq}^LW+1BRe~GgYALr~u$t*5jhOB``s?=2 zfM4p6gf8i-AU^daCT1$D5`B-l{m2@G_#A1J5HZ1!b5$&PD#5xohs!XB0sEp(*_~-$ z$lbJpfimM>11#GK)hfj$fec_!-X$~1K~hR$NH7bGl;L#*y(!cWgt%iZvm>Xwx2|0K z@XqZ&fB%Dc83&`mg#)B58y)MMngH^__9_?r`(=+^<+kdY zjuT%VaK=7>c>U_L58waa|F{43(nlYVT%Kkya=)Ce2G=)X4<(T_)F8G=O@J?XMvcNt zjw8#Tgq`jQ(>;X)gSF~7N%N%f7}TWzIjb#;0;0(IPl=JOdS}>p(8w*T$h-P5SnH?#?qu_j*F8L!y}(;$(oEd0n0m~r2tGJC@9koHL=7* zpCqzl$fnNB(W?v2ErB>M4&p!?(@m+bUmv~jN}2-`O}z_v*ANr+)Fo*C>7gz+P#l^c z+pG(Jtka~m77lA%>KKfMSA85#;GPf=Fdd_Lz{Glhwvovq zbO6bsbh`wR>+}76=q>g&R{h}7)BBGeJ$rG;o7pO!Q~UnW(Rc5dvu1rQI5|Ff{P?K^4)EIby^rqQ`|jIszWm~=58l6X^V&75 zwqLzGxO40F2X}7YfAG*ypo(?)!n24Y4iSa#e{k>pdv`zo@@qW-R!knNf`!CUeb0~* zZ}yQ?U6(|bV`Ujp5^<#vvC2Y_g{6}(^~mgMoEUKRxbg^0rwMW*wjhdS*;^!}thw|!4Zol5gEPMeyVMQTP0zfbsXyykM`JsGNB$ zruC96iOrLTAb56FAHZp%Q=It53oJkbEwJI_06Wv=tl2jMI#m%#(Wuu3cX9+$abENxm#cDf;qD0IOH36D76? zJbV4RjzM;Z`MPEscKra;o(G-~d5SkZgrpxeNq4?S(n%x%SJ@K)W#X-rE-;y&bvZsK zp20f$$SuI5kC4wX28GxKwgp6&nBpT79z2lv{qBD38b(U+iiJeL0A z+wZ(o$q>ze9vp6t5g9H1@b2Ba@7?MxA){5`ustI z*&F;hkUI384xH3g@ob``xkOtKs?>dt#Z2*qC}%b`rre4s*+_lAvZ5%tnN~QH3blF@ zeFQTe%D+%6ki`1QRu+~d<1@zNcY_+d* z=XA@r{dV7M?9%kyz+TciV^XJjG4cuoQk*Ltc;uYfK784p=&C`_^hzw92p7={EqRwd z>~v!zm`*rrE_-E;>UY@|?-fHWk_IoSm%?XR#6mYDFqc6Nbzl}Mo@vO~$gUeBgRtYb zaawgZ0Z53Pe*DMQgc4mW8o=pqU5@qibUFq@<$FWi09leR`>-3*#gb9$isie*G6e@_ z1j`(cQW*t?6Ssd~;jKhaGOb04J1g9lS`3#ZoGqJI;^O_NaeRqd*qqq@v;PHKcupGjta^F^##P6CT!l*r>Z5tiFmM0eAHZ}OHn z$`4M0jqpk;SDvWqR!@2$Q+U1tY4{(D(4JR)0whY%i9PjHLYdQ1W&Phk6 z`gp+J{PL&v5PfmryI8nR3ly0=VG+loTdZG zISN%PYq4`@ZC!q1R*ZCb^B3D@7};az?8S4no0jI=bYGQBzZ;jppqr89=}XUTYFI(F69TL|-vy4Ey1gyrYyDpbo zbJArtFvaI8l})Ltp)&Lab2=WW4NWMo3Q!n%O~{3gSxO6C(iuz;ve!ujq~BN)cv z2>had{nq$~J42Q$Jb?&t3Y!+9K4l(Zg?m+w1u)W3++POtQ)&%NSHgu1+W!4N{P^S3 zXU}2xA~3F*k9ceo2QxEXnO;AfHQL?EEj`JnPo5H$$%$qK;Sgj~3oh;Lef(js%Cho_ zF=qoU9XSe)H}0 z&g-U(pxhi%dj$FBvyVT#acK|vX=e&XUFQ0hLr6PSm=S0Oje&Gk#u*@}R|apDu3Zf> zQ2nN1bc`*LRAd=lTvy3DC^88QM3-h)M6_?YOGo@zQgl^%N|l{vHv}f@X$Y4aMfA!P znTeS76Ey-(nvO~Wb5l6?ms(tb?)ND2ag|XQH0eAGAI#(1~Y1Hop)t#S8XOW$58*yMGdvS z;Hv%D(2WdaNs#@lt)k9ey!h7gHZyC*$LJuC@sGo0yeStOBZZ7_PJCrvfR6J@9t?K5 zk+$^qK1iO242Z;+Lu=NjDYuZ3%i?7elw2WCRMCipVHS`O2YX^->QtHMGsU7rZKAIfv||= zHsu)d${VPG;XXi{>#>0))K2>>%Lzc@4A|H#)Q}APVR*m{&?={^kye>HkGQCtTmUGt zWTwPW7c~UxgOi9MkWz-A@(hqTj(-xAr;LiEnZQJwKg=Z-m(z`W*M!A?bYUUXQ-Dwx z8Is0us{B>Nl(Udl=}IBa)P%}am3>e1n=nWZIr9j=3_Oqjr`CAxpGPf9k&G$v)H`_0>T1jpEEyz9+&GJ1WaCvm1po}&!-Ac%>7q>Z7D88Tp!D(^6!3bMFlFg&N_MC7Q= z8__oMs1a-~NuItJ##b@5=B#MBV z1EMEK8DD<=&Gr2kV&4}%(rF}1r*(Jt@|nFi_B<8MCI~e3AQ6AbgG(dQgr9q}{-K=! zoeK==8#R@Q=r)|QKUP(=aM+JO{^T?P)&#Q>6_UyT}M&C4u zz1{O>Nl3MavO8X^=1tgR1PbxZQ9@hC?b3w!Ub<>%pJzW|-)3LGI%&B~QdnVIs&oh_ z>!09-_b^spt&cP2n?S2`illfGHCl@50j9jwfP^dq8{l>3Aw;#J#9&9%s`n)*W+LnM zJStmrHxW!qHQ zuzbc7EZgfc>Gs>yCo~9ZG$QRB@X%&w^6-7i)_hhQl@I{v1 z-NNF$TKrW{Bu|cxj0CX2q^1=jNN0ng zCK6`ZN=8=1WW3}s$shr>Oa#2d(2v`Fp@bCkg+UyAb4v+$vZqSUT=yYl=uzb|G&r(B zx8WU?{>0T!*aKPCKAW8K&UG%0xd$=h&2L3=|D%^iuNhc1;?0|n?%ut!yZhwOnBUMH ztN=&e_V)e9Pf6ZW@9%$n1S*)rZ+0JIp2X z9+l90ToQBu#R1T%a6nXA7QZgWJP_a2kTnzx>q;{q={fQGn(w4oDO_{+LK%}chmM31 zvJpguwsu7SFAiTiOVr6Cg36qf*40M9pvA4D?{#4bfzg_V@zA!plkUJ8mb@vfxfA0F zfpjF^jaDY3l6jJaGEF;A&46?v>-emSj5$U01Wby^*DSapsGD`+r&XA={EuRJHb)fq z3vRV~Y{2~lJ$LwurYA6bh7w2s6aqCj+4E->PzteXs!mRB-FWZ9xr-1s-2$`5pk#I)1*WL4v?l`#3`z^_tB~F+fr=LfG?gU$Q>@!eoIE_#g)4@kZ&sd zsYKTG<9FYAg#OE)fAZ+TgToikuk2m;?9)#kK79D?S6>~yeEFBZ{mWnf@)!T*_rJF^ z+xi4k0&EDPSLR^`^))qUB=Y8TzehcnF0tHsqt=LjmZ)Z;Bkmp+)b9eY?-6$n|3ke?aJ zbyH)#e9Skif7d34FCoZYogu`CT;h(7p<4;?m7+yeHFK^&RvKBOg)$2e9$dxB(Vt9kfRdS=gbO@ki$)Tj#axsWmfjfAqL2LMG-|n#oKzf$kKo zNYW{`0`KHkpMLV;om323<86UpAbe)Y4TeT?)(m`XybVCVcjeOFPJ{lYjl2tdFmU+{7wu>j`y zFXf_NLUdFe!-wQjGPwY)`xT$Mn3YZT$yoj}lr>hSU?|9qbp^&%t&&TLfvm)%cr+oawIpr0*bMyMG>(?G1Jeb(M z;$evhec|oL&tH7|WVsv4_6s-E=Up5FZxKA@$2g+a zgn*{27F%r0rjnF{$7LyD;Qai+u`D~k6-fasM$eJVAq5qPWY7yvsMD%n3H1(erbhy| zHg)_;#DP_oWTtddjA9ANE5EZ?Ppq4UEsz8fpfWb4wV{BtU!s#TgE^XWl{Ex+m8)8B zvcO$VH{=Mnd|i(pE?YPwUs8nkJLgF$lsFDN()-B^K$UpXeBA%+2(WHl##E_%e(^G+&^N zU)Q*`>LkU;H;MqJ>XvG#uYwOT#J^u}4+3xeKeWwj4ADHm+%KK~8Cd={dV94Zq6`h9xvAuQ1B=x6HpY}?D zw;CX9)z~9$x!LHPtFk5}KUj-b*9oT^{t6eUFgZ*@GKXezNFBY6$ddoqgd!T2HGriE zE0>+rjG@VBJ~$HMv1-8C|r$;*86zIv2O*NLZ;n&kj{aIK$sc=+V$Gx;xF zx%BioS5qfS6IY|%!)Laz%~I;}#Y?E>Em7NAXxn$bK#{jB@9pVo>3UF|sY6;cbFZ%6 zx@w}4tuQUMK(Sf=Hq-my_N_nt;SaU~w3g3AfA25*_M0zh{Fi_6>!1DX<3IfA&##YO zQ^2uaLkl*%=NN;%0UBd*8rhc#1H4StU=dP&1MmeY7CIP-!v4-QOUGkq#&k-27q zbV+O^)?jK)8rAQ3Cmf;Gjowt?nsG$tK+{DF>s3Jgj8+fpi{fGYkH^u-wFsi8LDK|` zypjuicudHhF37p8j2?%Rl6jRdwk%)anrgHv!jiiT{>V=a9FG@>f#S}Fr}zS-KFN$B z29hYsGvqWSZ}27_{nmW8Ceg0eP~h@?%{BQF^pBiRi z*gN(8JMaDR+xrUR6Nmnw$nbY|4qhL9`NNMNeEi{qCr_mZ z&X?Uov`je^tVLiq$kaJpl01Uo@#Azl7(;~O2{GX$V@!w7sjmU<{zxT?7q0QQ!S+vz zz7`2($RKhlRZQ1;78I0)hIkv(NBQV|vbV5H~*AqB}XdNy||ol;{Wl4G%Be%#W)#%-Rln_+Vo(Z1EC z!R5jwq@4@^-(?t9;ziapJNE-L*~m*76gAe{@JH#W?V%e^rLxsp!uns63x(c`0d(u@1TkKUYmuGtXHUB7T~Z_gWz>{ime2esIbWsP#M zA%b5mBN>3pwZ`I!2OZm&^lIAx(TI3cUwrhPn&M#J^0pamt1r(+M$9cOd-CkDO*Ur) zw^nDksYz~H!_YXqXTfQ_1M~jlyfa5y#G!%0*&`tfE2&CDo@2S0ebzn-S*7xM3uq=+ z$isUp*_fMo7piH|{AvV8kdy08u`+jLZA39Z(r@PsT_a8iZrU&G;=w^uNM3OgEXlkeXB3`tZWXD?vJY$0Sib=UU+lPlQ?%uwp zem#8jD00v5XuKI2WVb%L_u&2`W_DrEhDvW9KYaMtzx&Gz=YRVTfBz3Z{_s6}^k!mi z{QDoi*I?hebNAM*TR%K}JXDXn%84U4Ydl70c*% zXCSmkb8P^wC!DQ*&^0@3_>2g zf6eQ;OdS#%6*_@7Lt4u~YMr4nUx};q=8WCKdp-{_W50jwDeYCzU|*|keaD9>DDKfWdj<^-0{7y*JB-vt@We(Jg$uqHua=V&CS8UG2W zdLY2UL*naVrKGR!Z?X|U?Mf9*cFA!Nj`)y1@KSoo!{q#S27i6CRbUF2X>#^ER3NSt z^%?eg3ZQoKuU<|Uvnx07(U%+TRN+nnR_Vbyz)E>{5BScv>etq#J-c(V!v<8q#E4!d z%U_$nXUbo3Gh?q>%x)nT?64z^`Y~ri8(d5L!(8-1K>1mpjDty-`9;Al0FWYsQYv;e za7afzIkT2b%bf&VTwJeEmQ6xTY3kxLEOWqk{HCD?6nloXv;E13AD+9g_rTTy&tF)m zWV+$oH&+bi7+FzXgY@#{k$}&v2PKSdoZY@~_s;Emckf!=1%x?`)@j+*fYHTuEF+O^ zC0#7kch|=?9u*f4_&wYJafZot>8~Ym8?^y5yY5 z!H~VEf+2X_V9aimjll716^k(?Q4K|GhK70qB#Glukz`9_@W*dsik#7G6P7FO_PcVp zVSxs=<&=i(nf*)Lz7U)i1RQVmGPLQ$bNk~?lBlpI`onw!y8^;Rw zs#i|E!aP%qQd4>N&Uzi#hyU(B{T^h4HX#`lAR5f_AoR;0?%(_I!BcyHPGzAKd8!92 z;pF7ma`{4MR(^{ZFky!{%-0Sqe9vZ%zt;hLms z?+y+4rJdfzgW8esJJ97`qK2mi2Y|3A^B$}`J$z|Sv?dC@0J`rczGs-eP{Zqmw5;wj zlsK<+4`H!MN(^3=OG;7Jtvs=sF^dOk=OQF#N0iA?&7c9nfj)ZW{AX_CF0=QVJE}id z#Y1B;6=_Fu>~~}oqT8Ojn;%-UoK&z$85rQ41V}k6^(LjkAna8$tROOWXps@cfv3h2 z%4v`D>|$6jTO5!aW=LcMBol88}^%_ zMJ9(rwY7?nP~xxa$R}PEC~768v1B$$>7@)3l*&h?o|$f)^H2@gmMCBDT6uNm%$Dai z<_XyEe;>gL(bde>Xep=Y7Gm{CjB$SZT%)`NE@*7#L*zjiax{c-DJUZS_ebmngye(x z(Rpf@HG^M;Dj5sJ+L@d~zNC`6q&KGvH@4vk} zEoL@X#78XB5=vv1g(bB&D*ae+#s-F66|iT=P!9%#_SY`$ZF$Mk*ucTT!SUAd?s@1M z^tNHZzCGJ*ocVI!?4TUgo44LsDXw*LYqp`wwkq4cI!rz4{6~C0>9Iz!#G$;Otfd?= z;)FjmD8wnc~$W%WmF`1r}bp3@gY-U_~ z(|&r8px{ic$vMtPbrh8APl{rcIk58*+S1*$sA=uw?AeQZyVm1SoP8{|t>>_D6_I*2 zLU+?KJ4a9#FI`k)n54BUdNn!>&f%;&=ws}74a5c8{(6$^p#eTT3M0nD*11eCAV;YK zQaj#?^6ayQRZOW@jdV`~(CMtI8u8Ic79>_rVDKb-*4f-D{cCG&TKnPAjR{Enk~k=L zGr>}^8ALwPOIfBqQpwN&t`6n-1KFn(60BylV<};)4_H=@9OGG3iOF0hC5yz?t#qKS z1xQ^(cmJRVcB;zYb5_k%L`4h6pq(5@bXEC4V{=lcno0M`;eo1aMOKM&;$-dBhOu}7 zqLQ+OWX?ht$Sh{#JE0GC1^f*Ic5%IMo?;9c;92KatC=Kg=IU1V{&KFbuARw3@u&xI ztRCdtmTL~d;LIZ}3?adnOtTWSmw{>y&PoMj*TZDzf}2*+1q&8(?M^mIXLvLat{aeAyGHYy=nHC zwo}1#642E*;KM;9llhpv+LjFMZ=F08QPbogF3zLPp5rdAn#M>om~*kzNTo$V>6Ogf zO0^q?s=DaTQr_7_H;Y2jHOYSV$;Y33`2K(S(--RSh_UiZ8h~-uUwre8#lDwsUo{C5 zVQF2CZF?%e_~FrGO*9$tFF_^85fjLlFJ3hNhqlxU%0W-UPnL<cmhgCC>mo3 zk_6SMvV<2)lb1ca5wzj9wBc>o>#SO~z)#R3-LdNUe9OWY0K>vo(Y^!fKe3|=aR}{U z%gnzT5N=mK@JX^TXKKNejJPTz>5>29aq)!kXq+!6RK_x8iv~O;$9HHu14ph2lyiA! zKDc|wb6^j!^I-!RD0HOl?Ph0v{r!*LW*@lhGy(=(&AIi1cW&Mg)>MU~-O&Ru1P%*m z$~`RZ)CUc5w}x)nVOSm4%uItGg3(dAqENzoChLGNPszR}F(=--OlYJgHH4foefGk8 z*RPHAlX#{4vw1R^(gb_|;Uh2WynX2c3YVDDO>_@%C5BRo$eUy!YLhHmfBfW0ThSQ` zm1(gwMMtZxUAunua*wU2HIJ@<;)wZ+7`KnRii5>rzDxr99x?I(aQj+@7CG-!jKC3L z^iKGy>x&qC$%inOzY<{3+F5dv?B>a_{r1`<;Tn!4zW3_qpLUX;)Ei=b-04`m*a%KPMSa(NiPxV{W_{QW6 zLoIb3+uN@mkF`~`FkI@-smrL4x2dkD}(pu(u0cxW*gr6C|CDIx6? z4>UW&8>+;_=g+?0Iz9hNG*ldCUXYpPy`<=3L4zt_=liG6OpMC6pEs<3n`|wL%<(2Ua#7Y9Hy6%veeJaflrcMUpkl}Z#fzWy!sEt7DSw`O zytIcH0rB31sgEdvOc}-n-<+{gsPu{r<`Q`@jChXIg4O zt&s8g(bF<_wV0-`22~v>}@=6FH`IgW}Wh$!=Af&{o3=Vy^C{qw~d~7dtF>v zUZ2pHu8ZXZFAuj)*as6R_-scWJ0#U-R2^u=VgpY-=%N`1XRS_d3j{H+=L4_onPe(a zgB?@L^~bWw+!ta_&h41O)6+EQNjm4uXo$E@24a%30{I`qrB!fY^^s8pgAV!?vQtr` zj}%j6@TTf$wQ&a`U%e`f6B3!2rveyIx3&T~o>wq{3vaw)`J!H_VDK1cJ-R!#uRi6P z5HaIc;vZGqqG4gTje{)Zfj8jNFMC#Eg4T{LvrLXSJ=)zh70~9e!t@pj*=VtDLuQkq z8oWRD+J?pg$REum)|cv;NDiYdXfjy`KSC0W9Hjac9;UUZNH^`ZC(FflxFz2zbLLq5 zs+TYxU?XQ&LY8U`xv$WGh+&BsxJj(q(_e&;6`er2x)y-fh4C){snf~VrJC|bq`%G2 zD05&rTYzdWs-QUxz*BV%93IOdabk)5W7Nq#lmxi!au|9Jf&WHTs%DmbG)gwCR0$(x zVGCd=eoAP(YM_G4p9K5ep}GTR)PkdHQ%ol6;h8avHi!gAz1L*7Th6@ap^g|n#-DN_ zbt2=bBad3Jxmbt-&Zm2|L8HucJkpvkr7Pj1#QYE`a~$V{;H00!o+FJdcKF*^!_aUH zdetMbhM1pa8y7BKKDG`Ar#t5ivHYN}^Q`;5Xk71>sSv-%j^4{?_^NgYpI&VK^3cOU zR;L1}$ZX$eE@!V(@#>W&?}H%3`8dQe2x;n)!&oo+#(TO6UJkgP(ngzIIpR2>j=&Nd z$;W9o*9j(Al5B5IZ{E1_SAX%#zyJL6F_N%ns|x^;p53wD^z*O3{`BtOY;9YR%6OP| z-Q@4^`{W5aNiWzV-+Vif9UtGlcju#fciBq>O~spL0TpSI7uW9>V)o3P02hHNY}0xt z`>}Q8FPdF3gOoBDOB`jOG@9de)vV4|q7#Nz&RWkzB3&^;ZUDepK*Jv5*qGY^$n9mt z0|`$GM`IXQ<*rzV68P(Tq%QwRuZ*E9v0U;s)7As+hj!%|$BpmqJ4 z(GGKZBXhCX9cLcl?C6#K8AtujbA z7!*J6n9BJi%^k1meE9h3r+06`CJiZ(n^=(RN2YM+$`>V3NUu9~`@8?(@eto5J;@W2 zCBD2~QPa-69obl$id)?eaPFzMMGl=(7^oYm7$`+qUV~`Gf8nVPM8pWj-3D{(+l_Zb z?O5WXEaIaZ5ZT94qJEQ-dcdDxak8Xlcq2SMhdHOHabq+%oVp@`ad%8Kb)aqBn<7yr zkp(B&1k-oFf!mN(Vx}lNsmE<+(_*M&kTqmDoGX|Ls>I31e+eB^XbED_lZq?s$|8r- zE7$zw&!)^^s%S*6FX2l`@Fr?ZPIOu=T5+IA1((Ie?hPo3I`{dMu(LL1B@2i6m=W zIuvZ<_4thaw2=yi?ZSrDoR%uH8+{nO9Z8Fd;xw5hj1I9wma1ls;WX}DRWr7sU{yw` zV#vzXXdCb=KPwu*bp{Byf%xJnISqwuXoYR?L?;b2)^IB5Jkh-Yz&J9oxFeVLSbLA$ z_4+WqEu(K|4iCJZrzMHKn$xPoH!oj2{pjv{uWnyIxAS)Y*;B^TeDjlIJ27ZPNX5=% zSSG7Px_#!(dp9237tp^kOmlMh@{2!ze(UCq-~INtUw`|JmA;RjKEMCtkC(4py>aWN z=a~Xf@;^O3q9Fzx9NYqKGto;IA3u8J?zJnI4KaJ3qZ`OX(Z6%~(#ym16r@*Zz}xtQ zL%1O1)qgRHdga>X8`rN18;m~|F&cj~0J+m#V)H9R6jl`I5DGK2;~wO}n<;w>ubR^{ z(ybS7j|!F3ce+cOMP2N!ef^Q(GPjoEJ(Tl)GoiW#5c-~#2g;F3fHu-``bOkqXMfoR z?|T>K)ymKJ#W+ytZCWr0HAKo931Zi?UW+B1Z9;-Bf|nv4dHU6KFAmv^C{eoCzCrJu zVpdD*3rSZu3=0EVxZE&q539XxLG?d*qr--$rVeF_tp#d7P@iPp;ssUPOuG6lrf>AV=eE?+vCxh2FoM(@m&D%Gc>;wyG|4Sr;Ga~~a+XV)tC6qvxZw_~ z_cImC*Q5t1p$}LDL_>9H{;i^@8B-dCsUWW`|2vdU7w$<7G?9WBJxYvrraEQ1X(|fJ z1moNUxxZ*}qn0*K!lI5g*G6)&cdFF@%^ICD#8AhC9aZ0{vAL%1bRAd+a(I*4bD^+e)3>-snSNCXw}-dory`@j?EZtcrAM+M{~V@u;Zu#g-PE zD}bM2Dw;)@V)`_GS~S80Fd4J8pb2zIE%%cghYR0p>?S0`T*xRVk6>RmyqFxL^EhtF zAg0wgkDfl`c9D1K;>GLNuityGw}hyH%TzIC*noa~do*eEil$)$&A<<%M@rUL6h$!g z=Jh`%U>Zqi#WiXW8!3GQHez2YDAr9NrLWI~7^94-`@=`VNM~eu=Hk}YU;p;k|Hr@i zzdZ7^_rL5DQv@Y}w$-YqG5%q$>ON&Y!>j#%^3*0fr64q-u2agZ7h8 zKlS)daqGT*M5AqfA+00P#I%&a%mvW zdZKR%)=!$wKI?V|)>`trw{LnoQk4tkvvx2tlI)OziMb4{VUM1$1w6dbD>*=(#Sy<= zRKr2Y>R!I5yuPSC;f|x7-+|v{fgo3VcKhkm$DW;@$31YNw5hfy8}D)B2OPJ`Is%n{ z1l}(*dwOv2$FIKrn_vCBX)|&p32Hc&ZkO54&t|;9UOlD;659!Zx9uQAV{0) zTjt9bF5JF(!w8iUF5?R1KH&{ZR4#JI)Dvf^lh$dSR>&!?E6;G2UWtxK_v9E9_U2@DtPIRN# zkHpG_ZYrPLz@oZxlns}VZ)q!H3=IG4436VT$rH_!dWD z29FdS7%4gQrLI<%L~{`&P%K0eOD1uhJSVJ=Z?}`#yL}nm64Oj>rD8~qLNZ!vWohm1 z$P#scqeJ*6aG2*g&3|hoR;u~kdQ+4*qbsfdG5l*^Bhzt>`634=vN9OFoo|UC*CC`}(UdfAgE)3h52o-??-1;p1mUxXowK`Lgz4&*Zwd zXRN#M;2~`blTVIczS#fxqYrI&WI?tEJ@4JSYXLXr_%5>o&R*R1j*Hjzn|5?)bPNPY z)MHnX{R3|u@O$W?*t{(Hf|2iW;msYQ%?q=Vqz59e+763t2{6n`Q%eLetky<6f_Nri zGtKD%q^%|q6HD>2pvxBIUY(m^78J2#s}9se+f5xDZJoR^pU1;0y*%p3NHcKLK2Pim zqA$wI*~Ok|Si5@=HWNz_sfQq%7M**va&cD;eS6&SlNzzJy-V#V$G)LZh*+X(pzre5 zCQPTzwJ}P5Zl_KF`=$=MHqP3~z|N%17ZjK^E1`FrX0E=a>kX@JCi>4>O$G<)*jN!m zo~g*!@c+#Asf*KkB47m}c{CsbR~LNDb-Ytvlf=ePyjw$yctaO+r@?c(84TXqr9sRa~gPqy`3n_|%<(1;jp>a3GI&#F8_5@g*J7ZqJF zph`%X$~%>B$epTP4PVD(-DH}YoegLgCuCL+(JBiWkf8CDw+*sX#h0_JVE$(0GxjrI zvXc$+U{pHfP93JNcurLeD++J+8n!)~%jRghonH0D`5=C1 z4)^+_)y5Yu*-BkTn@^kV=iTGh{m3&9Cwr26&{E%hbO;djeQoccmwNfiWrdipCx&SD z@DC9@Mb*hHAf)0}?3{eejGTG{Yt~rE>w#w;c$h;SIgKGs-(WEu$$0TeShBohqCwZO z9pFZug}*1i{`n_A`|#fHKmV%PwsKEV6{D_zRd-w4fB52yX9q8DUAi!nPpg?4VebG= zJ`2xa%5&37d)00I^3$K$LxHO$QP~QZvVbTde22bX<6zw$`ze49RzJ%;)g+9pqZ>7v z8{&G9H~uEeilv&xRF_;I7B5;TygIySa*ZKoQx{%Ib60pu*!uQXE>9tmTXy*?UpL-urj% z{Ql2hNl8Zi^icw1c+mn3i30|V$u)Gny?gVfN5c46X=^Me%U2Za=uA3{K&eKRhJi?Q zpYh2um9t$cQp3ar2(6?bo@;ItvVk8>Vrnk{=Ej9Jb>cl^IrH}Bl}mSST+@JGu#XpjjP#UNr1;Q6Vl~7# z<4ozvt?rGABBVmZFp!q*bvx`!*5mFYr*gnLnF>3A7CQ>yp+cEHk&Hc8kJESM z=f;o{8kw@Ca3;WTpc*JSrclNGPX~B1m7ev73^9I{R*(|pE3HbqZYi+fsC3gLt9nCL z_y@t{PgdapX>J6)OQmCOh;eJOu&-6PgmQ$z;XAsyabkEUEfEXzJUlp>*E$_|CU0d4 zbF9rYh&yjuG&U!W0ox)d&Ad0Qm;qB;UyBg#1$o3dD8pCvBXFf6YQB?Qj8rX@L1xLw zAM)Jnw2SyCpK~Qjfn<_=rHyHvjC5hH<_5ilXY0gTDi@B{_ZD$;2@QEwl}a80at`Y+ z(xfA1KBL^st%Zc9wkV*L$>+A3)J&2$!Zj_0@TKwJS1$#Zm#^M<@77IwUkSYon%DtU zP7`6Q#?c&$naBuv^-ILr4WI=U8}I&-P=bh0q{uj@Dg~T>;CpdMQ{TN{$?-?Ox71|5dhy^P|F67ls{Qs&XGINA>bIE4bcI=AUs=YV z1z$MN)P${ORe5x*WYe**`pdzHf}aE)0ufzm(Ez0S&l%a)9WW)SSI$ zZB@~PvuuySs<<;8Jo`^ni!*E$>a68hPIP^;K?DEi)5WW^YLu4J#Hr#iW|`w;tQ+vG z5sZ~KH-JhFle4;j4_*OF)4pe&xe%!2t}}1uDqp7u)kLFO=mzb2%fX2r9zr3ZFcB^7zkReRpttVkG~^2M?b-e^Dzmvc;W>CpVj+d(qhC-Q62E zuHU+O^ZJ!5w{F}})6`*4ndn?IKni#<{+pIyLqs~}=LMHTDvF}a%d#wHrQlo~=Jc|+jPM-8cM4kUND zE(PQl|W5%l*uy%Ut9;p)H`7BIo;lUCTEzSbbvh&%j}8?DMX zTl-~%=A(OeOySJ#C+LbE#UN^Oo)zhdZ$uE}SPQl-me1b1bDOhQP)jrXwu4GX*_z}* zvz=$yOjUzIM8hWf00ZmI>1Q9_{h$8TKd0qcq!g6Yw{G6Mef!&o&+P2XyHvXpp05!uvd?V&(-&WT{>}Hd zfBOp}#W8^_2_n8sj^TTBl1!-i==AO1|KU$hd(7;yR?Kxn-5HCnM!k9a!M)ozuU>Ud zOo15nrxPfrLQUqlFT&Y?NU(y?YJI+Pq3<^fQC*YQz{r}yITY65bnPGJp=A)pp1&-p zf-qHiI`I#}|3}n)K53q12c6H#%B;++%*ygrUES4wx@SfjBL_l43#>#S7J?TnYyQIB zR~xXgKr93VLW=+)7$JjZG`8LJbeH!oE6sks=a)4+mGwM7bMLvw-*de}21sC_FSr?R zGnrttuwlw$o3SiX<6iFw&aX%DX(=9^O_~|_^CWhpbUBXFD>OvzB%V?iQsPRm;pCkh_&X$Ke?9jJ&`7XU0f$5tbeFQ129}N1#s<6) zD4(9w5&>yN5_6DZPGpW;wFESR(%ieOf$*fhdS$Y2hq`p%I0ozH?>Y)T^i9KRiL zc{DwepLR}Z@X8P1<_b1>xJa_iY#Y-1=cXm|Ix|6gAY(~Aa zxqfE%j9G%1S#|59(kbqTWO$~#s{P~>|I?_siI0-A*~wC}Sjvh=i3UN5i!WuFL{*}V z#Y|M-8=NqoU-BnC~_AMJSh;<@R! z`qW;3^9rD1W3%o<-1Tz{_MWuiw!m;>4Va7D&4|2m_L7lL%YFs_CMD3{=T9GPp4<^c zo;W&flJAaHrYCpcXU06~p2oQIXH8iVrRh&mpQDCnpoq)G*-!r|$1fX{Rx9{iECr=*SzjpO9b-sJ|jyU`L z`ST=57`nQxZACI|5D;2aE3hS9-uUbpI~o}fYrCiYgJ+nOSR9_9pMPe_BTxi+Mvvq6 z*4_>T4MLd4l3Gm&P!E?6HCLxnNmhhT6ZMpXIM~?;R z>JD|Gu-k}&T0=&9dEAwUX++@&ugIdkxfV zZ>nBT=?P#elz1F{)}HEtJZ){3C}K=22}aOaWZ_Cw%J~<>;7z=;)0ACw*E;ric>Dqc z1)_WsVo#q8ldXU;LB*o}xtlay5t)xRO8-*)x(+!ES<>!#tTk#&4*H3Dx@$^LaU&I1-bs=#V6C5-d&+J%yGBLhgn?lRxO9+V-VyNVhpZYG0t zUQhTtR$YopT2L4WQVt&I9cBav9@8?2HB(+cGXfwr)~@qWM!~=_j+Y1Ft!V>1eaL`6 zS3baGi#y3M$ADtgEzH6OLw@`wZjX|nmj%;d$E(j-Y+_|f{!_+G}{pepGyxo6k7lmtAue=JoDR5A($FALA6f*t7*fK8~HacE>rZ@_}mLd>8vaa7hgN;jMx-Y18+Hk<(;$L`R-q8p*k>f3S4mMa`Nu$DSiLV>(1*DqJ(DH7otUZL?+Kc z6j0DTALpymYWJ^7_mMc3^{>Cl(QoYOB}QZgzIq>^1{CK_?<%u*@jaqkPh99LgvS1d zm%Wk;qT>k5=A+_~!K;`brvdp!2Q2|KQRfH77m(fuGk2S4A=GCAv=+Bb?jtzO_GoY!BQ~|B5 zk&0I3$u$km^JmY#e)+QK!Hi-Wi)1cTDyFKQ4A#oNo@8CNrfq0E+XNFE0P`E;)9*m{ zH3-emh;o?fYq0lz_))?odEBA{__l~RzIipEl32(tS|~k^H66`=iHW75M8kO<8!c`z_r(LL zHP?4`&YU}cLFcU~O3$~g)6E59WEU=6bWospF#$+C$i!}}RO$GNB<~XcE2`l#)gM() z+ly{0CoRAnwCc5hvPV4+VQ>bT^sWiPqZvw%pSU!tSg~xTk=n)lXD|5~tmblvn6J%| z4{w@|sO}J%*`viw4(t>mcIxr-XJ-D0svkWP8Cn;*MQD16pFDq}Cg-M4ubr?Vj~39C z^E>*m&zc&wz4P(Mg_j0Q#K2<=J%9H68?Rq;^4H^s53%vmm8*Q_7hirUFhJPj$M)?xg$d7|K3&~5BUs)Snlig* zPKy+fY~3%Fe*DmuZniLe_U851@7y_c>*g(7+CIJW;E~0_E4D0K46)d~`LHcsH3{bI z`SYiD&iwStn=7|(KYa2`;7~^$L`A(ihD4}DwfccWh0_A!G`JK?t(Mi?-oZhk`MTjpVV-4!+HH*$#3k`B=xPRb`=O^fXS)E+) zmOtr;(OQ@t2K;*s|L2lK(|5i5Smit~m0`#gYVL^hv(j`E~RMg~=>jc5>0Jy$TT zg;||P-Hlw&hY>49cYH(^u@*Si6q~om5gpLs;7$(3WDR_8-jc>CYq0>&(F7|M+|OrV zNGVkr`V4kVzJ55=%ztd`2{*2eIu7q9trUcF|BkU%e>3*OQ{l&|%O$%DQu1(w4!P3`6HFHk$Wp&kto?rRkE03PP zxclhQI5s}5mI9;U>=_1nnUD+3daY*^Jb3VcIX!-&?Q6#_dNmD+GrOAC5uCV&u`I2L zf>jz?7YVAZf!1b(pu6sgbeImU;cuueucF`6K6!Cp3i1l0Fa`^b$8#bJO$Q&o^UhnZ zzxHPzeY&=(Q-2TvIcd7){tmzUgC9QoAAjpKPoe5vTmSsa8wTLmX&5gh9pg>-P5`)m zc6-}!Rz-(s2vj_NcAL3s@n;sql|uCXRXSGKBckS&%0GtH>+%JFd-C zOjP246in%6)gOT>juUBId7ej!GJ~RWZvjwXh8XEzyLiED8MAh6pcQ_S_y7@ckejT@ zsfX&|_;zr3_Vnh3-Lne|*evxCc!VS&47HUuG!3ZLTnNuookA1s!vK7!U*7acK2BfBwCn{Pbt5I1HLBxn>d9P%G8OIZ8lP4lkz3`rr_@vzUU!M38 z{eCiXH4~W+edSpI^qu5}uq8tRfQ-D8cKp_eI;F#w*y@TF_^v~s*lb#FeibfP1^2A) zEMmEr++_@$)Ll-xN#zYy8(sy*CCixC68TFr<_hq}Dis9F1%%7^(0CdmeLS9LQCaEo z`1|=M45MX!kKb12aJ=9CV#aW^I?G3FBc8i(?%b727wzt1FPnlDJ}hfG@xta$+V%L{ zY#56RZKWczZT+I)p~=33YGo`8TKE*_D>=RlqU2SQ2!ey;{S4_t5DNR9qyuqD(L=Yd zZv2i}un58|gWK_vIe?)ZO$l(|2M>GkjRDr1RU2RDh+Rwy12Rfv{G4gorbZkbK&A&Y zzzhyCmIj1V08Vwf7=>vM+a}xa4~?afs*7+Ql`S`@rE?Yd$v2hR zbSd=;0PT>4Rz4)exPP(d+KKhTR{he}{94QT(pE4BCdAiD+&XQVzOxbSaij@oB8~Ha z@z=&OXSX-aJG^jqd&5TIWOHg|$Jtw39KvPC2i&}Q$A;8Ge4W{P>;3T+h(Dua?%Xcy z9zA}fK+{XG?(W@tnGx~4+_MJPFKq4i{pqKlymsyN%U7>_`qR$|?(x%y6x1C3)suH@ zO=b?iz?~aV@0&$0i!fGkr!QZo&cK=dZz>T2zr1nd%B7>-GrLcpYPzTeQxV6Lk%5+j zEk+jIbLh-EtI?vLpZ?U>e@fA|@_K3581~Or4G~aU)3Kg)Y_;~89s4bk z(3I7B5kEPR!K%t;_(2~2+QO zp&w52OI4~o(9LeoKb9O5THFwJM65H55XhAExnjnOSYVBKXO7eZ0D!4 zhSbpUPljGd7ij;K8S-f8GkFK6tg_607DoWifG?X%uY8si3!)&s`Zq6@{1q`dIl1v# zlS^8@E(R<`XI)BTiBN4K*@O6o9dPmMSW(4`O&q9OxugHXm+`NMOyL@UBYT)B1C25x5c|0N3c0 zqXE~}_0ekfTKj7}PGX9#2ZBEWp#i@Ra9Odm{qCLHkJ@-E$6nJT8HEpa*|G7fAAIF2 z@4dIZy+gb(@?331qoM{9;+awL9H2WiQ%=v93S|U4Qjpr?@RhSCiDc-)A55PoBMFxQ z@TuZI*uQ*k=NCVC@4x-zqamI%sDn-N*dxu_>W@D8?2B7>-oJJkDlZRLe*Ed@)<0Xd zoibKjY@E{7F{=(3?u-;LdaPxcQ_ zZrV5jq~)(>UWwrLa{u)=uDpHi>SRTQ6ax%iRH4<$g|;q+vbhSOQCBd0mgsiK&877= z4;w;@sq$VV1VBj_)-+E<#meSWN{RWSWQEb;SndE!6xREad7w0j;xLvb=27TmQWn3% ztQe{=CC%$8+1GedwMl%K0xB+vMnobEwEuX$ln|;tDM0JHa;DV5r<5L)wk{fuP@2&K z(1NqP40dFOE87G+h`x&sGq&ZyD8GljQkoK+DKs82ekNC}>6Xs({Pb?dgCy9<{tq2ux6N6H=QLQNO)&d$~+pL`0T*RQ{+ zRbU@8D|AJzZ*6az9rW=hpWM84`|?GTZO@oN>0kz{*-w8Q z=udz8+4VQBUA}l_85}UtNSJ)?{CUxZEruRy**tr8)iDm2F1K;4U4uUUREe4>^kx60 z7GFKvn5X6zWJrGE%&Kik?XdIc*;BEx9exa-7&=L0BI|6HGSQ&C1J{wTI&`lSgWU$N zDLR|F@$rJI6shhV?VE2;h;;>yb7!G8S?VB0Ymc8lZSND%EV{5U(TWpkSh1~6PT9iq z<>SW!K!%yj0a$Z&aSxZ|Y|j@o9To@ZN$j&DnWHL1uNfQn#CSi?< zDKk>|g$1=i8=T>VlnX-y!^0j}$wPJcZR&&K`ew=#ui2TYX!PDO)^fgn?b_>ayg_Na zdh+b)&0Dt~J$xVwbFjCU=MNv)5Wo>6C-qzz+I#fyp&4xE2ANN%vcfGVkocV4J(Exr z6Kles@u|l8(@CHkC-kdK5;eqMxma~WTClJiNnsoP%M*_WaX+w+{Gp<&Nl0jicI5ii zZ-3*T|NbA?vdeZ%viGXWi!|#)zJ2fUk3RbB{nuaPmu)TZ@n@eWfTg4?CHBR`{rxkW zTkl?fqh`{A4+vRGNKBEr774X&GtEj_Q1Z4bbcA2!*Cml6O*Xql4^0`;LPc*Ds+WsJ zgfS*}B!5?<-7I42oobYE1-fCh&_f!Vml9n|T)m|@LZF$o80%h`I$WwL%)1oUSD*4M z_(aA$ny+-f@;!lc@hkKvY1z0NB*&};|COWNo$c$dU;VE?{0V&kBh&yH zry_SnA2YwQ-NBwkUcGe5f|H&sDA^RE-JmBoF)c`!9tsp(&x(LUKbAsrYvn6v&=of~ z$KIp%XH+l3ExIm2Nhe1!!4$b9o3ttPq>=*cQXlNUe)-~u@4oZnpWV#L1UGSWxHY9= zff<_nC)WSqJAd-d^=p6sZ~Vsi>AEq)J$NApRCU(c+WqHy|M$Q8H-GWtk5|^6i-g-e z!r&C!@d;AqupWH0f-zid-no1F`nJCRk3arYkMfmkZ|Deo@aWO)`wwp1 zxN-HWqk*s9xN-CD-P^Ez_Z{;euYUH~XZ#J!Z85JDmSg{8uZcW%*l5&z|jm!WM);p0y~af;aG%U5UJa2r?m>%C0VEyf1>HD4e9p6&8-EC>?3KZ~P!>37`|D zk2(763)9v&by_hrGCj4oaqir%eb2qP_u{eTjr#tHTJ@3-8-upQ0zmQoz%1MsCYhOr z`P5X;w^5EJgBw0*G^Wi+nc1_w0}TL%FW_3t5wMpp_O$_A2&rq-IzzzO$s3kunBZe( z557{i)U*b&=wBM=?dc#e(y+!OXve~ZvKe6N5LF+tRv{K*iDep>*t2-z@v|3?o;(xp zIdvz;Gj*iASy$$(d0wQ%!Zafc-dQ}YI<;7HtRg*!1P=j`M~BbFf4r6ody2|g zmNTcXJ*Ev*e>oS$>yOjMi%_askfEs3XBw+&_f>9QuL{)xHg&Et0n4X~q95O{NXsjS z^3YwvQip6PL{@0aj=DZj&PAhv)Bs)IdvW{T-TNBWRX(haq!KTSZi0bYbCA zWVjZU$k@9uiQw)%*S=FvlD`C0B7t~SLw_)LUrsQ5x>^1Yw%&wENJVnk1scStQfNZu z9X!Q|CA-q&_>izLv)Y%Mr8F8LviMhllPqN6304^auYgdQFLcL_ZkPKh7qC~9CY;_a z{}S|w3$!M->MZdj)G4jiBQ{jkuW7OTWJ1&;yh!YWzzk@?Oje=u(X}g=jiV;kFvic0 zSmy-saY5_&8%E<7m|Ib1I3=TihQG^W0`Sb$68%RSMx{f?U`ja{9230c0w(nYQW=s? z;fq%|I@;ab_|Q|nV}p6j{`FW^%Y$EI|={*KmbWZ zK~&0POzN%nW{F<|I7ONSkO)mTsp4elXENejk#u-^L<jA0Z{Uanv6JG-Uu%0ZB(i*xdmoSU>&)^XSzR4325(+0a-;~@Pgq-HY||_NWqEK zS=JDmkKM9$TfJ9gOc@@Nm~#CeUW~NEF{;AX1Milf<#MQx>hz^oT?OqJ$Xpd~4BRMQ z;NuYXq6tsT4b|@tSF;YVu~ATck}?^*#zW&bj*FSjsx|Ku-rcM}R8EZ8Q zg#tlG@{`AQtiO8cqEKGn+~tesov}qS6v^M=R;`dmn6|deau=Sy{q{SDmsU*WHPGML zbZ5`qcIxNtyEgIMIdjIforo@9y8Mg3@C$$b-R}tqc;1VbE~&-=zyI*wjW0LfdFO3& z1bpD+_$tBZ^Sx*M1NMFV$;X$kTz&7IcYpfXr?o3Kx6bXJGc`zC!_=UM5AK^~B#6J` zfVcY(p4oNq<-zsq*UccHKR^7*C-xu*Q7qDs^_Ex{I}9OfxoTm96mrViqIVT#cu ztIjW;CnL);gtK%A+mHWjXSDYsFF?xbcl*C<0k5j1MV!YjhrwczCB|A-LVtTaYJgxj1}Hbf7~{npjhA$j3LYMrl%`cN9%JRu%)|^Y z?~LM6@93_;>f}6w7mI6V4WzGaK_v@(mn}h0ngi7G>q1-7if@ZtGMdGM^(64U>f?90 z@z4jGr}(hIi`db#NjOFulpf&f@`s8(3}b$K>uxL_>;W)k0UkykOqr87hursck2ov@a>!gqxl28=ux9D9zgZXuj zo%L;1yE%NM|I+Z1X_sPl-48ayy8q~b2DgccP~qokCAHB4?G&&Y0}yQEHY?jiIigY= zfgpW_Wc6CUH_p*btzTeaSDi6$kXf9)h$uZrgXD?RWn_*U#xpR7h@oz-2_+7=PJ}7r z+0$F!_~89N|KU$ulXFY}6F9$3LtFjs4}N^}-orPqy>{dF{di!tfK$Nb8Q5*-lA5znDlM*DZ07=4@jM8U5h)UGNgVA&?SetQXYxBvkX zKr#ErGy*pI)`P)EYHy;`2MqbD!~5<9(f{%P{EIJd-2C-l`o{IEmkiU#OyC{u`Q-WcKmGh){NBI&!$1A=7poiA z+~E46>aj;ZKnT-a`0s!1-SlVL&u^HY7}d3w?T{*kONf z!1urIfFYTpDMtpd;1IbZ7s+sLx;WeuZ?B?s{vWe1P+4KU7Czn5Ellb@RKle^{cvZ# z{qGmfC79xR7EW_hyMv<|92W8hp_24je&a8i8)1f*)Rj3fA_DwL))QI!Uq)n3p%Qb; zO}>nK@pPd(AQ7-+7Zhfh_a7+M&PWF5%^> zv8#IPn!T*4d`cK|;&5g=JTnE^ewWqM)jY+oL_FUL8iW?EM(ILf7f9=Ezu_|~xg;3I zlyTVane50I-1_P+Zg$oGVMD5eAvCE#K_yVv@l*sH%D@dN9~i0|sR_Dl5&d#q|DyoI zmMm#z6L#nzJclP0G)e}iVQdw5Ijiv`QAebdRedvW8xN!x0yVXsZfO+^&rjCpSokMr zdKck}OEqR5w6wWeobYimltCr4i9X>mW0|O7>e#ui6;iD|)JYpi-LT;m@G$Rb1Oc)D zQZ#!?1)Gsx4cN;_R!cxegPUkCi-s#%c7`T7`q2<&f-n;q(A3rj~M}p6P_S47DpItbA;hW$1g&+Lr z$DU|vym9rlJL{*+;MaL?4?qV5edI7QGx68rZ@nU;*J8|N2)ybk6Qy{^a8a4iz0@`06X;-q(PJIzs^e!b zOlmq56XzUA#mVb0UOavJ+$j^3MTjEK!4Q*n&TQ{IKijzkwnd*PP1mIhT5@=L&g-hZ zK0U6D^#;wbOb5ZjwuS^a<_INfCkdQ}V1d)|CaMBMs)Zt^adK;G#tJOnur}HzMVlv2 zZ#sZXc-3O=Mrbm8ABxwUj=H(BPZMg|t;9d~Y))B%I=%ZZn%ZsrO>kM$S(GQz>j3Tg z8*1uK+oO>X0V(J>1G54*zPUa$_G{Sy9gip(lt_;mBmEt>p5BK|4|Gxb%6i6&W`l|` zt*It}#z=c9K~K1_RJ=f>@R>r{qhyhnpYb&dY~UwB<;+!gay7e`eZ6{H*&cg|86(q@ z`Gk*hA=PDs^di4mF+BaT23<0%w56^9X~NPC=%(Y;w^7yG^xOrt5bt3%?TmKf zHgU&vjVx*t8y2q=Jl*Fy{A^o$@@0`Y#z{P@1gaC8XZEuFolFGE^>qXV@QSQn8#?-` ztPmVWGQ0~-XwZPr?_bx@jmilQ0~D$pQ9ZJGFLTqKG`?}_-FM#AX1;Ux;jO#(sE5rF z&z?DVX6Foa`2y4XEQitFWiaey83eF9tvQ9fD^s*N0J=C@{_Nh??bohewqt{>aFNbR ziJC8n5O%0P(ah~-sxP3}8G?ZgN zdJ}Z1CT)n|cgQRl4z0tYX;92V)Uu=fbVI7cqt`B6xN_nA?a#in8VmF!d0e2oxRh$n zr8_>{tRcNw!`iS>n7^FmjEoJ3P~Z)N&SF8Ja@;dKpckUgU>Z}5C&mzMkCz% z3(o=AckD0Akh(A|P|MTso+~M)jp)rV18}04&2981a*tDzGAUro^`xJjW~<8abxJHM zRAUb>!~zoQ7*rh?^CG%>);W@@w5h$6JRE?)g4#k^h;~0HOG73WGE5M4{CIFbrs|ls za+JaUNfdLNMR5#AWdeVi(KpTsV&*&|j=e#Ec=Yp9tAzUSjfnDiZSUm(vffLxP-niuP;O`6@1`PxfS?{DxN3HJQ(#NLTDlY-34-a6Rd za1tkcLQ*ZeZa#$(rlACyi_#*!@>2es7!=)&Zh?OsF2Fq`hP6W-kbWiZQ==k+2?Y@) zTK*P-W7H5BM?p@uMgkf^#=(&{|0W3x*EdY|)4M8E6L~7s^}+h;p?wQ5dTaf}<#Sux zy3fz-UOBgGK)}j}c6gs4Sj8-jjrg*WA_*{8ceYPkI(2{L;q(LBLPk|4!`l6a572Yr z;+3nXxAFbXy&svV^V-#G#vZ1I-+TJx>ACag^x)~>7pUU+BhjnbLeHM-_$ z6LGu=^tN5n&F2KKTeoh#{>JMX6&C(dVxhQ}$(oJkpLgi5&JQ~aUA=ty!o>@uu18#n z4tBTX7w?Qa2tQtX?V4F|H@>`K`=gJ4@~QndU%PVk+G|(zpa10JPfhgC7aX=zxj4DV z@9@Owwgf$0H{#4t(8Af=KhkO0P*V-+j6|;K5>L2nuRZl^1av~Hp)1?+v~eSqG%GFb z6{4&=d`&;-*|TTNYQ{_&VF{!Td!72v^vJZnO!Rv4#ftX8=;z9c$yZKao)EY0eE-Hj z84VClpL{8tUac8$Z~rmD?(8{JY+VSR*EPrgr0wI8)nJZRjVPa?x3v~jG_2Fzt^x8E zjSQ~%ye2@^CQ73cxs-QtcjHj0=ICVI89I3FPOkxOvcLGPQQDhU-{;FvQ#LUT7n(;%hju-^9o$|;T}W)^QitRujWiPUFkgw3_0e_B8<}> zOe$d@=NwX-QifiXAu8hPE1fqwB8S`ah4}hB82T8fSYLMEbA99EYrF7;Qq9>52guCZ#tQVOfjv^ z%Php9ld+vi(C5A9TboAcvZRN;L`;bMTjoF6`Y^k2XG z{cr!$uYCWbPwfG)Hdy5cVsL6nhtFysR84{S3~NINK=jn35WMAFct?i~84mVPwU6pV~64CS2#g z4D586e6V?P?Kgk*+rR%ufBeIr-85Q>mR)P>)-r_-7%)B8w8|;lm8*_rK@+mL$F_PW4m~n3{Oy z?Yjy&12>-xk!=7gHv}V}`tZ~((5y2;8ai!|)XGfRy$&*5lo{P~6>8Dzb#(M*Nsthus$S!H z7+G~ePpvFf5UK^j$rlX}m+KeL9Yobo7WvTjQiV@L*7@Ewj2nINrt%_TI91zsh)3Y@ zqoEWGQXDfAGqpoQGVmM0X6nmgn`G@JOWow3|=cjvTNlX^KA*N;7P@Wi|takhCPTN^J?S~Mvpw$sk( zEj%#q9Q(`9y zoS(h@8+tFT0~8aRaQ?BaA~pn2w-t*x5D(u{wy)E}A# z8T__dKRE|pEG%#>-e z?EcZyl|2(D^zR(BTlBik*I7BtDN72KvZ~uoYUUQKQ)!wwzhgIELkP`dEtxP8w7-<* zhY5tHbLW1mTADXE8GWVQ)3cKKEfxwS#-V)-AuCZ(itK0lsz=3Vdy}5^coIZSn7r~w zEWC(t40(p-$NVLbaI(d~Fm}Vj!hTAbXQ9xW`SEyOEv02qOP&BM(5Bp=QCEV3Tq9EeZTc0~`Q{?byXDdQaaXuBLlRZ$OHM z`Qw-t`hmb!YZrOXns!(`7D8CEYb!V?LQ)99aQuXaC-rrx^v6wxt-|MhIK*in;^v@t z?AZpr9>ly{|AJ{0`M`P@HyN>10DeG$zg}2TRVM&Q`Gkrbp1647yzRkSFY@9A53~gR zn%gZl-?RyV&OvQjTeKZ8Y=0=<+}yB9^R%tCQ`Zd~y#M}t_9eE=Wx6z$2GA&!b1}HC zsI2HXKS!?OwtrHfGASgYMK;}_r56hg%U!Y=(a{_HPYN#ldX7nMo?qNO^R;*1{oY5P z8IjX-BL!FqQvxyZiIacwy&wF^_kZxiPd-&QE;Zuu4}Vh|Z;-YVaa;7bDq4%Yqm&rSn|z zbYxN2rN+d*7LHJ>leI#kJQ3rW>tU)JAd1FPx?wMPgC}wZsMmFthhO*0|2X9{kWdlS zg{pvz{s(sogL|JPb5~)|4uAPst*xPVz}ds@lxeJMfBdJaR|wGp#RjBYp#ZVLF1r2M zDv?|7DsK94h4x>rA1-;y>P@t!?frC-9t3RAaMh*G(~urU_trx_qSe*jE`)hGf{Ns_ zW}2g3F+S*+EXi?~XuMj)ra>yl!B?(d`#b-`U;FI;`ez1RJF|0t!D)~iOadS13sTQ$>e;a;+^$VBy7O|(dY@(dJTlU`2kJuKzHj!*Qr1S zV1+UYc_U33xNeA76w2|Icl=>_r_l?9jMOiDMma)sM6r7hcUrE_Z=TRXcOaz;wRCeP(FY>Jk7(Kl+k#;6!r-Qdy4i|D)=!SU-9 z0b`?EdMwY?M_atcXD}BL_aI1KR*uv?NX!yXY3_hg&uPWv`^=v@b+}Ht2d56K+Qn7F zn5&j~m@Q#Mk*u+$Q^jyidQFk?LE3A)B zzINacfa>N;rsQs-l{dXX4!5_VAs~3 zp1*KGNB)j;2AxTA_nvCU)|QVSJ$3-lTi36DdE?fT4l1_8&ARPM^_#2f(dNnanVq}$ z?jzAPF)LL4Yw@YBbkv`=iuuIKLuU>@z9k+qgCL3AxxH=Up{>(zzWJ6HRwI~kaQDE5 zpQml`YVhIKy?fdf9bvU>DYJIMnWK)gwJXW#&hpx}{P*#bhnjshHq|Go4YSw_rs+!8 zgH9kqVY5-R77Zk5+%qi{Z*QI6U$;8ec0ea^i{XluZ{4{q+963Q&J&_;q`zX!MNdtA zI2#*6J#tse0p)<^s`IV-OxU>{PEk<2bQmVHl|$)Q)-nXyvhKiYE+KSc9vL zN7pHRo=e*WP9W@`G-2-L#>xFHw9o`$AO%a$SW#L6t2zeSinIwL^saUGV=x3yRU0_) zA)HQYYl|SQ#2JD#YYrBaQ^wjwRdV`dEa=x;hjR=kJ7~BboC!XaXNG}66T?!b^D00g zA)RuAv=|5U=;PFvHEnYx9-ywyUUZGWDtPg(&zQoJxm!g|hT|y8+wpao*v}Kq|0{Y_)ITD<$AFyJEmY@tNI9@T{FWm<6cmZ6@ zhNU~Z#va-mzFv$X|N0?JLTi+z9i8l^nbql;z_NU7&BS&p&gzLlk)jx9=&2^m`(FX6 z9Y8CW-}75OWwqm)WmfSG^X8XJsg^mW?G-bkz&`-S`yV@_ympOLbVQH;c(7cdEMF#^ zs*P;AmbSQdDF-KX$HZ(VGfLqnCX;8}2_{;Ib5qC^EmV=*B6=d{e7(WkEl_WWw zh(8lUArJ5Jz1{_bMXYb0`1UV+_^y8Ziq_@6uUVCIyU zZSmX~WU9pB;ahK9J9B!AvmJ&(cPTUu27;=@)ovK1`p?5LQ%|JnGw4VY!3+%ZEZJ0<+HSB~V=ALXR44)J70bFyCt}}RBcyHevY)`w##gMGB0LEibs6Fp1(T$I zKm{7p6zrdy*x2pyzg~iX(DVkSbEn5Lv|a1}YtkTNnC`6#Dg2fLBHiWT>WTX~NXLPl zc=PpZXSTNPzOcJ-aYW^oB33CoU(hGaI4CQCReMFjaSWEFV@NT7=F7&O5dsXe&?@JF zq8dccARk{mTdW}Llv4l#^@5~8dp&@rU`b+g)%!5S)lg7-VV;C@Ktf&Ga5_A&7s>DZ z=C6O}zy0NRzW*aWleW~<9b`gi(#Ey|qedUbo3Ty6MGwl9FX% zcetXqv8Vn9ks!s%xs@qoBv)@W7!~59_BuZ)rs}_AP(!^=Gvw1AMVkq1i0{LzV%IjlKuoejDm}Hge6ltB4YU(BXJWvy4=SZKwD(k72MuNb^qJ=9t zhWXTLg|&i`rnOqGGX7AsCnsEwYvsiXpi;IWvwRV;ki6K^Xq@N6VMZqCFZNI4(-}|v zlN1{8w~@t+g&gS$Q>UP!bcv!pG&1KHKo6!74l3|QJX$-{NFZv8Axt}MD)m#rR!M4E z;|li8q+UZuw| zZOPwH>HCiFE=uKFnxv9(4Q#;Oxb)+n$GOEL-MT!n4g|Xi(NY2E+M!P6sa3%>E91Zt z)&2E$xn8jh=;+j{9eYlh`~w)|3MgmREYj7;+ZQgF7UEo7dk+|IHw)&>&MsCU+zL-> zt6opH1uO{RdD}njl({!Z5Y+CTJ@>}-H}BrRZ}z{K_5AKR%t41~LRelyt)2Rg>?1@x zS`ETL6L6kBesbaLg%93;|D%sTvNU05dIl9LekELE1Lw}3+uq)`^AG7a%f#e+alpy7 z8gE=~k+F_#I|c3UJ=qg+2vs?Z`(3o1tCK}74`3i7X;|CQNavji6?Y!o7hLTLL!H$% zCu|w9Fk@$X%Yxg})!?fqH7oGHTIq02$TezDzW__P*x5 z%dlEXI0o*S5y$Fak~7uc7|!#9J$t)8dudEci`Pyw#$Sf&#-UmQxw*R8nGLOuiS~}f zYRfD=7Y=(rSlLrpv{VgNFxaiLiiqr(YV)P=P^iK)QD~F(*H<{-;-n*U&UPGs&EcBlLg(2WKv!|c=_&|Z(P4}<;P##5^JJ9qc3wZ zSFc;s^JjngeFGhXW#_LA&nLypqs|AN<2vjzQ*i(KcvlS&nH2 zWYjM~^JM%+R~Zo$KtmZ(<6`C=Awz4bl_G@Z7 z<;cSjXY&}~8EeZmPGzP{au;M&k3!h4Zmykr`}Nl^o!xzK>u%lT0=r>D#luCSNCYK? zw@1_4ufMiQS=!?nuCPgV6lu98(6W`$p}e!FM=T8zoTM(==Ex_bXrj>OcJ5-!U`z<(~RUZxHb=AMB^| z!K#od+O@?a=SPOIk~*->X{%-+1at6`5)WfeJV`4m_@wiv>OQi--%E&36+(Owtrn@@ zfC6+UJ^4mRVY^z;h@u23K1-i-`Uv5Wqq9j3(zM;@7T-9DQ3xb(Q?*mB% zg1fQua;4t8LK1i;mq?7>Bw~t5>_2+_(uH67>U$!@y1%BI#ewAJOl)tgfB5zr=Xafz zq~P3XLRl4HoqYZBrGN0Ze(Q-NKKeAe*^GP`HZ^C}=^q`zn~lIjo-!)9!sFLbYqY{) zS-u`aljkH&eFLdD2svGnz~+0VdKtTS zwy5r;gQo)EJ=;AUIa+9{RSg?F!_b`1=zWM4=cUex{1uA|u)T_HBohBHjjG=MP}EXx^}=o2ix`*vX;R=gnWT+JeZGFaJ3m8gtEp-$=se%|zxhnqXQr}Q*? zXn`4dY9>^aY@?8_NN1T=L3!&!4T!XOOrqQ0v0sF87xr~W@)8&r2^@(!gQ7NvkZ60mdQJ{Xsp;VyI)D#`CvpXap+Cu2E zC{(8|3l(##fALZmFK#eeA*vpIK=pmw{Xv_SQGu^(TzqQ@uc}JKOrdNBLwn5lSM@)5 zbWiEF@qX>K*Up^Tx$)(fUK;=qg;O81{}mxvKX-oj>9bS!9^BhKch*AQ?|tvP0{!#n zFRUDG)`?zaziRP-6E18hWQ%ak4ElgTu|d3{f(P0-ryN4;sAFuT-VdHU7YlSi!OG#C zdv~5Zdt$-BrSliW;(`+Mqt2fl`5x>)6sFtjv#9H6+l+(tQ|8q?+q=L1^vTBhhKda@ zX$yJyq!-8=$!ebz-?B+uwhq`1JDFsiFU-TlB}*+dUD%Wp%A79u*xB5$)9V>iFSh0&@Qxm%KrAVz?>hMW1v^aD zGEQvaHtV@%fGa1oI7~K?F1FaPNTYDhXp2n*!JXzi^FbUgv|WRuWtwv-?$ke^2oZ}u z3p+1Cac}kI$_w@x6{EZvc#T;Z!6g4MtT9jMSnm(;Mi}buXcYjusXGx#-dE-Ub^4lx zhS|Kylyae7l~5|lCI_L8rOxw2ULFoACQR-n|0qHIx&oeD;AD?%$(9-~o6Ycmj)mz$ z1?8}(uun#A12E3*ozUuEfbtmDNrKTK~u(>BL=N z9P(9iU?CM=X@q|@ukww3j&X=FxsPE;I1|-^(2`~N8KB4>&*nSQhA&{cc*Wh`RAWs_ zJ|mo{qdu3~(;wLq|MoqT%(Pr+451Cd5I(WFe#+^;e1v8tmjrR8tO(^P``B0p|E#KXav+^y~2U9fO-`+c9&!)QOl|*f;h3 zy|>@W%j; zE=vr-$u`smxMGG2T_oAER2L5mVBPtmlF=gw6Olt)H^ z(gZSQRWRQqhtyqmzrz>5Lm7CG=yCi?On&+r6&emc!z~Ux+l#(tCz6m?bvo>t+1JQb|DFq1CEJOH0d?!0* z)Wj7|^MxI;tsBoWXt~&@yFlbF`d%2C(OVTmmuqt@ObPYaHkNd%AdlOx$2#VuXc??p zK_`=`+$=$@VPJcLgNlGR{C76j&zm4y@NKxd-8v|?=HKSYl{c?mnx3n8(x3l99N{DT z=)~^n&0qi4FMRmkJK`uV77Cg`EvlR9%(Pltwemev7(4@kKlTdS+1g}&>^okd$qT%6 z9rGtdKv)dyD{fYB`jftk59?knaoJ-4APJiMuuE#snNLPkf4`#iC4k^f)+yE(`|^ye zGLFuK%-Qw3X*zmA`3zOLvUv}%6dIc=q*@PB6`RP8F>{-za3^^+a}ScKC&lwYg5}_- z+a${C;jvL|UP)StT>UfI#_K60y2tYbd)zort_q+cu?j*VpM&%1EYo48j3htSs21dE z&=7AiepIub5T)g&z!;GZoWgdIP+>_ht;FDy_$M{2@^vBMt|Gyk&Ih9! zK6$kB^y${t-r@=izNk0yk=@MQjS3cNHy=6T0g^(y51gFHu;72&v18Rc*zCm%T}n zN1C^@^}BYBC@!9uce8V5&GdZ?(D8v+l%M_Jj7-NTrn6h;9Q-SIynN-dnG24~ym;|~lf1tC@=Kff zT)lGj^Ur@Ku-x5s@_=bb&(LT>j14+B&k0_)4H0Oc*f`KY0kBvD5uFg66R)7p6zRx% z{Iu;UFP^_3tcSBx0?7Ts+4GRmZ)jQpUa5>ezq$IT?JJ{3c0(X~9O;1_?Hy^LrI$i# zYU*l;Ras{WmhFr{>n=`7$tvwcVp>|VVo zw0rpAk?vakocc|v62=m?2@dCWcT|psrzkb18|Y`X&sR0lGB&PAbBoPzqA!mgI8)3+ z{%zLxh=|jBh&RWp{#~OJht3|g`H%feEcr0QX-jRVz_cYr z(v3Fua4{}QEeYK8TLtL^eM4LNj=@ZK{uHImaas|gsBu;N%Lqp0h_j$%L%p8MNS`|o zg577hk)bZGV<1ktup2g&YTD_zl+~~@KE4{5ZTAuobOmt`IWBG~ci_*Hg}Qt#3hSGz z9AN}Z6$ZJXtR`X-UJ@q-mCYm@$j3y3O{!B1Ra?@iVqH&}qB%1z292;W zbF}!@&Ym4(VftFrnp%awzO}PcDS-QHj+w!0=MiqKUpS`)@62{%h`MHnve6LZfg|`t zTZDqn>QA>-F$+TwQMHhp9t~Kyb6gJYBWOY!x*5sC87$$Ak5l@IrCQV5HfaCWH~#H^ z`p&aMhfvs?Wn`Bdj#tvu#7`9q=zR)k&;sYf{r8L>T)QR)fegJ_0-}Xdnb}6FFrvVq z)iv<5m%#d`f=zq@4Ha5Y7pOV7DkFo8DniPb1xr!!X}q zfaigjQdEQ9XvUH@U|mIpk}De{S5JO+>+b*gPygRDTN}@x*v+`I9z!&I2v z)#;d(ORSQy4B!v{^e<#p)9^u=Jpghob-TRH4>>fO?7XwCrb!m~<>NT_T9A?!u`CSs zt>g%vaUP_&ns!w#0y4cD(Lf7gQWH>b7BLR?ULC8!LcyzyL(GyLh#|dVBfionA_0qN zz1u#u_P2lQHyow?umAmja-x04GtZT|n51vig@nw}Jea)_S`}C2)@eRmw#@rGIC>5%Gkp@f$7>D<&2q~10WucOimn4fT9El8E@ZJAm zh#1MlL{mj=mP*qYM19><@dO2-DEk&KKtpk;PcsJtK`Af?I~eF<2$1 zw6j|q7k0M`ZUl+tg4FVTLgoKkzxK`dG%wpH+irb2Ikl+;Mkj5dx3vSm&|p++itN*8W2lz|R3;)_>m%D)LL>Us|8yFQ1sohN|_Z+C%Iju@J)(fq*cgo{Kt>m|hv=9)t>8#4$rA z#A?&X9_kQ~Xo@fSagoS$9d^A+8{pdiOfPrgE97f)=VJ0fqhq6I@_3j)T_K|^O6pJc z5z<8N`V>!@TtG4nvs7n9bqYc)z?0J04s7MB=S|Ho3gKA<8NR&`&osDH(_x0-pfY|0 z+FXHfzs|G$7LVaTGeLwc-1LcoJ@!H$OD%xmcwWr&T$z-aS#a_%CI9a-uWxEm z0Z9^>m;LWww2r(2gwYjK*Z;ccU$aTb4_mEkiMUSPHT+;>C+5pg8O_CdnjA^n8I@v- zVnMt9JK&c{Fv&!qD)N&jwKi3((PyOV)EW=pzTVr1XV0G7*tE^}%Lk7i?>)G0oKKxj zBIfU_4{T3=U@Fo2y3@WO2COKbEBf-*O*o!Cd;Z4l+m9bTvEEMHp@PJ9ZQm z^{SNb-+zPwdek*GE?v5$E%Da%H*oLH!v~L_BBdE|_Z~bv?Z{+th&ep=?%CUmmzl6> zLe-jyN_I0~c7N<~!#3jt%-rUf4vTY|8a02)F@Gw9#i#_Fy)Oh#BXH?)yN&sa^( zIb8v}$h}d{LXf0c`V%K?RVT(4{;_m;K<@DqO;F*x0Z8mTWg}F)z{4+{I`qVGexT4T zh){rLqS`m2q(%_*cdVZ@n^X|xSS7s{MIf{xTd2<@%dZuw6Mp4*Vhw&ZzfS8<;5jWH z8Zd_Gw(h;^_tvJW7G)omR|za;0kYL<$H6BDLM^HGVJbZR187}#DyZZPMBP&vYJx@U zSr%uWN9k!li`v%G8(Uy8=vIiJmNbhtRIq!2C!iunK8uk=gz2bMF3bj#uVt8A&>n%GgR28Bm-li1E@4P)NQ>#i=;4_gBJ&Mk7)N|}=v79%^31>f`Xpa&7?%(BfAPbwym9r)_dmbE ze~v5%$kc|iE?{GL?gFqlVzQejUwP-Pv(6b2UdM@|irhwJE2jJ|m+X1!p<+Rcdnqsq zHC(RN0l~^F%PXEV#K$X*A#m#EstzY>CHpi^hB5V)Q71dD@^toc*1cdrQlHJ^N6#D> z_;>z?-}v^|KWN5HJO~>;q6<&vcOHjtkDz#TbmPH;fBJj>-moWygHtT>a!~@x6#Efw zBv$1n(C~*NqcfvVk~m04RcnDAccxk&j*mzeBr+$@2&Cd(*-R5IN4X^G*=Y5lJy{*D ztdm20JTggGSw$p_lwD!o(b2{8yP9y3D7WnJsLECCH2|fc^2{fG;|KY0)G@Ofz@J0` z9_5-ViAq_&ndfo=41AD+aSYN$U_A}#{<({xxYWZ*HjhIs#!vBuqe5prSO4H|{|zMtIDM50BAcs~#o!iqpZZd-!8WQX4<|1!U^t9O0Jb`Pw#uWSsLKxYY z^cdPOBAj3q!V2f|DtCE^$32QE6Y|htGm9aHcSEh|VC~S;by!DfK(K7dI$&!)^+a7T zXBf>XvVz75G-V?PO_z+#UnQ7-Z5lL*@uPLe78!{|#=`5q-PjYWrVaF=@EMHSm_)1M zwA7c+GkQ)%%PoKupfwgP-pD=8GRu0P;3Kx8M7BZ5c}*9M3-Ou;D#KARgK4oqNbz>U z>cp9N5|itpImsmu49q4{?1bZAT~j6vC1nYYv&C!g`oR?9XgGx}jK-8`v?it|2?Y&v z{Zux{xe?HCEtvEhch^DCt8#*CJ=j}0+%$Nj0?0;YB8R96JPJTD^uy64A*Oml=uRer z5dIJ>@wO_%d9Nhw5HS7ZmdA=AVJgT~B@>SV4Hj&Qu$}z%)X_iBOkEEl#?Grxy=-Zp z2|T9qb6leH*REYt@!fa$?tyJ{+WQ|4CJxzKh6KO*;nz^{GlveWti1Q$d(7lVKmL*8 zAS!xwA`mjIYw%o+He5Ynx(*C?&z{jxd-&u<`}=4cY@gA8eeeDwk`OL)ULtP-d1_g&hP9E`BfSAp0}hxIi(lNQ;SykAEqdC5h$4;oIiPy5R~-T#(i z+mgsYhoOj2vYXPX890_rzcgjKAqBH1PM^MLZANoM_U~IVJ@FCbn3}%-xMV|uQ%Sk` zhnZusA(Ln%FXVIU)Q28<+Qx)xvAyC1l`I|SMNkV~jwa(Wvu%cBs)UOuB=L`O(^zjL z#vCLqzF-2;AbW*PW~O5?_X$+MSXl>?uiBe-{m{Hki&W@+s+__PR%1~qIHRD`xh+8R zKcEIzEL_xBg8`a@tE~CS`dHF2%8+(*KpkmaJSh?|TQ18iF1GI#}-|%auculXr zc~;^N0^z~1KJ4&gTUSxuHtgul&hQL%?hKXRW|0& z=lH7UgHc$)5SsB}E{^d(zK3~aPL#N0DcK~IViA%A_`7#;cl#S({mKtM{d~!C%!Bwd zuwn&0Su(qhDdd0)AH4IHRe@wVj2+17Bp19T}3XA7-78!J2U zd;jUb{N$5Q)Myzj%}~F^2aWV5#})e18r;O@?6PIZ=@F%nE{%6O$^QDv{@1_u!MDEg zwK|HRP)RvZYCKkv9vn*_-{Yf-(PU6>`I?q1dD38*a;vIFBdW_tH&bAwlZSQdh8n}l zNZ4AnhlGeM{12L;%vyEbEG&NtCs&lw%qeLuI7Vx6AOrv+9Ysj>`+O0bJx&{&M&!aOfw< z2{iLs;G|kZ((-Q9@mOCb-GHdT`2?fl<<24ToBLPKp7}?A_itN<^?U#Ej~+a8usoNZ zw!|Mzl8Q;OpH9nxt zEL17t`00*X75=|{^>UKOzmP`4LVh!>74iBd;lZ~?6a$e(L}exTGs9KuO}T^o`0w&p z$(=EYE_T2oXXtW~fw!YpY7kc=n}zajCUeei*yuLE0ii}?kOgP%d*s{0iCG8jMYDS4 z4GtETg>4d&vcl#QmZ24%y&bfIHoT4prR=@(6KU3nqWU?P$xMY3Scb^5FBB4Q4FKKU z!g51!y1q$c=8<+-E~^7X$)Y@)uZ6x1%VA$>9?xHt&6|kxAJG(qE%n%?*Xlj^D;8!U z!1LmOCq>R#0Hy{exUvo9!J5qTDAf&}TzC+|pQ)OYR9+ExZ!KvRmh9V7W#8HnBR*_Y ztxt}1Y$()lYm0juY(({RJ+~pFPw!NU&00^sD$~?pr$SG~Z2!@T_4`R;;#Vq(^pq-xI7UuE_Yb9n^M!SbzTf z)P^yEypZB$biU__k2AFZ06+jqL_t(zyJwQio|P3=h@R5&vKqL+@o44g^QR%SDr#>L z#l?%4#g)$dnqK5X-R+@z@}z0;y5B$e-~-opZT)lqfo8+it5-hz{L5$0ovOQj?%a7J z7LOj@H=ABjykC>yURnlV#o&fHKTn_A`~R`+WuT;WBa+ZALHUK<-FEI1N>s|@ho63I z(L*Ecjegf66_vt3=mhfqgU8R;UvRd|&`Ubs5<|_77Tgp2kU}RwHqf++2%wY+yTlf5Aol;EYt5-&RD{mhVp{b$G&ewMKCLD*YG$Fezdi=w@r~C z0)(19UA=e0Bx(#RzT@zDT{U|$UNZ@zN`5-~@>VaKFVobKm&R)LHV-z|4()q!prcH= zYa8npLdb=rj6!ls#*yd6a><;m{N&VQ}*)g)~Jn zFr7z=@$p5+YbZz6@}%C#Of>shzWrpSIMwsaH7fiRmiSc;bIRil(1Fv-64ng&Od>uCI&`a7;UCR%zM?GZV!-?&eH?^D^N+`w=YyUv zH0EVIE4$lL`JTWAg`OUh*oMq?} zd*PTmyZ<_O&`|w?fL@7nCj6Z}M`sH%Vidi?Y$)Y^dr;DmLfi>5vcj+mA+3j;yF$@< zGk4>^Shl2M{^OtSL&q+tVVLO+eK)wOSY?Sp^$2mj$&y^#~BX2i0N=2 zFH`M9yL_rWgAIY`9feWV=mSIwm>K_+{E8JBReT%tFk+;*u+by=Lp_m7GS{K)s^ZHt zS$flBx5Lpggqs^KV7mrgtS&)l`a)wnAQU^jAAVuk=wV;uedHi52~y~JskT7$CICNWXGN+pwkAp1JqRvz|*Y6%@Kt zBA@6L@poQ_DLq3d5|FWYn6#Uap_zsdb$l%N)6=o?98u4k1vE2HIFJeI0a0731RDXZ zB5bmS2t-q$iHep$@FAfkMlw&srZx3UiNhcQ6FF74E21WQo}4;ipn{qfy69cglc`&7 z?Z8lD9h!^z#sF9STS#Bjw3qQMy}+2^`|zF@nQBmS063=U;Tnm)tc_^Zog zm^Qi>Kdad=B7Vf?q^v+k)65dpG;-1>)pCxRi2fazSjWE!HVFwdpH-*Eu6MeQSh7)) zCJ5=SLnD{FzP54c;$>ae;F(A#Y`ggAd^>DH!-vC?MeuEE>Qg)Dzj(QI z`ov}veP8hSOXsst*tmh2J=zQ}4<6en6nf_KK>6m48<#F#x_={d7+gB8@`wt&BfM$k_sl~xPnWdw$ zu>e4G)OJaxBRS0g;g6mOH>iN-*~^EIpO`g$YW)d4eq^=6p3bSnWx~%QV)*f>)KzSi z`!%ucq@zDfjvfD*ZcXERsyrgYWEQRZRXGxFW;1Q<(p6KEv+9u{7)4=CHZ?kMbOKeK zAyh;qce<$ff^rNr4%i}~abKg`62(?U7YG_Dls#VrAxL?{oa?Vlw`r<^i~d8PEP0^D z8E2^@U5*fDs>xH#>&n;21ZQ7mN~;4C(=)E_LUchVjOTqXMwWy*9xe-ZIh}bfCR?VM zIl;uw;t(hHm>CiUqe1giMcgO84P;~>rugtxfe}4aO`ZkC1vd=yL;B~SxVS&)XV2j53P5`l)U5ks}xAU#`|nI1Mj6^ zcABVIker65ytV5nOz|ktxaXJ=EpCX*E3_n#(UQ_8$Am5&WRUI=o$Tp&a*oo_>iciL z@!F+JKl<{vHTTIr=EN!m&Tm}kDt$;{BzWzL-siK#sYgy0<;GVT`d`+`U8ZsrzAgz~ zu{{<8GztEHs_v}Gt}Hw3JAj;!nV1ql&0SSJu)9fd7DYNjLCaQz{o)6Q{o-eTQTxq| zaD){h$s{F_-OXlq&6tNAfJ^}X{nx%(EuWit?>T!|(_VY+wa3$_-TV#1@s^IsIPE+k zxL^4qEjj|(&4y5Nq)1S!Q0hYq++|V2Xa|D3S4s?|J(GN-Q)-tPtb~L%E?CMc zHJK@Vs!P@hWJ*=of{=J3t74lHFJGCXp~UIHpqg(X1<5>->889xa77i+Ao?H89RPZ- z?>@MF_&@#izyA3TKl_`1_wWDR-~Rm<-#t1#KO-OYIjNRS0R=g7`Bbd~eyiVXbLvk& z`|)4@<-h#Xpa0%_HxK6w+uSJ)e1t7=B-SOCzLyp?@ml=V5AGiQ&;P@})eTx*GzBok zX&C)Qym6z3C>C1?aF(b!Q7Xk1Q7vl_Wr{#>sx`R)7VoNAa3`+k5oSX2 z2hx!0Rz3GL2}%K40c9>{xm+VuF(M|n;7zg>B5iqw;B{YuMaLv9gCv29lKBd0xvCf= zrAf4)BoPcL5(I>{%pgvd1RcBuvto-w-?2Y819c%>`2>iGm3$Q@ygBfZR;d)0z7#LT zKX9#QGw#}ewmrhLutPBwn%CT%&(*h$eNKbxaed%Mbm&s{?4RfF2@$g4m#U8NeFgDS zZQW=;Lh=!{4Dv{LAPG?_$tfbMt*7I-8_PFv=QUya13C)Ymgg@V<(UMqLM~k{TNBqSL^Nbe1Te9}Z_4tVe`aF2$%0Y5w;KlW~ zy+U~Z`ZKR1J2|n(^|Q}DwaMVyM-MF=+IeRkUxo7A==sGXFVyN79Gk`t4q)VCk&2Oc zX9}ioPKV-gX%64^hF*XCFkOV`YPPdBBA22R zoi$S!v682`g^n14pw@?Te{|>O`74K@I?d+-1-d9NuD-Z-+Y8!yF_~_u-i`y5l$YHD zHxBqpr*E!oo$YiWq;at0vzLyDeBo#>UE0(j8=xl)xN1xvoLXnvmcO-BaI`8D!ne2c zmSx>)F#>h*mFv);AN!$r48EqGK}^a0U>sMLMr%g>)PDT@j)3b)p?2BezNxrKD389gA3HE~ep+VI#N8 z1OGboZpyZh2Q?%aIF1xi!kV;w3oQz2Y29_RXxlzHK9z%8-x450d{%!o6~Oz=;G?2H zHGa$3x$T>dc`}D9P8*NZ)_$Cw(PABddJzdmufjh>H4M@a(-hX?9pRBgnmCXM>u9HL(@1O`O&>J@9rP`^e4ag z>OmdAie2(0kr8!eTByLZMb38@KmP2Ko7OKn;WX{9;NVG#KaQ8HnMbR5A>Ih?g*IFH}rYsQf_V1R0Tg>fh~Mkhri4V>^+ ziSm1@jx{7<0xE1pK-tmc#?V@=@2&y7*Cu zsXRy9k_Ywc^tv%aBdN(18;LFT8p1t|%5921xeqga;j$lBoTX25FEf;55++t$Qk&os zNEsSZorpK*hud5K;)kDp_Tl^g&7b|r-~N|h{HI_3>dWsQK6(D!`HuSh8QCd9Y^osq z{`cIY_Xsvk-}}RzUT=`l0jI3OMdB;m;LK zV})n-xq1-Q^R<+aSSuqzlV@)DdM4*rh|mpinL^<56S^Sx=Sv9;#*9%q5iC*bu|$NI zfL6YhVyIVm112aLV2`P$-^z26AW6CL-;10Gz|y> zfWtr1z>m5DyVm_YWi}-)by6GI41;6E!@H7gzQ;zCo`}@dYR%YoKn4oUNC?wp(nG#T z^_?y#&1Qk>q&3vp3Z2gXnHm?8u&YY?kA)&?2!bd~37OK8|5%TymqTvsE|E+IkF}9P zR>UUClk$xJlo!BAoIiN8uvJ9L%EnO0v@jw${k-|L(UF6Cy>rx!8MU)V<*h})8|$zcfNW$jmP9*R{L_HhmaQf_BzJG!an?@fVg%;(_09<0 zZ5+d3rV@Is4{*X&rscXvCO3Kku;oAwV!qO`={7>ZtTu3%wsszU^x=o5HByzSUN#O; zqg1f8Z%)n}5bv1y)nq+*hj@=mn?cMbRISJ>9MzCa3DZW&>v^Q%Oo7E(YIy!00l3OJ{qc9R zYXfy*92ixOD#ZNEO(t#yft+;n4e7rIVoV6NUX+NfzBUIv0HOv3(Gw+@R6LouWg-O! zH|oP6E!oWa*h0r-hPapouLRQuk(kq;Vod>^S-qxunuxOfx8;kO_4-Fct$V%G^J{I| zke?W_ph!CQ+6f<~4q9&mT_q4D^PK_dpZY4Kxh3?7>N`s?O&d@nr%kR^&PByhvf7yY z#hY)T2@Xl>2uP^Y)y@mIXw2Y>({T5$Ke6KP=`)+sbr0=@w=T|Uxp(j0sgLPhb{3De zOJ48SKB3MMNLm$&0>7~IJDqK-2yYjaH&ccT zdS#R8R!}Wlu*SRcBd8RlJslm((GiG2kqIE~XNz$u$JZ&1b@T75F#b05Y8?UmlF$&Ve zh$-&P+fP4u@BW=zFTQ=`{Wc@50gM$7G~O|Aw{CuL?}$~R^m2*z?QfZ?rui)u=%gAK zfk?sI(4l>egSG(5F+@~wReqVHEj0PRsm3V24Mkyo_h!%84kpTe@AIF1_Q~^q{TJUnc>LAF2M-=UH61+f$h+b_IfvH| zZr?a`*3F0a?s-Xqu8|d>QKjCzGj6w`d==`TNnZw`uOU9OFdA0C_-G}NgTPAYYVWXuZ)3w$@-`^ywCxddm&(Z-ybOs2v)}Ut_M+ z7+jSXcnop@8cgsvJsRwzX3!8Q1lv#<#xGc#(cS@0nw88p7;Ib4*7+*EJwZc6UN*4t z>}y6^S(LkMkXJhfzOhpqoxKYQkoGFn`5D7DdJ<=$#V2mvQ zz&y)FOF13V1$JFh!#ZE_-0eUm)ZmFo-1Yj$$+|2NL0HzS@d_}tUAi~Ty&KZ+b+Xs< z%&s_Q+bCRb?KEab{=jK7lV7`RdsP+=n}yl6_}g3Iuk!m1N+O=(2q7kq9DS?pX%gR; zgZ|aG-+cSsgU>(vkrP56K7QmB$&Wt%_|cPRUwrX}{=?y|TZTC4pe@H=efQfRI2*wG zdJhhb-GBG+k;VF&My)%ou`%%d_ue<;z@+hL*ev6jCLSottZ{z(_{k%yf``~2mJU0bs?i;&*+@(=I8V9gDFUvb#Z z4WroHEc9ca=e_TAMWV^M;&SDOj(~||)V0!`I;S@1G};qY$J${AS8p92oL#%}>ePXy zCr&K1oQZnyf|PKAt;MUrE&PhY-zeL|A6QI*19v4hf|){W|53RM85 zGrmSO@zhQAbi6@w=!KbV06oupr>a)|xO}6n<)jRUD7dI?PLAfSiUZQp#Dq4=ilW)o zGSBePGo2|~l{>B5d*#A0pH^__TFv-H)glAgu*@hv{FQ;=j19FNTjU81)j(9$704-^ zyu-)9sLar?jHQq&pYOR!P|#GSxhpbT6t_@TUJOCZk@?o9fgh$8gelF4D{&990@1C9 zm2e?Y-jSX>HsOP|$4Oiq>*~QO<>M1e=w(64I_6Z=%31 zbw!d>GnP#eS#)PVxawk}s-%z(p6WG<`eJ#!PcZ@X$*R%y;?q_5OCRxzV9eB1k9Q&z z&=?dr=>qOCc~--+=q)(XuQdTjNgT4}oJa>#;6*|Tq_3~AY$y8oPafom&!I~Su!nL zW_s>gGLe?W#3Fxqo%m^|a68OvT-duQ+R9H_QZSE<*`NL7#~kc1WL`C2>efD zAHr#=N-J<;Vy8rddDI=^%n%xBTUn&&ztOn{DodaG!q;ET*_f8Ar`EUC{Opb)m|~u^ zx)P4~{N%m&e*N`#8mI(3k{l%^_zN?aAKtSD^e|-T>I!(YHGy`;V7a4G-o3Gdt=vGP z#F_EChc`cc|K6t`eK@_jfmf-H45b^UqTV&R{4Dv#LUshDBf1!pzIsBkEZkJLsaA`Y z35eTKh>-xpGi4==MXW(u)aQ0n;afY`ZXVtK_~__Qe)c27W|`&Gpms|=s3tJ?t7Hrz zq#{05lUNTC5(qX$;*9886qDR@gu&@5i%&Y~KT3rmC&{gcNjilUt#Sr$q82AZ^6*q} zs25Kw9B_H1TzCqcy8*rwJ)F`o}n+nI}II_Bfa9 zK`if>z7VgdU-J0QVA4Jw!Q;0l8HglJH9#6|Udm|x}AY4)s%&GNk$Evjm=gy`aO{K=X`f=K92A1`Tt)3C1J>YEUFIc*+ z6(=&=nk?aI)Ui^q*0F=(Ek&`$SjDK&a#t!tVl=_c1SN~nFgH+N8h&{yk}KK<8I5#j?X$_SJck7w;mYwh8|br zm4Q`LWUDn<@z|8M^)J8r>gMe`M@RR+`S#m8ckch-lTRFa z?+s$_z5l_FfBX|G;2%DEDCObJn@^v=(6nRxpMUe4AOGa1GrQ6&Zy!H>&INZVyVKVX z{ocKIbnyJ()zeo50IQA|8EC8=JM{X&8>fZ-?59Vc|L8}LA3chyle6o_^=}-0{oRA7 zFTb>*;qK9qF==B)OYDTD7|DxsWPSgOlM~v-#b7=1vvN`hRO?$$v~9Ew$*wKdbx!t) zKleI|sX`|o@?AMBU3SRk{=KJRwaOe+5y^6&-)V0QajWv>Q&LkY!G|UXD5Wg#7`Ktq1T|pPA|)nRb($pgY|ug38iC7POg>!E zWQrV!+M^+4nxHYpY9d?KPUnq1t4F|dk$907@G(G2ZIvB_m-P6Jw!SOXD$`gI&s9GB ztd9LaQ_^XWgi+_u)9@U;4Icm$D(_^Lngxzf0OfQO3O{|7Z_*Ch2q>RlOMo2Ao*nP> z%U5*QfjGq4Ah-f7WeJ&kLm~d@6RB66@j;Uu|3VHOpTi9WF$Wfyk)Fd}9UL`V&LBB1>^JfO*%kLgMKR(+(yk$`5 zouhdlJfOk+6vGA<1#$dkeD=&jxMSwjd!ncyEfU!BI<+A^s6)!2xZhNdrem)48Ubt7 z@KRwYH`<~#8`ou|gxWL_@t7v>I;fQZ`Q?30>8B){Yc<(o_-DhBQaw$t-_l zO+)ZI>Z)#=>~nKT^(FcOnWl1fm#PMMk|{AVfD&Pq+#sM0Be|xZFG#Lkc!s{}UZgoJ zM8!Ga_#@=yC24Wn$c6w%wr)*+xj}QlKCIIGe|&0Zk|y%!Zcb@F;pV+C391z7z<5**@z;;w;n3(sNYOkO1e?vuGHt7IPBE_%FrCrr{C|MLlkR-`tkuU@n zG!aw712ElBzp_Hw^GY~EpPm0d|0MgqPylb3-^ByXgP26VXH+b% znZhZosRq?b)?OphZl79jDv@bKDlACx8C?ryp%aznqSriKt5~NX^CBW;(KKSrGE8eO zEtCKmP}s;ovc>1(BH*G2(@Htc6r0{Pmb>HuuStr=t_Db19ywX}K>9GPy$Z;vrl;z^WZF)aUjR?UngR(__ZyzX23l3|cbnhE= z1*fJgD&kYReiOu4;mMhgKTsD^4zmVWE-m}q^Zx5>I$~aZe6ziG$m>tcGia@Se!IrN z&3^3Ei}9{p5RKmhYR&!e@k^8O^L_;mcjdq()GHt~n+Hz0;xmwL`M%w#kDopq;ioTOK6v=>{SQ96dF$vO{_&sw;TOOB!$17vAN}a}o*y57^WAsveefZM zXyeh{yTASNEB=6XTwbyuP3w8+AG7Q2lHWdfsB3oj?!6ZIoSqOuZHn&7tF!msd+(zU zKltqzUp#vJ$m_s9c<=q^&z|LZS_{ghHi~2H|M-MB9KY}N0Bc=85fU2+c@w%qq}5OUcN=a(o)4*y}$kd(m^SRX`_i&-7*P-M{y% zUw`qB|Mc63PhNO=F!eTb^78bRvex6W4yfsp0c(|3`*$6w*@0Q!d9ve$Vs<(>{=ZkS zHy=Rl7!7?NSFWE@xYWC8g|Z3khK|6+UiHPc$=ZnB?ZshuuXx0n#o>dT$Udw0bB4gG_I`2Siu|>ASbe~!GVuy&_>82vtUuP zNe3K)Cjb2U@)xy17e2#;gU46d{FI_=60F;ZSlPBuUc7=irx8oMc^z01tNBKeJR1b^ zH3k@iZ{|%BL#9M6isIjbm%$lAGVeHt1b~|9S^pzv-suq4H$NewqC;{&nfyzY6=d&papN5P1&%|jL@@$fYIQi^e6WK97*NO@t&HrC>`%RMut)O)KraJ?q|kF8`ujZIg_j+br=)Ks*Xc;|GQE9a+R z8x(8_bDa!S&&lQ24?enoPq{X-5;*C@Uy8Rp;4A;_zx>mS*JmehnBAJZ42p9VuJ4&< z_6C5T{NUpQuUv;Zm~p#+Q`p-Fdw>1sf84CUsnVL)!HEb9%%a_SgK!!)92v#*!k!$u zD$|iH$yE{_t7$~q`HS@%CJ>i?N2HT2hf1oj0Te6(SOSW={DXccu#%HT|GLnTdAJq@ zqYZ@Qw^XiC1Qw`HXLVF*#A01(e-cam-43G=OWLPO)omFAzlin_Y~gfc zrQLvE%36-{t<-LZl7TS3=7J?b)DCO{+Mpb)5?o3P+F={E@3RGZs9HZn87)&!LMg7? z3qop}lUnWTpX?_A7*nbex?*BK-Su;4O> z#KX$Gu$9aIvgi?Cq_kP&LE5mi0_h_{1Fp5p`PT*`eRzSJzSDWmZ~E<>T6s$xz$%I? zjoUfx(TQt0%?pXrHXWbA9Y;pdL!A8k8#!w)1Ps<15dBsJc8_1%n4c2M8(Gv}@am_=kHT1MYQXZ`@s4&!-+1e6nh_>?tOAmjV^q<2ZSHJEun3(6nm$0x zflvt${M!qS)VsF_2_N0TUURZZN-E#6a(Qi}{h1+IlYQ21M9BVj}a&=E_^5g0W z)Y5lJ=O+0}S%M|L60BnANTbwkvJ@$egdUW>?%uP~ftDJZ{_2}=Kl|a2 z_Vy1>zkArLDXuzt-@>AEgD(cWA3S<=|DJtYcO~X=)>7!P)Pa5T@dqDM9EamPc<``S z0vi!Mf4$e6UhY1A{Pg7XF##JZvJhUc1@D{*Rfg}OQ`i81ma9W!t*Dvzk?uIhnEsu; zJT?khV};KuQpMs+AwkO7(q3|}NZVMIr8RXnTOj(zp52>{Rk9GsF3cen{rUjX-n`)@ zD0^Oq%BQ1g`;K2aywUM4rlKw`EQIFfSn7T4nnh?{iPL^F3&1sDM|bXi{PEjg{L?>w z`R&8!r?2fBZXRlD`_|3fd%K-|#Pc;OuaU}qOY0-7vPaTeTQO~MHrr!Gk>ld3xG8K} zD~*I?5No5SdOe17(-3_;l_wZJ^veF`)Sk^@MQ8*p){!X>u+*t+5f|L<3cebS1QR33 z1IkintIj&4i!jw}1xTAEsS{$ru?ChpB0p$jN>lcteyd(giL}?c zd4MFjOT4xKDJ9GVu1eE)m2lmD?5!yg3}>LbOUwr z1drI&-0WU(qBhxf8yjN*`iYo9H1PsZHnc2&?ze48%4F&Q=t*(?TMi=&0svaX%foc2 z!^;zMKkVWsv94BU(GhwkCGo{r1Yv;1im62`upTSdOY#Q%NIb(f=p#4!yWEVl>Sc77 zw;*boR_QBbrnY@53b#t6@U1?yK+thLVBhctyYrT~w-M_#0|mt!CpSc47OG{SREMRi z;(2C+=XF1J`A|_*({I``1osX3vTCD2N+?c{K^hVh+#-7fpJv9@5G=F136HB_VWxu1 z3`3{; z_{fMFWHy5`L=>8$dwyr;PAmq-VTnG_B5EvsIIGEhhbVRex`>r{Dna0a9F}h2UucQY zo$Q3^C=5Cv!z@lo7NBZCno&}+5kt+Sl*lh=loQ1CEr}Kac)bM`AN70@LnxJ9hG2Fh z<&t;q`^>o;Uw5-@Wc|5}N}^e9PgLU=tk{BA;(sQ=t%CVZ{^> zZQTs71@t7rP~1&Sg*YFrgwb1zm9=oCATkT_I4Rax>SaVy@d2RUaWd1LHpmnxlbDc( za60^px>3(Es!G1)O$!z&Lkm%kHZ7u7nyHP0YSjw}2U=io9?38d!AQLn=hT*nFU5cx z3tY0~B*(%bAVOT5`ZU29Y|&JpOP#nc87`^N4N$46tB@(+g znUii7WtY`ax}kXkCEkW#Io$+l_%3?I)h(7Hi#o7=mnvs5u=s!&iTzw&3r>Zsq8SCu zBcTA6XK;K}K*`OLb5Np?+2>-N*CeDWh?j9L$X#*w{qK-S)$8P^Nv;vaN;f;Y?_O6>s1kNePrBTdcvNSjvudCvBm`(}d)LC%n z%XbW)bLP$Nt)sZP~ivJB&&>SoN^T#!EP^ zb&CJaseycM{eSu||M-*N`>7Lto>}MP^&ofd{N3OGm%sb_|MZ7{^v7xVwe4>oJh<9< zmD`UWJ?7m!d2zh`?RPhB-nQmfGh{C;#XubI*#_*Bv)3jEtf#*<<*xzdohvYY|Lt3M zKK;RGPjpvW)%GS2`{KgEp!Nz|{m(f$`Q{t5#cy7nQzB9|C~n~dcUeauDURd~lQ;G> z4Xa=Kn6v)Rujm9&2j+tZlIz?!YpPcZJ`W(0iHQ-5pX?s7+~M@)u@j2!+`gsjM8Zl} zK{gTa;<+Z_$l()PJFlNUHoJm0#k_N6hvSUGyLWDTF@>TAO`FG`ethkzQ-j`Kd8&tG zL-TttPyYFfuOB^muD9q^pxTmr&9@Fr-#ScO6|3$EF{I5^_I&k=b8jF!_J)LAXFz$q zYHlnn^h@J=;N?mR(uT{`O%^qpma30Er!jR;iK+G4h7UH&2D7F0NGHQ6ZsrPLuXy*a zH0N|jg&Op3s|}l}con}^IA?%@j~rv78Z}P5DarO*q^Oz4^(z^Em8?nu2A((n{q@R8I($^aM`|VY2E#W!iPH*|GsP z)Qq$Ov#KZ3xLWiDm%p5#M_ zV)h}hK=d25fmtKaanN>;v?5F+L&+;1$Nw7yz3x31@2+?~2ubek-MDe*;P%a%s;Bd_ zyrA1=0p*R3*k&!%Jg&cs^EWF*vmf38m`cD*kKbtUA{6~ppC09AG-|&1rf!OG9F2_| zThdB8sl@Nx$Bjo%HHt+h!5E(Vc>qQ6j$v2M25}V7`NwyUe){>Rzx>UY^J1(-Q;w8x zn5eGx+OIpeZ+?9LUNpWv*8rAmG%XzCwl2-71d^z@a6{XA$e|;YmD~3;i!DKwU&@Io zOL80JQ@B%hlz3GZ*xbk$#B>Us@~Ew+$4{RaGCAGP5f3W--mG|MfA{e-BObwL_pf>% z4l~-2m>QM|Z{Lz;)=5kZjGab~Q#Lfv+L=0Sdxns_^|A8nP!GKz(F|^8Q|rtg?AS`( z5%Lz^4wrQO-Hq$J|J`5x#lQKhzpMx0${@tG1ZXFQbXzghZ5hUg?IQiX^*R{64oTt^ zdxc6i&9s~wOE8R66_KR1qz!1P{j{AOQjy7=fCha6oxr6V>B3MqC`yLP95wR?QMX9H zq_DWeJ!R&8SXxOyL86F7D+V{23STMXW(lEWg*Fv6z|GHa6pz9N&9I)72T7x} zDkdpZ3&(2p8;Do~#Gjk){lYxA0}v*EB;SyUJ)JE~SAkl_py6hL^IA2In#y&<&bBdf zX@AKwmA+#LY(E=#QQCw5Mf8-e1S9TA8Evx3&%i}UUN=`71=NDuftmU0)pesUb&}~# z^u`={pg2JWU@}NieH=pKmFcP!)5ws?sjtp=A$3l!NY$QQ8J5|nj7tZ2a?`L_UnEBZ zOIzE{r{7O!Ybjtg`P_y%f=_6qa<2a8w}aOXw0TP|3L5_Hm`eG`(Oa`-YF<;6gTlG5 zEaPA(UAH{|YYV~*TqSMU3t!-@w38rON-hK7u*RL1}3i2eAZkIm9r?2c+@j1vA?@4z*Denoqs z%`k9OccZNr7Y-A7#;g8j%R9mjoK0k$n~1J$>8R+gJbwJ>=FQtb`Oy!xQosE5Z=Ug_ z$-q0#_`o*~ydoga#L$){2^D(9(JYT6f7Fn-Z_PaLzBtgj6*5xE@hC<^YujZ>sw6CBIeYr#xfMIN@7#viA}DLDj9YVMsExju z?#;C;w@srcN2Bj{Tsx=qTdz5I`rOHLN8WmHboBE0>|0YR&z^Mve`3FC!2Q+Eu|9)V z^`O&J_Vh@w1144N6vpXA)sK1e&RX*GUe1=O$@||oD{d{)zOhy+m+9h=&DasG%1~G; z=aVl#7IzF3PXVYjNa*M!jdi1sL#PN7n~F0Hd7@5ABIPQsBePy#^VS06y6r2-7N9ck zAFlRxw<;*8%PmQLS=!NOVfOojDMMz&Hx^ieZc!3!;@?ar3|tr zA3@c{bxphoF`1YNF030N6}Nj{0XdChsvbwng4Jyhr*3De<9`5)PqNWo247P<2XJCq z;H9Z`O$nE6@Y*&Ghe|nl;91pwK>EsTL}~dOZ_UsO2t%9=$_69Egnw{H69zImQ!<_y zzo=SxXY61wWp&Cl#Q37JHV47qyd}%>98=$pe?lsd1ju9ym2UzP5soP zJ~a*U`j40Pz`uTXGiE6-U4828&*#sni@K=4t|FWf{{wn-el za)jF7jAO(@b!I?Cj|60jw5j4k5Ci^l4@*78sLoBTqq#yoK@{e17VJ2Y#_n}0oy^jSOLQdz(rHg zdo_$Jl0$FwNzcM)=$?jCN{~Ct#4gXuy&|L!Z>}|P+q01-Ll{<2x6(R;fTxZK10;g1 zHEm%oT06HV1*`nfFk-HKa^p!;SkcrC6@ZXRDKTc<^W_x-n9<9K&K2VcBH)t=h>dlU z7r>cebsO9YFHR}}C=+35goM2tJ(^di96B9=D|8wYYaOXv9b z1>Uv12HN!&d4Yt_#A{eBSxG`yoeqTe+(h1ML|(gUgYVnplUc{nZrWe}=C|*?|GqPP zyvX;%4?l7$|9|+8fBWlS|K{l4J(Hpa@E?Es?e!ZsY#4t0^w{AbFOJWC{hKfL4-d5} z*2yRU3-=ujWO$qt$?L_710MFSUFZ1Tx_MjFH{X2o@bQ!HzI&t%{7#S4d3rrbl!| zujV{IOgU>kt8Kw28|6I>pSDjcIHMm3)|NiEzWLI z4J^yC-_iDK!=&wWXw#i{1bsN~9cX@nXKqV6{j;a}?DbQdm5=V+cAUD-z{}HP)N*a@ z?Lb!rQUBhHpJd$EzEc6U!?2vm^LP7J!|`6(KT45G_Gp*!-Y)4PSCgHAn3X-$Z#CoU zh$BUD;Uy1xe@#~4eU9vuJq_LynMxpwLf z{Tw|Vo2&$C{?)E3`lxNutAfN0{fc~=pZy+UJeMXqG7rrFsLAKfzckyiwl{@KvW-6v z+|&mdx}ChMey9|46EJ2d%X|qs=ph=z#tYW4GYlwdGIl_y@nsHXA|9b+NhWNl^`)ju z5007_#Nda0L;EPd8zSpXLEod1`5hLH@Ne?6P*n;ne^5|Lw!78RZ)hG zg~Wj#a87bBI}JxZ2U4tM=|8hLF57vXU$Y|7hj(W@j@(quhB(y^;0^`HDNGbu_9 zoY#Rd9B=;c=Rf`Q!;gOT&36{v)&i0eIR&q6<)b14JtKi>Pe3JOSMlwMW>^f0l*_ySW}f8pw8X;Yxq z)eVqh#-@*O@TCYwufZJh`YqWa-%Z-p{~~o3el3=UGj)Fu$`%4J=-*(9%*|CbS=7hE z6nOA28C>_V9ofU*=&js|S2|2ZQE_bo3+Ko{`VoKNOFiM;%$?Mg%0P^b$u4%JAjL+a z!B_V~QQUx%#9ag~c@7~1Xo6yG;K7bz5|lm#h`@4Ar0=lwHQnzaDlC(Lu(BwzRoF!O zEqM4{BrK95y-Z?#<7%qOJeIFGi2CXmInmJ9WkiaXo|A_^Kk2KRDf(6^Xi77@ZS1i= zrKQCd=0eN$X+~8Ro)WL)?JgrGJW=4TwwX@CoAMiB0Z$P`BU z+cn`|-W&~$vY*rvdMpgl16#(xYtaV_5way#RHrYgxc z-v%8mY(VBP#1|SLI_fL#3cFEu2~&tP3CG2ZX=xPMy?+2=H}SRLwIf(v38U~SY!H0n zG_{n8aHgCS_>|bH4w2h*yB|sAD0g{W3D;=o>sK7Pm-;!DQL^@$SHQ*Q;_DmqIs@k* zX&m-6pD$lNZ;YRp{qCw2IQaGIDGNY&hgcdf%~EoMa-A^(k?eW}kQv6d0bXQ0V}Jg;br_W?%%(6@8tNoAx?eHSzAQAo94DJ*^rxV&dtMp3Z-Xjqd@cM zIGrj~-AO6lI{Dq|ALX+9x_7jj(lI5{_J--LP_J~J_+|uv%bu?s9U6HUwsnu}DQJUB zM}^@ci#IwcH{r_^6cu~{O zvIs&ad=ot=fh=KH*fQfaIUOKFA@Io{$yGB0$V%IHlnd*30|$_aSk)1X!LOVq;Q~J2 ztNL!fEx0hGExf$G{MSe%!x7P7$Zdx&ix-7%rL2oDU`$2XBPU4y0<)41!b{oJLlT#{ zBro<1@N_N)8Jua+l6Cp32MCC3H`R&U=%}YLP zM%jQwliR~+G*@$nWm1>|2@*mDjs+bCB#43eVevDK?kD+3R48~$DlM4BJHEQxBk^1{fviQH8OSRCfh$!-IQ_40sfTB_yq1v zTE4sTZ~x-Y-aEQ$#_H(Y;|M zw448wJBIbFD&BM{g%Lku-<*P!h`Pk!P@F7vj^-gzQHmuOgj-K`bBBMSS3WhTVO_hWy`xso`XdxNi{|wm`l6^e|l~r61%8DDLRN_ z1&}uV2bE~CLrbjcyCl9$*=5Mt!~rm(6dy4G3M(Q)pG2gIsye(2*-|qI(<3o*Cdx7| z*polG4xE7+*yt;{P-bDU$kDCvC|Mt{0?=QE0B~@^FhMI|0V&_m6rq!TQVpFRtHO~K zL+<9QR1q2Mk_0zfB~>&mI>Ll;67EKt2v~^vQR_;Wu}py9=Nx21b2%EGnauND;d=d# z7Ovf4wYB8aNC_GVx~YFgkbtnlGj>?o#V2nZ1z^SwPQ(zS>099Z>5URy;RzDB+&%F@g!NbQ!=qv;}x_yh{;(}J| zqO}U`^wc|ltw%Ekx@G!cds}zH%N?Ooa@POo#Si6I_)s_Y_wI0&c025d22dlQ+?~|t zuo}lO=rIU4_ODws;SEe)Se<@mN~;};7&ycuo7#*QF&l>7DG1C#NrSKHLKrv%UX7Tl z*dqRhTbn)WMG%xwFG$XY7vJn!PzLPS8_CO24Nb{eF(5LNh+NHps7!sMeQZD`tA%t` zK-d*$6`(frD0zFXSLtf^}|Qck4?|j&UlOZ z#l^k5cbt|>Vb5OmGGJR|R3siQC@N>gk!#~6_U+rGvgc=K_wOF{rWR?C>4kd6`>(Zp zNb8^za5l6sWNX$p))N$|ATYU%ZxK@wCo$3|QFS~S{BEm)iT2`Ig5^WzRxmvAB=1O-gaMd1cZ*($Efm`hyI4P_9Ap~hyW0-&Lmbb<%P zB+;ZSP6Z{ltXw4qqxj@7(deMNlz(_Ai-AXP!K;Dyg2gvHSy@7?Fh}>`r^Z@!h7k+K zW>#aLuVz6VQZ&~#@)&?H|Cie=}?d)o7m9EmNMd@$iif4x0=P`G=sXsBd z$c_t#SZ{UOn|)s$4woR53-zdhuNq6#W7^|3z<#m4N;N^Hy@_I?CORfZX31r;Ec+a& zvTKfg8j)~rJ9LdP&eJ~BsOC7X3jvYMHC0K-*cZ2w}Wwy&(2QKbnW_q{dxL- z27_%*6)epAr}*+t>u*5q-8*vv&t5#!7y-K)uchoj&zaq)yh0Pr2DbTsr_aqc=)7&e zd;Q9^zXNkCLuU@lw|#=Q%Byyear<$v*(>;V-)fd^ zdi?5&1vxQc6PW{lDP1aCDvftS$cLeH zG=pc-h&yka=*+br-M;ha@k1l}ukIX?7>9;7^H0!XQH`@bzkBddC&l_3n|(Rs)Lu`K%n2!FYf?)Q zOd~5j4R4%ZfIj?4q+J;)-GxPZV-u96&dsdEm7;O;s;F93P2k}M83nM9-d-`F;Zzsa zDuVR7I&LrYGLvmzQyW%m-@R&9rQBFzddS4-im^QYFLvtvyU_HGGqeC;8?1^mt~QlC zLsDa^k(_!t!C{-QvZ!Z0=L-{LBhm;bXt~_Emjp3x=5jkC8fbGL))f`|p5|*IfEcP( zf2A3g4LyW%5|<(Pv5>*>Vnn9yXjX_=sostxg?H*&rA z$JQpEjD?qpoL^Y1Lf7x#y<5-QD-U!dtfe9Gt5=LG9Nstp*nXhrPoGM_xHM?h2$AaB z+~nw7tq___T(Ro)ABin4YvkNiKAy$~PwNrQ2`2u15)oQAVM{fop9D)F?nmaq0#i)A zP_4J4fA{zP`@i~EfAwd7a_&J8G*c3+1QS5_ccFjd<&j$D(X-$V(jhTY0=Po^iJp0i zX4|{HxhAZGSI|`Sl|P;ZZ=!}NHCq0;s=Rp?L94u~Kxu37!Iw8=LvWhY^F$TC?gW;- zxsl&#drGnxkQqAhD+6GjkmE7opk(68T3Ap4isrV~574x#MA0Z$c&Mf%C3HTwxrU;6 z52NiRnxq?Q$A~0`L0ZgYO`9d0htU9!Q4p7v(qMvP(o35b%bd5;XEL|1E@TlH*2QvQ zrpgElXaWxRx`w0t!H0R`^CZvsEppWMg3#Jd$Kd3Z9gcyDV9qg=SNLNmLS=x6;V5XeTX1w2(85 zW_XqvD%GMZL_LTcsl<4kmA>Erdk~snOB3P=ySyDqC0SRJ?0JBpJ`h7w-{J?*02ws{ za-fkj;OS?Dt7I9rDbm=w%rhy&KEzPGiXiI>XBbyGMZ%QPN(Vrt9_irt|Np@QBDxpv zl1UYR!+Tj^Ah5I`yhWsbi?T8$<>Wdxw8Gk;61HV{Ok;m~+8Ko{;cExqxaWprHf zH44Xm=ZsePVF)kSwa_`XjZm~3FYlivy*LNQ$etm%v8PZH13CklpFj|OB7L0x317~C z@uh?zOhi)+2Ek04FkcpSjLt+um5(%w?<>&dNI**<69``9a<(IGj()5!137joDrpn) zP~eb33sl%yPZAFujm&BbtdM@Q(*Sv+@_c(=L{b|9rezMq6Yi=TL2a611X@;!S<|c) z?hIQ3EDMM*^FCX}&`S7F@Ri#8szg=HhL#(HTm^@}8p1kxeUwp4L;)Q`7dMDTF!9At zFdFtUCRcIh&qd&JG&UA6j(SZF!1E*lMg@qz?L|>%dFZWc=TIst^i5R${7ak z__$Y5C8^v&n}$yqJ+JJ=Xlrb^6g#`tA(6n0N$;Ja^Yrw_-Fr^kym|9x%kUk^dA7Cd zgw6fkl2O;cCfyL@!a*Rz|Nz=^>T z@o{iW16<$9SDB$kixFNQ9Ngs1@X@VswiM~?_{`bXCIpO5%7IP;|77*-S=|y#jFcYV z#tfYGKm4LvgO(U==`0zU|HdobopE&U-p$*$o;BWJ%-ZR{o2yarb!RMKQb{`S z!WQ#4XC0toNsra=7Ds@}3u<(q)wP$yI}4IBUxGw)ORku)aqOz`eJ6eB*M*(hmk^B94tASCnZnx$FSSgs1AOaWPyeaNG{={1{>#h4lcQk6hZ^i7?1veoX+g`>q? z-Uz@C%}dX)0JvbvQ6?IY=FtTB2J#96ZdJpe4rq+8Us>v6l>l4)_{<+r>p|oY?t6^f z&`h+rbV0UY20Hv8gI!cNq0ZWDj}-_`h`aC&CqCA$Ik*5G7KR!r>XBAs1u7K9UUo!| z&)0)XOg=rJ#8xrH!j$`Pyre!Hlzvbth$>-(lvI}Lhv)u!>>+A=BPmyKf>XB{Q!3gr zIA;f__D^1%Z0%m}RN`w}kDon%{M7FJ6H0M-a8oYt2z=GP6XW=`QkdNK2D|Fhz9-}V zI(N>|XM9=iN^KPa3)(Whp|MQ>bpYrPU5Y41w~|QG8_o*dI4qSuJhILY*(lxNul6jy z@aKE6TtYyk7)X6VQ3*SfNafr+xSk9~x@qQAnXA=DwY;`)Ao}hG#)LQpj=;b!bnaY{ zH1SMBoOv=qy0iy1S*yt{&qzTrMetBKB)JccgaP*_ltfa??^zQARaxrkXAAlN=3+e=G zap1eff0-v&bzNncVy^gF8z4mj9TMheY}{}h75OQWWC(`!4Xgru20q;#Z41TfjV&2$ zR3I(}&?H|Pb*YnJL}0MKzao%CE5gB1MERq4epYD<&F?8L5nU!pAaezIJ*+AyP|){+ z#IyEoa4bCT)0YuQsgk)wXc>rr5JbW+kNBMEK~#O1%zbSrriH^*$uo=VLAGpZFgMkw zEJ;5c1EyzaR68(@+zlmFj?zpbSp$%=M4P+hyog296cvr4%8hUv6tpIoidk1_T{(Q}^{l8srM0kFA{Hx->gy29Z<;nj?| z_y-zv3oGa)?5k`fL587c-aq1VZ09V0PJJ9}1-R!I)&=uD()EC~zmt>l+S@#G=4;F# zGY+8>g!Fuu>>D>IrlO}FlwJh;8?;`7D#Q&2-<@weS4BU=3$id>mw4+9nkGSWAC<5- zm^H1}RYA4|W6B)$Is^5cEBr9=%H}q%DaoiywYic{WWt&kpRx#?#4F8G9M4jum92|a z084|E9lz4hD%CQ~&$rVxB1y?xJj#o7^rW>c4!>`;`j)MsUJdMwdY&L};o(E))i@N$ zA<<_P@7Apy;?W`t!9fk}+XlbEB}${$AMNbg?aI|QsE>do&p5XgL5_3lf)M@H*I)nO zho8TH|NhqxpPVR%{|U6o7hl$e)06hO91VuGfS|y zbDbQN*|R6loDcfJd+#BGJP8)RUX^Qf{?0AiXV2{V^RLBjaww(G$UNxJ-SWg6t9z2U%n(3D;{3ADyda(mHId{-ebJ|e5zIl1}djD|$ z5T-Z4>peM%wRO<`s(=cduWwhsGMGvM@`<%a=Q>TcsOO45dw%kCd;8GFKhqjE{pg6y1?BXw8kFh!u(YJHV{@)45k z6ITT&R07-cXD9GZBpFjC=AE63;fKxGSoZ26*ptC`HB@#qsDk5HET{{CzyPNL^Hu$d zPARLj$_ocs^UUigO1cund6=)MkP~uACm<0K;JH)kRn0<%xKz#_fIJ|{$Yx?fBpphJ z6f}q#iO}M@Wu(UsCeVamt`Sg_sEQ*Yp_!FIEw^}I(Y;c}tzu8aL&``-e!~`ZP9>VA zc~R{tVzV}p)m<{0!1xyGv0mM)*Ip8?PbZYlc`NW2Uw-+;mtQ&R$1=Wj>fKdUgfdb# z{Y~|x)5Kh#41%UA_+-;Caj7@bkMY4=i*5n_I5 z>LMqCt1#(asvUnL330WbxY?@LpJ?zB2BC54lge%a%bJ|*57Qi+%N1~B%{21&(r%t* z`C?&7PBO{me}uLfb{4+Km=yUXo7B;a*kl-DWLF^vR@G}!L?w$G$fGo%^<2@!z``1w z9s^QHO$X+>ACGs!q{y}igd;snZCtg{ZjxKl7%(MqGsO4;X#*3z(^gi9P^ zMv(@3a-4cT**yehaxNx999T~|nbm}fSUHzV2*+j=x@;8^=qYk*#HVgn#DiJi#+14@ zfF8g4=D|bnQe>!Zdn?L~n^sL|zf*?cxE7r&tceCgYYtUZHtAonTl+Jc{&^CfGnI`_ zaM4XUAkv2A|Uuhd{6 z065FqC%Gq|%5l)3TwK7hWrTelVQA|zjwFFjbJx0>7JRmPq;Jazw%duA8)88MPyhVY z_NxI=VRJ0_l&{w+Xk23r7gc1Vyg6WTS@jSc$1R{DAHeAlN0mp&x9HO-CBn{6PfoAz zpKj%YTE}wLh`N8R64Y{`N0-feY0Y9@X>~PO%z>e)tL)gIOeB!bxP(;x2-)Zj?|Bhl zWmiZEEImnx;~}K~DrB^0Z#ICI949WoG`0>4i|lo{?VWp#z3aQ?)e}2t5bH2v;qb=o z-SfR&uiE6tpPX2?ecH>$yC&oQWO8yk}0fPUi^FewlRNN zbM>0x^f#~1PH!9>-o1Ck<$eDA1wt*YdH2Ov4x%u`&No(SI-_jf5zvu$@to}7o^HbwrxP$Z>-+{ybH@1$EmUa{03yZ zyAK}QbZ|wY0t%*v-9kLx2-uDWpBrJh|BR&mUn*YqU>dT$H+YO8eCZ`d2Z-*TYPMxjD?yV-6FtS9<6 z0CC`T(8C^(ky@q=p~=EiW#(&h#Vi3XDUh1Gf>1`*3X;5DzIGr zgvw8FoAVs)9|A-TvbX@TJ=uWbG^$gn(5h5fib*@->PtkrCfzVd~4C!3+B>M|jNtYs8`W|TRXk?iL zhGvU|-}1)Vny?B$QT_N@sX`+wM&`zC@6up8`F~=nCw=gkkjsXM z7?vS9^$Fern?Z?;Aw|k!ef6B&N;Zk7z}8c62FYY9wuLtnmkdLgpg{6iNe_S}$$?zR zV2ZzJPQ2y!IJy<;5-pP^2@~c&Mg-4I@n!Mf-(qj*6;WP+1B;~6zs-g8(pH3NjGy2{ zZbF*z;esVuB`!fCEg8->N5ZYt8!1I)c;|7+c#>DRgJsYFK9dK+D)%M6jB~Tr=_mkx zlZGbMpq`V%zxwXsZ@&8a#qqJNiWW<7bvR^|NVLWQPsP##Z&o>EDGk(F?L7l3z$2HX z#WxRR+fcP7@8ils#Hr|rft}jxI3>}3Z=3E;#Whk5gXG**l@T7nVKb2*;iU$_mng)I6au6I;Qx;ARM9=hK z-dZqP2E-NFnjC7@0W7CY0qW;e77TCSL*w9)0W!ntjSA~v<$%gcj)Ggd!@J?wEruCi zI!9m@MT~#KQM!8N_;~mFi|rlL+DK@@73|tEjS$vUIIO(cwg~Cnm0p%5Y$mHmb&_nF zlu}IbFT$W{M7S!W>@BoNJFZw9ZRaV3_)>6eT-qnCXM!%PXa5DH3l&8nxNp9qC0@mA zXF1K=TZg?UuDuGZV%uT4bO$$j(bmcFi|yTW=RR8ob!u(L`NggU@pq1}WL%%~e{g6m ze$Fu0^6cs>_vTIbqE@Ey2KNu0x^vfAez}@0Pw(YU2+t4R+jmaJw+|l~M}G4B#lDp~ zH*YvGN7qEpmgGr?U@+j746iH+>0O`57h(a-9+B$z8o~$4IK2I)=WkB;JHFYp&VSVN|R_eDyH#MiBZJ~}tygRbUbR0P;&)k-+E$+8fLAp;{w~_~M z+oi|i!tPRVWS<%~~skGHTmue#4CHJ{E z5~Mj8N;@pX@Cv5us#fLD3OGA9-?YTb97dnEgi~qsV~@D3JFFrl0k)?VQ@hQwGzK8B z`2oUGZPczU8+N_M79H*6w0KCT(a^7L3Undt8I!D9YFl-P_z9Z|V=FG085}pzv1fZ+Q(VcdIvnZqFf7Hqr|1*g zcel!*()WoGGLwY=BUUigV()NRLjv_0FQI z9h0ZJI*@9wkB;s+5Z*HThT!%O3^rFZa#{{_{lH96j$_Nktu?=Lbm#7iWHIDh)A z&Z?63c7iPjVs0JWu#Z_`wI#qr5XW@u+NtgZhn#cq^7JJkj0;yDb(2XOa+;g&L(KzetsR8{N`--(8$G(lqXmJ}$@6V^-f7w#;;R-mA6i2H z;^m9O7fvB^zRC5bi9s;q_lc$PkxsL^0 z&&?pbG)wS`BV^vmw%?jAmEp~fBly*A-m~r+DqHr_a=z^w2Nn=rhmOjGwrxTE&N`1c zl}vrIqXX5HOqI%eCz3q%m?zAbZ%_c3DU8vuv7AXpC&#=Besd&VU8r_xuBtIh@@!k^ zgmKl;JLB}!Uh4wUD53@>dGq$BV_>tl$(~+O0x1bG{CVyy| z2exEJ*35*>^1fDBu@W>(f2ky@ghK0Vne{}w`Wt{WrpBgfGbP-RF65)O;@)DMwnUY9q5*XJB za-7ssMS>JPM|Egol5{=xRd9vDpLR=utXN|~O^P+iysd}!=Jd?l{z^@`m{@1xveLlM z!FDo2Z`Q4I+63#xG35b1f9>?ErnuxBT+v55W7;aXXjOJvPbXor{==PsHH z99DXeGiSHpc?$f98I(dN9NrZ@=>oN+y);N%nB*&Z<6;b(VDuVA?QI@ZLF5&NL>nHo zEz*)Es`AzgsZ#otxB?IK#w|f)NCeQ;xeH7z_?-|aqhjJFb3_b~$p-+&XjC7t)+Y{> z`p^e=%~|WhqHi(fZlzw3BtTX7h05H6`^Lt=L4Qcn&w;U#q%;bpKBZ7J1v=vAIc9?u zZ3UHPBDxzarC&G2Qqb(hpr2}4tfZlj`FyN-4)B;hk)y{VMTf3pJv9A@6@;)YF4lnQAN2p+9X^!r?7_D zg+T6rj*R9uc+m$~1qf81xm%!-8ntvHd)OqZ;{`9CAHV$So9}-0#TQ?H_mDf!jNUST zddtF$hV~{Wr~s8Wxn+si#ljj!8D@BkmPIsP*q?RG+g@_37hy?33+atv4MsXOTu135 zKUgs#$%5R^<(2szRGakqmO}J1geGXQrX8kvL0{P#sPbX_`4^lF-GtJOTvAr-U+Nm@ z{245?U9pCQ%c1vIS(6a@)S%_GaJutaXW0{5YuUI3A+LLHRcsq@@=mWrU(K1qo(7wU zxmnhhkO>BO|GEOe5(UqUBdZH=y*HS9MM}qwJI&q7ifHH{I|Gf!{Qm3L`C4lIuGw{> z)4+{BTj4Mh6dehjL$VX0MpZWllhgws)bwl2LEeD>KDB@{ryg$TVzTu>Ev(K`wdP>@QEwVxAJ9mzlHU(=jJ)e@pOw*n{ zd!hgE$;Y4k^4Gt%N!>;it-fJw69_Z`t#x_LSy51=Qvkf6uU{?yh1b#JuokC2?d!L* zHkhw(UChD$jlEmD=80?sPFuFFDm(Hv^nT~)j^cK%j#5@CPPM#y?`~dxU|#Sz=RBv8 zIp|g;Q$rPBA?4B%Dw2>-1E(89_O11jt2ref=-6h`Z`|1nXmh@Iot_DbGeDtx@Zj68 zzy0?9#VqwP^Ky7tN8#Z^lq(2RZ{|%feE0~1FCSJ zAD_KAePvSszt%FWsv{TO;Qr1oog@=Wh>o43iYy!&ywCS;bsXKslU$t`kbGiks-8Yb zePd~AHLy!}rZQF*@uHH(v#7R<=nL5h(Tcfuhx--9^bp!{s~o*eA+=MDL(&DTiwq0Q zQ(VIuj3;73JFbepRGkqX5}H}2|BtEr+OBI!5_3(079dE_liO5R?Jgaemd^XX!nxAu z3#nTgQgxZ_=m`)2Nf;96`y&^2^<2PObEXNGk&%&h=FbD2N%oz8qe3(pp`CZGm|2&i zs~q4UZd7G> z%#c*|F6F?P;6^usBP4)B*)nItKfOe2pQgrVp-xd;cp6tAQEL^ zu!$qq`5wMHI5lj2yCfdm)42#%`Buc{p-2B?X9&ae7uaI*QJ<^5IbJgwC+tgzuNO}O9 z>SS)#8ap!u;44-mSf&8I;N!vB-AGl`I{jnEqa;x@2}_XinW!QSDgBJq^zD~~T$Set zWP+rL;$$XmauJY|I((IHfc7;Hu{)e6hHR5kWYmlftwaIz zLEWH>qyaK;)4B+sYXHGBq)I%!p$S|$3kq9oRr>?0$wb!lda4VtT#Aub3Fk5P#uAAe z<_FM5d{W?6_>zsAc(`OYgkc>O1Aox@SyJ6}g;;$M@GufvSw>9K#iqo?Z%_wut`uw| z&ME5vL`gnY2m`TzrP=X#4}-xA zLf;ilkE^xokrTjzv3$n#yixR^B@d2lIdu}qt9NhT9~^QzIT@A>eE0n6gL`+i(be(P z4HFvraq`bwEnr8KW)N_-QgqFDB^D7d^JN>7Z~#N=r5&T(a*Iw5WQe6$5=^FvtejK! zwUYil!kBC}I#BXS3VOPeI+Uv92D6Itl@yagttEst`nd_U*7S5D;vp{ar`-4Jc8H&2MZPqU!50uT{{!Y9kO6p#`Y(O9fzQk7EKAm$NUDHCjeU}2Fp&3 zw~NuzidFtgySo>mt=8e( zabFvUZWPB_lFnoG;#}iiH|-9nguL;x9^}k~AtR}z1Y=<;#LE~Gak^k$)(R{h`6*}FO3?`quBJ$xH3%cvn7UDq9a zWcD2WH+If1ZeLv7xm&JYBynSV4o9-8zCL7?=87dEx164e>s z6gUAo0QN_}fB*jY)A6nj`(3*sc5d(NP{`&h4D)YqTdwl<-MeQ`pFMiy6vXQ~0G2I@ zvZ+DmJO!pwQ-`UdT-HE5d;IX;ojYP?+5wOnF1wqL9irj|ZMD1e@ztzGCz9W}dGlVV zEvfD-r{CnP;BiasW0?kd#+OA?=^76B`YrOWADOI4J2hYla6eKP8UVig3P2*U;LYuN8 z@F1p7D>ue$v;tgz{;LCpED@V|r#ya!BK5IaY%EU8WGcG2MPn>84HGujXd3I7$8D_^ z*^fqKNXe+`3F1^&!yNg`8{?+AWEJw$8UrbBmD^`(?D|Sc@Z!r~tE^JEd0endbEGF8)sE9F=G&8Rfmthi5wgj_02w8)Z6a`e^5=`3B6Hq`!6s}SSY3tlo zvq}f|{3mjhjx;wAipkF(vLIBOv_|S#(_xiZaNA(fftmF z3#;r(Cp469OCKeuom?m$$_I$MVI?4GKpE-^__!}`X$ORadght3+Q~7qm!KdJsm%d;+z|Sb#@sMtDBuMBNdLs3`EY zvJ}*Rwtl5k4wDMg15$7$PbmkC$W*_3dFD1?8MWYqcK94cwyxXggYhTCh)Qq zRhbrrl8Bv{A`()am&sP-fJ!5xxeUQ9**sqYWM?)OYr&ZCpiI)9Cd>RX8?K5x5a87f zh&!>1iFMEPl)mt3XhbHsp;@*P1%7Upzz4u3mBNS8FeiYaW~4Ys%gy--LuH7PNXbJP zzikjr3>>Mf8Wg(l^|k&tcf-fViUYEwQ(y_>a#RwSB7--$7g|K1t}bHvBD4w+6;hUd zUF)~BqdXH{4gr8K(lWk+WUEOGTo@yt4#$}Gk16|ja^j%F{ez=lU%&nJ)oaVC4Hg<* z%oBCm*p2J=@7?{w@4ox~n{R9b&{zY3he)p3WJ|4mKQT(8y|gZcAFL!`W*iyFIE{v= zbdL}OXn+-_%vzSQHWhvBcloZQ$RI9Oss2_V!_Xv28r5~sCk=tEUpf10kW8AW^*mAI zgcK)4RDFOzP3@V*E=+#o=Jf!swS#taVrZ)ty-^gVIJn_`0%4sW%$N7 z;dR1{W!5UB;a4kRqP$&*`~mQ-U5MG9HQBw`$uan6eDUFtoM@TF{K+~d2jEn(CYTwaJI)W{cJafMICd#jnk=2P`!Em z(ovdZZh0@5PoF;7KRC2_-O2qnuQ-SF!Grt1|Ni@5UcFWTTp8lb%Lf>V$j1Om({N(S znHj=UMyhKnD{18Lc_F}O?-d2}_O$>&9*=S(T1c5f-?}8n;PEj)cJ}Od{1$V<6oc{GVQ)m-zOm&tx|aYL$a&s!>Yg z)LV*v`7Cyaxx353&{F`TFqaYOj$Iqv?x4-C>9voVvbk*V?%OrNce%U`b z;83g1Tvh#ZUbayHeN^M5zE95%2pW8|eWox-pjeqg!$CIK~FqOM%(SnjnKTX0ikyAQ?)+^2*%m1>t|N5;ft%&>L%+Av@ z`;n>_jZ~NcNTE$T)PC67G3`!}28Ch;JoLmy|G1+%F|A9SBFKX)9ZORIR0hf-!?A|z z==Tb*T%_}y{%(>^9U3&flA|SnCly^q8hXpfJK3Tam=So3b#&8kgN?BiQ^?(t93G%X z|KJM?WrQ(e5)FtPNl=&hjM33h`L60L23EH*Ui*s;*?|e-NCKn;n?_GyW_cp|7FIv`{aqe{Dxj6 zX%^KnSaYbFwud>{+r9JX!TsCYx6!WO>~#U13zT7=_jU|xn9smp^o}}z_QEmnID)z% z9F4N-}^e{p`Ma7q>GN_dfe z$x4VzJfGn|+^AoarlspPw38LsbsOfelyZLEw79~LM@EtBvIvcQL)v2eRY#>5T+u)2 zK`1C<{kZKX-Jx?Uv;^*^Ngq8dRRSGg($90IHe{2w^eLs^B(1rq`HO%>!XPVO&)M;k zE*Y#+m9gml0x3Jgr1LlrLMMG*{!SpW zx>njsX%Ta&vSOnfJtD+Pm!Sa{gfnf-M3i%-fvYk2(H&-S#FouG z`uzFT+kv7h^3JcN@9T% zW!^O}K&xgTR|-ucd&?u?yphuqwzL9l-PG=Jvl9q9LUh^SCaz3Fbz$8MvgAkD3#}xB zbgWinZa%esF8&<5p1Dhb4Tq$%LjbtZG0=#Dqb=|DNduQ35o?pMEd(iYW$eewJ6~Wi7_6EsN zHNm%|R3nInEeq^v0|MK|GF)c-Fh;FnFR3cVu|H}o2zAL^6Ye_^lRa^yQS(!&B{HKm z0ZPv*os>TX=tO=ipv<9Iz7iyY9vTMS8oVk~3R!FmgPfMsbi6}u2=K-&&L@+8^R}sg z>pW;Ht2oof{;3K)Roxv7~j12va%1}lB6f8s9NuE{^jWKNX-R7NfNTY zq)sl~u!Y)ifbImGHflRnDPK)7+laEGOq$VBF=j!!_HN7iwhvd#jWow{UthI@{?jen zeiKkvLoEW$T(!)u;;|;M+D?nqIhP7KyGN^B((C&&G0OVr_>(~jm4$JkSs?ThsA_BN zl?l{D49&U-H_afVX67d|^&C^xTp0C+cUNSdt@*IH8AU@^xHTZl** z#z}Q2MA6ZTYK)3!Vo*&y^#bJS@c`l@Z%{fq?5a_AWUL+!qk*_gML|+!_%yN$lpdNRd9W$coh#xYl=2#P->Ib;j!GVhIBc zZFE)ZqroQ%6OE6Eq8Ppv)lc<#&`i)^E>p>?|B4pJTQ}?dB%w*HINYQ*0OL*xxQ#c0 zBX#++Mrc5QASgoImn$MR6=zZ)Tz^xYt>K{=YDu(Sw3WstwXa{cd)IoaIj7@DtnEOz zY29X{uW!Ga2Y}ac*VL>2(we1mQs^w0gEJquGZLJ=;4EG>6uCr^tSe`HA|66y!QA~& zS0ZfoN$NxR@G(3NyTfrj#To!HDmI1eDF{Vwepk8@Okn8(oK8#;4@hYXEX{^=PVt@tb_7L_?wplp_^Lw};FP(UB4rn)dmrL>egx+)5g*#pG~H0P-h;MbO+y>(gJF zHQ7mRl`B9|i0h#MAqjV-1e<*0Z+zdunmYPW+^%wsz@sKj!tZ2Gu@DiZM*RHfq-9?Q zbh%2Y(MWc+W`*>%${=8cB$7TQiVzk-gr`z7H!IsvQFh_(OBiW^C+@Om*GXu5W;<`+1 z__2^ilxO8lJ}Y~G!#T_KOX-FVTrCTTk_D9jn9;(MbApPfPuWwhXSc^B+!_2x1q75a}^lKp?4EjB6TE` zB@g`Y2uMXZAKapEB(r#3+COliER0FiWNMuMO55m8Bn-;X5E|tz}im`{tlG7}+B1ZVIdw&qnvci(2b`{bM^D3Zab^1!ufR)FPtHDkbf}F}Z91GJx6{IsE7qx= z8c*g~IV0N7{?XyP_lMYZ45;h#jvc5S)oMuV25H>Pz0>p=Bh_XB()@%4uKG90XEk1= zXcA!6?Lg_c>GE4#s8hkBEj zLuqxv%IWgzE?|;YLwz}%x-j3s44nH_t=#~XA&u-l0vcGw^&l#QJzmHbqD7v@r z4lD_NyR~1RYP(Zq3~kS(elHqpqG7f!tfeRs-O?Z;CMXLEDmX@qH>_}}ktJLFV~Y=X zmWtBao9#PZWpGN!O2Kvf=~SV$@Wpca-gwZwS^G`XV!ASo(%2D#Q_)lqZ`|1K4Y|70 zoEq;QrmJZv?IK++wB8~ub-ZbB)9aOVM0K&dyX*M>gG0xhpyb3dzBdQlccb;Ed%HW+ zVYuM9;=A6S9BcoaWW2M2js2?__z^{;lNAn?v-UgtbRDYK$PSYtxN1;*^J!LTJE2ke-`@C(6q5M-1 zx*5lXd(`RGDVg3c#UHIsudpu3m`$qO@uT_GEuN|$rI7AF;6yMyPeoO+vW$)v9x! zIWJV*%!bGw`H=y3_0jWKIT*B$P&U|v?&jYzOI-+n*@XoQmFC2YU`U?f8?P|N6bQN& z;X`z!y9OK7BCXm=Y{RAONhqX)Lprykc|3wVgdRgI>wE`k0IJhri#QM_8#L0eg>`kR z%Q}37K*QC4<3)LsbJkdLpZ&{a8@2%*90FoP;9#k|l6aSNTB2kHO#xG>&OUrN+P<}8 z`Cv+Ws=>2WJ}cxjsyeS{=ibuy$$QBeN;-RT!Ut0-G?dLs{#K!|Lj|g@G4gJu}!+HiME*MOyW2{sofGLG_ zKVh5vBMkSKSDNF#oO3raswjdd*aiEDuFwe%mt4$qEdm9*EcZ$KDo4(- z6G@ZPZ?Vc@BBk!vUpLbzPo7jFgd#_8NGiBPOEHrx^Dv~g>V?AI6bVGOGG`=5R6ReAgJ7~v9mpfSE}-&Cyd?Cb3bDe26Utf8EEB8-jI&emM=L1 zD$k@J%+Use`dD`{8Ve3epy%Rm7CaSZ!7RGkNbVby6CkjJVyW^J@lbi0Ch~`sA$}x* z5sEpP67!T~^ayEOIm${rmwVSx-%T`?aAjT=YT%b-qUHzpQ|OuKK*ir07Y}$fsq~i_ z$~t5guylvX*TbJHs`{&>>%f(96)f@1{nQCb5&>~0h>;Q0{f=^(d>{_$7d{((0xJ$r1*^8t_N z;ONs|e*Wc8fBlP6HVC8?Hl*^c1@W&D&)bv-TU|KY}6NUqDVqd4--MeGm`mb zjdyqco>nOT|N0Frzj)}g^$b6wKMYexbFbWC=o1Wq2-L|!%M?Qzu0S>!7_+SKo z-`OhGAaL5`rjCFbNbc>or@VisQ;-k`cq>oq%-fDmj7jzGsw-FD?pvjG#Pz#*bfg<` z>-LU>?B%$9R~(iDuY2df=wewUq|NQP6(d$P!8J+4x(5r7x+g)cg%&L+E_|>TYQ>a8 zi8mD^k&L72W+W)iM{2d5Eve1HjjPgBe))1-A5hEMZ)E{Z2y3E9FKM=<#_9v^zR#C0 zw{+~cY~gF7zIEAizA2wAbk_P;iItgwUmI2Qa$4qeYo~pvpFW+xef#*yv-=Mo{`%q< z-W8oS=>C3x|IwqTrfs|)>)Ep>zr1+Cn}7Z4jV9Y>UFFUjF!1;0-8;Q5-C{%d90teO zHz@q|{6=fyd)b!x2BojQkeJ>YA0NG3x()`avm@Lwp6z?4>fU1PeWMh2Z||5B575V;uQjI zxw!F_@6MC9!4cv7Y%fSQkGE>34!}`6qaAnH;5d%(>9kV7vH&!4f8X+YESrb{pV##E z-M;ogsS$~ujLJ{X`Ab$t^iG41`YrV_Pi~2*cdu>h1Yw5?pdqzvr)7Ow z=#n#xRQ1S7b5uUWAA$5hYfW_xf$ zh31IVTxb6a@<@gnj9nxXIHB0 zo-~w@m>wxXkNyN~YRH%{n3YJ8Wy+9bA6)i8wxTs=JRG+5(LR~0chn#Ok~yrScx}`U1neWV8Qq&(j^|kW2@XX z>Z4lar*=D@jG+m`|5;HCFa$oaY7rL-B70}853cD$!*0!pH z7P*cNkTBz~gNQ~oUP2wz}h}Ky2@tqh)p=5OEu?C-5V4G0tO^bOlZZV^h<<(qkM=PG!oF} z8F~)WCVEpJr|!1SN&1-UMw(IoVa8+pi;Iydr|2w7Pw669 z+5S?1bPAdVlRCTruSvs~nEBtN7bMGMb&I1+#E4IhK$kx0NV75zc~KJAV*GY|NlPO;R98c0fwfZz*)5M=Vy#I&qzWwIu(>n(4`1K87@^n&BN;A@_I!qO-ee0AQt^~Jxn2>75MCuV0 z&M9Q-wM~Xjn4Y{Iz7#?&6S;lutLA%{kJq|-V;6=?55xFI=2_qNo2D@AJ^kCb>ytki zY2h^ethxq~fz$M@8K&TNjo_w7IVvrEgQ=|1TnMNLkO?t5t=cha)KX;Q_qm4MRz&Ug zHEgE;&^U1o1Yu(}p#H4ZxN>4$)LD~U&XYMgI=1kqV->x5()elF8iL)jF2hO{1K7rf zy^yHgjRuK{j~_fof*ySmd&vEDdc7HdqtBnTR#y&>w%@&T7UKDxJ4D2raq^;*9Ff86 zGNkqK=qO3vxbAqITeof_)l%d{Fk7}P7SOsa5o^AtQ`6wW%;>KU6Yi$k&+^K~3aS9g zN09<1Eq>LAG|ZpCB#ZNUdTB_>G^qIu?UEc-$(V31wN0EogWsbMfL!Av3dc3oIJr0@ z5I6P@Akvb0Z}Osniia0@bGl3jL0?k}GX#C@+Ofx<`4pK}k5PvhEYAFtoM z+1=gK?!S2b3enu$k4_FfIe-27HAOUX&HI1y*c-$4UcG)}8_dboEraglsV9~{qae;s zw>#PLz5+@^uR4VEi#_tQ9EDb7C3tm6ZMw#yBd;F1z3sStoiz-)&(5lVc?$IsowsN$ z4uz5vQ(A?y#@+}vP!q=cchab_e{P8o>%mT7y0d3%^c@|8!rCajrMiNkiqTkZjYUib zeV|xL-5=;+@JAUosCtiG`0J=3DRoJe#H!h~B{5s*v892 zCN@Jq_DD5Q4B6!QotmYM&zSfupBFVu#Q6Klp*o+jXsn?`>f(H_{Mp0WT324Mp^f@c zV^W8b4We+CYILA_wxJ=Th`xvsOQVV^qkkVa)8C(L;-=C{Fn@g+g_nDUFZk7BK~t@z zLrh<;z8BuG^)P0D8xAn}WFG_6bV2hTM536Cp__Kr^P>UT9`T%Mp7wasP3P1b8{cuf zIfL(jR3N^`RjoV%9LM2UcZjXs$ks##;WSmGtH%gj*q5d!&~kon%X9K@i|`0zj-Bwr z3!TpyyCUb7XL`>^rs@jQLp!ObRgCASpD1o~BH6nzt8x%l=P6dzT8*j?5aVB+<|cN8 zzGzQ|BtJCwHUENy1&vT6cyh#1cagAzW~=qw-6rqku^_VB#Xu$6YDQ2}b;{=K`hM#24P} zaHic}B`2D}+9X1IS1J50LN*~aIV*eOAIM@tFqfc7j+abDN}3WBN@$a8TluX3Uf6trr0hO%gb+mXMMnICnGM}Meh6FwI zM9uJl#}d=W8T$xYDiV2{B=Hfh(vMsw-{jeVoh7%W9{~-8?iZ-gY`AWa5r9oXiNg?$ z8NP7F+VsK2K~>H{KEd)vK7Ck3s01<(1=ACZ;vpb9R(Om>I54?c8}6y^3eIpqo7Bpf z!RoGJ0;&7CQ)`q-u|wbhOc2UCdx?Uwj}XJ=mXjC+U!)ovWscGSrzC;~!Q>q-iVsP1 zV*^_Dh=`|wAK-$*GXb@^0L87~r5dH^zd{9W(KX3KxgrA-=%J>&-zXU#R-(y0DK}rE z+R0-fh;6R6|18Z&V}M$bGtP&QNN}PGhK!w^emyzV09-($zXn>(a zNJgF|t1$}hP(+8u%@~2tS5gaP@D$%Db`>O$Y)j@P8{p6Qm)M|588@@E-|$@H*(KGy znSS80sk-=Xdo;)W$Ws1K_@>&bH^GWEE^9&d(Wln_kee}Py#S6*f%e^Q3T?MB@*CH6 zB=z-P#N|npO+as$MeR#2eH-tHTs8g6JAU$HV z2+p<@vU1y`6=&(rEuC`*dfp&nUaW;W&G(#m=^u(;n#t+`~lZpX z0ED1qqKN4ufYW0?ZNXWay*fWv2VVVY0;gf&Dm#@MgF_2vWOvlDik^1YK&QbH_AuY> z#>UH4nBtqn>VxNT_i?HdR7c3g;cXe&4E%F7hd?}Gof0+Uh%kj&<(e8r@xqq7F*b&V zOiL`0;s0uEGoAYt zECJ!g*cjJ>GkXyn5)lIerlFZC_@&i?N100{t&JFVN!`>b>~-FbtfBzOQ~^xWdJ#4) zlBO38stcIHaDpHI0>Vt;C>iVtCn}~W3YD-TAhE1=0K_7cz-fQegYOR3$PH2|ajZ5O zvBsrL!ZigJ18TJ%qzx|6ZaL!WHoKs-^xzpp8wf^SF+T!H=$(nH9IdNEj^3++fAfPKP5#4Qp}*0 zcKYl1Fo_@8snEnH|B{tveg{KQseKULjC(op^$Ajf z0ob=Q#^zzL6dH;B6b14y2Y9q>uF%BE;El!MReG70G7YMU7bhp07llcv+u;fmlV?E! zJQ)UD&LN}({d-)%oZc9`5fzsxX_dL{c-;gcDfSu0Jx-Z|R)l}oqzqyvSwdztpEH}O@KLx5WUdac| z`ih)vZNO@w`;jNvLmsFoOg;OFypc(XCL;5_?%oNU?jaxY^!!#0?}cOQ z!C|DIKYj8afB3_87R| z_2d^>735F~!&pW;NnwV`&kd4ukSfp*AGkbdZ9l;0&vW4Bjq1cy06;B15zgUhiSN== zC4hgFw|13CdT{}~)U%y!8oyjqO>$E#`g$9!SBg^tir$q-6BF6`C#hh9#<{9F z*W05ArKu#nSjrHrbNL)TV`cd@T><0k#^4l&?hzl?m>Xf3?m+bJ)?Gfjo)Gfaygg^q z&9(vO<_gZe9H8YkeACpdu~o0d;y50BIDC4WB<|9AC*E)YoOADhpl8pX8k|kw^*T;v zz%mt7I<3aIb62-u=lL_Qxp@2j9WtPb!KjO5tujOHA&K{sFq$CYp=4(t9&fiqKlc1S!On>h!2u( zNZKK*YO3bO7(9gjrd6*IothpkSQ>`PGCDkt-6W1uSGrH~?qyP?x@=+B!_h6N^Rqeu z%l)6&h)-T@NA_fl!DNrwU#7$!%L zjLlXf< z>2xvy4r7A2dgu1-JG*<|K0AGTaCBg1FJmnoTOk$LTYDCuc|k^tEI;cUDke)~fyfwJ zeP(ly^BO)Kn}nla1StDz%GNkm1(|B(AIvFO89tR_l&Otul~N*0JOr2?;cUc&@|e_u zRuHk7H_B4NN)foqbTWt08+YSHU&8_Cm`V^ zvETy}nc*2|i~JClYw-FRty}>qM%4-vco^b+4axHxFVdG|U=P8$FSWswGoX_sFwm5v zq~zkUIO7||E&ifU^rR72ZhADNPika|MSYAXv>=q}kZxDK_SVAUf46d699uQOc1JJYRDy76F zSa0l$5+HIw-7XZtYdTxVn3Lq3`%rEL?hc4m1dEJKzIh1xLcY?KJggZ8f{O6Sh~hlt zo};D!q4O=G0RJ0UexitpS*BwoC`Be@AS~xrvQaldKC(n*(kyFalzhkukGtsJ+=VW= zYQ(^ZP%-IN!4s+ACb677M{oo7-IXpr=2J+kT4J{}{-?Y$z5rw>x=Pj8{P`+vWDeYc zjlj~Uk&S!Uj+XXWV52ST)W2YK2U4(^dfy6`AOaZmlyKWB2UZ*@e?L zHmkd3W5PGXTd>WQl2#rgn*jqU{TXo)1Dcwj^jM{mGHS~yzmV)f1!oP^73ab8q-$&H z9^!>Y<#M!Mo2z2PI$8V3$AY!VAxXMGWaAsu#(?2#9DJj7+T8z>&G#$(?969m3V zVI7FX)wE9Jqh%Jy$DX7e9-1fy`|G~hV}2ovZ*c-yIQ#(Ji z4png&O-Hj1hAx8gHM_li+k&EVi)421>=<=+#CxuA&F(c_729E*m4d?S*H7)en37zD z7}G>y?B7}x=$f_|7XtyRD*SNOx_N34LoKddYLxf3NAZQswk>t8KYS!fjVIAz;9icW#WO*-?f^Of|FTdzcm@T=@@th-dXzB5+ z6RtF`JE!pr2-Ifhme>1kJ$Z0nW3LlJBdwu-_x?aHRV>%q7C~+npMh|Aw#wHauY>gO z@9mjn(D4w)MYcu;J*Qt@bETaw$T_8Yw8YqSXNG!zmK_?OI1itR|BMT2v^rsvry{M| z&cJqyf2dL8lcK(UZ3ityXV~kMDT>B1ah(foL0#TtYOHJTd{UMArx!22{qBP^i0Ffr zKV|@E2x1KXb58+fmhax3Bmd9NTS;@xtC8)8{QU0V;L7pU2M;gyc9FkhAyt}ucIpkk z_O*V!`uX&$zT(N1<`weW=|+ylrlAr-Gp)(-7PjlXron8$ts9mz*B!p&Ynyrb%We=lwDFm6!u67haGRzG@L*iKf$cm+Y zeKI<*hhFp5h?wvS0Yd#}3wT^lpFH`0zFh1deFT!JYu&$bl@9E0`6g4EwKZA9(wz1F zM*5)|xS(aifYg3EoTV3u!@3 zVl8}~=HQ^aB2THJZ&3E764ZbOb3#)sR7c9#t)VGTaAl->D{Xl<%x!MUF0|bAa&f+B z80w|ZeQ5NvRB3S>_)e;V3eCr2V^e_nS`z&YkHA)zg{NGlikbxyA0su1MqtTma)xb^ zPU?%VPSrr(3hfrO4gcLwC7SvqxolIpf;gBq5ktWS?exS-7%yR3qv(}X6b$lW{l`>k za->BpB2=$PpV;zmB!{SVTe2i1DiRhPw|trkctDn$U7H^&P>-RsU6HW4s#qQd)Q~YG zNH7W$nsEinGNefx(JCGQ#pc_lLyOGdj|G~3z?jhXu1&gNlyqF%eDVb%VP;_$_I!}i z6uF@`O>JN@PFzI};7I{uk44N+1O+JzRX_&DqyLpInU{w)N;qO12m`#YN}~UPA|TKc z>Nku&zQZE>W$jC7rClEIW$@q+t(ycE9qzG+Lge-D8jowSH8rh7!^R>|iU1d+uB#28 zjO?OOTnI+|6NUG}=&&t_Jww1J^^|fy3zS81}Lx z#+pN;Auoxf0qhzJf!wSDf7eKFeh34~F>Oz0_pSZV5A2-f*3Fa|t1K$7)7lQE_+qP- z(3N)8q3AeKMTi5{G-i0&unTi+*0-R!tb`kAQ}9@1gKjjkqeFAc~)JNf*6|GggA!{bB4(j6F@ygLwPHWyf5^pR*0n2sf6>qfHY24NdB zchvROUWOA?OSOrt=>ZffOEdE$)T+m_M&eU}RWLG+HdJoHFd3hCz`xh+%u8yCrsDo= zWVz{xUeL)&x_MKhi*adefVQKT@LQnr`P`5^FY(U3d$%2$q-)Au*mArn>{$>_dHvOE zTMo95T+9FMNxkaDVY}Dgzq66rD7f`EjtwJ&!A2zfO&!z%FMZ+l&B`(>kAs5 zr5fI>pcAE+OZRPP@zttAi;F~BLu5%3`Sb2?^Rz=*?V=2`#IUx8TrIcZgL{KC_b#p5 zI&nHvb@bX7?@B-bjp|HTD(*pLt4MgDRpjMX3a-bHQd5VnszTLZY%MlZ{jdds_nRRI zRQ~L2iyC=Q-&yrP1z)wCvhOEjJ=Lddu4Tl)1ukQl^@O>4B{rlfOU!y;XoN(w|LFjX za*Y2!_AmM*{b~s64b-> z9#goDcZX&{Rx#1A!D>S18QYX9Vp zie@dP6$}fj;f|?QbA_op^+hn?#p74bPmc8~&oo?=No&ML(|8nF!3Q=)LUp^^SKTvF zy?QqIg1{fsF+=urSTD5(M>Yg|w_yt@tJNM*5wfa`~ysjSj5r>wCqD zmLPS#gf)CtQ5S*Xc9jfGumB$cgKz;IZey|7aTJ2JL=}~DrAq5AxsLrm=P7hgs1T{d@Hq2VCpJL$0S7$Vt`V~ASvs>3}~=o zs>L1M_b|x^y1S(UWteM{l%kjsUHJg;H93omR@X-9Mj}iQ!opv4IVOE$AM0UL^`cM8 zNf!)BSE)vEafw0@rWO<;9mN}70vp~qm8Rvl2Y%iGV6mnuzTcN(UZD!fOHqcbVd zZk05Ac4xP-9syH}v}3Dg6UlfUt2z+JG^kT5z!H2OL7D*Ytf2AQ+8n~(;WZoa42~oF z)Iqd{MQNIzfDGOetE)*F4A>EUsW!nQ5g{sA6;=pZoZO=GD z&ar;P=x|EIiQswL$8Ekq*H;>x-|ZZH73(Y$_qDf?B0g{0NUw5Dtuui7to&Jv7ka&z zbfZh?%-Nk0Wy)lrQU>KkXfy|i&!;|C9( zJ$Yhb8tt@o;mdii2Ua>(t=zr;U}wk3`wiXGzx?>4j@JIM*Uur016OV2iA@iHj?jpX zwZGvlsXQ%rIz`vFEcIza#kQ`C8IR_TOnNwi3gyJo(yddT6{BP`H)mZ*o`1ddRaw*_ zqk`AAc=hTY;vB}(aILQGCq(wH24lK($)I{8|Ndep%&XR?BQ_J;hNHUZs?9pRTQ_wx zvYTjh=4w=rl_$FlK=vWE%_P^nUM>f!t;md|_dc1>DWfdFDyb@_YT&CjqlB$Z$uI1V zof{J~aDd8+NSpB+P=4p8?}7#Tnran!8*4v_GA5hEn;SpDZG4l(19_b4zA`i$Cwn3? zg=oqa8;=0SN;bsSeH^PC;)Ijp&UnF$UQZyDIgL_?@h=058ov{cZiJ;15fzQKLSqic zH%~N?mm|*SOa!6W7>sx=A6Ow903JLQQWFX_(=N(?MG$z!p!rZ+vW{c5M`*>OYnlR4 z>Vg-vw|%Dp7;ExAo&NaCtDj%LYl!2TK4|N~RAaq@?OVMpgC#QFpwwWq{^#L?`#J`P zhxyMkDb|qy(d+0BRCPn$x3|R<`=48n4J4Mt-*CE;ug;Z#K!;rZk@g@6-iT*% zt*_-*i;jTFQ6}~wqDt>Cp;Y)2;5QEi)bXF=T*+xM<&b!CtiH$4hu+|hAW|v-dzu@` z4Z=l#WC0(G33BwJ`$37Y($9CG${v(SS&EPsz5za* zmI5RZ8%ieK3Y~OOGc{v~ni4J+t8-t*sW(BJ|Cmq}7BS}GYTg@)jD;Gyg#t_K-Is{~B@Z=r-Kg zGL))?(w;jM<<_U$`cw{>zjEd9_1ky3>51&BE!f&Edk+o|Evf%lTg;3a(>H?V#P=_{ zVRnLi<~FtPm26s*Jtw$p*>7sgkYA^&7PvQ!IwCLmVJ`|Ha34$|(L5Al)W>E0ykTYM z{W&VhN?ObQk6Pu#Q+GDbheO*-b{sU>TA(c_%v#>9q*BqAZu9St-ya;@xqHv@eZ2yr z(N!flZ>gKfg-+(SAl&SS0eK~0h0vF;pLB@0y}YeA@7~+_`smT)AOH6Azy9mL{?mW^ zuTP#n9w_wMei z*RQyfmhYPx+yCHYdHWbdKNs}M6=(A7fM)bi#|Edoi)OScRFXciLHCB48>eA*ZvL0; zTQ~`UlNq)URHh?U6g`nTe|Ie6nombyma8 z>ZbKrmB<9SvLp&^s({p^z}0Z5A*$#UUPVY~V|nU;EBx^ea(+UbNzc6*OhN6YtDj8$ z)QqT^OKr@qMdxy`$9j^9-{#6jOybB~BIp{Q&XXl-q$5ly;uiRYPjO{1P16dk2%}-q zH@EV-hRxz7Sd=L~W!R8l8Ooat9#s~S6)RI_V(2Z2=AZ%ojr@osCZ}T-V=>Ec3!Ii$ zwuH~*+A+osRf6R#Xrc3i0A(VZd~8gIZPgFs?;B#_xAX5E@#{R<=Ao}$SMbyVsS>at z7Acygwd6<5pl3Skj)yXBfE6>Stvzhz<*5E}nfA#!(%8GZXNS$X2|_!Wnu)7ZfF+|{ zXxVBP41SbDf|<%OQmQBl!W<^nh10|uX9a8`CT!A9X-}dump2FtwA)H#AS4J+B`5yi zqy>R2v(J=cO;8Y%iw68w(1+x2sea0>o8n{yQSF^YwU=R%g;j6$GH0?-r>N2RB zqPWCOo9cI#xeWmu#F>MDEMn;ca(kSw2#!CI(SeG~*q&lef?Og0@m5k)RtCbRRgAnS z2%u|0nXIJ_-bV79Or_hnW<_u6-7r~dVT%%Mc?j|b4ze< zE+LkG5N!k=tauod_}cIwU=R#rBYdUUEJKd#B~e3jDF7HRcF8R9f|IdXm~j%C%W})F z7Xw8H08aN zWGR9WcT=<>m9D`Qioh<;n_@d?<+M>`3wv+e-saC8?C<~l>r2afkYEwH17`m8*S{J^ zeXw`;)!VoK`_F%S?JV%G7o1Ip8~*a**Z=w-|M>s@m;dtk!F`T!WP`=jZp^WwiRgxC zxT@Sx(b+S!_`y^;BxWbI#e$epyZ5+ZhW|9|D|U(-J1AGSP$bC7pL9k75UDUJ7{4ZS zZ0s)C7~5{NEAt|-C7GtlQDLwQuqEEk?XR3i$|coc&Q;^-v#tqFF~d2^5GjrCw2PS= zl>A5hUV%l-14DX_b&nquyR=<9|WocpSZ7hWan=oYL*!f%iyIPpc zK@w|%LE{J4R%?SeOBFE9(ULa&a7BBydt#@oJ{X#3y8r%(Uq|M-`G z`Pcvb{F`S8GCgwt;e*GD+i$JFX<<*V z(|Y*xi(lXG9}>vpM~`@--XF}V(0R7{fV* zUs%0k#S}<9U>%jtIXpYlQTOJ)H*en(CXC2vwb#|F^nrcTN%4*{W1yATnV0_Fv+(D~ zpMQ}Ua@vJ?(}RaL-HvTh=18QV<6d5Mx~EOO4fQ*ii4gQWrWEvobca41eBfUz5&bJ? z{KH~;fhaijNMP2ihGHnpA5bQeBOr4FC{Z8jyW=_OEgec!99oMkz@kpo)$U|jMl`t* z8l7%8kr96VZGz$Ynnt*Ks^49=LzGXmle$KB0<!w`mhZrq_NfG4M|xHtf$h0 zCYQU)ox0P#Y}wpZZG5uMZGTainCy%p%gVl2v`ps z0UPE*x}exxIII<__|TI1>=x3XCA&-ovK8@k0OpI-vRzfsoMV#g(HS`XSTmPVQD?gb z89>Q&2C=askT`OgfR`XdQ5d#_65kphYK@jc#EdLaSfbQD3!90YCJ510hK!vhpQy>A zVRs2vpRk?w33|yppz`Ci0VAV~{Dt9v1QF<(v>{zOC_SRk9p##pFpVYnu!FaEno2`0 zU`(4{w}yp*Lj9M6_cpvQl-^seeyL+R3AfFFH;Y;er1`DPw`w)%5$+f+!$~fTvk00l zq)8(t!bTiW>oZVhBA7x=e&a7C#PvV2@i1bo97f90PlMn3E^#Y*RMRcs@RlQ0nH*vC zuS|SaXwd{$fFvndw^D5iQLwx-B`#UF=-2bMAa$rMVEzIps)r;#CAs+fcVujcS_nAw zB@BA(Wz{xzt$z+&cFiInevgEat~6lswv6ptkjP&5o-o+j(Snb;^9biPK^ z1MD)hY|DKK0&>bexaK+YIN$;wx#?~^3RPfBfSQ z&%gT}Y#k1G@EJ~-wCB!H1ct;k9#I`U?YY5Pc}4SYAoboBt(VkA&XS)6e9f;k@FsKO=IdKV4Xtw~_6UNQkTEgqjPf8~< zBCGNwxKYr)F+2@>-h{$N&vXSu0BF#+!O)1)v|HbQW-gzg%f{c`}V*6_~X%M6CKyyzJ2di3sws4Iv}WXeZWz&2c4fC79#Z6bc!SnpAa2j9Z#}T6c+lCRQob4yTVd zp2$f}A8h0}AwBN8;q4?YXEscBj>Z?|Ww4+3;!R#A?)Wgq1AHNs}jxd?C-ZsIxVVMspDfdNsn#+ptEIcGuW}&1#xy?;Fmn&I0F$67Q^|CFU)--YNubUb{OzTDMJOWMxA7o@yvfwJ9kL;_jiZ#!pw#6C&!6p_L?tAk0L2 zMr;|gtQukw3cm2-&g&{fj4QQdZ{seSB1{Khq~1gAYJRQ!K?qAS2o1XCvEOjdv`sP@ zArTh!iu<8lFCrXmu+#WJnS$ znF{W&6`q46ij+FAOj6$?{^6#k^T4ZNf4v4+%YIcPRf{= z*!8^ukRlG1k|Hz-tbS%}B(Jjc(-#0MxO)L32N{Jgr7T_O`Vl-rDx@0s8$1!krdTeC zbfX7)k5h*S!b&ii^CMgXF|iGROXN|}Js50P>4xrjp5VBihY(HlIPWm_G@w_x#K+{1 zpSklC{<1HA!V)r}u2m1K5fxS`3M{MV!UiIOLqEU=;9yxC8$q3$1TgY1|2i}ixB(jA zRl+6NAOSsG!)g?@pL!w%XPE|^T(L9Oy`;cJdg&KM)yx1fKmw$gkDCn_d7XnWftE8i zz#rvw*B;QARF0E?$2QheCaE(U0ov3QNlH0jVq+>p<}tXgBBLUR3!p%$V_={p&>?r^ zFBMghU}c%_5wlc2cvL^5fB02BH6W1{KsYAVfL*Ld&_)X7!CK@b67fTGZf!lQ$txU@ zS`3Y!hKTcTNR{V$Px@%_L3*(j#PyQT&BsNOSWWr+Q&cG~^t zKmK6#|Ni^;zx?{ELB1dU@Ouk3_&VB=xTq`b zvH8zar*ww){mz;7h3CbPz68W2BAWnixi6F!I&t2v&a5pm%OJ3uMXhGr7z>N`S7Vtjf4 z@WRhF5QiiyK$r(D%Z<|}lD+bi9knKM4i1lC(lfB(+u;8;2kqmr25LGD7q?Gt(pbw#w8-{Rq6C(4 zuC%{Rk24py=Y6%-jvyo!I*DiXMBI>M@))8ro3+TepHa7NQq5K_Y4C~Aech8UPS5;g z@sDwOgT$~aFHYtUzyCoxUaXfS?Csq*m`wSw;SI5!1$1;gYZ>&AuKoPt#opbI`ZmoU zUC`=;V-DZib3@?pItW#vZ9A!R>$8b~vV~0wdrGBd=E*M_gXV zm4y^)LQGM@vom|0otpv_FJH>*+t&@4B45d=;j<0ev_}0>c7a985z*+&8v{T#kZT)2@buyU@>%_7>Y2qw)VFodR9^uW%Os%a7 zGa!t})O6Ozb6pE8rOU>~vM8yt46;;inR@Atqk?a7^s@U^BEtY6>QQC<^0}&7zy){xukVz8f_zDZy2+|HG21=m$W=&w1nFC0 zcOT7-C%K!5cK3HJT>@%TdE>#(ATo@XMh)k)ta6eql>QJvcM*=!NuG@tJoiDtffOaT z+R!y^L7HUu%T{J9;f_SB!D@jH6KS1NqlBjqfGDLBF%gLILZ-$5Te*g7u!IIR6=L3} z^O_{a9I3lW5?55|ZEYJfp`$O{&VA=Fm`VgNhG?gCbJxA&+PhkGi(LsHOM0$pWX+a( zO@ZpWz*EOcO})45jOHGzcf(!jc6&S zpx18%A}CZu5ST*B>$aL8Aa#;v##XM2s+SBFSwi z-E90|_aIA|b{4v#6Y(_qAj(;%zwgM3v1pvnm5#VxAAuYEgGFlEwRCEu(Wr<-GN1fC zRRX?B8*reDr6{sas(yo#1WGp+U!g7eCgM$y2GRxl!P#uAwah_Eo8vFZu5B7HNKDc-1 zhwr|5{OIA`TQ?k$e((OhAO7JV*ivV443C%11e|BBEoF2$Cz^c2{H#6}hW()#r?o;H zx!AgLVKZ^7sH;4nC?0czb(;+upZ z4aC0LPx{TyEg2>}EqN0{g9f$`bIQmuB7!pieP@}v5^xDObU}$#r*di++TjShksypY z$PnDGR))6f!Ah7G`B)fD3C1_7n1~1}-6=42y?S^3$B!0_wJ86tgl`{Va6fP@8ZIFRy4=_SM`Z{EMfWDB;gTrn6f-1;KTy79gJ44FPg%;FoiN|NDme36$`(N7&V|N)XImoh(IZ15z?TMALUBl zRh(A;-@9|~?j3qf@+ydO;Vm}?ex)7``jjB$w7l800|n4cakLR#HvnneStrM0c(Ho$ z3X^*4QEWflu?iX_23k#fzoha0qS@e;P z5qPSMf0(9vvk0mRdC0C(skZ(yd7zS7!wWfEhIc6|(W;R;8TMPZ4`Gh=F}Z}cwmc9q z_(-9g5~c_XA89vZGlR9o!talDa@*4=x#W>Y=17uYY{7gXtWs=Gy1J_xWmB@fFjD>s zTotMoRDak-g9c*N9QLR5)p)>x#U0s%vI)$Qz-Yzl!(`F|77No+4=_@hSdi3Afj~^7 zDLf{!h(wu1T{Cy^iZ{GJh6Fkt`vF-=;3D|Rw?)GP+-<)V2g0}x5#6_6z($~y8`Lc5 z7{h^7uePI2zYl77}`Mqc5A zuB`A)+$tRRg|F`@QsvD^(HQdz66-BHWx#x&?iZ~fd@m%ZV@bKw>>L2ym|U=N`P~wp z+9Jwz_3X^-WMxBEjIm!uJhj@}f@NYfsobxxUi(qyr%X~;eZ4*v*%a8&m>q|p>Jd^R z{Y~Bp8GP(Rk2Xh z8_LO;>c>Tl6d1}%W$s2kLwhP4Z>Xv~3zMxNVf_Se_2m}-E6n#m8)kYKtb=E97uG-x zZF57N5;Zm1z>cA!3yeTOdMGS#i2$DiV4@ZelZXMBB$74rg~tUvKIU%(v@F1e?KpFd zN>)L~UkcbT8kZY!DwoiW+$OTG#4z&6Ol|m$v@mZ-ocpNlpHvrt^DM&>yQcw;xujjm zlx~3cfMvOd+pvLN=!j6cCW3ltY zSiF~S-u?5x{Ht$TW2jPVJ;#r%)3v|+;dj5=y<=SOsvZK%XxI==Cl)?^^x)}(`;YJM zJ$v%ddqD5)+_vuD2wr22H=ROrN$n7X34O>w($@jt5to`*e zpd2mZDx+=VeNH2m%~~3w0IRsG*@Fw5wEdE0kV!-dWH&TKonTyBRy#Jk6cysgS-ERo z8>{mJ$NZ45+-XGWMRXN{ykprL>``{BBMAB+of57zz$ewUebeZu!Fn5Qg;4oxhWQX< z%72#5b9n5mh#jMkr_H)^j5sK1&DATXjw%_B8vOJEGCVl+q!l~1VI13pX6*+V;!?sh?4^Q&W-v8y@I%05bqfACOK2Jbs7QaLjnd3dJz|(ed>+`|tDAX#dWaos%6C zK5JCI<-oisiM*ssyNNd=mKr-sH3i6f=~cMKi$^6b$kvmCrLX2ODyKBE=qaBV8y`qJ zDdzm_(5SzysOMi^zI?fVu>bhUQ%kEA&;a*7NCfB5b@M`OKy{RaJfti};WZ#i)s2IK1^wo2E>hE#I4pxp>}8cNOb zGib`R6OxSdv|0+3=Iq|yesFh}Kdn2+#@Hz0ctXWvCoRwA==6wQ8ThBJ6vH6o^Z|MY z9O%`jD_c5{Dh!I0cB9@_dR<&RefkV`{V$G?6HaYKc=_^$g0_JF?%o400%sqbYU0=; z{l!n_p?W37*AAuP%vZ(;r*~L!^uNBsb?%epPYoK`vg`<-TYHb`i3y+K8lNsakB441 zWl%S4o>5YDLu=Ktu|k<-p=Iq3z0$#^dS#;0@B}1T!`WH!97eQ^Ygoo68mHc0c)=7Y znyVmm-DRn40G_^|E;M9nFHoT+djbcH1;HE^vIilm4}y@?yK&w;T%~KzzYtlYK$>Ys zYEDX?O%N*AFs<6SD}_8L8^IL>0Ces^@f80xotYwwJhdrcAe46T7L?%o4O!Hv z1};YJDu+7D)zEh%$&GMDW!$(@1&JaMhggEaTq#<5^~hXFGmsn#J2PsQrN z_w}n#ik6t45?>sRaE49cQ+WArk^&gV1!;k-z;lEx1gWO{icv49H(Pk z2$W61g;4qQB*TE zq=-O@HxXbC^5}85(iR^X0Ql`^VymPD$ujN%MFS=hC)F&TN;!s1xk+o}QHH{XU=W6$ zsZen|5YySw4j~D1)!*Sb?18L`F3{N;SI;Oxcj3OK!He|j2J}-Fk<+r65nXNbz?ga% z5O@=S;7y^2Boey>M@kRWFI zd=qfj(8IUmx0(p`XUMj74|q)|23aB%nG9w*M&GpUB8}sYIL2Az1_K%K_-7?qSM8A! z@4YbYg|7Cr;3)?dL3T|WCmFhhCiQPLER17%mRuwZc7@ZRP>ot8)|f1!5?CO`=jMnB z@<1rf)HlDiQ?p3b0)}2sYXyT=Yepa?SN$C_7mZ6zcrV-->udBFpyG*jI%jjpXsdX~ zAFLtCh~UCn2~R;A^G+Ch>xZl2ci1BgSM=hzS|Zb0S7g%()jM)R{Qkk=-MzhAJGXTv4OQw*>J`{EeEgTcX=)Fg;%RH?ty|}B-w{!| ze|1aj|N1|Fdj8EfE%o_$@_+wqcz##WSkOy?iaXlS;>ae0>CQJ&59~GQyi@Drl;io{ zT}SSCMQ&T<&6Aji;d$TLzLVE_dUo%@{pVZHOrb#5D9N=gOMrf|UjEk?YEmvvr$cgDSIti`h@ttNSm;)IC>`b!6JTuAUl3X&m7FktYUDfM!w|XSOfCbnP48cG6zcygN zmMqDdMbny|>9saivAFMp)ag(NY~|R|i)YXEX)snr$hkUrwPUk|*_3VWRPz=363vBl%n=`oCi}EL z4gY~eyNW2gu8hG16{TIClqi+a^JuM^p=b$4wMO-Fk=xzLt>>?yLMqFysB!L_5HPJY zGgWfp)nveoj12Pg9Hvcci)(dpInZ_2kv56yUwEERnkz}wvzq9)BtUXZZSYlN2yE>|agutU=1-Y?M%IjW1WP7AGWe9F==Vhw5^jOT zOevdJifbm+s-%UpDGMLp@t|a(FVpI^;W1CM4_124f|{*-&xGNILWK4x*#puwgKk641&=Vnmtq_OKR=bz%Z8<*q5{ecD{^|2CuU@^v3NZPJ zzV;x*;>?nRhiJ%q6q>$7?Jz8?B^nA#MN@s3eZ&4xSQW`BSIj=Y3?tj4z{< zZ5kxlF7rVCtxpA&b@Q)6h8HOtFg|R+ptq9Is|8;~pP}e400t#yW7IdG{s(5UmuOKp zgeJX+TIFs46toIWqOy4@zfdWbc2p)4>{Hn#UlvsjoPbasJqQpOp@Q(gs18VhELE6P zefnKN;1wOz5^NHB{L4r3_t3K02Kq(#0q@ z5aJc;dq8jS35iJ&S&PICuoR`o07LSt*7QJCClNg9EkB4d$17N2aU9`++yB>4j_&^Nc<=u-P-MHb%k(aLy6?SL)JA7yI zO!r&FY@r2bixo*rXbUGX4IGB~RFXcE@?%>!-!#J{{D+tKI zZJ+jSYolApfj5x~<+?bzp3c6Jwe=NnHR6@PH%#I~@#_r5I;6kV9d8_?e)PKL+JKaW zG97>UwoRJG<^x)#i%)~F(Q-R{6~|p8G7CH&TP5P=`<)Az(3p28fuN!k<^%!O<(aVgvf;Uw!EWO>GSa)M++nKe+Mi#q)c24Z6ST;G~BBkE~X|dFv)A z7-F*=|K;IRM;PHgZ<|y3=;;G}3$$B<`|99uF8!pm?>moVlxigi`yP&u`0FPfa|%zr ztrzdQdEvr2?)Mq1;;nGgX*e<0Dx>`Q^8@`Todidz%4LB&Ug*32>?c2VI?&g5zI*oK znND0E_Y1lH`T4UqPZCo8G7q;OuIlNN7q_+#y3v`p)>J-m`(G}6iihTlEYGuE-zEge zVxW#ULF1`q2t>w}hA>68q>AY24G(80vg(*QkH_fd7=S88s-?eb8e!|CmHd`7{rqP? zzkdD3ci(;MSp6%PuYCH+4^54Hef!Qgckk=hU)tT(@6vra^;BQpE)z-*aE|+UKvfSm z;k>m|n{`s0cJnhw&mBqh<`JE6a?#bj-HR8`(*=hV>GCw7+H#M&vqNK(5_1zm_8_&= zxy<#iQ-1nn`%=;zo4g&LYR~-XCe1)bRIG5#+0+_Jo>vZ}DraP2TYYS@K}W#sh4tNl z^jh7lE)IlhzFv&3Y;|yy8&-@8RyW(Vy1a-~?K$lY9ei0FPxWQQ0ac95GNaG#m??Co~P9 zmtv?v7r<4`(@C#4!P;$-Dy_1#4SKNp2{jV2At6YGXW7JUmM;rA?u6(;M7E)zl0zeU z1|Q!cFrkCjB4y-O(K;k)Zs^--4eF$kGls0T=Y%eW)y-$SAAF!6FTa zE!j{1GVvvuACOc?g&SA+_Rf8$1ly%T4NjduZ6)ECFK=Uks>Q;jDRXl>)z2VqpRN|3 zJ!2rjutcYV?(bhUN^ss@9~H7raaD0h&<$04)7#^dh8+I>H~)Bc=iIM<`ByYIbsnUX z-3W7u?R#2)00i2)+X#s?U5?nY@~Im`CW4}UqVX$2MNC)^USs5jfpQOnsp8}bgQQlT zies88bUv~+j^sHwK_1QlISBP$A0vc9DV5p4jMI~??EaI+Ku}V1@F`HhHv&$knUMG4 z3&AN>83A2@sa}oG`&VG!Kl<%C6emo=@tz8p!Y8U?mRnyw)GBo0H3bX|rKW#07t6qs zz_-s3RDifhMto&X-+xogWURoFO{Eo4?gBP-VPZDG#DWS;@G7BIhYJ^)yvCj^bTTqmSjizRi>sAqpa#T)J4veM2=JEt5N9zpzt!5_`Zr2 zHdvNngYOhR5DFONA+_YYv|FfP6S#p~-6+W^ z*n6(VsRSUX&lOjD0vz zC4{--rtDrgf19K8@}-XD?)kF{8P^#2RTGkQ(iptrqs@GFNj`sJbotxwzT=UF#*FJn zXiVrEVo=jD;m*JiisKDyh#L+cMNS^I^O~l{+bY2{wXi%mIt@@IV*^ll9-9PVQZHdr zL4CLnx%JinuEdk1gz-7<ea_dHr9jG&MWE^#u}fOaU)x&~r! zDxIVUhgE*7{JE-5s3ZcSHM-xGG}h=e61dP(HFMdG84HmGuWh|evX(cVI_b>Fhym*D zn^Ui&-4sBSw-B&x9TDMmRiv$3f@iYJN6+90RiD@&eTy%2^37X5y8Cxc zYjoTYE=A)uZ2?XDYL_`b^~cUmoLmthI%TTx)a^? zd-v|W7Qvj_)>zluoQse#k)G#7wCvVajOm8fFUweA52B`iZwb(E@* z&ck?XmRd&+Ry*$1dr0Sh*3? zROqC`{du`(#-ly90;>AyDy!M*Z6H4@IOdakar^e|8$KU=U^j*Sn9YzbF8=J(Ppy^u z!=JwR=FWF!2zM^oEO6ciXorKgj1SrNi1U$7xym?D3v_2w`ez&trpC%pt}_%o*Y1gDyS0|UU2xQd*nCDf)MNJXxsCDBwA>P7!Lj&I(EB@!WWv0FME7AsLUu7MWS z)R_kq_}tD89dh6RB_kI*S}eRPlHNFFxkgb(_Y`%Uj<`EDEF*KGZDX^B+Wv{J*WxHe!GsKmBP^e(Rt=XOdV`#~Xy{S9|ZHRe1N zqrab$DQgkqjU_5_AtHKGZ3ni*X;z+&U$bdw6CMbk6k`f1WF8bFkMw<1Oesc!0)-7F zQ*TCXCjTm;WW}yTi<1Lmp%o?*8Ict}&pCl6qZ;6N4`SBK`(#r4wAYeSQl2r+# zd_%~lYR~KDYhm^tl9Yq`VwUfrta5{}DacP6^IkT+DkB^gA3*T1xRR<&GzgKpSP`Ds z>`QUM^$=%-j{zs=Fc@VErA@N%S$I@ZDf*-jw~$v&sQ0TRHN^ z9#PangV2zE?pSl6BH9(}54A9^1}g!Ja9g@{%w9ctH(Z#6ecFrQ#0*|w& zy9bH`c6oPCu~uO}(9yGK=!E?qR)~Z-WogpDjZvnn)BSXWd!)(R7dr#S!p&(%k+P>< zBaYBANHKTHYb7;&{0BpdJv3#+lRBl1jD)sXcN#XPr-1&dR`XJZ<~t4!UH)YQt{01? zELKVYyu62Z?r`axu*q?@Eb8H-hxi4eL0cXux(ru8di3zx_3J?DHE}L^YZ#`PwvOe& zg9rOB_HW&~X~~Jr11`3U59fDG&N#pbF;xW2<-j$xn~~O=cyK&A5$lYd_of1#@4Gno;PiQ0HH51) zCob%qSLt8AjGwrdQ{1YsQTmfxFQ31(8bO%t;~{_22}1VqzB8r3rLac--t+ta_|L!l zSHJqzl`EGW#QNQxJLZs#`d_Ssjs3#|vY0M_)77_L^Y0AwojR?z@bqP~CB_#VJcO36QMllEu9nOhm^Z6qfsq?2 zPHlG!;YcdI0rSXAOkC>XWGuuZ9JS;}6=AaUPLr3uGPRmXEyBQ8$>=NXlO-0bk1aT& zZ=`TSCqrIn5C>L|;UJ^N1`wk4s);9ztU~er*FcR+ZUNV3dSR~jMIhI?;r0^x5Vl;> zXL2${I#Q?*s)5wOMq>JU`TRus|7Ha5O`6r}PG_+DTnRy;vu6|<4LlPOUJ zEo>@qWvfq=U6Nw<7gS35GqGV4#D0d1-aMv*nL>(dT%;0fYsNR6mTTw>%49EREcX*Y zzxX(K7au9}>1UHfm^xV!Ljr;$Q@(&y4}zR;GC9Z-Rcu~KpcVTT$Ve>ZVy#CduQpl2 zAWXrU43yM@R1SORm`gWa`B_^;^(!_a`a{)YprFa#&$l$=xHr9eiEZ* zVKo|n#(^LH89XV{jkHEQ5QUeePhmb(d&ItZTUt}N6bs){jsBM>{K6(I%Uj5TH_3X5 zso^_Cm;5jB;ZocxY*2;m_qA2r+it%3(q;>Gi^g+YWF#9SBSD#NVDNt41+=hRoP7rHy2iP&t!sT(U_B)v0bEl=%I>K@A3~9xTGMc-dz}h zFlLFshH~{7|C-$i6_1mV+frbkDPahq?|74sX7b&Lulao^u3X)_a!F%6PJ@elZ(YA? zQvgrLhNv&Tx^2tvRX6?aoZr24@#f7NrqSWR53_+%QJLW!)WsV;UPDc-6t|e$Ip|-0 z`PDaf?l@k(qZ{q>vRI;nL^Q%JF3(eJ_iFcxc+fD@kU?=`)F7Cnbv2R`g&j$dM3p?8 zd9Sj=0?}E?tXa9r(nrczLyEYfSl{tc124&jfs*)099OlwBSZqokS##IMGe4UH^`b` zC6O4&=G03d(7_Uh-Rp_G9}Fok1gyWu{Vvx%`N36_>vV~uii zzw|#|zp1t~Mmi0E^V%DU;DN$a#KzPx+4pqQsx_AHv(K50D896kZ*QH&Ir`o{2@UEw=g5JRo? z!gvN2qMz7uOqTUapZ@TNKm5VRyL*?Nj&IFt z6c6~4`JeOdZkuWVK#P11-#tCPdgTh4;JFDBbUPi0H;*QPdMw|5dq+vPZr)@bx{cw< zv-1}(b~@sTlONr@Da$>VrMBAP^$QCnV8iKmF6g$g`7`HHewtRarfdI&YrM@6I6%Fg zz3m|$n@VE;w@>LpINDYBj^nKp;fNHoRn(q_Ihcr#E`jwy2Z!xyFypef>p})cx^`L$ z1y%+14X7^G`L?xy_m5wcifBlPOQt)qB&Gcw@u~yOTu}h5pupTZ1*&#Hp-WF7m!k@n z39S^U*0E(JYB4soJb|b}wdxrXX2$EFq(aakG-P8(U?mypl|3Bfr$Hhdf*7OUw;9qw-z@lXYgh+2V~5bMifv) z9bVWm_gBAo`E2*(p2a*@x6hovUp8*;v;e!;e_y{O8a|W-;zpIfcGkZ-2T;U1V?bnGzA2IC6KOEp&Aa=hKj9} znB}2}(v~JH0)V!8jQY{5q9dW#@bgxR9{pst%k3|ImURK8TvJ}D@&ZJZ+4o9X$*b6b z{2sgt8qw6YDqd)n-4E>@!Usr75J-Q$i>QP+>vgaQcF%qaFFDJHTeMHxYaq=!%`T;g zBclDKIt%)8$N?T~!Jm@rNKiXvWM;B;i|#=Km)u5H&lcY%?SH&4Zmff zypHUICAxt&i6&WJQTP2%#c#4i@q)J)08c3NSDsP8Aq>`)4EG`@%K{2;rM`!e`zj#J zB!g7A1Z=1qoH9Vc_BZGbE{lLkyby2Y=qO9CL0?jjzag;T1$mY1$3q1V*vTXe^H$Ds z3{g^0l8T=6C7cv6m7Dbu19u6kZ`e6_CY^K==YXVQ_U9cvdivD*YizM*nw>G1&dsv- z=?AwyxMNG`? z9`NmVcfb1jo3C%*cFdwCd2n=aw*?g$j4)%!(puj8j-gh^!2Kz?8T=Y% zmg5zd<_{ARXdI4S&38T5j;U#)Nr1L6a+kKPY*8<R3hu%C3=@+!ApEOU2A(+;nhj~aKg%B9dhp11UiVI&`F!6fiVVGVcTy@lDS=s#V zEF$UvgiJ;<_s}3a-*e~O_6fJn7$lk*08AR!myfD527Qw@oS`?6-q|^CHiDnRb3Hhd z>xpAHY;%GR!(6|5#xU>@(@A$xW(7sfqLtwtS4}9EfMG$dlN(Imymmd-`5hzWW(IQm zhYf9C?iu&oBD6iXtwJucDIX@}&tj4+HXJy##|~6|$d>&$m46OE)nZ~AubSwoQ%ZaM z?$*s4+_T^R@%QksXoURXp@U%4xubZN%S^*bW7OlM26SA%!yDJGK78=tg^k6}F5bLx z-HMiPzPqdZOBc==6$Y7SX>9xc_5(idw&8z!OkMNoE~+!9_jZj;U$G-u-?;OXS|oL( zmtbB(y9Ir+wBAN9#s^m(>jeDM?=4ghAcFtKrFNpggXR6)M{29z~tPeg|n&8l$ zmj^FZ8OjCow)NX;;`QVccxA@9D2nEAfJxtp5`0-Yn0BH;j+ImD$+#i}f4u!CEmQr|MyrpldaS=(1t(@Sm2!l%)Y84;N(poiL(#PjKd zAbw2(sQn8BbtaMwRzG8E2WaztqqzO1?#yl)EeBH;qUB}Is|k2v5TfL3TrPiqP?;Wp zrxyC9CB8k$JfO1zSrxfLVLsywj6b)=(3_C6_1-scCJl=0xCHw#Koy_hAOZi-OwCkR zJOFv|DP*Oq;K4f>8w$GSTo`SWIFprBHvh{#Bu?!QDBu8>CJ4&Rpv9r^E>1*WqaWpF z*j7MxHBN4d6*Yci4Px|^RKEGfDo{&V{_ioLr;<~$g}wxOc;%@`Vdls_{^+AmKKhtA zZ)@?~K+dy2dhKfJ=}gxWsXO>??v(8?9_FB84QF>!7_&Kh+JFJ=B(f84ju@bx+dD-u zD2S4v)LfOZoxlE9zjSTk-~PjIbWs~BH@)7{Kvy2VWkl)@lY8w;s^Itm(qeRI|0f@R z{Nv9Yt!*t*)1oPK!w)qu!`S&rU{hQVv0x}pTk}xVKO9P5;ic)wA}>ZwL_!u)(yXt} za>KIT6JLU!C%>un_t2*Z5QEEljR6~Jss4=qSh+lnX+Zl1CE?8$hKP5i<}8|#t!=XHZ{$x` z(=y{qHFq9G4qtqG8ZIvnHZ7Y%hi`oNoIhD`)9m5^_8jb(Sve&yO^n7$Iei@4g3T@b4ebyww}G%|HJ2B{^3ubJ37N4o=aey}Pax=I+7c#?_@KyqJ#)q!M>D?`zY zT1P>6nJ{X-N(E|z`ob=*Hjt49g>yuQf}ehA=Ahc+ybnpeCCFHlPEPqD8Gcj3qIQ_F zzKRNFIVRo0x#z;ia*CPSFbK#na5rq9Y@t?4MQ5~qRCFU|&+Y#`IIyl)2f$EYt?&tJ z3C?|V^Mg+xJ>mHrJ+;hXUnfD8=8n65==80d*LQaTlFa3uj4x%tHMp>Q8eE>!xA*RU z_02bCI4!_<`F!7~3XfC4xSu+;BN58$0>|1S+-8=HB)mSUYx)LIR+r5*BB(dOt?6Q+ z@ecsOY-Pm`1QCyyx)sAeUxrT&YOAw`@~|agGpxvp_*gi+kV~1(ruZEe7hQZj9zOLI zaTA8j+>@j<)YdeYa)pGFf73Wg?@=tajG`A6Hw z93m_+2Ik~VC3x;|npZIo!e=x{>MU41B<@RNnCH%>jQO&y>}pJxC(r_(E)+O=b>XZN z&$Gq)Sk8iZYvj{mGM0b1{i`9pMnh3_5}zA8?f;w8FP`&|&hOMSZUL;us!5wBr8_=I zf5e94X5)G9M)nyEjX=zW32sH{2vwn^jpVY-(kIRpPsX=^B;SlGGp5$na0rBqrq?C4 zj~+cbeED*3?*g9>KU*jB<~6ux<2!O=%T*EXGP4$(qgeL<2+LGVXIWhMcVYbD5GOj5-U*U}*#P{#kpi6pE*z!Ud|N2yTv#nt*;ZQU-OTWjf=q zCqH;I(zEkqRF(H{Z5wHHp|<5rkO$L*&04AXaqL8XKc2BCDo?8;@|omaE0C*N$Fy3)z0UxdQJhz(%|83G4s|N56dGq3jBKmJLtyFNsN zRF2Gjw{yfIABNZQ*mSdIO_UphId;|f6BCO2`?o&0e#=gRD_1Cgtpx4J5+HQ7fi*c% z9Djo}`~Yy&EH;nOhcjtPd>O+dA};&cWU7M^oMC)4#Mb1{C*kE)j}RpTs-}FhPnr-K zq@X5sa-2wkwSb-{w%@_kTSSBqwH_Q+{^F-t)>26LdD#3c+{k`~j4~m+*hw`R#Z`Cu ztWLzJOZ~{uFLCkn1UV^H=z$c{%2a;(BB)Y?4XE+}KM4iXHECsyzKEMtRVb-t8sve3 zL12f0D^XPSG=NLJvc%gda2^0$2n0+!nc}ZXCz~SS1{2Sf6w4JfpVABCyW1&d#6siGux~6t3;+Q@W^x!XGExNm zqIQ+uLv|-ljLL=6I0EZokP3{1@KNugZ{qw^^7l4FJz0?ZjyGk1M1^lkE9aDp%uSG= z6%s%ED|0|6rHpWC3P!-DMl^Q1$+QwC1hGS`2Mor5ij1>bq4+rvS3;?W^L5EWFvA>G zz4(hu-(xfp8U4^%m@(qv=rdqTy`H`IFsb_TnJhY@7y`Kv;<{ZC(oKm18)We?>~Ct#Ku2=_4UI?Pb{$Hg`BjM`-~(_tQgmr#u8gw ztke)y?0`1>H;Q35rO`cJCU?O)V-MVeS*F_zxkfUTNIdm!<@PuH9Ikq@!E<|D!5BKC zsd}og!!8xIMXbZqxSBDGLGi4wK*jDvMye@iDp9r5ba-GsRz$I-PD)Gk6N6R8TLU7K z)fntNA14ntu#+Cfovsr?l^Gn(Jx?ChzB@9a=HCKT3yt4e-eAPW#GOH22UMOuv-R$b z!(^=cvHjPGZ^tX{$iYbkUD*)Al{_m8vaP~EEYa9&*)fv3;Jl(&+>A>W*YBKt#@Vvl z`NWoqY-itDjB0?uNijm~Tjl@30V#mnB21@vVTpt<+pxI;`6OC^XvkSvOBkp{Y(rc^ z!d>Lg8@$rI@)=#xBVgPy!^$5GC}-=~nqR?d{$9_FJRlM<@7`P9I7*gvpz1M&$J? zeGIPU)=p|*%+KohSxnwuj%Th`K6J|MeJ2QUrf+_5^T$8>?9t^XcFSnT>AV))q86qA z5W{~OSz5=?v^cRPRZ}m9UmdzYvF91Hh*TxGbg`|syB97P*w3ArlXk5FGfWxlPaNyZ zktc=_=Y?aJ=~|s%7fZ!lezT=KXDxSr`Qp}FyM3#S9ij$-}+@;JaLQ! z`<@-!1)!UPA#4>N$Jw$7=v%g95{CSARmtn=v!@69hxgvN?eyK1OS@(voR~zmyRTl` zHvCM-W8adhQP&ykPvwr!l7ID=Q5qW50``aQsQC6*>ibg;=dI7FQ+kHTY%@^QQku8= z9w!XNIu_TdNo;&Ry$Q+s2^17X82K7$Mh`4`I@mJMT3?H5TE*snhNW7k*iqlCP)y%^ z(q}tw8sTQb^1Y#&;%;gz!I7Ez8P@F*pYEJJ2@<9InX_^&aKf}YDtAKzxo?11ucnj? z3nMdJ@a16~|LL2QOhWOp58)wF$J8(_F7$$dac* z2h%jf6Eg-)DU~euuX+menzfQnD;_L=PAwLZP4w;GYWk!BG5(Z6TuFToQS!xP|1!MO zOyT>0_Ubcnn@2(kOrZdPA{{8}d0ckUQz*t@w=Z!O6~J5``%R)}g92zn?c^GCz?CMW z8zY^eYa^0FIa`X>JuGjvaP?Eq?KFt%STm~`=ROft_xam5t|@VKfQ6E(N&saoObZp@BZ|;m4zQTqwDJBo7b-!gVP4br;*MGM5Vx&Y-<9j%=v`GBoY`? zV|&xg$s-S^&xp%+G8H^H4=*LD6&W_9Dxyae0~^$lRMn?!r2>2<^U&Ks4!}H24#^a$ z=r9@m3%#(OH(wB_6U+y8>Pc+BF^7t$sN;bS&848^&7!sjFw$wv+naY`eHcTk#c*`qcm_$Gs8x#^6 ztiZ>So>z$2YR`o9ouol5K|k|Q1`iW9W9>PxrysvO_`m++|NEc++uyUn?xe7UkyU;2 z!rG6gMgyl5Dk=nvRBmpIqqF2NSY_Nn^-2l=047i5ph0mAY z@S>)JUOYv#=bVicDWX=&Sg|QAe)@GssS!%J2!T9s*W*e&B;sv3%1MO@w8wj^R6c`uv%b zJ&d?I^{h1mwN2`Dy=9GP{-F^&iw}>FE$}^8t2!fR8pZ5K2sB2A9?JdBagzrI{(txR z=YRb3mo64ixE9&=JU}(jnrM#DYrlDY(Yt&<&f!uJ1)nxIn0GTgh+F8HF}51HcCXYn zP)-e8<<5==qgT4c`ZcrJnQmyy0Ivx|^|DYgWieg^NZsUCz?iC~bZD7RjPD^~daR_1 ztiAB1 zNyi)xpSVDWowI5_z#{urgKV8~4&|1i+4HWO<->!-QG#?!Qga6y)71swK6!=p8RzGi z33s;r>!SwCw~pQ2pvBw0aL(!anBfA~53XJ3MOsfk)U_VVk`;5#w2qerO3s->F3rnu zpz@tgXzg;jYQDm!zZg<2r&q{p7Of#7FnrG&r>s4f!$js!yc0DXBJ*00lx6IgR*1P@ zqRz_UzNN8y7tbFby|PWbVc-kr-yFVpefUC}2H{R_y|yFebecFQWTvbNzkcNkw|dl+ zKmF!|>sO)pjk~(s@S~mIK7Ieez5SP;{Oe!;(&m=W|NMo%&h<~P=m!7c_kWHUND&*ikzdd4j7!^$YCE^^{CS7)a(CSA;e1k88IY-UVnDQ>>FCHnfH^&f zoa^tLJA3&DAKj#RaCFj+yw?6Zlu3nysbxT~-#AVGgqf4;*RI(Hec$xQ`|Mdg272HSVQewJt74 zjEOpq7=uYPWue{LGkql1s?7jfCoI*li=ayt?0iq@Vq^Ur9j@HDZg)0+WLhQ{FjWPT zrYrXKqCuM(xZk+K3PZdbNf<51|DO(Pb?#0P`3lf{Y0( z%x_7BO@E@~#U$k>D}^?rfjzNq;K#nfuq-)FEWe0w=p6>c6koz?5ffG+Q0X%HFcmbo zdeRvH?K{!k^tpcJs+}%wBfNFY+F)Q58UfI38`|Dr*;-`k+*a9~4@q00k;8>*Wq*wf2!D-<;G(Z@tIAq08fdFrz&KlhM+N3$BuNZl2g>++uO_xVib!_nm4PBq~_Q{jy zfByW-KmYkl$FVb*K5C-!wXN=WnRURmI58Zyn(u{LG9_$A)PIiLta-ITnG<%{bw!w9 z43dRrLUllvwjd@3#p7wkG|yFkjX@m4!`SGb_!u-SnH+NAC8-%flQ6atW+p@0a!Oy5 zf~K)m!l|ctuX$(#d44t}NGiBeiW^cuFCcvfRTu!1b(ZChg0U2fGl`00ZTyMTk@Z1i z?wZ@8xGG!kjPh74^!hbjeCP0%H+X8nHRu1W(|)#3U%Yrp%LP8SNh_rkJ`ivo3{dg) zt;go6GbxS_wGM(1ok|I?)jV9zvnRLg?7n*Gf<>u(vYXehS^>iO3xO2Xa+?-UXe&8_ z_;Gl2`10VzxqVJun-(BXnNOReU|F`z@?d%YWh8Shu6`Lza#vd0(EJkd)wPKwv9=Ld zGNn=yq1M#*xig%4`+PsTdHwal3(Lh@ns)x|+rzV3+g*o8p51x$@E&KB&;RQ14Q5%? zMo3Fq8@)y7Ot6%^6SGT|UhSFLTrSZLU|DG-Krdce$PimaPBhyg!V(>XT2B-BZ zXV2X@b@ir${=Re6-+lKhJa};U_~9el4D4`?OSTanpE&2(T$G!H=zuM1*)pxEnxvkR z;WG_Tw5pINkpAEr#fztyN|$1ltJ7ZYn_klcnSR5un|<{aov;rImddL{CvjRBRe!rl z06TphtE!iG+7c-t7WwR)bJnUZ%?_d2jiIZ<&9LW?DVPOcJoPh|FYWHXd~tMmU~b4B z2bV^hhbfD@!}ed8Aavl8#I`Yh^zldM&Y%14&OO((S>NwC)cU1rUVm21Q~0(5cZ8B{ zFYjzp-*%*cJ_63X)_cXG>N8VTrv(AN&z(M7k7P&FMb3>njPWGK>XRy`YjAGrE#(x8 z91Yxy1GKz&p|jB}7)(1-RDSsAK>F4Lq!_C^0WDKBtWC26HpuH`w@;mU^b9Ha8@R8m zG`0#}_hXqegYXH^Uxp@+C~Jb?acl^z=14U0`Px`|L{wCaDM*^JP->5l7oUhjA&Www z@E`M01ml^Bs}^}kQ>4l9m!W8UVaD0-m?L1V6z?^V3e5Hn$}LM84HG%-7rW>_Tm)q6 zloPz#5x@gg^WEcdZ9z;O3>TIq4RRREnDq;-s(V=r@f5J&Wow7fNE5rhB`FTAYhuw; z#Y@m&MeuPgx2Xmo_zMTJ5jBnC`?4lg%#|9eB7TZk=*=VGfetB7Oh-*;u`DIT&fNE`b!rI zg8K1st6H~r46V&_;H{f`dmn#r!(AvU4t3nUw0q;`O|yi5{_6IZw{NSKFI_$M?kU0A zboBK20J7@84!dCq#8M?-lURI-V;(W1!eU&louA|!i)y_wC;s*4t=ObD5AgNA6h_9F zH|bL}sr&@f86zjW0z8fi!YleKgf}}G>VFK^2kIa-}4A>Hd*a9MRpn97hDnlD8hvWKF*gs)@*NUy?+0vKkM&u6&^l%Y7{2zCpaz38Ajm5pIou|0qhOUv&=4r|YgF-B*LvAcWOK2y&*s=Z%_dJGBB@N{srl`ca)Zjf; za`hR+s)1U?_ph6W5y1v&QG;q=2rFK8wlyE*nO4%>UY1+5F1LlbC^0C;>1X@y-oIzO z!!VjnxyC%L_^-9jQHg0`H)wd)5E;C9^yIOV{vD9OooPOSSQ`eeu{8GYyYUVlRC$9i zXpJ?`*k{+L7m#)t;IvZuK=W(7`{W4*dDLWJj&EA#4I>TwLI@W+(*mLWhgq`1zF0iw z1((#U{zs%Hu7;X*lCetdnW_YJu`85<*C*2?bY!)F2YF0i5lab*9ib|cJUB3YEsft~ z2QdjGIA8t~Rtc@SaMtS4(b+d|&XlM9xZEAXZ%$Nu5{2<;v? zs@cefrMHcaiQ~DO0_Wgs91WuLmGw+q*>rVxM?5FcEWCSmKfZaLBK`S`yH=jG1K{0j z^co`Oteri*W!?TL!14ZzXLxFEfA7+V7W`|+U+up%TcQuZ1%CE&Ut@aX+7+FYCyySS zx_W)@((X@x^5ehzyTAR%|M8o@`sFWNZ+-v4eG<8N@!XFueERL3yRIIiD*VxwXz6Z| z*Z%%_C&l zbyRGJWAe`7!mFZEz>pgU+%f+*uABPZ!x;mhx(d$PcS`TWhY!taaPh$j9`;0d0 zH?br-c~z}bc7pFa#gs4Zeu-*Iny*^Kh#5Hp%{m~xI32C2epPD9pxHuNo-rkj12w9% zTn!s$*RcSCKz+XsAA4kmqKYM#5mlXCT~-~o`fQD3R14dC!XQc1O1=8J@TJgvLL9`! zrX#9D*4DLVX-y}n4k0@Rt!|nEs_TKveoSkaN%p=P!f6qpip05DiTyj zn=lomu}^)C8V=@zUI?-$3zbaRIu6$_{u0jqGaZ21wf0GfmMC z_6vEh*~#JFkPldJ!pURp{^Ws^%wpKTzJ#PRhY>yXK_%w-~9HUKL7f*6`bwOXBs2x z`04)3(+E4Nj^&mY2fx8d-v1C9N zmhw;T6(?fQ`5uagV*0=^(o*!yz_BCwA3c4h(X-h0%AV1~3oSs;GdBA5+A2qfB=Y0D zI{m~61bmBDebo9y>e9W1pER0WBN#>m8A)rH4^MQq3^miZQqQ&7uV!`fY2(9c1Q3Z! zbvu3CsGkCOQyR!zW4gWZ7kB}!+MUCND?SoN>thF^hE>C@>IJs5j^S8Qpee+T4PtHO ztkk5Rs>KYbghLEkU=oXfYCwz|9;Y9mq+cr!h1#TF?@K}A{$#@>g@?lucs#=hl!jXc zV`cJihic>Z{G>OW@}V*RqNz*NH5(AAZJpJs+O3Y@dEJcwe5%)0ho3ridGFFW9fN~?ovvnDG-3@@`82zt z#bm>XM+>-rX8Xk%(+}FBVGv@_(D{J*P5pHu(9BuE8kMaFtS?j5&RE4$2e(qhoxyCndHjrW?b|NRg~XI%jnOxJ{XI4%H#`%^}B03fn5A*2RHsUgce< zTD3RA@B&&qqupHKS`jpr78>;};+tBkNpskMZm5m(vv39|4xWm4%8Dwp5!mW8L;iez z`Y63|2_|YOt|F?ARg|<82bYGc^+Ym~sY(a7vT$8+Rosto$!a=V%WW*(xdgm(xh6A4 zXg1=Vw6#u2IfW=EtC(T*I98(q9Q5T=D=yHC5MxDgpuBJ*0mxRQf%bl>dxos#`*qbD zk}XA54JEP1pVXO6j}Yr2oW}l*QL7dYJf1|475Ke%R7j;EmI+?<9zY=rr;NGSd?JBT z&5ti6)w~zIOlJ?!2RzL#SHyVs0Gq-+)Q=JX*C8ZIN*l#JPh{vV{HIVrhJDh__-N5S zk`8VjgFgZ9VN8BiP#}ib9j>GA3i6qvKVj9|-rki13*`>>>k0$wSe~7;X0aURW2+DY zt&>XSX`3#&Wtx{SUU>54*}ISFKrr)cn8C^<$K_VUc@m{Lq&2lzMRvhANUbs=dgC{L z|Bt`@fBw^dy!ZGSBYSS=0_3QuWs}s>@XtjP_Sw#+Ty^e;TS+<%!1!vy43UmLMXyz2 zBp$K8`i&1|WBr$?hB4lYra)=Xq+FSX^Rkp{LIjj{y=2Ng^)7rPo3d0PlGlh%98Hk0 zhiN+#b)x@Ebojh4U~&!Aa4vK{m0T3Ca`z=B`Of#0Hpodt^Hv)ka>|k^P@|gCmnBd^ zPW{(YNSAWT=%=zInF2kf;p4CtJ`rRx?LasVaf98;vJx-}tqYbeg;%M+Cv(K21%^}w zRCbK(@x4bofw?ZF3E^mnij@P@!Hw*FWS@9QmNv1ptAts#vBV85X2zRk&Lq-5y~kN} zFf&=&5cRfh&6Gizrk_RM0XD@Eh?&IA7SCu4~$m=uagcHMGmqu zObWCN9dWR@3HKHLC1o6{LMvp$b`gpzuIi4h5>@XTr5P^NC3p`(-H1Wq0Q#uYSQu&5 z%t{H?L^3@!+hF*TcKMQne7FSxeN%Kr3Q*nhk2cW#h}{$x>&AkWkP3uQ3C8yXs&p%8 zp(>gG5J`fm%Z8&BA3l91!BP^Ly^>@VYDEr&^%_bOhl%esruzqvo<7m3Xvv8C zq&gl%DjJ2ru2HY>L{zRgET`NxY?w4W@zZHBkm(qotKKw7sG7xbviU#JuDx~+sG>-#~O%L1% zZWSsHob@#zdz5zC^~Mq5lJr+uYaEl;#qf`_o^#Bfu-J3^b?%Kyns?n=c~N*FzN1eF zsAUyz&QscaGQ4*IrCo?|6FxaZt7Cauu+RBC zv31c2bXKcy#;u`fbhJ9v>8oaj^fK@d@0k*vCoZGKvHNEabUzYE{g^QH<=v#hc2~v) zLcgK4EIQ*}M-^4cVS_kJm9%%~Z1&}3I?dCCR=PsAH9YUU%L~8w`7iWioo|ETmo8t{OuKLF-N^%xt^5T%o;zFf z)G4=Nn2^GZXtGL#)byNQxtHL?)vK2Q<_YK(NB8M%)nROH9_mteT-W7LwXCrK(90ms*wt+v$~E^omRRIoY?tMAvToTFNef@2pYp(UiPCem6s>w{2<9;LW>^QMY#VUINjc0e|IK3(v=i%$Ky01@3k-jIP# z7jbd@DYf&0Y{1=aiQc(G_n#f^pL+RPIlyv{4ZhKfv~~+4p<&?<=Dl%UfNc#_1X_&6 zB^MN*ci+HyWA<&WtR)(Uka1v(;!M>EB-&O*U>|h!Y9U2Xk*XKIWku+lneCFbSgLkc z<66I1rmSK%VFpfBbQxb~>eiT7ib$8s=)-0q81?~$_zbBNz5i35kqM%#TvDi_(+lb3 z9c5reb4D}o6Dz-%F>|HN9{H1E67-c?jJX7rfug&iA(P$5E`a^B9{I%qItqJ>h<#wQS7WMJ1qJG=npuKY!#Vznf?^9K<8Wj$+o_m~*nCpl zwCJ>1HfumlCY*P3`8d2 z_y}q$odU+cSQAf&Q_AjN&WXTS7YCM9hYb@ytx^}6G*T_e@iuXK-4n9LdPl2r9*p~GABvg=AuE0*BaS%rn zl6^^}ELl`Y8$H4n=>qe^Hn2wfWt#ZPTe*7*{KTYG0#>U(>X+J;V8NVhp*`<`9w6VN zq70u2l?f>;FFpmc^u7Y+sbmo!km01Kzhs!`(v1*3~t;E5!*RWg(D$L`CU=JG->dyoq z0tZqUuAIG&?##pYa_>6FDJWS36zXi~B=%4Tq=LLp>0`0bn=@IyvM$AT;E13}Sec2x zqK6g3)C!j%ha^i8&J`ugsyn^sBukd3tcIyQWA-Nd@MCFV$Pb^%U~JVK(g0?*C8!fU znmfb+IiqLb!F%HXhuOdk6Bw}B;kT4C#?NyqGsUm8UMe|W47WOxEft6nQL!|koH2Va zsko_0sY#4|aq#-vyB0P)<-=XOa@q0TjuT<@z?pU}`&6k}ETe*{Dg}zny9#B;<>iIaqOC8xM<^Gf_31~)M-~Wz)fU}hd`GK8QiuGw8fdN z93qbKHI3LpfmEtOnI~*z_=YZDWVMlerEGu?EmT*FD6GLtG-v?@;n#8w7OsO;EK6>& za(G+-kp^B2ZYGA*W>}wYFfl_|f=8e#6P4@3CE?BJ4sn0Ty!e(A=PUYe2*<8XEP@=U zl1OUPEk6XkibF2`Nl0}kN03lndKq3F?>J7d$)xnn#?bRyJUr+?aN;{ORg z{2{p5@}`@B!PNjC9_)h%A=MMpPo2AFYocO#`io|`10Q6AAREPAzclYjbzMUR^`LZEG?)1LSTi*Byt8<;NZ<#-r`mDsyU)W3B zQgr5qc$V(bP7mBt0PErrHnqTUGZ}A>U$yu8^}&{WDV;u|UYOxXd@yv56^bTf71j8l zk4qlFku)%E0y%9fuL%M0au2%#?3I1JSFT<&%76AuhnVxc>*Ul-p0q2N$MRvHTh!;y zci&M(Qf&CTqXOPMfBww$pD94vO%YvqY;-`M%mxSR?4R7d_w>*R{gyt!+pQ|>>vu<| zPwE4ldbO_Z;!Jl>u(8wjQj?>309{aYA0Bpw4^($L)^+E~E@=|DE5A%M%_5#RZ=)ea zsN$(dggv^CZ2&aOBoG9ti9m^5G7FH9+{{Loi(6dQIW8?EVu@%;b-)xE4j$Q7qocHH zbWLf+k83t@Q#&feALgz!)38Uu;&LquCc02Rc*t~~BWIYMTX_luH>gU{3}r&S zgGZuFl0 ziW-ook%ROq32CT6M3l@=Q=)@JSm;>KfIYT*ji3|9PW$|K|L)(p)#A!k#|oK(-20oq z|2q|f?X&Rc+VyL+*k*nj!pt=a(-i_us`U@_c&C6`N)P7ORtzeJ;=V&?3Jc+Q1G5+o z!_E(@p`XkgixLcO5|!UyAHUQj_z9_(3hl3~eUlf$#Q};L-hoY^lOhIb6l=;$A8&?# z0K}zDU`Tmdlh47c zpr!2-z3>+@B`zrjw$k1nJKFE#4?q0pKYdBvsSQQWYT3Dd{PwJ)056>T!N(t40YZzw zNtCf5CXVC*Q5K}b!#5>e#f?fQ)rF()p%PhySgwej2d`2UI7|$&#JB*)r%Idm$}M9d zh$(rgG7z4Itue-n6xql=rI@E#p;_(;9&Ys(8w!&S2LhiOF1STiHS@fKS!`g7W-%&t zF(6@8E}lyw;eeDj24Gp~0$#dN@L`6a%Cz)v)$P(qF>nV^poV}9qS!o2ITt;_i))ou z;QSXbIK_@Z70Ag!tp|sxiV{yv3-&6rrX!P^Tu*2%?>&5c`_5e+(cbQ^6FiIsPW$L! zVaE`ge?NQm^5vU1-1+73^0V#)u>X$-P~#c;>IolnNXQ1geerz%&fR;D9zTVavBRZ7 z@P(x3PVO8I#L-3Em<26ui4Z4uz;?fMf13c)k@e_)W-YO&k_9Ow6WN?Ium4@k}hk z`=v57sgE7C3K3P=&4x53G749PWi1V`*Oy5k!2oK$EC%P9fy~8|w}N=^?7Ecx(I5`b z+fy!G;(cp-4Aom5G@XQEV?na_uPV&JumAx|Neuj)IR`s=G3abWwslpfobTfvSVK@U zQ#D>OhVjxaoj(U>q@H(vOB+9E#fi7)j$&-5fWw1H(Bc_k=kr^*VbXyAYG7Pr^u`j} z5Q?>ASmliqt;8VW5LFuTb$)cnS};+cPHG+z%$)JFV5{3!}hHKxgXUE3}Mmw0bOBd|cJ9pyjhc|CpGGu4yXPGI_X&_lUg?@3ebg}3%YX8nk+~^mzkzAPIEo;raPP21RaML zFW*?_Nfg%SIOx-8P>Y%EuYdK*VTVLvbA{o>k3RW<-j1TV78VBS1CWvW=`t{-VOI@0lPp|%obCWzsX0pLqC+4A2JX4qSehs;%a%=gV?M%*&ePv18 zoS5RgT3Pj^y4r$9o$eH+{*U#dk{Qofd9t-q>>k-+l)(t1rPBIG9phyY@^%_Q_N7gNABC=pu9jj1uEbd-auH zM?QiZ3Gf|C%Yb0XRQqKP^%J;%wIO4ZM5E*@ki z%Le6T&jk;NX>2NDkzJ|*1d0R!>p>WlhVTr{rVflTL4()9r+OiAuws_a`yXLNg+bsE z9$vw%Xu|lDD_3sP#G|(+zmF_{J+MK} zwqTP%TQ9a>=pGm;b#~NXtcGuG+mNG=nSnDX^XhQ_r9D;2wm}g!H=Lm{OkI!Rn+lEW z?JhYucx^|4F*K|0%(A)Rk#uEaY5wrXKhs~kbLeW<00h3|Ih2@2D#rTYH7tUpqYD)JF!JbDg!b@xce!Zn3$`L!620>3NRd2$F>;;Zd zp1eV=_%v#;lXU%?qQvzUe|xIHl~9&>-C(uh|EdNOz88z|a4X}=5F!IeyI+$zY#a5? z@_3CPA6XQMAR#r}XU_cfuYU2nKmPfj{`AF*r!1QtYgti)D(BCf`^gVK`s-i*oMl&> zyfrHZ?V#$_Z@h;@qooz$y<$r0J%?)PNd#B?Mr*>Xl;v**GuYt^sZd*qhk(iyUIwWe z2lg|$6WMUKZ>kdM{;iU|Zc=UDC+S9Yf?4^=TzO^n6EbOBzZ2|d^nW&ay(cHT!0)WA z#rZ)ta18zur0_{|K*=5R`8QcQFfyXajn$()W1vOIZg4DVZpTmPHU=qc!z!cLX8ECer1`3(Fk`Veo z_Ej$JsgW7^acBBR{9#Qj_kY7LEL%YP27G9Xk15-*JpnZiKv&9Bim4jKCX$UF*53-# zg{vaJ?*#NLqSM~s1x*%^Q`4Fiz(i>*_)Jjb{=AmNh~})oOkVt(ngiwQ8F7;duw_vS z&HK;B=iqyveRJ8?3_+D6v^`HQGauZB(t>4KOSQ{a&n)=O> z!7|zr1-YsjTDDVC4Of(P#ycd>VZgB=;8zDH-Nn>-J~{&JVK7K%iB2;r^%5i77QDm# z(^%A^mqY8z&k?m|+A0-&$ctC6*wtyk?(~_5wuWB5{Nc$DZhrgS9Y|Y6W0i|sHu71@ zOT#&gb_ zT>MCaGs$3pToFWo5ei!U;n&*@x;Luc#ySij=Qd}U3dFTpUzDKvtPUYmG1RPdLtD^d zFsq)V(!yKc&N$U!lSB~zn}%B*K+S|GLf@XV%=K?|RcAMK5m=RL%XMbYT>DHKIlay> zY?%DDJp*#Fs7^iPvl0($##&3Ac@0mv#H44I}sNJ&I$ym-lv^6y$(H z=YHyoSfc#Vt(&ipzPhArK&zqC3831Tdt#V zrG#4Pwn5!9sc{NdO#;vo$d8Y{w!2Ul=M&hqU=Bi0j5*H}#!K$L%-l7H-cGF-hP`w0 zkA2E*GU!l(+S9l1h{xq#RXaSTtm+m=n^-!Ah?!^VTLw82NKi3*>~991*k_!5RqN6S z(zg?nHOu!I;bffT9ed?Jb z%%aluA7wvq-QVH9MwBd(O(Mx8bHqZqe#RjQ2ZWi;t^CmPwOsxlDA1<0J89qjfBoD4 z{ty57Tgr0lgAe}AuYdWApZ&~4BtnhVeE!AffBf>RAOH9#q8xB*)c?ly8()91_4%WR zhHz9=_2j{mr)t6{uU$_BOv)_jyIdMCrmrZV@hx>YFnlRYy% znQ;%GP=24ja(nR?J^k~z$vmu>w@o48JXuRr#rG+vYVA{9?zh|lSz!B(IHd}2*?kfI zRMtT&!b;?W?38Wm_y@Oc{)d16zZxU?yHZ&q*=zC9aUByW`QmCv!aqyTDNL3k?DZr9$2^O7zGi6%Sqa#BYU)}!p?tO075lg5I%fgG-1i~iK z*BDJ*n0|g8B2ALv|6C&Op414@X<$uffYTs(TLT3)9Fil;;Ya0B`VyY0RVC$Rb(Zs8mvGY;`fWWm1Ugae@;)?q(Y;3Z4HqAsZ2Lmt^*3VB@rl#Oj6XxvG1fbtNCz)I zgt1n#J`Ru6dXBCl?%Nj4w|Y1BfxmM|Y5g55sEC4qA)~i~M(ah4h}js@;P8nP)}t#; zzXn+TB#Jn`7U!JrsBXQUGmf+6I6Ae4W9%TmYuB#><-y8xuB!H(%WM-%3%B3B)i-cn z0>!oi`r5T?uin1>;_I)T-~Pr;x>v7Vx#T#^^A}*rM{ENA=(WSLE+6eR|1ea`Ye0S( zHZoNZ2CV;hT|Y3EClWsdmOFnsM?xZ{T5aTRW=A#Tf>H#n4`%D+SQ_OF5eK=>frLvs zIv5S(zkb8VHPXWOaI=SlJ26Lp2~6T8WlOoZ!BBni@&)Ig?@dxZ9q6=wpcmo41IKUQ z{^qmKKEq7I`7k=8cXV*za6)>dQ0TbJ%qR&o?bequJqXX1 zJ#BZlU8DN_^S^G8XgfgM9LEq~xY-vg!gPnQv<{WU(wK&GnGSjt5A%*{6II8iYiy6G z{?x~A7rs6&iNxx}qYu!+E&ufENJ0hw-lU7K2NM_zkd`dbT02+PF&VwG=ejwCZ9eGo zthn@H3<0>n$H@#9>VvpVjl zpaI)AhOoowbySIz75I__K;$S#3(X3RlZ)v!&SUmvP$>ptU;a?03T+ziDQliH&0HGz z^1siL(Bq~gC7`DWO@wi{j70`c2u7uh0jY~EZBJ#BXrPbQu3`o?v9&KUF6^ea zG0x2NFGed!F)*cCB>Cx~^?IRI+<+A3e{%2Frzn~IphCh984IMxQJd2DIte3V1@tRS z)`U6qp?Tq3)?x>fza;NziVb-A1*Gx)jbIAcfLOd5E+w*QG?lf<$C_kCy-souF|dk9 zs{}T`Jmn7m6|y0=|?)TaVsnUolFD2 zXP;sQ+HhvQBrL^L4mSE-Dz(9|@XX!-ld{CVhm{-fA(KrXq@*5%LV1;`kq|#Qr}%|O zks?O(lTf*R<ol+it4(wX*C##bPXL62fIsuPh=$~FZqkIPw8P}G zFJ%3U>P!I<6k3atwA_DH8j=u;XE+_wlaGiP*7Eidt4Qhgc1 zhD&lb+|`Je1~xCV4FSVt002M$NklX;lK6;oVwMyf^VFbi4nsf>#Ok|<{}GxR1B zGe^ik3Q0;CfVk2V zun=B)1YHL`N8Ia~dElWVN`CTh%}%^7aaj3Lmc|C`R*V?tHZ{dSv%5Yv<;It6_kb4pgX9_2(z{R?mIJN&h>xx_{o*aS6$j=?2j`J zUFzWfy?ggho_=)Zypi$UM~|OaN@P{a-o<_Uk4-G-Z#bOj^r<}q);w>qb8dVdDu$3O zk-F-~aumes8ylT&lWqg3oHz73*^lm9JYd6picP9qSYB@0(lQ^C#>IF{QOq#VP~4#~ zQqUC+3Z=J5PBk$%Vi}L!yLJgX$9yVCQxM#k7PMesGYRf8bG1=Zo~(*H3>=jD_{n2} zGqYi?!uDU6qnQP;B@DL2d+oB5Qs3!5RaHpH0O;%2mPXuv{KVy7L|}B?Nn)?w9$&t4 z^|K#-a(?H+-3JdOPc55Xew8+BQ<`dCFH;tLXH0LRH`t?& zv1r_4L#GnMN_h2Sq5b*?bB@YGx z0J5L=z_0_lG-67Oz4iZ0(lH0r;_-_E=h(e?ZULMYqrm{IHrCHxV3_lUGkgkW6K53~ zlRXAr3-#v2;jsY_H>!XJusV<`DQ7+q>p5PCB%@RJA3phSfAfUMtt z?MtW51@1k=-fORUuRWfV(;GM6v*PCQ;|HJJy7SXtesb=@B|QrN^56dLmmhy}>GG8y z{pd%ZfAPiLd1d$U;nVMa`=h`9tG{~h-S>DF_3l@9?onfnU9R(|zxY@m*Sqh&Yo`Fr z>fI_74gXrHg3=fX2Ut~w3`d%A5qI>$CkDLF#mG$2N!&puR zVB8>qupEpcXkhr6lGcL_<~oPG6wUQJMC~kzp2qr-=%2*R;Isky3Hg&)Aa6w)Otapk@`Rc;% z&iB9b%@5yyx8-a{$L0m;{WXS8v2`l)DO0;te(S2_9^4iSSWGx64>8?$d)=73pigNl zaah1Rm)L10{D$1}Z+J2^E^X@;7lh2kPN}E-aG0zs*OZG_?hR^#M6gTja9x3$Od%YY zWp~TbPNJIp14?3E(|Ovm^1B~n>m%vLzzCblh~Z-gr5>=b-y8*6l0<7&L;R^WNt8$k z-;S_zECq;x@@yVPYE+bS;7^u(O^_Q1#ao=w6{pcH;B_75Ospu znD=nV^;BuGV#HpDmab1D_~!N(-6CXtst8LySon^sb2F>Uhi5u zTWBbrYTYL`t3FICx9S!ZW?v{=C@uNRS7!yb^-BaHt&~6^ipbbilmf|ZZz(`UrV&Tr zRXO@sTcIoW9E$azLvFx|)SA-(HH2Xwt=)a)^}|#R@a>kBj}{7kb}5&tSU0>_UjVnW zb;*Pf3807}0#Nn%x&4Fq593~4XU#;f!7}n`yx{7!tKJf2y32PgBEE3Zrhn%4v?JQ&kX7*lkw!J8vfy|}TM651cf`m{M(hk->}<*nBq7e{g`3TpPBmc&Gb3btVw?;xJ#(IN z`=67Z!DepE?eW!}jTKOHNk`rNsz}ByaZRj*>nJhEV8pyF3n`2RgdMyWLP{nb9F!bZ4&04F`tHo*PiP+QMd2|zPMr|eA=ivm?sH&piz z|M*Wo`RPwDUb}Jg=6hbZ$?Cn6^B?~CUqAWXZ_QKmPEFOx_rCva3yh4-sTD3ATzLPj zxBs{Q`d|Ly&;E?FHqOXR7%612cOE?AjF`T6?-}3UKX+Ztov7e3l(Ko4;unFmxfC7F z075>7g=Y%l{z8_!auE}kAi9fJAFEkU-=`ks2JVKSQIcB zCw1uu}Zzv5yY#XeC>bk;9 z98S$u?kWsM<9l9DnrDt!(Kl2B6SdOK_Xdxs9cVN2bbrS}y8Y`17epk=$<4(5wLlRX zL=-YX0@8`ho6Dq2qbe?xM%W%!+yF_8J?sW+S!%EWHgMvfr-AQhmBh?emL53Xls2$R zyZ(dhY=8s5o^};7Nh7EM3f+}3-Ri=3rastpEmN$T%;jDA6Lnr7K;?`Pkrmhh6YNPm z@o-;K%Ot}9jdvL2UUrv%nw;_uW6B)>Y2W-fZ(Zt`B!($>XsAndJou0!@xpnN4Y2wy z&=9w>Hk@8gdGV#;HP5EwVxH=}I5m^^`QyWpK4SHiG(Ps5{{485d`?4$dSC_ z%{Y^QB)=j@1*&{%qAt!eqT&?+ib^^}Cs`GYhw5q>dscu{M5$(< z>!!g1S0CyUB;LePOzU?PsV51tZ#Nbfpb_!I3}@2B@yMC; zs=JC-KNn&SHsC2(FZ^>|%|#h8C;rGFOf%~Q1G6eO<9_aCa3(aNh#A^V1cI06Oi^rd zdd6g-CXvL;+5rJ;2h^7@#Psb1RPY+>cAod*YCWXFzP){Wf$G@KTRlCoH;z%P^RG{y zsX$FXxqIjCWpmnY-1zbjEaCRWg9DTJnS=dlZr{1{;BlL7YggcmZDbUy{?}SFCjbR% zZQM?;Gq2`{STA8nz;0R=2ZWMlhpxfnlWD;;aFG&<9*$2Ap`yB;&cw{)%yTn#6T0~) zqS0nS;a^&%``>H>A@ayaI2$X3{_qkdBN>ZQ@U3vL9WUrXx`o ziy!dT6n$}zyk_4&K0eV~YO@q7Y)n81etPI73CS;wwqfjx7uEuwx3jG3#8>cRtM8T(VCkUTOGl?VU5L`Sad@1g>p53+)B>I>0Z!cx4edhG|pd61R5L zh)FxM4&~`}#*etA_H=jYA#Uotn8}8zsW(^G{KETM69hu4{>dFVfM={VpSqp0=J#;A z1u#J(t7TDnCWH(-J*xTAog8GkKUO=QBM*uun}h>_l!S3 zR3!n_;4yG&dE0n#M~{c8k5oj-tS`qY%PJ{`s7ve}8ie69p0&}urX$BO z4kZYK=`b`}pcX|{>0{!3wGC+a4)Hy#)ROu_`iA{^IybvD3Fd3uD3JMUd_AoT!@3DW zA9YFynWfeLGy57(;XDt(NqFM1`L3Df_BiRl$G}R7aZ}*%AYYFqZuPhfldh9#5cRPk`l!~535F^1ju-+R|cC#&{WUIs^o#<;bgZT*i;witVn(@W5@gJfb}dSv7=Nm$z7 zrc_G{jh#&+T(AYZVg_Zxk(babaQogpntgQi^w#a$mbF{9Z=jNfb;%H>)&gUGYO+_KmYZw z_xJz&`jrD+X{l{wD}RKTSA@g#^;QqZ5a`gjNi!*>mpH{AMkV7p;#zEU+v(zY*FgKJ z!wGrSXX|U8tn@)hI-$|mDm3x9+c60+cbF8kv~iA2dF4?=Kt`siRv(A6zL*pahuxGl zoeP_bjByf|G;Ak*Eug8+#%aZ#A+0=QE6MsqoYTo8?*`ur3}_G6^HB#`Hg~!^Sz~wk z7p<{p9d8P&6l|#Y{(BgRH|K+|J2K+#0=0R%E-J7x`td(sl?B%H&am!fjszeA1Q(8{gyz=Y`e)OYu77052*!TLjgk7K|Zvpuu@K0ZSB2-ns-_1%K{#-{oR* z(B=>zOoXK>tg7S6H{ZE_y-K$z62a4iuKe_}&ag$oLJvkEUnHk~G!^*P{RdW-|MHXHK016Vkg@2H zh?=Z%=9wsIw!VyxD69GecSHt8t*5%C=-h;TL;fmv!v-`e1SnKblcz2WnX>Ycz{a&x zSf7AYvo#2yW~olI?}$jFyH#D-4v#P?Gctl|CzkrSJ!|x7;^>l$%$7NXei^!*O!}Y8 zVUW9nQNPd`Lj4wft9DKFup5Jd8j&lakVt1@DJ4c=Q zx}8O(p&D-MTn&_-BOzdsup(=>!Anmd=S3jKu$!8hiH>7PMpoiII}xO-b2&6j8;^ns z_LDhpvWy@~OfD`aP)moI<+#TOQ{zWwfdHum}Q%P&9p@I&ha9z1x| zG=SZ+HnD{wHG*st@WaHGm#0;(CI$4SEn*w?%=DF_*3-i~_aF93v>orPHC+y%=B>4C zaBbdya7hDIGeryB5Jc>>S77lnE|_DJo~5~n5sHvW@=MU3q|~-)j#}865!W;tpXmaB zt!Gdh02j_5nDTS6S^H8G_lBV(KcQKcn` zKSQA7&tKT2iF!-KQ_cQK%}8o`O)sajC(-`ZgG+|H>XQ{d?d*6TD~;N^d-wQ$y8yu? zq76{Z`Xx-{(+yVi6dGNSZJWBKXiEm>|6a9}-jT=X8c7^=uTu!lLfnNrebvezS%x`AZdD778 zZcRsOgEok-XW7@BWeoWgdL7!z53bWZR~XS#8fxvd30AXjC>YXBS$^bh6+zAo%;`p) zB9@WmA5$e)IY>wM`&k5q!eBb?4mbowKSn5Bzm=snks3zpa2kKYUg~wPtM$)^E!4{7 zUKLFKC-)wkz~-f=x~X8S9na@`V8zyjSoE!unVwVxAEKVS@CPgi`?{QPbqen z7u0lY=TQ9xO-z#x?@Y>Uhzj`GC;Ocwt-lWRE{oU_=_6zfaerpPlRWxRtSBA|Y^Ql6g z7DeXYgHpkE5c5Qdx&tb>!fxuLD6J!dp(Zaj_@ntK<}vCi!8+yV!8I7FR^X_-ce<((@ z|B=9AVGx5|u}gFnHed^ZhlgR5P~|AU-!8%_6L7196&u$kU2l9Dta@Iy^3G#7q=SQO z(^_`nG4GOsPRdC;XFXXN%Dtk09K}n_>`9MvmDS^&bl1J`DUoASeD{(-Lg3Hqf~wJ7 zJ!M5~y-%H2KlcC`YHL+?vxD0qyPCCm!)=veQg0RO+$s6WP^LO`)eoApz&Su8VJ{Ok zuU9w6N8k8{ojrvSoKJilNsdomr}5|C{}=nx9D(fV$W8bNr4G-O3z%v%Rj2EX zHA7G8;k0^8FSgz=p`={)KH>L70CqDwwUu64Ob~dI2*b0BB+|Y0Nd2SS3R?vZ@u*%2 zO4(87=o%~rkr2wiR;hqKO4MU^wqV7xaYw2rFKW_zsteI%3L*tdR&f9k%%LBtg=az& z0Mq)Mp%aJ=>%ojU%0$snK9+)G#Sk@RitMYX6S;z%h+D0UG*!k7oB(e?kiX;+-GZ;_ zsfD@7eBI8K01fW-%hie#8%@+UQK~jTWJx5UXq6m=Xrq4VaC1aV@?P5$^*StU zP+jg&PhGC51Z3)iY{aHkn;uBDXM4a*0{~!Uma87RX&{=Rkg9|&q!q0}RIhCT_MD#; ztzm_EF~;gO7^7a6C@Ka(?`jt?Rg5Uc45QaOPZN>C2(A^Y0*ORw<9V32 ziIS+>C!qNr_FBvTKe&JU?>=BuBj6viG3LJ!Z~(aY38XVyF$2YG@qc zdocrgR-D=B3{A@U%%X&JsAaN0P|??17E0^70Z^=msjnK!}FS)8CB zHl;3R+!Dzrb1Xsto+FY1f?wqP!U)w70z3a^?7k0F8q~?GU3nqCV&FftbLu z9~$E{TO8e+>W^-g-HdY#DNFvn1-|LwXi#Q(O|jJ$r9V#Ai_R9 znxY17&@l*TPJF7Yc@ywdzV0BZPgw?)K_`^l0xL%~u_;yOKhPp&cm}mxZbS}bgYPT8 z4h}1UZ|QthK=*S^#dZ>JQpOue2FHrUAQt86Jdk(@@};2YQjdVu)2iihq_63Y_O!O` zE7NEW&89Oa=XLM)c=_^}s)bxtG={fPr>6Ga1rCCafI>~YP=s$Y%4H`7duvn&j!#P` zhXNTR89zoumbv`hkAC#t`yc-F7r*?sUwo`de0+GwU_W~Ioxk|upMB%~_XYiTZhfV4 zJ#50Z$^TEz?VP!E{@gon-T3mWuRs0lv%}*fq5t>Z{mu`*|2^AvfA#fOPY<7PE=u{$ zr=Qsl=-@AZ*c-e@*yOJ8xQ7tmD|qQd^5Mw*^fe+=eN11GYw}5K$ts>Eu{cZdJ&<_r z=K(tLU8)i6deU`9A|UPYfEn{xnO&DW@s3{1S3+9%J&f^Fu_Xgcq;OD5Zk05D8P)NJeUWSS6Nelif*xheaMkbuLstBZ5*+k{LU#GMv&kU~=#QA2!6LxtI*W zkMN!jM518#!mf(0{EEfT(5&>F>Oy{=1x?`*6lXHM{7O6obG{ZF=c6ivspQ~Hi50O> zsvw{0q(Dx|3)96e=Noo_C3`kEVavm|9HLHgqu zvQRqALUZW$Sf;$1)O}%0U;U(W{QCkBtjU_BiVI4_Jr-~kxvHqdL zK0W{gOLv z@@RI6E&p%YP3*??x2|8ev7wMbeW0LdPNLX1x+QNupUbbz^~7U1>Bzi$|KV?c|M@4s z`;>!H6PtcfFX{8=5+g|1UB#-ZD{RdB0y>1?5nFgQE|E^v$Rfn*C4EXOLp>@J%%&gK z0`~w4g}07U&F|nDwsKXc=bY_}ak^upo3plkv0b7N)%KO!!WFCwhs8bQ#IK^FSZI&h zrnA)J{KrHw5Y3aP(@sWitGwV0A`=LQ)X+HTU1b>*THmxEtLxL8;E8QhjHV-ZiwskZ zSf%aJ7X3wU!bjbu6Izl-e?xHCK|(DlP?Yrz3h?0IallU~1`H#73)e`MO6l7?Nh!Bq z8f*v{Ix~17xSYitwxFpTkwKe;;zFJM>{lE^iR~RD)RIl&vcT0z5OmAb-ZL51lo7?) zKGz|VwMQ9y}e{qy5W%4 zhVOdKl+c?Zo$e}FeIAhAJ)Vi7)foa(lSY~hBZ7omBU&7MF_)1jjh=%<^6E7rJA<$r@*B=ax|PZz`C!ARrV2k8~KDDle#@Ji>kR ze(9mW_u26gkHfkZM(!N3{8J!#w-J!y0ffu;m4?0GQxj?nrdHv%tx((hTSt7sgy)lG zQSeOKvg_5HMMPMAGTZH2X`t6a!^E(}xQCkSWBZ~N z2D=y7%#daZju(g3UNpf=5l;`jwBZHXyvoh22m=wuP8vPoXv~;6KJsg_172x8u6IAz zY#>FEz4sB<0I>eTVgrsePl^w(&sr_r=2sJDx10z^D5IunUYkSC94W8v6}}cm)1b}J z@>rvinKvi7vAo<)H>ypusg&2Vv77pv0$U-eVhFrrg*z|GNy9{HY6hLsG_;vW-~v(4CP7I`SIyN!Y3HnB zax_!dmwIaX62qYsom8_}l%nhbp%Na-idd#XcDWd)#L0wDgK3VVhmjHR{FUlr);+rs zj>Yyp%-HfsC06bb4n_!|v71pirWfZyg|@a=nLSKOu|m!_K-?G>jL!?OEEp8U;{dh7 z&h6Rp>hQ(u8u|u6&t2Hrx0FPKT{`^$mg!ht7eHl^-wiuwpFjWX<#TNX)BLLK^~}I> zR@G3f?XB~$TCmvJ(NItHy6oSszCoOs9ffY%zpt7rgb#L@#6%r7C^Cabv0XkCV+Xu2NSje5JuNR2o zz!WVh%ODw7Ka@=ALz!C{%@iMnqmZ5iwT!+e;Ek?9fLJ$JY@$=SIwt)lM_DTnv?;p{ zB| z6~{_1HjuARg%y}hl95SNgySk{-7gR5x8!Q zi8b50elj6T&sdW>5k%(;w+;DFhJ7<`EE7IsTa3ghY=v@55iG6LxKW#rRen4qCDPRfM0t8p{-tAYdZH{ zoYnk0`+R51GS)LfaG*zRW~MS5I6J1`{lMNcR<)75P&+}7kafr6)|eMCg~Z+HgGauh zHh}7=hYYQTMY%N^&}C;6n7Ap@7T@JP3|Tz0D!4aKh4xDe`EY8_i>nSF9ce~Wq&s)5 zgL$%d?t(VKeH#EhdVF?k_rN^*OWrcpHb>Afn+UF`bxFxb*x4{^qNq*k9hQz9jA_)T z3r~YtLIwgDBCyO9A|UIZj%|rxbcd19@WC6g8P^-VB1|b!C-T2&B!L-kZD|H*4B$mA zC6c4v7GmnTFKp2mIP-GHNJTF`g;`0A;g28L&FiVGhD1_%GLPwixAgYx5$rN+jXf*M+G?|yx~>9>jPuB!;B8Ft zw7G!%0rR3=tGBzIt509bcEN5_MZ$u~Qvr+!|H#7Kc1dk!qt_e>=XJ>9{IlmLUV!xA z(UXTyy!*&&0a-oc7+vg*fosFh#9W)C?(;t8V_D2R`kUJ4hYT0w2#q+Iep(okQ|vGE*c@3du`mO`!5pt!A4YXx>n5|@tccXoINC9! zg%qzRv*2X#Y86lWA{PR0pEZvN6Xq+68VJGZJa-VWRWFpiH306W1n&8T20f|vi1wu8 zL7nixIJ&GM(!)tul5s){y-7k#uqkp^>%V#gG1;7)G7jDHjP5rEp$S>QQH%a2!C z#>7N9W!x2Z;vN1s;|Tng75#y zx2|3Gev?=43F_auVK7cHyRUZMfA<~xaDV*iXEez>3;)Z1`p<^!fA))C{nL;C_0GL} zwgi3m`n4NZF1H8N>sL45y7Ap_efuZ>_A~FoufDZYWF&CD5Qt}b+_tF+#WzD)ki63bPL z;j00^96i9BX}CT$ec>)-b{7DZ#2D%n_aq8>_1n+TeQ5FT-|8e-%A^WCvdH+O9? zK=P{UgeU^4x=WDZzmegp2q(6*t4aW6n2SuwTs4ifAi2IcPYd|vQjN&xyd;6I%BgDX zxi62JA5+BzUP2!{&=LX>9uIxDEewV=FSyx8-f(i_eJQtY-}(FtQx_i?=+x|V86ajB$^p3COF-JocD<8gN{tKTb&3g45Y;UYD|JGimdP|PH5IUu2g=W2Bf)Z z-1=-#_dG78<#0vpC?b@~Ft^LKNs_F&Q|l1~WJWVi#Xu%As(|YtK;(?~q`;%KAdIj^ zGYZU$b=7u5EX*h!Tn3uYa$_riU{vTGRUjESTSsC0IF+n@zX>CeE z=TIHyOP`|#p2FoGwZoFsli$Jy2%RnBFtH;V>S(VMomw?tXX}jkJMF2=_}iG+SdKk= zM8;F;q~6m2NWZ<5AQGhqHJnVPh_CH&6i@wO%MhQ3XbRBTRvIh8hy{g5AUF{)2b*T% ztPm|v4i8~&v9ECJ+4F}gcI^?3q(_HGug*LX%oujD^|nno9^bn2_{p&dT2=l`t7+UQ z9`5e%>I`UOPZAgn(%PWGbcm0MQ3PWKK=Dp$fQ6Yn7W9C$(6D{dOQ{GmlP81pM^b`7 zcOvLc4i^X_ZH|iRL#tPqtn0j2W}-8z|GAmTz34!Gc?I-1)OdioM%Yv=-Wmh2ly%bl zuji)V;|Sf@dj5=KcfxHHT_Tj6S-lB@qr$vywx{x?7P08&^UW_iMP{1WKIcDzfPj80y2TQ z6~1vI)@Qn)Rgy~xm!f>`%>P^u--e&*Pe)lt(PfAyo3DjWtSZPF^rxsHJnWLmFo8P1 zfmI4RTx)#L>^eM#vMFp>8jkWuOsrox560o>b~S1KrgFQrSTaA#+LW6J!4G2}RZkF0 zA7+y&yqr&G-|#kMs@XZ!jb4v*m4^wnJ1Tlgs7&`NDoV!TDw+BKQ992zoJrxc>W+=y z#RO*p^hSKe>QD`n)Gr^chJMoruhCtVuaXgGsMhp?&b)^SI=V~Y>JF|!;CYD(rl$qnp{jp6CrU6m)pz!$#Mc6tb%oh(c-RDAr$l?psn0e8d4|-|F#TUNv z>c)Qy&xP&k?#fZ3Vyln}_VW8|x&h(o&nr;!f{`-GFG;>GVRe8fuVdJp2nL=9a7zwp zqQv2oEc_7!Qihp$JtT=_pqYhDEr>zbj(a94^MT%q>g7RC>{@VqZ0lhs-ben(CJCon zsSppPu<)VLY)FY?`B<<2i{$2jiObQ(Jj`&-tA+|E`Mu&~`5;h4S#0H~u7aU%7wOEU zh}cDZij#Uppp!*t@BYw{C6cf=q`5#M%O)OpbCPy}ogC{fN@W!k+-i88|2l!wxWZYk z09qbrPPjDiHz-$5O483_>1a&=SBel?G=vjan>aEu zMjd=Pl#&joEK%n>EA8z{(J%ycGbJLc{-A+f{D3XNQ(<9ws!#&}08!xLbKh`^$`*cl zDHhg)GA3Xih-odgM57O=QHo+bGO3*aiab%3&r2>(a3a2eJU42~hBZ$lcXqz{b@#QWi?A1B#Q zLSP9({~Gc_f3-I5a1j3N(l(wb*=vZ)BIX8U1*rj;dGKZpqzv-!?3kiMwQPT6!pv;r zv!(TpE6_fEeJp5!hUPCpm}MjX?y;@q&j`42ngERrnt0>=0H)fT6|`sb{CU+biCfiB z$l<*WH6dP}7r&drzpZgw&-%0T=j;k9SikUkj~YVWQyjGUsdvw_Me~`Ov_oH}%R^WE z3~uahUp#nmpe3IEX@C*T$9v?%rKyi9h{^+Kw%#~bkQ%b`8Kfg%RC8THLVe_EqXn6}IeOFk5p?in zcGg!80b=klK~e{XZhsi3FAB>Iw7bC~IlN?!59@TXN?DT5r>7IvdlEWbJ1g+Y3j&O= z^5e?b&ah}|Q;B0=MJ)bl9l%pb6ZJGHkI%1Qd)sD5a3qy#1UW($w#sofvT`=GqHv(f ziLhY3>Rlo4y?gb>i&ww?^o!f~A7O){Mt-kN$-LO=nxWwDkB$Nx`cU zN?mMQr9Ju&KmM^<_<#L_9~ggxfr65<(`tjTK$48Syjq^WtLmX~03gJcqB+?u?N>@> z>voIYA3V7C_kaKQwj=uG#~*9tny+VOU4v>h84aPHck|N0`Tz8He`~vsTet732Og`o zpB}w+Ir9Eft1GB&vt3nZW4QcRNZ z44Zu2%5lWavQh#*bHOfGd<_q$0|a>hi(GMgmn-YgPHM&a^HhrgnlWyuV zFna*7M7|c0!_7gfJWljkcw{a^T<*1Lv~|!FA*T>!)~W5+r#nL+s!)bGZmwE#hRB5~ zD^~Yy65d3if)z<8?-ALHCG33SxC0fpD^3I%Mca@)#N{)dA^3!djAFEBg&K@c&6gL;PX}h5_BA;Aa%-ZK4XOnj-K;yW=q6!uxPb|UN(Xh0IZM`Y zf9c*xwyKAgNL*40GPiy*I!sW(i8-fYZ9Jq`p0+Y|W-la-79EKjni47cpBQ4zz*RMh zjiv=f{$ok_NJ1gt_O{V)0VuQTpW|1dPEXI^1!K?XrP`HXEanF z)quBB7HcD~q$9g@apkKTQ4%xI zu2)#}Icrg)NZA@e6aQHRB4U1Kii>gigUgq#o)_UYMApo=9nF!aLSEg%^QvIG_~<6) zi2#f-5lch{uQb}CTb822xQ`X#O75{!YfhdC8Ar^B#yQB~8y67&hn7?$fQjDY80sn@ z#6r-!78Z(C^V25@L+=KD_RKs# ziX@2lK$Cz?7sAvUtlcSSR=OL@K|<~1nc4PFl*R2R1%Aq#so!mn`N z%FNZe7SrRy{)sIX3Ei@3Iwz&xU9}9MO-;mjbQ_&E@-sb2T7zJ++4=($-q?IMkmWD& zlUgUf#!!stbz2IVGY6Lt*X8maN1Iw0G}FJIX|MFKUr z8C@WuRUKTv%AAfz$fEL#kuY}Xd-Kclrgn|1DK2La{u>NO9S6U01hqV>%cz-=%gVCy zRSpaZ)86PkY$?C210>smM3=gRxC_>#jCUa%Gc4M28LE}wyRZ4p zr*`Jq)T|~nHuH&#fY_8V>~qnmg{0jZ6C;>ZdRpd+s5D2CVaL@_)qlbp4W8o{x4!=B z_QS*73m5CN$yuJEc{cjJdijcGxr*@qgZdt~&&Q#5P}jsot^o?rj;aFVB!% z`z9?I2+3-)<{U=9~^}5at2^HE+#yF2d9LUi`LriljHU%v(eY>9tq<|spYm8_vdgxee?mQNG5=x{$k;RY+cUpxICnXrT zw86s=aO2H41Qt*nn)_u6-u&SdTUn{c?QCEz+$N{gbaNFlPo)khrVOuxHw>El8^%oj z_yvBM5?LX~pa)ofa7{OEf=r#nDjU^?f~xK7f2{M}Ue>zhm-BIxhwWZIHdkw63Cop~ z>E-%KqKE??RXhq%)vhl4^i-6n+E(!oI`{*nM4u;&y0m68JI|b4Lp>AoT__Fd_-0p} zlB-$uR(f3R&FYJ|+F0DoKKoKk->VvDehvNcg91@N3PA!G#NwAyTv7e;__Yom&IR3XC#bgGml z0Ly^mg|ML7#Qh06$xAJG|DiXcR%;3uhzT^X`BOEn2|o_LPXC)_e={j|WK_&MKqj&q zcyd{Ax7C|kvAB8)9`TzlShZ&Z1xkQ-A%aTSjS4A9a{@zks`iS?MfHw~rYQ>Ny!l8+ zqmWcVdYLHzF5^2GA==OQ+)9sSpn{H{Gs)XHfbM?++NFtY3Dh!Y>12mpFsn7%9+8Hq z8hVeIu+v@VDSn(C4Wj9l=tlC9*=10cip4S$VIsM|=+4~YOBEBT_>Sj#-biF(gFL$; z>YR`g6B@3E|FOB^+R9I{quct6g83T6g;V zdBmrItZvJ0jY2c$bQdv8?ZH~n&B!76z>)#xB4We`Q^@v`yzx+ioOb6i(4|Mzy*B@PJb(hT z>TZ0V&%(Fj5?bO?F5lghaMVwahZc&_fpIQV2CP{uMgsx(lpY>@J^OV=09E|;zU&zP zI(vAc=4s7buciYT6|apYnn6alyaLd^``|OJWyIsHM{qjtdq=|F^W&X64?D61EB*uKP;H$tk~?dxdp!aC7k^mO(W}czX5V;PUq`-MsnE7hit$&p-L;uYdQc zN=o(2i~agHzcoI0_HX|B?HkvKZR$bOO!t=ATb%rdFTei5cmL$v+5M>+B6%19@cJZj zxBMt98AuU>HW(}&85S);j#xMR^FwTX2F4rgkPyg07*naRBo@LO1H3` z5;l4U8F+ym|JG^Dj|GuWw>G(^U^k<7nh_qxtUC zxGb6?Ok_1CMFA6~h(e z=nX(sdt{ZgrfUPS*aMP#CPsgC>+3IX-O>P%2(p5XI-ZBg$OB%h!OlEFbB?PHO`U3C zm;W$@x_;6cX^^Bbsc0k&)%ED=aTtZF^e`>XBwx?FKTfx(*a|#?Zdgeb!~U@OOue3_ zJZW|b?9A>>4^8K>!Ez%j@vpxrY|4a_jCrq0Czf%>seyUK+5jDTZqu!n6v;Rk6mj=b zQg>gP|JqA7&jV0kB-7RUVD00|$Fd>U(&#Vc5`GDwd1n5Ew30K8;A;Sm_6fGK!DGEl~N`bhk0O?MYxlBFk{fzS?Zanm7hfQ-_y+6xj z9&1X)@dhMbU%&B|%Ja#S$MwD$IBIla+tNLHeecfQCuS8!Y%^*m@-pMd3w*YPtE1&g z3#~Elo(no5=9XS$ZAH0A&om}0PU_{*MLkv-jl(`UVc1241Pz_>b-Z^Z&Zx#lYvXI# z&?#UO$7>{sMRcAImwJ_x%=%eCGGCuJb^ys@>U=qU=_Vx?&^JWJQ;sXe{I<=RJqWTi z1?a**J}$^%V(0cRT)b$p?^j>`frm8xL2QiEddRrW-W~Gt#D4rcJJ;^syKjN+^ZjSZ z;Z>U+X>Txk_Di!{%PY8V*KkGXhTZmdc>(?5;i2Z9f});oSqJX@a*eRgl!xuD2ism4 z((GuFOk)ulN#h&h^2g>FKJCS1p2{I)zjSa>-wbvkoK~Qmi$I*2Jdd9|uE}FtAtM_d zVmYza886%@+?)CU={o#%WNaTEHq2{Z*0Wx9aQ1xrzkA8Uu|ZCg8@-)v-f%@+?SQMl z1id;&>MU6Gc1{2zNfGb@1LYO!<^nn&_dLIDc6TJ~dmfIQ<1tO5#N+PLR`M<6E@bIy z{dtUaqd<8M#RVQj**i%Xv259G3N|l60m%WwlEkj6VV0P40EQ7PW%LalFhL8Hqk2M; z(N#PoD#Y)3;Qx#%b5Wsv1V0>L?V%D87(s2~LI?-Iok2JBgBTsYs4D^cs=vE0-_dKQ#38VsC3#1HjxLNi`VF_CXsBG}FwcRMl+N4>2r4{53VI&P8^p z|4``k$?reY>i*Mje`GoDRj`J# zD5|HWkg{j%svPBqTdByntGPRUy~ToEhr#@o6ZlYscFV@ni}ftQbO5=*bNp_1`X4ji zT$Hj@L8?@A26moSJ8d{Ica&X;;Z~^~gXZ7_M9<&gT?XY~!Q4vY0Y`AA+)aq-q>^gG zf<<%|F@+4OtH80{@EatYbWn!zJ^^NH6WK4o4ZSnqjDPH3$}IhqYj2v3L0GO<{h|8O zjbFk#=~u{DH&1aF07gT(1+{qFZGc%UlT%SDGw;gNs9^{^Sdp|SEvc%khyNQLft`f7 zv2GS#A%q?vm8N3*>^^d3`PeapzP`f z`a8?a7+^i69<&+&ni=f076#*y0uoZ!IrlA5MJLLf+cIw0NCi9xG`L6rkdOprW4IWk zqfTet?(9~H6XY@m9pr#VMPC)w+0q=yxu}@@LUDXjTH;A1@X2P5$|3?H&vxUV5g#$5I6`MHg%CSNorq`H9dGmmreuRP-gA0^2E+W?%0~)TVfN~uY(1?A$ zh~3i=S?0@od-u!$GV$cvtKxfmK<(_9&vq=iE?_g-fAiVrcOO1^VG4@CSr@lA7jJJL zSQSbV71aSLO=4J33Y0bxA~_XVzqYNnPGbvXwJ6T*9UL48Ed`hSX!D*@uTd5x!51lu zw2edqvby$ClOb9PcS#g2B&i_xUFXa~DaPxZ(P51>n#Z}-x-0`l_{z(!$2(4fvsTdJ zdd6c$L%v7CB4Lbru~l=L{hFCCsy;ov@%Yh0HA2xgav1rpZARZyRV33Bv>H-LD)@R^N~8YZ;z!E4c0-Ox)FWPEk{?5UO0G=NgK z43f<>#RTslktNbj4{H|S|Hw5Dl=~}J&0wYG-CTBy@^(8Vu0O?tfTs`4*~L;29v~+` zhW$B{QjS!j1jv~hvrszCoG)TmGuI2_an*j_Y>dDB0L)ve6gCM;J1&~$WyZenGCaZo z!+e6%B$zka!d)&^4vWNIVxG)`A1u)Tz_L8?5DDTsDOptZtDqrZQ9RkZ?|;-g!-z96 zVpt``6h4iuJgM6QX#>Si?q=0BA(L>CjhDc}NrE~MipqF9R>6T(-I&gyB?RJ8++r zl+qVIRrs)ZzFpzkB+W5~Rw=Utzy3?!KbTX|t?Ts++9P=B7FFq8*5SIs{mz%~H$on0 zqYfofTEg)s|8k>P>wI#@opn;0QN(hr+wLe~fnQI%Uy+WbD3iUK#B)&7RcAx}oMmlq zXrv?o>yuYm1;R5Ju_qOt$Mcn{mBd9rE7`fq%apt26*e$>02v$!LUgbEX=4iIYvB4W z$eKGIZ5%)o2AVt|g0FjKUIrzEB76S5xP8LGCUF$UXXE2IHo1A;LVrr42O%=zWx-uj zTsjs~^8}KkB&lDls&53Qv@5;?sK$Y7vfn z`qxaa)icx-oF>0|RwS6KDRmKLQT!ZJ@5N$LgF{7kgn;Cui|`&^hNfG-+=-{v3;r>? z3?=K@kM(y7b<8+WPve2_jZT$%#0d@~a+PFLNT61<;<3X0yElT)BUDE6OqEGfeO7v` zgbkNVE?vxL>QeSS^H2#eFaw}z+*qNCdSSzLqHYYmDf-@tV+dvp7u4$Yku|_m#S5u9 z1Mp@(2FW$LCXX#ri1R1%Z%wKNzwt?XkoMw@9>V!qTD#{e8-Sg+ptBEJH z*nVz)4X-ucDYjrq@8W5d9)Ur_B#ftLMFFh`O{$m)`uNKD8%r>ynfqq=3_Fz*2mN4< zDPn>i^8tWuy~XaHpCy!-zL8NZ1|8U|;r z%B2RC$S;Ud>s4g&6r?}vND~zx9{Zl&3qu?TtDumNZ!V!WlmFmUY5r()?YcVTsizFD z&F;`nF_X72H*fb^m$qpvMh9SFX33qqcP|`V(yM*r#*I6+lRP|}ZUakyB7=4|bA^}L zGMLaX0C(i7`7rIMQ?ygFk||p2eCEvKL(Q(!big!A&q3k?G-ez?Y>oG{ zM-5;ioOA?%`wVLT)V`f#8D=_Zt?lhyy?j|beSC71nfE8O2P$v-!rGbEYVV5FpFDc< z`0?Y{7Y|JKiISO;XN(St`j1SDcya60wukS9a~FB9 z2M@kh%vC!Wnow;(W{5g(9{=v{tXI}?VQIGRDHDWh(_%FU&yE>N+p(j~pa6!b&~SbF z{?F^aEq*;?>A}k;_xIMOqI~i~dn(uLj>bQA37OO=h1b0e&*~G2SM6d5RgT*-I({Q_xPDDr;N6>VzjOV1@9F}0NZJtW2eunRV?azi zbH-jAzy0*nuWsFX+px@@H*c2j<~whH=iA@9a>Yx3t6|yX!NCPBcnc?qn|A#6)8Bva z!Mp$AJKup-eib+Rfmb|O3?1;@kGBz8agj0UN57)Xt&O~sudIVud(KI#k^&AVLpDG| zd*NYV$`^?PyEL_0T@XMGxeA)nrRX#H=O;9(SbYuTx;s}=7%6fv9V&8*Wu7}wSmXq3 zB`OzEqGBKcqqdY)h93EgNdH|$Quo6;cjLRdeIATG_Iz=mx*?X%9jFi9~=G1LhoIZ*(o35zE3;qi5y#Jy1&*-?9r5H{XKQ_;G!fPzH3C&zUFM%y+a}RAB zoL#~tDb(*nFDQoOaT=CxAl#k}{`nM0%jPD9mn``4AdEdiKOfL{6UY}^>U^3W^QO@w zv5Ja4RPm_YTS&|2RwJqLvyutlyU@j7R;4h;@_ICxyHT4!D44?mG~5-q1misP<#e|2 z@+6YyaK2>0pygXvn|lD&Cp^N*SZAT(N`tt3b$KdaaA4nERajzND!buEZ1KlgCiV*F zXogwR3m`P|Vp=j2RVM3NX(M~5NytH}M9ewNuA0f(vDQ4hokHlp?+lcwh-N@f{iOnf zA+^a{8wzL)po?P2r50!;u3AMgPgtu+iDpv^TND}?+&M5-e(wCGOIO4S+q*}P&18If zWTz!ks)FR)COof{Xh{!{<_Y21CVH5QBGgf1q1tN^T%3x%iY3AtawJYGcxJm23I@g6 z7?f7zQJtn?3A0lwi=FUQPNhLyTtRo7r1mg5^U51Vy%0!*w=@~|SB5gmSNX}R({Y=) zkcPsf7@#k)f8l>`xsr&d29-8-j5!u%F)iF2O|S_}h_*UG0(PIBJD)7584geGdG0LM z2;~<{ehK@rl6Q&RLAMG@5r!IU1f-e1Ey~oxsL4V1OO$%V+c@6=qb9_R2wVW&z0ti< zD^e2U(Iulec8L*3t~z6Qlc~arrfAwN5(8N;U-~qx&QIpycn-0eojHUAm35tj`8Wqrb_L8CYH6?bwD0zEs~3CgXsnYaJuS*rWM8a`Fr zLN8}OU@LILMNi#~*YwNIq58N?@_X&Yu%7ImzksZfZ|b1>omf3LB#eFRt(x~f{rb-R z?|kpOfA_b4dvN*cPk#1ua~C;C=+aw*0o07w;-SW*?46W6y{k5J9U{_u&ej0V*0%Jo zOdipzd}7G~;RrECfKb6FvJivDmIRpWQkR9jS!aAwi(w-h&Mu?1?*u&cTy{i-wIgjV zw)W6u+I51bYFVA2Mhhbsp9ZlyrVVG>j-rD8+-O$j&UED4iS&%A^irV4BSp*(;cbCP zg$(FnA5ct(Vo@*C2*sGjQm$XWCMLHNi3vX_(?Y53RBPXOpalk^?jvs_KJ)4t3LhUH zK6r5d(Zffa3|VV(L6B1;iFe+9n{&H;=Qd6$9hQeDhxhJ1Jvi|OvWxnE+Kb7yW8U)n z%!Wla^5-=g_k)I^170x^Lrk^z%2sPPIBX_l;{bY}H2-PshdzP8I#9W&ZP$8JF-tAh{lJyKZb` zo$!e%Em#^(#BX!~gdS(}e$WMd%k9HQCmRU`q@b#eqf!eOD@-@xFEsTa%!oaNgYJ2T zkv&nn+LJ+LeyL^W*%O``7*$t$G;#bgz8+QFIP!k7AjLHz^eBJplNNX6T_*bPRwTT^V?FIV@QUf#q7p~C89{3Ffn0G! zkl3WmxAW$`?O-Ma5sA1s1oJJ5QF`=E)-g~r85R}Uu$~w=9e0Fsd~#?GoNe$xUpq-r zh&5-<34O7ec z@BXj<@$cS!>n-{wFWSa+NFOhTkuf0?-`cV($5&t7*8it4?Q3(2jPz?reD^!wxqNUi z0_rU%-}>mA-~IabZ+`pfl)L4f1i$*#Cm-CrX{`^PjekMMf+f4om5_Yg?;>6CGY{HT zY2+9hBP2Y=1SSRrch?J+Mk-ttVnv@e+yPRW+G z0Iz7Y>}Lj8kaHOkc7Y@7ZV0Co1Oe~r!8DVy0=K8*YFahPs4vKGR#+Iv{S=RZS+`NDPb*xXXt8A~EHH5^i+MU$+#?%Id2bDw91VV}~OdnX6IF%PV>G z{G>UR(8#}x3LG4mm;RV;K0AK&;O@P5Z@%|G|4)A-82#+aKbSv4WpzdC`Q@rfo3n#k zbkW*`*<<2>_j*6^?W#KF)st7(u3h`^!w)D29#&A*)9WXP zPaiybq~l1rR0xtZ2g8N-m%Ny*;mTfBuBv?c)FMnfvqQ~{z9Iy-H{UYl|Uwx~bMAER)zNe0H-P?1S= z8x78N6TuL!TVz&=!AIu>Sp4H@wZ&+8QC9`0(b;p%n92ANIkzh+;4T-2;o_?Fkw~HD z4x`P)-7MK8EeGV*r{sw=OItT5F|-DFw;(b3^YV3P#WU>SyL8~TWl+PifcHgmVI^C* zj6qgo8Pa<3@WDgFg^eo2G1#J8UKfL$SIw$ElTf`@}t?lbqul~(n{w3UQ-@hj#)&KOfFMjj8-~aIY z-^;sfG$nu}5&to5co>VTz^d7tDq^lz)i+X$$Non?o;jI++MgD={G+hm2o2)IK>3Hq zV*@ciKSE3>^SDBb=2G|w&0>N(Pr?^^5j<%JD@ib>;?}abkotGUDIc+W7>IS?q?^^| zZl)kgQhvaLBJ!vg0Zxsn74P*UIBVksE0bp4h0)0st{EwmfHmT6LA)OZ>3f_LLO7i% zsNzwow0w7-NG_W4<_M`l3%)the$rTtOyN%GkA0*N9!?!yH653F&D8XG(1r5VHB+Eh z@6W1(3*B|>7OVGNy;iKR@>Mgt=TZWdgl(tmvKJ`n+o{%^=u&A@c_}WwjcXW=rUQd? z=)ns6`q^+eOraZPJxT~mo&izUCpQUH4O77h8c1ru$b>IL>PV-;a<)rdXh&i0!XW-l zb9327sXPFn-*_5-Qs9&-MnYBZ-B0Nks)(9KDJ1y!hH854JYK0>)HDj-=m`wMlK5I3gx}WQ z#(@EvMqrX^Za9YpHn3drYYYPdri;yp#7c8>wdY!W7c;PP+=_Et?yd^opye}*h_pgs z5#~*r9HpW`;|nn_L~54#&XwZH2~NcfY^|yk_L~l^T_Rjd4^dF01*;f?MFwi5dry`F-~T<*Zc z!Sq+E@??#J+Xgk7fY|iE{YJ^b>$6lRjF9hS3Xqw=hNfs$wApJ(?s*x0-2`7dED7&Hp#4RM_M}}u`=;xY9Rz#3_F@-V=}m6W_THS?CR^G8d{8AI8iC4= z`5CJIl|j)w7!kaRwE7H<9o7=!|!OaF_yji*R3Ht}&gH6Np$C36cm@ z9%A)?Figs^6_2-!i`#Es@UmrZTdpy1boA7;Lp2ZXnfz0`@VN@*v=Ib%ch7P}kIkuS z{fKukU8u3hF4}g*lI5+l2Rr-bEn3uZ^@=K4ce5 z-a(^fH30g~161dfDcAu~N4O8qoQj;O3WGyY;97_@ukytUI-2o9#r&`Sqz-dVBS=um z=uEY7ueYYO*-*Nip3T!CxC%l_xANfYSQQ<#5IQ);j@(2}I+k$W%BXCp0|wp6?n&Yp z2KIQ+k#njl?f#lr!N47GhN1;gZiRVT3a)kr2;{~HH^bjxt*9$x$5Md_us$ymp}Zo|LY(A-$#$0aPQs>;KgKb-?-sT!+hze?i4Ug zPj|tfKsEVsTC8m-5#-3p+w}^ z2L-It#(@LLFwh=WMI_WpFYjVcs8lOVDru+b5E4qw5DJL=OU(!f3HK@Y9G>t5O_;Cl zn{`{ss0^ZF0JPrlmXDLCP_d4Z;x^7geTmf-JTa@D58}*RWM&}fWEMEsN!1TM@m0XR zxf|Z%Pvutm7PLjN#ATApS+L`gKR05@+z&){3YhMh3ABpa+Y#BfgL9wwUIpHeXm>oF zwORC`A?_3oh@@Mwy@0}G@#oKm-PqNouM#+2;Ghpc(cb&v6Nw)hsbAgq}Z1Q47&YNV=( zjdrNbkxBIJ0-vrTif#|bQYpCtzSY7(YzD#f=8Jm&uU@s{50}y>910(V_KmcK-tsEl z9(M6l6kI@Y_O%EH7HMouOKgGw`k)PvKD*_%j1r7?=SpQ?wxTe(@=0YcsSHE^ppfR4 zfIiIkr0y}+1HGwo##qMmDug}@$eKsd2t(3S_DCEPX^hxV7td~c5uNe&f;6onl>sYf z(#9zB{^>38rmfQlg&EkM72sX6!bF-P7S|-~1zqXE5CL3To$pd6lZtdt(qAzicQLMu zN)+<7LDYCC{09YNat<-W!C&{*VdKQvc-mKooUnJ6^ymWf_N?}cH2KGI!898IYrZJA z01W-`fdF&H2@3eqE|+7|)ty<*eCK^wG3`BPZ=5rA#Nd?{wK;Mo>+7n6+Y6klr>_1O z!$&V)nl|Q)tzW-!`Gk*g~T%>h%_5H4nCT)|yZ?dpqA-9$?+M%==BV zJ$(A~-rYMomJLLSLG}IhQaqvY@yY(vbLaQY9i6LJKt+N%-cFnHbP^M=uP+Twl!ye{ z9F_d4bVel;EBa%GuU-YDVl1@5P0YaDTzM<+Ejvb47B3D%yLv>7?ke|%Q+7e*QEaBE zq7|US-84VOW>M};!~zyfwcm?p<`0|ibUsb%6$fWczd1S1fYdTdIehYjn(klNXLo29 z#6z-|X<>maO1I7&9UbcXvM)d9HG=8)JJv&{pf^lU*G zX~GgOWhY_&TN{_%ys}#q{mhIy#^1U!CSDVu4|X zbvGwT&8LX;R2ocPymS*16VB&>-Hd@joph5imVoYvK@zM&=VCz$<-90REJ;!)Aj4)* zfKPZ0Pa zPFq#g%{oNoI0F82?^W-%p-R}z@;nD*h<*9&@aboteg5G7!}IUH9lC%f%fXgG4qfpw zMWH7ISqBx%BU$lHHMEAfDFhQ0pdt*_E@yW>`sRn9eewBEAN?GP>fz5m|MJKG`qTgV zzx>CmmrNE)Ndp>xh7Bog5!NsyaWkT+ArkBahQs7p$`p#x?GxAfbi~?pXSkXKs{kp6 zM3ID7gIbbuUss2dWgTuTfO8#~tS&A{8N;3VPc_-a_(12(m?ca=pI6fg(my*D=&XI^ zA;o&F;Ho-EWCb@&$Z0hsUz$o26ND95=cw2U)hN)?DK??$gLLL%_h6Dur)nhTpwT`S z#y@BZW)u>{j{5TwV!{}$bgw^ksfqU{Bfrx3S7?;t_vDo4}Zky7-@($eKMTm3r7A>TVC>;rfhf^;(SXaw^3icvGrwsROz_ z_?Ez_FI2R&UeMs&jeO#OU4;3@m_|JVBb6b?G3}v$Qxc8@XUg|8(O#fO-(7YqM^!|w znrveY;w|E27xwA0aJij}bm)*N|D6LY{R9SU?8S|&T!+o{xL4PeX54{}B2Kf}yz1#> zL|Th~_(=`OnjmylE-4o+L9YI=v4h0k0wNWy%}pw!nK8hmIAk^49_iawQkr_%U-r7d zV^gj4n(&yy54CLL)bazJsV1wL^pZQy;}-{oYKonH=6PLKq5+K5k>)5a`DpX&gBVrvs zE4_4?CdJ90k7d2kB{&+FUG&!{Y0p`vD~}P3nk6HCgj$hcg_h_v=b}Iwm1Wx#aN5S0 zZSs!}#q$@hT)l1z$Li*Mqrw0OeExDYPTmwZLW8+U)~fKP(VekXW~J%LH6TTi&pJOL)zAW^(slh zvFM*5g#a11RxNt!qz{1{Ms0@rHUVgeDm5q+toy>k>s^fzuCI+{3Ki13;ehXC>$^|& zMTAZ(Bc|1$EdXm*p1o%M3zb_1R49d}3geSiTbPwO;X*Dz!-juG7l;*uH7?!Y6xfv; zxfiS98>UmJ)QGd9Nd(CXB-9zgWw3NLf})h3uRVLtYkWYyf4_~S^omriA3c@WrDP_? zlibxym(5dpaR0uc0Ob{Iwsqy6MX5uv_;lDBm}k1UiNww)k#Ag3^&T zp%vaWe{y;mYCu99a%ASz56A!v24el?Gwpc;ukQR5uul0Nn%~=o*p8 z^0!2~-oA3=jlDGHs7AuB4CcdYUzM$$ksQ5{h&`)dOqS8>l}o{~%6xv~`rGFYE@{Sa zisr|g8I$5E|I8Wt`H0i6ZyPnajzwRed@T?c@a7 zuUGVpxhbYeanNclMVH>$;oi~lQxuKY@|&XAz7x|AE?&~MSG``ov_q7JV5~N_1*11O zGx>?R&v^YfueE(=TRo}ub?y2!EjOG|3!s$LG)(?GQTM@QS#})wy?mE%KL$erAeW>< z3`r|;#nnn1`uGZk6rl}i#obsC00F|xV5X_-7Gs1wj*fafBLOsS)6eP zELo-}YZ6;W6-FHf{Jes$0xEU$zg8<8(li|y)vc@gTm+r{7y8rfUU{#q0us5;Mns)h zn_L ze)`FR`aiUs%aDXrw^$+XI$W@;=jmeUG3^2R>UKJ+pb810+-Z{bz3_5_c_C>qrvQ7Np5W)Xn1M_&f1lXV^X>CoK! zH5!O-X7vPH0t4uX>BIyUrk$RjcTCXRSA&5&^P^?3g?9L(f|ech`Q@9}KmOuZ7no}n z;*(j(4fB{s2Zu|z8WHCOmg9r{fAe>LXZZf>-~J2MF_-_(fBrAR&42j&zq5G&&nNT; zepX$@*o_iHaCqe(2fa!Tt)uq>T!79iVv>e9vhXAk0)mpUcRtQHUcMmj_`zsQwiEbyn}|?(HzA^o7JQvJe#wfAX2Zz~si1=^1Zau&0x?w@Fu=;* z3+MYfFNjBn32uU28q_<@qKV?}K95 z?s-k!)wdGB9AnWRfEIbz?QlIvXFSQJ=)EwY62M4BRxjU!Dsk_664TqrWN_C;TWt5X`@%Ux+U&BVQq5n56g16Z~Z!^JSL zjnNs7aEMwy(0e{c8q~qQaAF9dfZ)bcf*UM!kw37W|NgoJl5SnY7)%1K7I0eITxa|K zn*YWA5&cn~GIxS3R7rNq)wSRUV!&#d?9S1V+|`}|n~sP(M+MiCG@sylw{-SND|z9 zQGFp@YhSPxBWoa8CDa28x1!a^*wHd5!N7ImV~g2ShUAR(Q>}Yre?~J_pC<#_Y7#Cw zvBwC%8ZSpe_L;zOq7*eri6mjdJa%Bw(eszDUp#*)dQ>&)4!}V(_~;Rt2Bw1J;dS_C zd=hx2O0}##S@1BWh7{9Ab3U;Lb#+EgLR};9k>RUdYBT?<8P;7J`%;V1t&6&Z&=lx< z7v?n|6pVP$h;a3(zO;*k{@iTSUv?N@m?|n|Adu-JJS_$jFSmwi5<&Hn!=!FP`_^#Z z*xyZcN^)47b#F0rG4fr{<-2wfypwl##$rQV6vsV%SUM=`#!4eFi%qsmyA~RfbM%;B zu+pVIJIl!gw{wdep&1;=SsuWPgb0BKV?>NN#Z%Z=Lyyhq#?tISQ;-G{Gwg%{Q6Voz zU^y0=>SahdQmAUbo-R1lDyBi1%tGhzL||WGh>pyIizwUO$EQP&l-=8BMQ9adNoP7y z9@pLu*f!?pfC2OT#Y^;ozlTd{M6=+vzz=wdaK}|Wb4(DB_1gNDJ?HLsh81h}-4EZN zcs$kF33Qf2LK|6ZZX8&!Ik982yEC2(9m#B5pJ0pm7;VG7jWBEr>ud<4z$xVzsBPO1 zwhwllyeVpE0Y97ImH`poi%XD|gOo&&4ie29J}?-lhXUT^!=84Z_8<3?0uVnv0-k)x zOlDckh6&nluEBwNSC=0x)p_+&l78dt>=Y#r4iDB5(4)CLD`0Q`!-;*oAIx^VdG*F@ zhd{{sd#27gU`L0CuU|fO=x4bYW1bnRLE?o|eGFBM1=jxX`#)MJMVZ+Uy8@mcAF!_s zI(~^ch41SJ>!+tD7cYkEG}$%&|9W;jm!8PBqsi8RpkJ_19 ze=ccYf{^86H>xpK97vHhD?YTG%Tr*!R%V*W@>DIKS?!w4K2=M{`MxT+TFb+sk$|b> z>U7$@0y=cDN>LMn5U0|k}ogM!yWnWr@)<_4HOxUmQ(?=9EIvgi}f)=U)< z9w|MRb;tCg+90#{%JHd*ACf6v2*N`3phRMJvv%)HZfZ;Ja{HJ+@e2YiWE%`T9&^)E zf&by}|ND=fX!gMqz_*>bd-VFh{+qvf`_>*}5GV*qIFk*(>tDQi{XhMW|DgOI{_sbR z7U2Kq|MedYW&ZVl^S57p`2{O8G+QWHIHO`|`Pz}&TmR%?8e9}&twb#;m@q3!g3=2e zs7J2bjS0om9DiLb!Yj^c#%@Lim zJ|&j`ZLZ&v0$Hv=j{*ZF$diFexDhpCbEKIIMF`5dJ+#l zCxbciQ9fh4RgX5Z*_Sfj0uQyJ=G z))JoBsu4B@<6Ks{|C!k+=SQK|AKc!_67c~gRwgrmY?Z7jk>bXk7^15pAT-CC3HNb3xz7|PyM%Wn2YlwPX`x?N=oIr2Qt6*YuHjhRqhu(`b>N_g?|)r(iJL@KTf^YD+dgelNMd4<8!J+Q_I zqMP=0GXs`+Os|6WEUMLOY9;tW>`#^=`ndA7DNCiAy6r<2=yUn0PWW9+O?vfFxk?Q& z2j#>$C7&&W0oiY`5}DHZwVHFzKG}va$h2S!vyc{9FqgHgMcOSe8GeQ|dDV_UkHpVX zMall6|CYI3yGTO_zc9~MZOn22ODCo8A8=q)AS-}wSTdJ4R0On z!&S-@mWgMbz0qrJ?8q6+P1cu5jfc1KRnzkJ`48jJc?xwGpC|S9>8N0?=G)c0P`0v z1Pj`tPB|-4?51S)v*(9LdrkG0UB?w|9Nr!}`g0Ef7$ft9yS{*bJhJ=`Q zhf0Z1V#r$dyaES+g{)3nQJA zH*elrX3Sg}JUA`%^vuY7`{>9D`|T~mg7Y)Or0bgm3o4%uJ{?j$ z=Gw!;JjN@Z=15JmQXK~+II{PbS@TThHuu_Vft18?d;7(Um-`$Hm`pO1d$CqB{Xil# z%K;(swyhcmkGqBu)mH}^ShOiMC$BPQc5C~ zE*)(12KuwIo=9MX$yRGA8lrWgoz&8Om?!qxv@yGB7p`_AzWA!U#bAw{kcFDu3xSE1 z+2>l(tUo!Bq`q&Ctts*f=_Szw7|k&lZdYo89c)!VkG(Aga~%0j8uF z;noyjvtWhXN)5m|5XF88Z$`aTnvey~h06^*#%60x?@b{_K&VHmQDV1|Bi67AK^l!C zU++VV3ilZx!CpyiQxcb|H`V6zjQXo8vTNl&!>L}`3oXq{qnX*RqfwVut#*a!jpVw~ zN!`QDRbt>m1p;G$=cUTIc6Xt9Z9?`@#C&pkVi>9!y|9R>Z~BY@{zq=MiDxZ17;oW9 zJhO#m;=%E}fGSHzShX?BFs;e!0Y(5xS8LVb8=*jx45zpEcL5keq`_aw*cCq`ww=Cf z8-M*5Ki9|~zx$r4hsV$U{LlXCFaMR9i-mQAw6ftzPBrrN*I)fF|Nh_oum9Ws{@r)q z3%G5A`49i`AAkG1-~IHPKl{l~zIJ%1(*j5#`$cYiz+u*(vY(ut810Y+B1+o{D}Zer zl(=BK3g0J&FOX>zlZErIKnpSFHJF4=sxvrsIhKsfPOufgi8-Tf=R^rvhB{dxTA4<- zhTauR7Kbc-QG-}ckEI~HUMF?qE~falU_{{%8G@Ll8iiwGe9qDos6$$_;Cw2k3{_xQ zNpCR-TZg43e@g6k{7#VTBq1a4zLZ!iW9aK(2DTBg+_n;mDC%1A1Q1vC!))pxjyut1 zq0%T-cjl|BFdgbGB0F~?P8w4S{xmsVNBKw}0J`qWa3b@kHS&<{&oNH!_qOFZl5=sE8?eiI@4uVF&+*h$v+54hQggt}%58Cj%`I@uLhpq?k5=xQz|IAdZt?6itRkWB24gTrQP zFTkmpgphO?|A^Zl7JZpY4#R%q2D3>ZffI0;QB2>{8l}L>a3qZ)(-4V~v0wnhzyh|x zhJXPSd~(99A>fq_&54aLdl;QOr9s=RV~#cmI8z)+XVgOwy-NZ*W}I2+Q2&Z(4!e>e z4b0z=AbQwp*o@F3V_{25>$MknM#S1FVWp%&`X(b1AE4@vX89yWoAx+qFhyIQ-IJ?D zYJT0&rM)m_LyP;GLl_*ktf;1jdwaqXXSTmMu=m$yJ`fMA@jqQj+4NRlv1@^{uZyB( zQ%OT)%{MBQB^@fO9PC4r^iF(*f@1=bE_K48SB)*Np-xTJPIX;g`MKya?Yp#G?w-5d zK*b72;$S8VB$mXpL3n87(U80yw#@@1%*0}*I?)m}nJpv)BX*Gfpe4RGlee|y>h|g* zZS|QM7M{^%rD&nfX9Q*GlocGW4t5WyS2%@0R)%@4wXk6An+XA%J7E{iz}9?DMu8HM z=Hik9=;#vx%mcXU0;*}37y~l=p{x2x>@8SJ3)E7-fQHncOn-W zkB?3u{q*9Y_3LBkZXa)(b%n;Wo+V&)4dtZf93s4D1pi5v`sCz2FDXn$C>(nE{JFWj z4?V1tNM8T|KmbWZK~xv(%pL$gt^U8B%Hy>}Bty#S=}@Z5O9a?_>-~1CbhfvC{PEq@~v(jadDxC6Uz&p zA0ILtrxKtA1~>9JOuJk=0mp;uyDP$qyEg0&o9|!Sop^Ng)t6rpOj{)^AM%yskVgd% z&uTrGPK{2*_V0dtZ?%<2h+SMcj~JtwS_O6ua$daxv&o#WS;S^#xRt}89#Z>YUF^}3 zNtJuYj$U40irgmYnj^n^*tcsk`%=7I)%0VDrd#k|?j}#~-M4fN(b5a#v>cys$3x3E zENyiI94xnpG1?Xfvf^Wz6Y+^hT+Py^@?<{VChKW$0miJpYcy2X2<~7vZyD89Rejsk<{Tx9??wTvnzd@qpEn~I^Hn)T;rn%6fJmXua^BF zEbjC4VkiSsn5|?Od0TLnhUt~p^AJ5cMJbsrgK2|e9&!d9g;Tm6enuF`;3IlilK8iO z^Ot}9mp?CTN|jqu$%f+i3i*+}2sP$JM?d}PH_x{J{r~YF{_o%X=3hQK*Yy73ci+GB zGyz`O&YJ!G64D#;@jBsvkC zJrKbr`iia5AP|#}v?8W^X_nxVFN>6%!Y3=bF|)nH>O5vVtDIl;h9+f$3}TI8MIPdw z2C_4(7O-dfv%~!ha!DW`8e1sc!jY7zw|NixSegtPF91HT))gQpS0O+Mm8dAy99D+G zfZj#Ly-M_i#UfBHCZst|RHIM+WdepCs~Sk6%+husAdUH~W!cFP>+492}NyXD2juxX`dT;wDBu?yLAAFr&S9bq0BLlx>A0`h6K=xQqO zpT>;`iZnbg({Wg3jP{X>znpag$~SsI5kRFIez!M;)E}>Jo^3rGlQ0Ll?eFP24#)8Yu6EYP;b!NpG?(d6k9z?x-cOi6Eg!W8s}l0(#@^vS&`-1Bf|pcq31;z%g_ z2;oR);sA9?ZA6YGBiG1LcC#^ccHhC)51%aFom3XRwXR^JOaXx&DV|~&rVU-1!}4zKjBF-OlX8t&*>7zy3&>zKVZyjRpzq1EaTd)Z*Fhc5~ztDSXe# z&^qqc{4%33&P1X>*eV(#g@QQJ9pi)rq-+29paVZFU_9(x^Evqk!axK^(h?W})Y|~K zXlV=%xVfab(UKfHn@_UlrI%?#kxU5`q#PBr8YR~O3Juj}ZTDAm@R|1sC8y^=N!+7q zAfaJ%yUTz8V@7)-DV)nJbqJQVCW_}Cs-rZSo}?N$(ENs-1V_kxQd zs;)a|{XKUxvw)LNrN6M|k z`|A%91_G3DuZI!?|70A7Z-eqLl<$`MZ+oEhRq)E8`?QK?&)Jp z;_}0zS%6__wv|giemH5#_Fj(%@q7c~+BU%3)Ee1YBw{$LHCTOuAel_W8Lmz)&O$aK zDuN;qBom4BKIXr0(I0(~o+8fVh#cP2qV|79Ut)!KzLQ)`_cP|nXE)P+-*W|vjTN5J zbH=djmHrWU`v84;M17k>i*M7~m~n(@Vv3iXv9-G?ZmOmq?3?bU*WqfqM%j?aC$>HN zHm3`uUe!p~mjYqjx`mJ)UCc(6s>%GMf4(hk3cNv-Z6;Baz2%B=XY5e;j}d@p&8t3$ zB4IyjQr>#jaxb-9N0GTwsYVhNP}SFpu>b`CHRgU-MPPnvQMXLkg3NM9h2Wf@siY13 zor(Lo7d*AG>I73jK72Jj+~s=hO}}w^#4+_=CwagB_T9HXez?3dQAXdkE#EbOF6Imr zCB+7~AGO=+TPRO#%7gmvPhvsuOp+)Bo=A_$?lNi-d2Q`rOcfq*?oDrQJW-n;-o0mb zoa(C#0NLw^F~sz*D-o$+TR75^a<>QaQ1lCQg)2Ig1zlJ|GU@v2t1tfDzx@YOX8-id zU$G`SWwCf+Px$_Dvd}Vt)d?v|oU|u88d)JyVkui6Tx}F-Cz)<2)XWYKs11wbK(7G| z5O@}8wHoo~1sr--7zB5Aj4YN3*eU#?-aJDW8I@+J*vzqZ6F1F3yEpB!G|F^dXFe$p=F)^Ukxxou^e+)v6vJ1IrD&`D~t zw~&J*IxRB*7$=K^Oo$d$D2B+RuA4o_Kzo$A1g0%cvdDY8qJQ4hidLygxtZ&`aY(i` za~`{UjmM%sd~%_KW%P}>qcg~i4**iKhFRjVuZYEG9xtB+9)Oz}Fj9uXy2<;0IA{$>BI&LCZ5?9$CTjxX~XXuFuS`d^PDx_ubkRImza zf+vuC|J(xUg^IQ9cH;N2;z`yFb`{Kmcxff1YsV{FsKL));zw?nFGn@)f}z`$$kz>A z-O-L`M&&z4u?EVNO~aN~F$zi`hS)vdkL>tGo(m>zr*uh?)K9{g%9h~AhQ zg{E>|_@!qW$~-f#g`M))R{q#xQSh&pNMD0m6&ip$W8z2itq+b=HR8{ea2W+`#wWZ! zBbRk5qY+iuhfcL<3{W9uqDnVaEx_;Se8;9(fn_CL6FMw`wFd>PnFms-k>OuNM`2&) zFUl1#M6?Ai#*j=2=IpYiyX$)`M!9O1UV{0n&ZX&myzA>rs6$Dn!`_A#E_6!%7Q*r> zV*QVoXWUH={ooL~P}hMBwIsgr5*Po_`uVc{l5y6tSQZaT`vDj>54b(Qka1UI{1>g= z*?IfswW$2V`w#EWPFdshX5rxwbk+~VQFE;Tuo_Vks)fM^mfDv#u)uhd6RWJ5Sc0`F zsbyET*ODa}eS_?~AKyJniQ_B;WTC*SjQpt2%3E~dNuHHo3!PeuqrwYJdk zp(X+-tjtNK(Sn_Avj;gJaj;DP$Da3lD&gbQe?Hmf8}>%3Skec>5c%?Q>juRg>2r2! z8f0=fq|oz-n~D?8JLnD>^M3+VlOaP@ssnjlhgD2`x7K-ht>Z6bCWao`>jZb zyIyu?H_~{L>GEv1aWF}!DA(*VB_|wt{RXIcvmrS5;6==!qd+p4TC)(dNEKB_6=QUS zM)J&I&8CQp5^&k0S}B}ss~34jL|GCBpV{Y3hn3lWrN{%%nxdwFE?^|>$U_$gN1c_X zX-+>x3`bJ8dF5ubl!0}9G+F_=m>W-z8p+^2o~09wA-zTe>m>`J)4=z(d<+gS2LVl1 zrYI&vTaeqpeTA26q6bt}q;yxF%}eDbGBS@En3gNK&~wqfz;`EG?y_Rt4z*sPN6PoOw>BM*M-hKPs zySuf{0VU{&+QAB|7q3ZuOO`V?`l?o3i!?%-(F&@lrEq{LX)4-KfcP6Ql-vz`byXpG z6EzdD zwe_#wzTNvD{@x=we(_Jg{Pz1liHDF)mJc|}co_`yAB9602!nAm4MeHUe_*)&+;rf!FQ8U7*G z&;S`K=Lb;HB(eZ;Xr3Dlm>gr^#qHcR-ELxJB05wM5Dh~{!pJY4k&XhQ;la2$^)%0P zM&-DpPSjI7@+g?`&jofMy9_n`$DF!l5v*cpp;r_K#qcHwX?4Y>8CT^_JW-u}9L@!< zju@;9)1;Rlam2a^Ry(H7OV4o8P$9TYezlAQnF2UyoFp>k(=Wk_*9;4ig4u+t6VnW( z#CevST7YT@SNfXSszGonku@3oMv zB?=jq?Pp|q1l~W|VIvXZBN=Fs%ZS=M`u9J)JG;8#n_aiQXxGlyHa4e^8Lpzn(ZE~+ zbPWg!M>2vNPe7CnI`k|XJNZPxMXm*}FwrdzNuuCR6YB_4jxeY(G6pqPZNZ5MpsIye zL8Ujmv%<53LQ0m;!mtJ<>SmGEC*ftuI@_`hmO6tH@ud`*M$I?00$~viDJIJTFo6e0 z$boUQnR-7bRB0GIkC`7D$V`+a6G&Zq~)3PKdA-G(iNjb?+n& z3!>CGYJ9bO{_)vWjOE!Z^i1pt3P~}_^9#UK3-Ar$I;dMDrZaOy1t)V)c`*4iKJ{CS zS%3@n+R{4^G67>Y6Z0wGdqml=7to0$TEof7k1R$KfVG|+q)_dLL_G3F?DlsL4@(34 zPG&`4UOg+!UG{=qnXnuE0v%{P)HIT&&s8vi_w#FEcJefzV4Tg*1BSgHiv(N1j$`?^ zj*2psOP5q(V;{F;p zlB=3&@qhMN|vsj6KKPf#X@T#r{kpYB+*$;vv+D)2$t3Faw?A+5GFqGLUO{reZL`3UkZo~K|;XH zv}H0ZqGcuN6pQMJ0ErI=R5F{_H_yR)etBikbL^aW3NE6}Gj-Uqi8KZQwA4h({N1}B ztv*Qu_75nj1#x^+ChzX91jp{q{)5#uZCJ7_-dg`y!KjkYXO0+pu(z64Hkq(-E5Rfz zePfl?^XIQlPT!w=up&sAQVV34lgi_vZOM2Pv$E-F&ABx8A~zT-YPY~Vp>D-Ff^d>7 z2)ck*c|$j!5rb(G=sQ85sor_Su{110pVFrty1PHU=#I;e(D)H!kSHuA~m%%q&$o=5L$3K z46E$h{P(TiGA3CEZv{Ka@GAV$m7DmsYI0RB7T5ODk3!R+mMUGKS0yba{1LAUTAoHC z_JD=cZT)%Uz~G0YF>)WTKA_n;E zHFtfUI)k^j$eDmw$GF0SPmcb(|MurU|Cu?>@BZ|?1=&ohcRTJ2jG%h&V6}FNv=+X& zCs7hKv?^CdvemghWeu|6C7aw@W=9L?S(8i=%U5!jNDwytBSQejApa)1imT#F|CaH| zgi|C5F`x)&*OQ$vGpf`|TR-PJq3RRT#Q~-JMQ$F07EMnIjO6EGQ=t)zk>8;tgFF3B zW&LPYbzx^CyGaxvBUZbl+^9-%jvkEx&3>XiJw|c4OM+o4*+oey$#7U|aCln2wgnE3 zm1w+eTK~zcWxJ+qU9%)3)298_XqLL~a4CQWFvjY%g%cCcrRaegH&6p41;;UWQWyEA z)=H`x^2{h_lM^sdd$_$98TR5f0q>%AutaQtHxeJkU5t3I;AA2ef%MEP#BepXZd)bW zL)~4GErV31(P)C_43d_)_)xBQiK38yeeFjyYB50l2qE!8je=1=yyyLS6u9&9copLZBfu=F=E_XsMp&2H zS*EH{Mf4Bl`c%37ovrC_xHAECHh6lZVxKl|E=MB}5DPRckB*~!nqh>C=n$i2y@G}$ zg%5pTD44P&qX&~a^E1gMCDB#5YpJ+lR_DVEz)*ftMXX>9(0YBsjW`8gqVZ(bS-;tg1@6J)nVN zUV)S23(ul4OqfJI(dZP31^p?klWAkb6Zb}LYIveNj1znZ872rG_#QFf5EHQ@V+0t$ zgGNCk4LJ)R^^DVzBTLjPU8B<@Y?6Cbm&&r9M;QpBXh5S&_;@okq^h3S73pRj|CE`j zj;OPcZU#mxc4W6Qq2B9K{e>DU02kfBxH>j~>7gTF16841SpzeAvh{R7_JP3aXb+)?7oxGj zujB};dTW7eyOR$$x9@kY!iVNVVrTRN?zC6XTkZp`jhSTAgx+=5Q^PxPfttdav8MGG zgszd;q$&c>DF~m8%FG@>4ZAjb40Ifuwi6_fCj$mKPWykhYlRI)3U{fMEc^AxoyKBj zERzCx5t_i?e4?<9;%8EE$feWA09HV$zf(D@;slG9x3)}=SfeQUd31d6ksLu)P%Mj<)2vFas$z7@wB;d*KUdhRA%x zn550(}G)31Ny^Z)!0|K_LP{G{ya zf&ovR7gM877XrW(`=ft!NQQhED&VgFK1Uwp^NKJzQxWnSYx3orH($JZ{a^j{U$Iuw zqHRAoJwKb7P>6BJS+ow0(3G@m0_STQ917+yuH zJ_9QsYZ=M_MXuCaK1U6AhD4=BhrTr(1ak<$sxx%QyHW+(nl?uhQCkCTp?9%bdTi8? z#umeuVu^lHFw*D8ip3Q6spPKRLzZP)rgG`{Fb5J~q4SAKS9{7#%PfGW@wUb~@*u`N zpL-hHfluO+1$5_OCSJ{9L3Xs>Xt9v~J2y?E3af<&1X#&+&?B8(e`wb#yHFhARngY7Fd zw%?T3(~})q>xuMowhzxU(Z5BOt(-NMC@BJ&@QI|%KVKzE*>dW7E(GlhKyRZ4a)n8| z2Q7aoRd1}|oQ@@{5}NeQKf*K=!**j!00iU^kcx1_M{APA&&(g2k-NINn?XQvtGj{0 zkVVqM5L)Tcm9pKkEr{7;VA20`xfqaWMIEC-s>zX?x{{cI``E*>A`_Dt;PrwjSxcA! z1lv|2=c*eFxh1Pt-ZbyYR-g{_2`D6&5NX?)T3B5$xGs(tzROj?7f4M`%&B6;K>T6U z2c>x+JsOYCtZ@NBBZTPONJeVPZ?Tdu=IJ0)hdFHr4has3BrZdh)gfG#8Kgcy zj`lEOcvd{rOvDE@_C=R|)fyr6JvoIz80JV*qGZdJ(Jf!{$C`)?i7sISBtH60EXp4k zMKUFEMpl);xUJZ@{gKoNSQ|-#U#XwK8JJwlHjI1}f-5r0Gc=8|sdMhYzrOhyfs(jZ z(kru@-U<6#WF@^FfW8m)AmOH3>Uzs0EkCvEVs7wZTnHP}1KwKtZXigH*pF~VJUvTPr&ERRgn=rKAQ|9KHphL1| z=S<*2CYb;j>wzr^?+g@R5+M@jyc_-(E6pY$b2!;UbYb-J0K*ErM9fA-*dAq=-yt_R zgvqXyEK2D!1TD4mSov3H(XC0p?UfvCQ7vn@EKVDK87 z1y%=k>5jW(vS144yy?bZ1;(5wEV0v~9%eyL-gNzDO{BIC+14c|oy76lap24j3h( zSL?^mU%Y+$*0Ly7Sib1}h=eMRPb6#Pzj`&7NdWTFUIIP+ z*7#_FchZ0>(qRU5GXvnE7>0HY>eCog7Hu{&aBJ<>Xswn(6O4&b?5qRe5lc(>y)j&a{8(}e>K1s z#nvt@eYrM;0>!TuIKlKW6D7l}1~0{%xap4TkcN4`6H8c%_tl#>)bJjB2v5Iw^9rE3TYS!x`8ViK06j7=2n>~agLNGY4KNL`Hrx@mbqw2qOe)zan4U=l{zXfLH{w5!;`Fp^`8X?~{E zT;9d9Eit`2iG#1KYI$SldIk7O?xn^>lTJtK$x|JnNAtroU!3<9d+{J}&o8m#pYG*@ z9X%#EO9x{NVwL9fDbS-?d*N(3Qa;!mQ&$RpVM?fm`XnkM%kT}k)KnLy`Gbw7Z-dzb z%x`7!?+^olL0@@Bsy%?-`dM7&TTzC+TFIDIS$U1B2@VaU&C^N_?khhycGB zHq(|p#VKgQzj+CLSziQdGq)wD!x4jVRtx>$wYr_x)R3|v2GUtaXR+tm$?f%}6ELLa zLoP|FX;lbqkJ$3BP#y&|(P~;d=G#|qD#W>g_B^1l9fZ1rQgWC=`&VeFtGh?rCa^rY zr>&hax>|Yn|U_ zRP0?_?=U{67qbE2y5I1iYf|&DerxVgf1V*`#gQw-$-@f3I}G!ly@?_7*4OT(drO(9 zP?Ry&0*hTAqD42*3~eWRb&nL1b-%6rBRH0&d8D2-D$iQ1LSP8{@LsJdf_Q#3G-@kD zUyE4nAGQ$&+T+$-RV4%B^Yq$>K8I6^=r3b6EMpNE5y>`l0AoKazG?~}VPG>RNS+>Z zp+77xW?0uY$-Jexkd&upkM-;FH|qo<{iQX^;z4D zR!&Js78A!9M!QCYR1L=w-^%*|msv3e&~bt%M!_Onq;si-rpI1`Ap?-c((h1O#&}@K z-{aB!qS8Y)-ku5^!um)SfAO?uGm{=Nx*i4EjKO%s;K zkH;3iKo~`z?HsD&i8}0;>f1XnR0Bl7&`&v!D?V7Gxq#pM6T59x_rI8Sjf1&;s7ry0y99Mqb&6 zhiAJ?i;cpB=fY;f+Z!oD+N~rDYE_|{woB25yX#vr%UAyJ{SQWEFJHYz5##^2Uw{4P zi!Xlln}7NJAHTi0IvK0*jG@kyJE(Z=(K%ox!su=uyhE-i90$cbvWCui?ZHePWzkd2 z(S*t%M1P5T`RvEF4$Ve5R)%#+G|H}JBNv5EN>y%rtzdSCjFCShswqIQ8Ye%4c41D@zgE7Qt6)xu&~pW`2j7H_1~14td$ZJ`CwF0<&=tG#eg{QFp0B zejQ{nUDGd6} zA<1K5$wRi8XHY$1Ln1AEMMs;b>MTmsR_M-KR2W_?y~dDrkuB>M(C zJrRRotLfUA8s4W$jigao+PbNVLYnB))TxT^TF!`6Y>64;mC7VUCpCe$W3lUgsv^4X zeJTF4oRU%4dv;^t^E&1CkUzuCTA*l%R8-evi6n!a5?JEXti8BL^?a#SadU)XtYMB5 zBdsDYA#P2Gb^z<H_}Rr;Nr}_4Qx>{*Sx+|M+kJ{$Ia+{gTZEQv}e;5D4r8W?;J! zP@2OJ}!teC$!z7`HORFE2#<_ zjVlz9rM7k1d?JfwvKD|0w?}`N*zN3>*_y0Pww@t2a?7tVwRk{?@efEtsHk3Hheskq zM?@Fj4xNTn5i{U7TL1>`!qQHBw?pd_BWqI<b3IaKb9Oq-7mP+7d zYI9XciW;tK84t9ZDx+Ff(0M+V;M?wIir7|YD?R9DTR9Ce)z%{9$Q$^I8I9|P6P&GP zX)VhV_A^KzZyXJDTft>xmthb7T6V0+#=6AS?)I);rJzgg7tmU6OEy{%4<@8Wd|i-H zSW9;kC*iB)nS4wH^1O&hT8CkfNm-KpkP{G-sYVviV*3QXs3L7|x_G)yOlEF2#N*)Tr@+I(l$ z9juPpYnPr@vdEz|sJEJ^CARw(z9Sn>7_AV1?S@mOTMMRjp41zNIK0Eu32ZYA3KG|> z8Fd6`A=QS5krARrI#W^ng0i0`|CLUwQp!0{w+o>gOi0Djr;<}Z&Tn(a;Cl2pVo#ye zRNKC#*}jFOyS~mhZ>v7%vi6l&np{#1dYR%~23l=Qt}0g?QT#4EtyaXFw^dgGm*LPx zMn&91ED$ovlO{o73=DkOj3!6u&#k&1EWT#V2Ik5%1d#vi_=48`Y|4@PW}mcO0CnY$ zaYcTVRx?XX zV&s=$Y?laKp)ZP!H9si%zT z_QhI924ONIWa(=wg=U{D=s`VbwUY;&ntHm-!sAeMNA^HMG;~5p)5S&q#x7U4)>aDr zqYh59nJrCEI3lBZjx!j!$f_{@!fTWejyL=-j7>-R1BOZ4F6T!R9(9w0QM_EWzw80I zcM6#kHhZ$+8XJFl`T@kw<&;WjBKDIf3p`T@5DKsbddcs!p1G*RiyU%Fo;4wXSsUuDVO=j zZ@2%w3vt+FN{Fow$c-|7B4MP?iP*U^Q5H+t zxz#co@MU_MWoZUFLYG5}JT0w^LZhZKQ$22n?#@%^U}-&KX~IEHm=cVk+P|@Jf#;jgLxj?6b8D(sY{I|(Rlq?I%SCn z7`XAHXpB4+5(Wu_&)*lijDKn<>?D=#?VFAJU;OHyiT>aI&0l@9 zMK4{9h%7rj32)UG2(%aZt2x{ftGZb`^NJ43Pq-9T&OtOTHT*=n{#roA&dMbKikb7U zKv@JhwSainLBg#Cw1bJT>L~Lk9b4uU8S#Zt&F&>vQJ7uK3HcBMqI(>NCVDEj^=wlm zLNhV0_$rma6UCQSpn+>H*8*d`C4*}n^WK~1FZVlOTUy9Qv@45Wv;czGsTJ;I0E=Iw zZ2H3;IS2q}OUCE9$^0`xOUiu6%n;`e$QD96fZ5zqM7g)4tK<9}-}$j~tRL-HQ%h5F zFc1uBJ!%;dZC1N-)@#nmRRTkWfkw^QHav3j>(+5O8#tN39ShzT7whZ||81q3B-tj9 zWjmLLeQ{+Rq18df#cWy9BYPw|zBO@TsFohZiXN`(&0Z+ zi}B!5aY7CLozmV+>=+(A!Zs8`^3Vw?+Kodf>NXcN2%tXEzKNqhie8ktCXEn2LYg43_EdWCAJB=7zzr@#^)^kpuoD`zo{ z$I`k`sABo7iuf`gRxNe6h78dQFlu!A56yXj!T&KaOZ^Hgy?ZCXIF+LI++@1N5|+!S zTGMZ3GjC}Ck?ISoTREmocDr&u^&uEneRgF&x~y6i2eL6j+SN*BlUnAXVUg^W%{%s4 zk_2o9(Co&tn7W8|0W2h{i^+T_!P9N9)s;DTq$`)9^D=#-OxF9-qGFh|7|5}}sQNs=jh z4$mEsFn7eSr_mORE>3A+&{=pDzXEKYCFPNyRF>w2xgK?>XSfzp6nQLQQ?tJNZ8~jA zM1um9GRr3449-23&tW^QUrSvAw5bUumuA6wWWb?4ng>LKU&HYhM%mJ8gAMzCcvRqT zYP4naJ(t7YamvhV6DCH+MPmRO%~1Pln=sW2hV@N*RPtdPTaNHo;K)fd(#{%v=7b-o z+MFqObl9W%t8h14dU|`rTQd624 zuC?3so5I<6lzef4%oEXJ-w}C~07tUhfFjRp+ndZRvh#kRbg{e<=s5)q8T@sG+BP8= z2-)Pa@aCgfc(@ACS-`W4Gnw_{nvbL+lXFyBHh~97j)pY1Ii{v^dLVbv zUQoKNaKpygnuBI(a#9%$v*bWlCSH=9A~0rPRv~Ablwzd7|MB+bAAj-7-~8_PCIOt@ z^Tn5M4Y-{+#%58QWH#Fhj(!s0@Fe{U`9d)36k!sAk=X1~@pwdQmMQtjr!>&a4+kSi zv-9nfn>LVXWV-whsh>tK}Hocx?dNfW)5r$6WpT+DrOdc*?OMA{4p>+g4FymVM*2{Itu*WHm>~+y^vW15xM%b{xT5TX zlbRg(;0U=}*&4gznwQ*h;?s^`-cc1dPFDEHj^B#?y->j3R>s(X!(TKRDW`E88_h`Q zn9JoXo%3kQ)w3)dnhCqki+i$Cw#BPuRw!oUJ<91OLZlm(`OEPME+k%ja2cN#fkp%+ zQ?^z;_|2|)QA85Ro|ZhLFy7n1S&AeM)?*y=R5CPr%O_Q|6f29FARxW7krIJIu zZsyQ&CfcW6Q(Bom^ciT17jk8Qr&?E1cdhxDPS#)zcu}#b(nSqV(cb-3pe|HW0loX8 z@LaD6e;K_oy2C~$!&WKbMf?%p4}26tG@Z6pwEp{oqqV?jxD*xUdR;A@F9=nDR7NHi zZ`vD&F#XnA#ezeV=qg%3UuL1@Kp4_|QAB`7L-UM-j%ub~1?#ukk*GL8YrcndbjdHe z16TxwYksPd3*<^brtkjn3l`O1TIq_Wr^qzPJ}BPVY>vHmH&zqR__Pe(ktZJ>3Nv;h zNA)?DCx|qs0sr8YZ!d`{{SU7R?YkM4$#RJl|H@b-jf+JxBm%;4WEl$-o#*Vu`zp>b zg=$2gBq3p_zs~>nAuqjP2Mwz2=^bwMBmxHV#q-w=YBUBA{zM_YPz1s-iR3zMPQHpM zLMkada*U&k*$~U}J?fYGfPKLYl5C*xeR2?wHV0g6p2LI0!}iO6zNp`OOBJswbPE;@rnf z;d*7NRTBx9#zkGo((r4l<4f@ShGHZOciW&c0x%-Me?|stc`SZ!$xXXtG}ny7#=W&M zrRFWXWTzhPXUCQ2TQmE&$*loxS|7~g)(Ns-^Ae_uopf}|Um`qfNgl`)6X+0E-Mi4n ztS1j2+xqm(v}6m1m`JN(h!t{3rr<1LvPc$0UPf79w^xv-1^~aww$Dt}R$I|Z+nuwz zhtCgj5I6{d0jD27$_Vk|j=dwYcEWO_kSh$Z3)!r~^XIZx4p3ts39sRutdrSh166MA z;NU(V=@>oZ*4a#C#+sK4&(>w+CebRNVDXPrY7)({R>+Rx4dXYhzmyLm)a9H=J`$;4 z&{!N%_SQTNlPrp}#t2Q7qZ>qxEvk_k9c8X@#-Jf^#Qk4g+nL5=-M_N(a z&<*Q_5~1%~CL?s_HHsaFGAzn>quUI(;*Iy`Izr^u6zWEGHR;oof^i*yAnln7icgu^ zsKaU3HD6d}gNLkrT&rq>V%2gKXqOSyiBFS4tZ_)1kUBniG#tHcw_@OtETP6HkKY@F zkQm2t${?0D1`vLD>`wtwgFK{z<&5LAm2sEyjr4fCu??raeH|XP=xR}nkK&evgbbs& zdF}Sx=`w$~{qBc%JO3o1+z7kF5D%T9VGV(n`KDqZ0gXPuV0Z7Iee< zq|tFfXQxsVZ#h}7hNFpG0|%0k;AD?m+AQ*umu#7KGSadolu;b136|r6&(8$p&P9P% z_EAaK7j%fY9$Rdmf_cqjq?SY!8`@&@_WmbJo7=347?ieygUK*U+p~ZiO)#`q zpc^)8FKrsx?K47hb0J9qtsDo*$YNHS_@EAIuzf^BX5M-yC^PilQ39{;*4h#*he`UN zN4`a-gxqjKC=Wb8ZYA8q*H)0`9hxyWBg|2uyuk}F^hV=VF-Y@(O9o}MvcVe*2pWSe z?75LWvYU@&`TaazlpI{YDQpIfMz%NalFbcDRM7pY!|S;oz;sjvK8#FcVHUe13l4Ft z4Jj*aHmnoaJ|UTMawKW0CO??%HdPCCG(X_8DjN0o0UOiF%93nRW82v z@~^TQSH24!i#z%@6OX7p%%gdCtGPO zMQfo%97sIEJ2%OA4L&$;+(>(dr4z#jvX-OIkSO?)$lB;R&>m;u#LS~(&vo8!*(5hH zh(e7p&~NrY9-Z3*X1FV$qFe_8CVt~Sb!qnqg4}Ofh0?k{|5!-BR`WxOt$>jkRV~Qe zN-$`q<7-Um-{0@-?ejGB_=_*UWSb;(^v!h8c~s(JM36!P9x*1zvR=^>U_Lb9H!yIj za%M#pZ8@b^_9SZ^^nNn7k$3b4wzD1v|r174&XPo z`Pp*tJ+suTPD5X|!X^txKwNKmVDkjBYnyki&2C{;QjOZ$<)5kX}-HMutGdDU#B=?_(7zkd|dX z>vNQoFc+k?eZm=~c*-*|5y{wynbuI2kko7xR~fO(AknH`H8ky0+;$3#Hw=Kpk~^=P z><11nh}2f{G3LRY<^T$HjP^#IeWCAq5)vXp{gxq?wu-p>5zwNSX^WDYJL5yFW&AR9AA=` z_MNoCTU^X_&;dzbanfX0g9pjAPb*I@;1MF39y@0q>=Xd z7-FxJHS*fQG7N&nQ-)8b3q1w!-X4El00gKboYFY%bS@Yn74NYVwp`F>Kx?+ObrBoM zSjM(ffv4aVlg!@-Z>AvLqq69@Q+=#iY(8fq@fJrl(rNXP%?9%zwh=mH(Y}`gAFnSz zoSxAuker;HI*$yMAU%OCN!_jU; zd(zJAgR1QB7q4Gx>%imdS+3hVtD>Btrf{=K8N}vuT;$2vCx&JGD2ZwWD?WCfL$V>5 z#*Af~L2O#f?HjqBpN|qWXKjl0N^To398s5%pr9EgE}sG5f;zg<26xs5$4e)!35ElC z-Tds`mC;mM8{Qa1UVudA4eHuAjIc?yu^t|^(E?kf(W-a%{g$4;K`1U1DFM=ortFe#Bbb!Z`rCrDArP)s6JSe!l6dXO~}k&LM_-!ox8 zYGa{JAZRtW#1n^cmR*hHWvce2$ki)DHCi?IN2W4fBgNf^OwA=a-CR0J%BQ!v@cEW@ zy6*);oiBYZl^G+&nA&bDxPm9M*_Yax=Kbe=2oRjJs-fJ+DO8t%qy1@q8qy|YHPgMW zMBeh%vDID$|4af%zi$#B9*HuFGD@EYi6&t<5U}*UhA#SW&7Jbz8ud-vgVCrFm6r9% zT}7XgL`v0`1{M?nSSKYbdX*z_sU~>P2!z>q;o_60>S1+e9F*Eiwf(E1y46+GsS`+= z{=vm`EzXVaQfFYB`%9e#uNoQf@pFMH843aI6=_7t2r_*|w}_|cVDHZK9|+x=JKmLH zQ}~z0u@23ebip+{Ix3xzlJnB-OwWuC*k28=$S`&XPBo~Tp>~)guP6X%F*Y=FQg}#V z(}88yrYf2Te#J_Lg>5`gSM$%INs$7|!gZiTYdsVUp#!xtv3}O`t@7O zH`zVrX^gy?$Q&#~(F)JKPQVkR4zo(lz-qiXNB=0V(Ylb<@XS&kr{X{oT8r-CLZ%&Q z4XBLRNqk z>$2oM>XSES%1YKt3bk)MnZyK#CL04l=b@b*(o^E>yLB+P;73oc2p-z}g8UXg$x(}( zHQmyHZG#I+&V?z81FWRn|EA~5IhXmyp0`dGGKy2!;N9DtHL(tK zzXWmlSFAs5O7hDa=UZIN2aR;6C-XaB**Q{@oGR_zYu*>nX0Aq}s!4P%8F4c>O4lUF z!59booxd)1(Qo!--YOj?Iu|w6v7+FEg)3ETgx5R<+4C_~0!dJ$a8!TPJQ?lU{MSaM zr==0vG*aV!U9J8+v&QT_8L&2qER)1^nrI2V#yaSKoE?bJsBV+Hc?|%IlZqkb7`ivs zHtedB`OCC=^{%Z6)%&V&Xe!XX>8AJWq&tJuU^qxF?bWSoYzrJ~y?d4JBak)Jzgh+f z*Oy@fc0e`zGZAA4TFuIY%*d5}rL%C&U{qPkYbl$ERXf@@U4n%yB&E5z+WWZOa|75| zEg@ilw=UAk<#Ol9WdJ}y(^w2<&kRFzR)f0Hm|G^oz&EoB+6N(?U@OWAz^!(~AaaUl zqylYXm1)k79eglcIs5p}zx?OlTEOQ4tX}F;9!qanI+aj`vPMLcGGdW;2Q! zp$y=}Z{!Q(>@EVOwM8D-qTeMLDYU@1f(&e!er7FAAux5KOqZ&G3tZ4oCW&f|A!wbfi)(8V8bJ3b6Gw%@Rg+jT)(JbB^o1$1CmeYVq-!TDD{FVh zp5ed$EyRk0WDYu*66EB1#1eOnD;SEy9rGZE#tw2u25)ULtIVukXeN5w%t?od!pK0<*)S<HN+6m zwxa+yy)>#3DqFD$6ETZ$@AS}}$;yo7P)5r{U3GF8B}aY=72~Q_CQS7$>!r;m1t$8> zjj57!`!^QIZP!A`?^Iuz!10vG- z)0n#fHaO~L0ia^jWB_Iw`%-rp+pT#E_KMYUzdF+(6EHw#go|yOlF)H7G_Q~@f?S$p z1%?_fA%`~S%kE@@tRyZi=c{r0110`85(;qOu+Jn6AkYkdAPy*_GeOubLy&bn$g zjprDJc6G%`)#`8ouZp}h!YuUv`~g^QAs4AD!DvV4O2j~mibG4)qJ=OhN!BoDP7l=5=C4f z?!b;9rOh^QJrmVG934H^i=dD@mgS%G*>2sR10`8Pp`(ofcb5Kt_4Q8ULri1Nm zPQ}+&j0|Wt`frB>Z83uaW+G79O%`+d z%rM^FTlv&pUDD^t!x~%W51t(HFEWnQpNt(GZNfP-n^{iSF>Z|{EIdwzES!%2qbbv7M8X1wB(SrXdy1e>7--_wgLx&Fcx`jC=4E=+*x4zRzW(|fTopK> zj9A|+yJw`}V5u?mO%)8Qo{__j8sFTyDUtLj&o@kW=&JdETSw#zWKPaLT5!v*kF<&c z&|k=2NThI!T&$haWq+yjLgc~?hxJFOQhFiZ@n%IJkc7hf}k`@XfMzKRax$L{9l0czJ`yD~; ziHi2w_sF+y^F;}-T!Mi&vm0j@`j|VDm>f|M@+*tS4e2x94H=;<#hjZ3Vd{k#f5d3XcxznlYSu1{SJw6WPd~2FkybwiTYKBKynR6jqb6VPmwj-eLv` zWD7V*HKLGGU0+_ZcQM$JHG6yJFmy5PG!-D7Wo0L~262$Eik8d6snws1MF-g92vgQ8 z;Yd)){P5JcMgxa++364iKAXhkw3yfCoBb-qkCMjLWN8l zn0@i)^{baJ(Z}XgPeh-2m|}7|4=!^}B(R;*o=%wk(H+qfoEkXmJdB(M6A$3jwKxiV_xuD;yJUiT@}+PALNKXzjVxZ4Sn#d8RqMeQV|q=sfJqhb zu4~?cB|fg&QE^}!)>fbzgXg?g$19QRGD8?xL+%QOMWVPg(Tzn)l|+(UgmxEWF}Mc< zOF5D}O}olasWxwJQJ^tFEgnYdg(3_53ks8Fk1*Ud08q1%gIc5_bB1MG_m-t@CDdX` z0cFH&9Svv(<|ysc^0$y;p<($1~Eux>@wt3ta@yZ8YD8gh;%Cp zTPymTrtmV?KU{V}z23kEDzu8Y$zP+Em_y90o=*Id`05S7u_b7SFH@yz3uyzzO?nb( z2hnLui@xLlT%OMQB!YK2DXHbDvcN%4C|e7wmVJ!Z89U~mLRF(||JBAfDke2PI zPI|2>JbD#^1p0ulB#_-n>=f#AWVd?8ef@+Y+4Pacl_`UFpY`FG6k#)iOYKc8IA3!A z_CTu3gMN3O-ASE$s#nqEZs!FTj}|v?%WgFXJqvW(QRbmktx~nMv{$aly7s07KY5Pr z?RJaZxf3B961j^c=vJ)kL@!N0duCRhivvE31V#iDVZw>i;T@X4U`D%8;@k*=7r>#A z9%+g2vl(~!Kp@#Pq9Q{KT#=%ckf6<{p)Z@=W4t@Y06CS^BLJ;R^dT|Vik{K0Rg>+s z3!>d!(LM3sT;IoUf^JY_Czs}y?0Dv)#aG2aB04)Fp_4u&iq0psK*|`S4H#e-u?t@B zo-7LJ^rfQV7Is#c7Z&lG1h~BZNbp29%_`U!@bK~M^aBX7?#v<1+*8L+v_zT51vv78 zj+BC3G-6|Hzcp)p9j&rUL11pu~Tv;k`_!)nf zw(Bzu)(g6F?v};I(@y{7lYtv-MF9~n{b~#p?Z=;0kHZ^f1ZVDR$eiX|x2PM{Rht0- zCJ?E#tH>hQ0xfO!$~Xmc1}%;T)KVZ3tn<`tDv*&rE|tmB2!R5XalLluEy@%$8!9fr z=GddbSX~v_)l|2C!QhgXGwIWLJ@(0MB=TVz?pB~JV7eb1{s{IEr8|`YRYL)vYQoi; zSSru`j7z4!d(~D89!uv6&W*0<9`mqITdFURi1Wy?K*UyC8ZDrw;kJ>AQN&_Qhpey6 zv>d9PC|1}|PqcT!F%gM2*ph-#OB?gqsjw!vts=tQavX7-(x>-8M60y*)Agf`k~QE8 z8=*|tuSj0Ia9J({F~6C{!>P};jQ>O&sW{H8Q{zkT1+_cdkP?W4fK_e6qAm6hH0_&j zzIprlwY>6&lhfaS`^S^BGbwG>*wow6?h)h7G+uW!e!Cu<6|+E4*PhR}yYu}IKm73S z2feV0hCV=Uelyq7ESnzyBT08gE||3jOQv*twQxWCkZtoaUe5^eOg$XMiDYuyFP}RA zK(qY@v*_1~RmOG>_nnhBudi-pdfZ2-MhnMB<}qN+d_x(z8A@IFMt#9k7-&K(haZuK zHEnQ++6`bfb~+EcogU!VAq2+Q+YC^!LJ1G{HazPQ_pZu8Tq(riQURS>gQSbuNNkxM zX=>#UF4qE4YcBaQ%}2oqqeGY#rjua9ElY0Q@tdvZtD66DTACwGEq=M>)xum7TTOxR ztg!L{oB`v3xOd2sny?2nF20nB=K`FOv(J%aG3FpSh>r|{8UW3Afx#*HEh2LwL`O@m z?``&^rWmuedEADR17d1K1n9lB_P~7F631Du)sOC^${fhRveO*2#VUKf`9AeV{iv=^-yzaS7vGiYt`y=>9_ZWCK-Ua__%+LtIu<*+u|bq#)qo10xQY~M-8x}J|>OONOi_;N51gv6;GDW8R?1X;ZoqL zVUocg8M-Lk1b}wzZ7?_*WHEU{giL$X09@b(N#EvwAg($a0Hf8QQHp2M36tw72HCjZ z76g-ekm_z`CJRpT87$s1NooAvf<9w)bG%IqGgDUAjOe#YWVYf^y!LZhQccMtLBF8z z4?&|L1Ze7<(?tFCg4--w^va(rlQ1aRZ+5BKw1M3$Px->syj`J23dk-aROnlAf4Okf zLka;2K}#g(o1Tg(AS3caV@7PpxxwnoJPS_m#GoKZ`8(GUcow%jZ zkr;BICw^!ogQxM_qO%i&|jObjqRE)a1^3rVHdBw*|tbofh+`=;MY(jbWz;xlOlXrY&v< ztUQ~Id}45#5ZhbllYh@u@mJj6I4C+pTAUN4id|o(lo|p(aJBoivpvvK!6bp^^u*;( zkH|4;y~jQ>?p85UJFg&E)?eC#Z+jcBmutqbj>Rorn zTD5x1Kh9n+F$3ixv}O7_ND4f|q*?4nSgT#03r}+$0Gbk&T1C)o8X6c~ zY$m=tQ7Hsg^wO^Wr)$83b+bo`qKJPQh`y0$+II4XQ|IO5H!OazJm$^I*I00|bG5tW z@gf~DM#S+=q$t!vjjC>}?by2G6|#i^M6EQ81o|8gDmJ2*3;l|dnY78S;J&WClG4{gKt2Jb5wzN*lMV^C9b$$Vh>yBET#m~P)bb&0SXl8^zogWKm7FQ zp|f;~!=zW=QIGh&5QR~XNVLS$7)JK4IB4ouH{91ln%0b*s3W~qVfV5Ll{d%GN3JLb z5}Kh1or_$`E?4CIs4kq>D!c-U_OD3wFtC044FJlcORlum`uFbiG5hk_&v||tV|bN{ zrlp>nasVJ@St~JgCoSNh7hJSD4GtDQ6J5#nqDARUly<^RPHnZRZ(K1lRjb<>e>)N zi7(+v>yYg&SK+nMW|7P=n|9oAJ(!h^%)Nk5LcP{L$HG5f2lmMc&EzxkliHH^S-hjn z3Zj^h3-__&#b>icUb}}(!Qi#0cM~>H9C(dRizq*)q>gwf>EEvH4#qmQC0>KrwZIA7GYhsP9mPP$gB{7iQ(s>o8ZIQ)TE&*2tWz<=LP?FJ zFOs9W)j|PoylpDd3YM>T3gRfVT?b@=2LhlO?HR(fKz0ilP|`xsGQsdtL?OQna2q&> z{Qivx5=nvvsEQ6rxMe!oe#33G4BtN0g%U5{XS~M9>m8tc2FutJ5FdSJJoA-cWj4iF z#)Q~`fS&A0x6dL`gY~AMdomzVI{eOQ7RabWvb8H%ra3QAj69JxW7F#CHn*&nPK3pK z%nKBjcXHQ^#S)#as~m!Uz9;z@6#Dq_z6t&N_vL}8!9OQ=c*eZ> zgu;4`K;3f~A0rk`bk0O)rPdB7Rtt?RMKG>pFcKMOclXpbS%8uBpuZ zB=#C}jS|bkj#9-`mHDVR4TKIN}q97x;3{IDC-jDpP*IWIQu*p_1j9YVK7RM;H0RlnqH1 zr8zbahMCqXIb)5y^%YldmF_PQ#De>0iXp75F|W1cI`%H|xa@We>fH4U1r4`VTa)Sl z9*%RSJ+{WLcv)#GsQ#}dnEbj_3Po|^Kf!2B{m&N~H7K(2!G)d~>j0$6Q4#cAAS2%D za)zrk9044-k|rKMrS3Dn0mY9Tm(MQFdpvfd*P80JYDqS*iuT-}>p%YRi1ArQAyyV- ziS^~sh2&d_Owo1>b^MF5*}9pk^+pN4inyyZ)=c-v*d9!ExB$&XwXlV1*W0+>L3sR0 zp?UEv`W!&EXKk2XAiGloaKnu3bYwns%^<44SM28e(1sZM4qidlUdtyP0V@Jxi$KH zxSCj<{PbQ3X;}^AEn1{VzQ|Kx8&@K~JPjcn;_=+dd*{FPvW~;>$~)d#2Zr^FNZ-DB zm4T(_8Br{xcEmN%Ax^)#DAf~ZJ8k{W<3~RjLg0bT&omWd*#46RpbiPR^WpL1P7(dm zZVGd5CPE0D-!FLOY=o!wd0+Wzx{<}eZicgc|7{+q$HD_vPt%}tJqq?5on|=0Kj(x- zblw(|Zp6rWZ5zyJu_ZaTnowI32i6!|2{DM@t~n&(@uNrNuUWF1k)O;tX`mYQ&nM&n zTDRL#J=ukommN%G69z%Di+Ur)$8~^=knsdnTKup*F3wLi0YKW&TnX;a;w>{XTX~s( zO0y^wC#+v;@FWr(ROxG^g!q1JYnz+nZ>1*|6R@Crl z-Ub~R6$>Yc&-ZUTB<|*uN7}Z9_ubDCAV|^bYi8f}qAxed^?G$Hhq<3_&(@Cr$jWjq%hN|C5IFvP;0A!8 z78V6{m8Lv{oxX?GuP|35x~f>eBjs=zju3}q7+rODIm(1QE zT9n>amGR@AHbMPLwd#G|j$Nr?)*%_WF&*JdP&X1-J-ygeQTrP)A(}{iu%opOQJ2!V zWtF((M4kq_@tv|ta`Gzg@B@5!>=WoC&swxH`j2|a2}v- ziBl2#?BMT){yx)e^tZ_DG*Sy3-E37OKtfw(g=jszZ481az16qxHSvNAVyyMVpyx(-B{qkGcxt2p!R=h6`l?KdhhObq z%{ z>Izn(jygD5k3NFue&l?3On^*b0Ve9=rFwvIm623xF+*FBy1L4?OJr=qIgb4hSO3?X^nqBGKfQ!U|hE}_@Rw03~pNr`EhRI3s7zt{cLMMCQos^ao z^)3SlP|c}*Caw=DRUgR6LX@LItO2GJ@50ziD-uD~s%fPDOAKTj`7gwCntg$k&-XnE z&7sGdzYc=BWGQy9s$k@+xUKmbE6|yuK&%nXcTGUOd-GZ}REfh9v%2Z-wQ36LHn!xWI zoR0tbx4-88+GukegFE+%S~B)Y8$tIpK0h0X(PYgXmq>A`z& z*CP*Yr68P}209AdGgzkF!>9Duh);<}vfOZrKeYj>BVH?lZQ_l2{2|W|6)N+%V)mEp z5oitpG$SuMbF9Vwio&IqBciPl3%vq$F-KCJsq*FKmY&+>RFP^HGI{LeKEmM&p6b4d zBDcY7k(l!D%xEo3CFTp(M0bhgeB@}XYE|h~$N&XNP%cM)(K}?sK9rVu-qn0}P@?3p zR3oc}VF-}Cz^iXWe=0VLTaQ2)6vvRYmvcjHjZ*B?^^q`vPYPf)*hux^Dp)P42>8C$ z^uE1DR_P-*d#mCkQgW(**F{$yL^CMLB%}bXSRZ8#-dvu9Y4?@;Y>#I9I~lR&pzv;7N0wF{Mqp*m^IT8*>O z#Lo8{J>-A(1iB4Z-4&z&o5p+@8^dPzrT29 zbnn0Y^2@^qkF=rh->vDYpW?Z;je_yIR;cr+2P2dr#+2A#D`QgjswCI0p-VvDoe_DY zZ8^px+N{@cSrix?W6~`a)%0s4j8lm^U}S?DF0#jF@n6gCjhdgc-HiMiWwPzik_xKv}pV2~DNdd(+F+Co^3jh09_SPT8)Lv$LT( zFDT&>!2DNj0uy9xxMn$PJdOR|;GzO}_4>y@KCwf6Fa0Z|;GN3!+V0;rCYHV6!dG^--sn_+HMfQfR-bJ9z1OGq^8b79sA$f572V|jy=tHO}jK; zVEs768BcK&YxW47kG0hX_^dCrhC_>xLT-Ewra?Tt6w*UFVQ1;8wjo+;iRrBfh|2hY z;(i*aC_${oX7$yLZ0`8&Y&UDn^o8s*gB(S(M2#d{`K%r`_J8946V4Zz12o9_k08ps zTD2OKeeHFl;lV6`ys9C3b?4%Q5qYZK&9vl?KvJ}_G?j=WDbvg-eH1z5YNwCM5zc%8 zQmgssh!nq6b-xte&|7-Wlrv(r)6mFVSVtpQb$cIqj1?dTDM5DOBhvk~AT2hB?Sc;d z-D$2L*!dkPO8HbU*@`l16}Tg@eJi^{N;^3!mqi814irB!2xG-*I)uEbBzI+wji#u4 zL_)yhY9D^(O~uw$hgr_x5qBMYXISFnhZSoFltcI676l~LcEj`;4HRo8@_ zLDFzX60Ba^8~Whk!{2`U?boNzUN{@RiOvRluDwGA8T#mCUr&j$gu~g5%%BSN{=IcW z|M4IH@#M)9`2Y5gfBfy~Z+U|n@fMt5MI0_}Bm@L8T%kGH7@@c5<@J>-Z{ObT31kl6 zY`(cvEWoRRu_`iV;^rvs-oBTcu$$tKmZO{r!7GmWt8RaKg>l~NqX8-lnGB0t<(PdV z!Z?HpZoiG|r`4trZ|QnD<oc3^;Yme@V;XAL`xQp0X!^1SRVam{eNy$bjw5*+* z)Z9QZxN#sml|Acf-Y#b3bOfD-AD$bk+l&1)BD%&-n=+kr}=D+eNw%n zcpTN5F^Iu3UU&=fA^!TcgE>sRwguUUep{erCT%)im{QH_8Yi`&BD;nNn6*@flP-5t zLa!=JmCRRp{CTYo$=5_@N&O)OSXrteB@Fu907a|BqqLDKH&uOisz8}iwJvO_u>Qdb zf90vwERs`=VXG#Vd*C;Cfcu`w7o6&T_i!cisLU%N2BI_5QT@$mMxE;-Fuy^T9!m$x z;i%-%)Q6X^q_FIXL`ps)T6)KGtaPFExDZ0ypZLz@&L*<@kB`Ib_?S%0>IubCZG{T8PjmB!FLHq zoSy-psAS!pkr-Ja@8bxz$kgQ_<-af>w4}?u@1-KhUt%1gh>hlr$pjc?ep8Rh6zG>5 zf3)7a7OFZKeezvG=E2Cl2qn=Pec#EN?Bf86nQ>BNZwWuF(deV5$zQn-Z!6!;C}&nR zwF~crHLV%8yjW8VKbA-OCY-Cn+-8(BjMD@c>(Q9%~Bcb;~H zUNT&=r-vH$2G4s|LTfVyV}h)uW`f5$33p*&`*V7>|?05h3N&l3+&#t!ie+ z1RI|b09zBIA3qw7`F6WGOy@wkJvbY{Xat&a`Bz{dv>2q3U~DI6-bvuu;YA&YK`du) zsQ5BpDgv-_{cTSx6;qm6)~-1q0o@j`x=X=Ep@8Z@xpz*|enl8RJ^4wOKv=1{fBo{+ zKdx9{bGt(zoU3CpH2HpZriYWw_;!xH6*Xp4TEyCrRueIWSVmw)r}*n$*}fszyXuesxbZAQ=& z*O|p=3kPP)%^jgq3KoG{Y*ixa6m?U7qm=<7r%#>&_WHs7hYXJTAKbt9{{8(n6V|Y1 z5ENIZ`P)|yaIlwBjP&bozYXLK#{As4sF!t&ns0BqqdVbuR-k&VpNZP{`UBw`r_Mu#?T!*UXqI1w;9Qjnal%VS_}GW4n4Db;-% zt>uFL$Ti_5)!}n*E=eUW==UpN3YK1*7Sa=tthLU-sWVF41${X~DuSH(m87inbhezi z2U#s{5!o_fJL|7r`|*b#JG!`~IDZj4{`ljMw{jUXhF=|2{OScNj_*Zc zp6=eA>#ts6zD>m+KmGOZzdCe(Y!P<{HqFfVYKX{nWVuqJ*X6S>Wsw^|xGJ13`dm2#_R6`1Dk6y~h+u>)J_qHy&nUJ2 zN4EofaI8eQLM|2Deyu1dNs>ert-|KtqlMggfljhP>G?0P$-^xYFHO{RHb*LTUtgrB0qtc9|)}@JL>oK>MQsZCrZ@RYCB3dS%H07wG zIsoE|xf}rL8WD(Bq`3j>kv@I!05_`(S~+CUMoatoqbU%x z2r`fyjoO`H5|ITM=RzCVP=dY$0y6m-DY{@IjS@6Wyn83qr4k?XDLBzE5d`y?9&3>e zXY2AbybTiuiNoB5BUW{RzB-fC*yw#`@#^L4moHzKczN{jv33?NlLLl3$#bHu8f!G1 zPyYPc(?9?Dmw)?j|0e8s=@~~&XN#IVr&1i|OmnmT@9K`H>>`tRi~}7QLPI5bYE^aK zX6z)?_HI`AyXRRtg!F9>g0Pd~Y{pCK*3GU{YdZmjf{l-!2b2%WE$#vicNzz;R`?{B z2`=ft(J!*DX3e+=IJ>H5Q#q+r#S<){sfVqL5S`#asOSH9k$S*A3N~PUuD*;N_8ced zCY6CZZ1SX^ewAT4b-WbxQ$KCCR{dpUtjJ&QK~Cbl*c9xmNw(v2b*S-f%?t;Px8{4| zH$0;!gWMRKkPcnNea?v|Bkp{*wxYANa>*}FA+6f(yr}=! z03X+`WQhOoY$w}b*N6rF5E3Nh5KKiBoha%t4YM+VNzNB=Wed*gVY&Kkf|rOqVdAyr zWf6lsI&dwNWMJTEKpTJ+Z^wz3@=`FZ2P2apFnHR z2ph*MjB{NbU)TLjwAp8eiRo-d#ED;_<1EWc@D&oKDg-+%qT|G)ob zK>6|gd#V&^W}rpS$4ra7snZxp6-+36{i+AwIx@&nRt?5~F#O;lf~|$YwQ(kM1zHfk z^8Lp*3T}+1DwSM{64EExpZr!XirTeY?lUcu2F)C_OrJCP=4mmTAG?Ct=N{MIL%$vC zcFP7La|Q3;zIyuZ-OCrwnD0=0&{ah&m_PB^>YsaY#Pd3hSWV+2bnz{6)>i7~w2#Zm ztM{f28aqRw9qSplUUYQUs=wty&hxG@S2{<#sS% zOz9X`UJ{5Z;vy%%NI5fKB`IL-l5pCv^_#6u^j-^CYF;lZuUx6<}hkNQ_2uEQ!zk?vAK_NL37sp$%@es*G?|v%?s^lp2^vw|u=y0#xkxM|BiNY8_iLb_G zf=-MD(83AM#`xLXc0H|_IaEdQ;HLB}GTIVg0RQRqz^xOvSsWg^DiNbtz?$ZSd5c?Tp9>6Gz;*6(H|E{L#!Lz4}|RXbLVL=0ZU# zl*fT_1zf`TT*{Q0 z?>Y8H!>(nWjLQ05LIWTwj#jO*K!2Y20@1HxK4KEVaI!|kn^JU9=!yEjtVAU>PN$G2+Q*Yc@jJDPJPK)$xTf8pIEoRs90><*y_Ts_9eh>r@ZPPnP>=W zLv&(M($EI$^9G5vuYc8jxe|HWif_gjNSQ?ai9rMwf08ssup3bp8{E8lyA1(0^8V4@ zJfmosz>woFocO6%16~8cP8GXz_rasbzy1E}t2b}81`g{Ri5+zyd^Kw4g=`I_J#4~D zXDR>t|M~B<{y+cc|BI#qv?ir`bfD1LFfquIOr?WkhQ;C*BC$%(Y-dOup6ka;bR~ZbG;h#KAceM0H z8S}}sZC)Mn77PpXEq*c}<*tD*fhLkXxej+COO2*|1bPj zwyu{gWtO*Tio3olOk!M2;>*=Po;`cUwto8QCsP5&{$IWFY|YuT&Dr*i-MDG7K^0hN z!jQ>f2e9VobNMYgsN%NzK!4AXudOMw3`(zzp$h1q_moVB1?r_VLP=Sw2?5tb6W{E} z3Ul^*67`%A8>}IMb0}D;zuZ7obkfP#lVMWI0FVV~t}(ZsRnOPtqX-PuXMI<}o;S;A zVNQ}0OsE;PbsLvv_B0X1qp83!^VKuPE_=a7wd1_1FJzuWQp+PxiJ8~X7SgQt4;)fc zSROCcVQ-b!wA#*K^CQS267^4wJsvi4NO!Losftb@zRKMbBj$(0+yqJte3w!pk{ij^ z?#fDrl}J!0Ku=)gfEr~0PsX9Rs8-?ZFLxwoBH1hOxvCJMWzCc{nEH&%E@BrS^~p*d zc_jutBx7YibHrUJ2R=GjhS*)0%NH4B<~7Wzi8vh^%-MbVaPRTs$AG-_7+8DAS<#SJ z{|yjdaiZ|u2N~}5+g7NUIQ;m>P0wW&`it2u&T{$v>C@*go-@Q>wv-6v+K*H~uoP*Yv<(crkUrTr4@^B>BBQbuKnQo-Fh1~EnUB+GvqAM_dP$n1%)=euqb3_i6jjA7E=;J^SGbC7?iL$ z5w$3`)E#ip_y)uD4D#K(H~1nE?iLM>T2pnz*)N|z)1T2AX><;?IsmDFNB)<0&F>`4xFS{X zC9>Q7F`^k7h{Kd&MS>g^k*V_!mCM}9k+&fC^2~MD_%&oi%?omhgQ7IU^*&~ewo16i z?=7y)qNRW|O?NM=&pR9`Nr$_i>SL932@S!FIC`G#kL={pPhVm6mki0Qps~slML=U< zcgdrgUaI_Dn2QRz4Z$jCmy9?HQ97}eC7DG^$&C&Ag^yOh560jpW`RWVPF(q*5#*Ow zC1XcnKZam97*eH|cChS{NFoZh5kod3{Q@X}dnHbU^&T#nx=5I5-)-f|Bg6yQJ+J7G z^De@ku*aTE7m3B>fwdfgkMZG4N5>Yh0z(4|$|cZ3s0)7)9TM+2JJ5ibWF@2oloI6b zCjavIk*A~&c>$cnXGh84C^#)A-@)Yf51w!mWUwm*n#bHI+{gPyjq}?B?iA zc*#PFe_E=UIgJ@cXFACHXT?Oa20%%8s;}VJ{~cWZ5UFJyLqScsMYQ1f6GFk(^26xA@UP$XEeNDf{q{}>>nGTOcG487vC^BEkYTMwfqQ$U1i?1vi}SJ-fv=^P zod8o1W&{Mc@P|cXptJ?hgVpgh%JrxXpp0DLOemxk>O5omciNUJ*NEW688u|T^A@4x zz{M~=$2Wd!o(w;st`nV{<8(vkscc1-1($aH%&dK!zDUCJV|qQ-R$ralTF|X)002M$ zNkl%H?`^#O_y6PHjU1U1f4l?GOsFK)mS;NQ$GmR4 zMWNQHDbNgjXJEEQfGycCq5=!9bapbnj{0$o$~CH@!57b;8V}TSsCToKy(VzpeE9Io zFMsxU2l4;QH>onIHM-N{B}0y@${iSJQbIIB_BtCrs?A@&-n)AJ$&XL6>2Fs&b=cBb zhhpFc2df$#Mo}MY%l-x6jLFcl8dAH}+LBoVoO}`>ZC_>xZe{e-P#)x`*tsS?X$p7!AC+;pHymIwWkf~T6=OPPveel3M#oKfwpAbx&DTrZBXQy2-Wz=2GF(%l?%IQU zxAho!8#DypG{L8j&a=R=u^Y93-SPJp`aj$0lBD1{!FjB`Ts9H>f*V7)C0B$(64nG|`g z^YrFrYUV|1O>%MIlMmMPA~WKQAqSYYHIS=4)duw^Tgi&n`SFY7s+yVHC^=T0nJpeo z>nszT@AfW^053yB`e}N1-H6>xeSrciJbh~g%kTBQmQ-+M&nLKrEhuK_NV(LStUfDs zmyuWT;c_KkCf$RGy-uZ)nqQIp9fxYaY*o#wirb1*=SF=n0mr?MNaN{h;z}t2k1pLk ze-dFkxQ67Q4*!8c(Q<&5 zBS9Gg$vM71#sAs!K4+>*(|SC zF*kh(D%5BSpf;Nj7G9gNN`RU{QzOWkJ2VBLIH4g zEJxq+AvDXAmu`}2RiP%*f1y$CBFyrMA&L+9m9WSpe|-C{Ihv@WvA=K!-hXu}nxk*r zDiE~Ekn@oX_-@Ts0=&ya7-79etLH!|?6Lzt-W+SCV`Bq#<`*n*~(IDY(m{g&Zx zZKAL;$x+9&z|;EsB!ss{+j*!W=SH1-0u@O+trDNMeL9EE{i#+5(3>6|y(F|J=us&vGKovBWtdHq%trh#_W zg;|ByZ(iTJ-55DxI!Di3w)MRa?%hKUZk+{TJyjFkIAjadcJ0aoV!eL#-Fs_+iltFL z13(T2ip#8?kXZj{BCGP+7T$kn$CbY+*lI2>k>ZLD3*2?r2zWCJz}R`1S`wSl;<`8L zWOlm&A5Yc%(qbKhUc$+6o1o|{<_yV))V|8dOS6sJY0tj^ipgWp=iw%us1MObkvbvzAsBM;KcNqFrOMx9b&BA=JQ&CL|avErNB*s;b zPHpjRoi_*Jv00~>b{+2Q^TbfaOgwLA1++fu;gU|LRw<>VQA_pCk+15W3LTJsvw=Lx{7PR1)5o_UmJw{IF^`5Vv?-Bg_+4i) zY4ajN=Nr~0f-#2>^Q=}>NR0822V?Bndwf6_*+U+E-OO**lnh2PgpxJ-R$uE8Hd^j$ zUT{p0$fBimJPN=861bFk;it#CM;g zFtg0q&=(!g13Ihrf3uTaJQ~b~BL6*4MGdcF z`?rF~mejXzxFfw$TD+$YK6JeO%_l!R{^{`}$J40pRwWwUwx?4!5N)^B)tfi(>uPx2 zxvLty{_Edfzji>Ug+n9|-#or)PmB1Bi}ysVPR_D6uE;_a_h}~Vu1&vYoAxlRvk$>} zk2GlY4_ES+fBgOLfB#LFgSLrAPo0lIbq|5MwRyBew)20ScwnvTo=14!MqT@=?%WoF zBf+ByC+oI4sD?Xgvqh-PQDwe+*B&MnlFqc5t*sZsM`VM?dK8~tynbohZif5CHhopx`s_~X%*t43Ud_=?`2q%I zzYyV7Hj~Uo{&`LHnNLwhiORYtg%-;=Gdjk(>k@;2#7w?BKt#qa+1Bj0#Gwl7%((iI z;hz7>x>HS%9?f!B!Y@2fNQqKZEQ*rK`zM#cgd?!ysZb`{T|l>T2jSS2zUUl;@(lh+ z4Ug)@T1jZ!55xg85`!{w`gTFw2`LI(54tRjCEAWO8al0(Ay6wgRjkU5uRauj#_L=I z7AIWkVlT4WvTi?t_t$$g$OW~URU@Oc?`kq9FqjuIBQE-C#%iTh;+PIpcoOKr?7G0> zI~}EVf5rDPI0x7;yT_Ko9N?*A$OD7l0Z#K!Wt3G%l*%6Y3im-t1=SIA0T8Oa!j7RR z9p#4_x4_IPQ*D(MG?>Orr1XOvl=C8}Mg}jm4F8m~QgV|-w<`}4N_;l59fB0eH<1eb5jB+y<+BRBG~RHOCf{5Kk6)pKnR`{Qi60;3LAH*_+nM_j!Y%4%Bbf+OdJdBv6>2(1*qIg)0`r;}r_yPezBG*L&l$$II{Q(;$p{K${_O@OyX(?LcSX(Z@lVaaC`w6c_SzYZG~Y#kszdn;}6GAs>Z!@!@km)53>I1T6BvX-4XsgEs??Tc zIa$7;apU){YuhL{5rgg48&9iAkh3h&1pNUBm zsgW!+U`ne^jVlX?&3I%}x1)f9e6355I=sXo`j{GA_7SnWzGk(MAV6DfZGw0~_pI)s4ZIyBD7s-UogXUMrb-!{B}+S) z&?P{5D0zSO6!Z~&+ptbv2QULT&>FzkD;I78AF4Y>-k_w^^#}@ zvTDhI14t3q+UD_)y|>2jk5BI#Urcd$pbX!sj7CaGfkeo3C;U{K<5|l?63L>Pm7yl` zQu@kVm0U%sKuyD9WM8?((lTvVN-N6*YSHpl=mS-j8mzL;r1ZxyCzCi|4K#t&pl9SM zEk_+W6|Ffiz!=40Kl?gP2Ik{Y@(59b=9c2&E&mp|E2uzX@MC_;MbMUjvMl)2arP3s z#WxJ6w)GSMx+7~6VfW|JGQKz@mHc}$^6u3f7|R(p%Y~ZLnk3;%)ca%VbwZ*sfT&TI z+=?)I+6kO=oJKQt#zl@kvvjmups;M*B7tf zpat37tw}o1{1fT_`KKQ{AjTZr)wk*3N^@AMm=L%+)Sv#Vh_Cx32FrE?8JZTFTZ{L@KLWLq3AqtKdi9lAXceNBAGjJ;11sMzvM)7 zzjZXe*+L@D&DgJ0ZE;YSbIjf&CVe1&7DgjL^YT{>Pnoc+UZy#x^wcb!tl39FQ7ooK z8D@79{OpWuDgI-IDXW=8cgdFpAf+MSKZWOZTx?oRt znIl@+`ngc8$SS9_cP-&;FXLUU3E_-y9Pb*&NUx@Q3t=?1b>4g~B@ve~M$@xg#T5PO zYvwru{4U=rN<~$Mg-us2J!f#OwT8&>umdy=yA}^?X;3S#72(pURt90Bmc>|*Nj;h z4y$fQjUcKua&|ZVfyqu|XfRsirg={xM-r1*@lu*e^~767LkOgYc%#AgxuH^EX+K`8 zL=3rUP(*$i6q=VR(@7;md7_&J!OTh8F`0mD6n)cB*% z0!QFwv|_D(#+0~2ibf8B_8n0n^#WaPL4PWCA>egyQB8}P3HEyP`jLz^F?zPt57Lt^ zQqk5I+6fIg7r$zatBAdNCJvp#I)tqOyK0ui zprrnTMMAnJbx0UcK(@o1Z*@|l0n&C1dK4NdLtI5t9N|&1CpLU=R-Z`2@;D|SVnGMd zH6J3}=@Byuu4J`u70-IN)wxvzmB3&2NLzuNTTs`?Y0-sJy{bs5vz%+;IF#zo2y-X1 zo*a|7Oj%}WeRn~`spTjezD>Eg8+FK=_rTVbYaYOsT7aQIMaQ@-qd@}WGgz9hGWP>< zxsa7SWJb}t8a8;?Cu5g)+W=tJs*r79WKBgrM%RA!KaFY`7r7KnQ#y$sgseMT}OZ2Q@*d2Fe`aD~OZ`2s%!BT^v1}LGLw2iF5`V zsqz#hotk&#F%%aFWjS6p4d%+v-9);d$b9qB@mmfA*7n@eE%kWCw|2v__)qUuMQi$b zGh0s^7~va=#nWf~HC&mVQ@5PU?CJ|}$GW?YCbB?Z zb$RI&o!@^~8F*^T_%pq{kK_FL)V9H}1JOTIU}OV$r8W2baK~g>Zg}NOUbq>E%%oLM zgp|OvO?{A>Uo3c!ZRfbq|uIwHcOjhzRRH(o6dthe<7s__SH5NkAOR-DteyM$>c;_T8 zFuUI$^|*vWaoIw-X!SV8-SKg`k;Yo)A6ej59M}CWrNyG*ILi4i0xlvr;eN{Bq~-sfoaz(twjIi-KQIwRYLUBTTRI{54DF%$@_g$Czjnn z1SX&X-g8%)G10Djyt{JlF_p&ALHTkvgX0L6PD<#_Ltqg=qvDUa_sixW9Pu(ta`LM&G$*r@<|Q`lx^N_Vw+%_ni46Xg98p zEknPi8mJBsNMF5vrR_6~-Ee76%-W%kU*4TOR~?V{;^oU9pZsXMaH2uAAo8=Vs4w2U zehpRHcZj}b9~}b9BxDm?liu>G6?Xe=)5Ljpq|y3T#bQN|f)hYSyalOj+9)~N)QzTpPRBg9r zl@l;(>Wy?gYoY80DGKw+S}^~yXUA6?QbIAOWwz%C@vS z7sz&kz?cMZ)QWi_hKj(}FaaTcRgchNWOVC=u27Pv;&~{Br9bSI;;Pq34j(tO%9#Zs zKhakM2r}2=zp5o?b?wyC1|iEG!6H3%_*9`<)%=hf`Nt~nrQ#FyTugp5^-~}7`Km^c z6?8m&q*Rm=vp?`tHd@zBsUCDYD*AF!-h*toXpiRgZR$o^|9hy9h(Q~Gyo4orOtPKu zn9a68R^uRYkZ{Y1(8kDk4ZsN(*F`qed;Y?D)52AT6{1|X_PO3~;aotcAs36#`w zAT#aUp44RwgI`KYjo5fA*S`D=Z!FH?%7CY^oEt6_N`93s<9F0aJQ~U$&|s9~Xe*#Y z5h~5Vc`g5txk?2UF!u5Kz1}v#+^-$@^*4j=#H1^LCW{3LUc`fet)=X&8TfiI#pG=} zmX4~rP1wn)nKGMeuibEz&7C{E^6Tf%opPhhojZl;;IPR2oE@&4;&h}gt*T{wXGewJ zyZ7LcUOjtiHsReT6R9AX9od)p1X|?y z^@bRGx=$qgRyQOjC#|E0Uo+V%igK&QW5w!GZ3 zRQ%+MAyfnpP#x zi3Ko-%hvT0u$)xzmr8pbLc$dfF_}pYwU53dYk!qE1OpSQd0zr9cm{thT)P8hO6D&7 zgLXp08Txo;BNyz&XK^i;nByV{vMvRfeSw@7&=oHw?)xN;#t7`c+raJrl$CRDnQKdd z@Cu;Jle*mK4q-q813*Kd-ntx#Jkh7D*hbupBV>}(ViFr@ zgmhW{nq)QdtGQXu++xZr*Bnr4_%>)6PLa1EqP_63uA(wu1V>I>w|~nrcBFayi+!<% zlZXb5%^Ga3s&RUwNosI$2_b9kTr?J$yoCeX;Oj>ZdRb&B{8HNSHU_Fb-u!&~%a_|` zi*Jc640#bzL+1QJA%ZvtTn`s_#D$hlLq;5XDvm*1Glgj+Mi0Aj>=!JABtncog4N(U z^Fe^}^_z{nj@J7@6#n3={!Hku*+S+4v2R8Em=%4aTDbS=Q;W_ifx*K0;`s|4ail#T zVEcLpl==><;d*ow)=g4^;s*N2R0(@IRjcup`@7z zi(6VkgVG32^<9?{d?LP#GS5KCHL&g*gw9lip?c7o$#tq4NwE?q+2yFog>HluCVM(G z&j>kX+n!NpLKnouj_y-ykL%H;D7Tczwlb~r5n{qUW@sAY2wLL$Co@5&D+`kIPtZ3K zPlm*dncKG--=whSdjveFJK{D>;kvp~6~_dIprkIck!S$Y@le%m>9r)WD^4$RF9<>*i#!NrNetAx6n z4azFjefP?HNPuRM*8+4As2bPCYTIGj1g&Zp`;8je!k_VZOjuJ&!ceM+Iq*P+$N(^0 z$*1hGF=J9D(*5TjyBZbwg!u(uqnvGpjcY>UOkrP=YTfEv`Z%CUK)djk6>MHC%js^i zmQ99Z#{t@lgQkTrLk|F7&4eiI7!N@1i6e6b&_) zA=bCRP;Gwj@x#3b4<9^yWETN7cp~r9XV0AiDIXw@A3t)s4r?*-Kru|>tjuXtpoK!j z?qPeJ{2w132#h1#UHfX>g6-J(B|%@jr)R5LWIRuyOQRZYxFJ^YtRhVIGZ@mM0_`cj zn%L{yvAh6rpVkWB$z|fIvN4vUT5`y7vLOhZ z?JA)}d%Y|;vuWJMao~z|g7%$S*DN53I9)*PhtlKl$`O=EI(Pjl@z5_5cG=X1a+L92 z{){IbLEiUy9WbsFN$^%Bzq92oTHNh3;=|i3btaU^Juh%qG*I1_!C!^`H*GI>B<;4E z;5F;1NE*ES4SzrDQytt9K^udpbiu;NIV8#JR^SeV3F9zcYWI$gBD`wpDnT> z_Cm!45(HiO=p8*9e<&sFWEd4mEr8J(SUS0&Y&pDFDr{8pZ^5Hq;Sm=y6|G}lRdb~v02c1U%kf=w~4mH6hNA@DrNFY zN6Dh#fHUT_wciE8BI3CE4n7n+H}Rsc6d= zo)$1B!e}V^R7$)f3iIqot-HZugGZhRsnrmK6bLjTLsD9mLJ|W#4Obr@6lpaQ;Nk14 zVJjgtzktSB1c?Y0iO#|peE@Hdezju$)@=uCcHVqkCg7ATTr?KdXt|@Pb7)sB$j-Bb zKaQts@HHK0DG8-)Hl$3AEqeM^BVq|iPwcCs_>BJw&`e7ZiCuZ}iPRmfXQJJ~cdy>O zAqVFGDc~SDR0w(BeST{NtTXJ;tEt7Imbo_c+SQ-`{0ocMLvT>X_4n`dW`BHLgxygb z)*zwZs-ujz{tr3p)R^ywek=P+DK;jSBUEHTO5x3(H`MKSaSa|>&3r1>ZJ%h*UZFn$ zlXAtC>%gQo(Ix)Y#~aRDdn&}ww_95*lhpzi9u$yguNjNtsw&$7!z;61?GxH*7jArW zu3d&ffk`z>WN+a}4Wm%HPLv^eJx8bK#YH5FSlTyn&6Z<(zFD7=n8_KOHqB+QFl;RW zo0N_8%-Psx-H1Pv*cemQsxc#bEWW8x+8X_-Li3NmBSfFUZI5A=x$|bXx&f0WdE8ou z>$9B$J%s7itDTG@ey^X2Puwta{fXi(#O&EbI$)smY)zWz0B>RL2BWH`dRF9BOKQ=w z_P{^U$>{5QH7BWvs?zyNaltqeKT$J8_Qz}pi z)Nvh3=HahgoQqhh3h#fH*-%y~sf;Me-PBSB`SbP99YI_0Zzak%J0&LM$flsQ9&)8P znqpFbs4nLN-wqeZU6=%DmC0yT^AwvW3ra)vBYfJ6$w~aek+XCDj^c%ETh=q!Mv-1ntn! z?SVgVvgS+PuHqEzK=8EZBEqbJPco-p#4xi z=!2eh##Gd;l&))}23!9A?)96mZ+f8Bdj~5Evafyn$FILSrPe|qCyElSdS>}emEN$A z$9;e7=})@B@7~&Sz?xfK$wN_upDdmY+l^0f+{l~uEF9esZ&n#z8k|Z=uGXWy!nIH1 zHx1VGCzM_!RrH&PC{|Sg*d;&-X9fu*upDv;Yp6AdLjRV0WTe8s)f-Al_SqVZ@Js;> zP{@+=GGqYs;--qGjuA%(^(Ql6r+f zp=BhHBd*z?o5NZVcf=y=$vnGCbVX(f!{s4tZN(_sZ{m-dqN;&Qpn*jZvbvy#kyzZ~ zPPH>T?9;Etfod|#Ckn{CqhPQGN`m@G*b*5|#pKGLRYHO~cVPaP+`t%Xa#w=8!%wfK zjZ^?4$%4u>fa!wtz;En|9a);D2@(xyz(S`@E-MRn@em~MlCG^RUlrN%2T=$Em(%Hbo`GEJe zi(&U`pt^k|5!JQj!SCPwS%Hg#@)Sv#!;%sLwd|)9Zgx+4B6IS+^E)Sm6o6Uwdpyc6|4eXnr5!SHB=*u0S47;?8zRMo`W^Rzy4&0e#sM z2nCZo@KHy%+hD#@ewggC|36}Z>s3A#5s~N`^V0o-; zkDC>fnNRNAn2Csm@;g^_eXQi^AQIGUu({0upYK>_ZERj);g^8e+3%KV6!3Y9T9K9Z zAUxtdrM6=Brl(wX(vGwF4F#jO3_bm(r8RQ5Y_)-Qyws5}KI-JkVy2dD3-Ao;+F)nH znQgz%Uz!R3>NsqTsfh6D^QS+4d*Xy2=X;Wk{kz7qo$~{x#XsUu&x4{(I6OEWn_iaJ z2-%$-|1baYFAm5yjbOjTYYzi^^V+Z)R*a^iN@U&?J3Ay_qugjd9W!Lo=#t(vh>enW5?_qs7m zMFWqQj`p{P>ol#4lbrC8E6ea^Pz-4{QLIqKIhl$E@|%pUoD=}`DNybA>C_a`82*a4$76z;jfPRKk(pQI8)t(}rPNDacSky#Z>6DV> zbC3SA4D0>d4nMhPam3w+55C$bz$nx&9RSM-=tZ!^03QGFgC~0Z{`7b2Bxqk9Ws%j; zD_7lp^TBh0ELF0mC`5X0mP@l2QAx8bZ7MEcsRHOpqi_wQA9$#9mdn^}B5+Tbly}KZ z$k4?+hAc6m6f_iFA{qFq)Vd`@FF|EX_^?XpCy|h*MRlPxPF4n=fgH^;r8oG&_u2)A z*^|N=9y@UI2`jNY!nB#B-g1xR3XqjRgq}m6(3dJNIm|DmdiNk(Z-YKvtkuuYt~uAn zhcc^h6;K}+=y(Zx-0a0a;2XN6U5pgZyj#iPzVta zIT_~p8j=Z0fA!Hk6MzSQN#SvKWOJDvk`O;=<0P!MAO%jrZ&-4-ZsiyfQcQ~%n^H?G z-V2@1s6TM&b8J>4=e2^hoTX0yBR%mWV(RTlEV)kIJ>TRc^B;*@&}C?NxCfisJQpo+hx*7;J}3!uBoO%UaEkGRn*SHHu+P{K9Hpsb8XCG&{Kye7K#7TX`*tsxWm=c&TlKkEaq5r>5_w~6Ze+y+ zkb0vfj{QJnh=+ZjBV;TjSQ#TrhVtyM@9NBvdw#ZC`NF#7u2;CU!*VyJmC%PAn9ff6 zRgDPTGWEL*g9v=7BVk4T20+=!ILWm_IRF@g+wFY^a#4dP`Dkq{p+j3L7ng670V zo>srHpoV@r4qu%3@cu*fLSDC0TLc-f{rL9XYo|y&eeqmWsW;Tj?gh)iw&W_*b-O*fGW7c8Mf4Hb9z25UQ;EXM^7FiwTUy5r_kS~OiOk?Hs>sKv86zymWVMfN(avgo`Szm zj)x&M60v$#o$?rX9O291{Y3xe%UdJ9LV3;Zi(j8UfBsxgLxcG8<@022Bk!j#BKxB0 zPPzD?$c-=7()0{p+|jdc34BetfAn;)=Xw+xwB}ZtaOd8VL{`{ZEF*%Sn4h@>GX31@ z8C!6T1^U_AGh7RWiTacId8+DR3snZKC3P_^S}Sbuf)bmp;!ybtonRCH&l@}cXD5m1 zU5If_rhoI8g|-doK8Y9Ro!bC9Lk&zOsJV}yx%~WDwE?NJk%L<^sID^8EEb&n)MgO` zFn3T%ZYe8tQ|ilX)HxE2WGP9Qk%25I(ihM&PcT!nt6H%S1Xqcz#F@4y*}d2 zLp0f>yZF^k$7UO@eX~#C{(U2&mgrcs(?d>g{m}kl%bl%UkStbi9nsxy9^IfDpwGaN z^afwO{ZHqDpKc^3Z{NIQvVpY0@u;`hZpoVUKOU>p!&p`B#MIzqkxC80yc5P-1F=ZI zAjZTD!ID0|4xP4q6JrrJG~T)qE6O-Cm)f9@_hvg5;TKFH#6&nHMrlItqp&QTgnq z>I()`@@gZq>vXToYLYMG`!qlk;49YLr1j2nT>k)6VGvNrQzy;>W0ptFn1V-)Ia=cgi%NT};k!OIanTS%lZ`B;@YKp&2-tD@2m+ zqq6Hm;BpTtR~cNd;YZ-`4EcB@(nw6SDa?KU<*ksU-8-0rpW?a`CSS(u8bI6~7{*EI zAt?v;UB$p;u-1YzZR<`lM}{{}7|i01iVhl>jYuVmfHL2zW#eca1T*R~igFw(6KI;M zt3908ECB-5!r69$woq}RXy^&n)deAblhmSuDT_%p@V{nB2VjF>q#-~3#U}IDwmG(X zt58#8v_%3McoQzw6$6@s^zB{35GOjctDn0J@~LF!C&`S)?-|FjxGNc%jahw5snV&#q=`A=Pvx^i$4dV3N&A9Jh*xbxjgPxkrE8&WKeNwm{4;3_jiF+71C#%93lb}^L>FOBML5#&D99&z8pM7pvbCHiC``Q5%|}=c&}cV&>|N(g_((;pkl%i#BiEY>>54ERdb- z?!%S>{(yTe)bQ-5yZ4{`^yh#5m;d(j&%Zd%&f#)}($jb%7gXX#nCxjSB3%QWC~S6I zu+4*QxNSnsQc|I?Y#SFtnh<(@jrVo zU|0BwZF$-XD9Uf_pS=j(K}E_|3yA-{C-?TciuVQk9baT*(4m)l2QoCx5-_jXC0on# z0zqxnr}_)ZDZiBP%8=DYjI=r4%nxbV?K+XHmg?#UT0SGQ9^#ylOxd~8^g~M7yfaGF zhN@vV$QgWCQ>SMHc?Ovsj92x$G7PgJu}Sis8Tg)UJ@l|6^@GH2US|~^?P+_4^*|3E ze4+N%K$zJ1Xc*Kp*Xrioy!FjvSATo@{LX#zUXQO{|Hs?cp2Za9R$~dv^eG@QUz|Uav4_cR0R|G zA6E=Q*G#ykI~i6p^(Rt<1-F}uEYKKt@f0!9c>d=<3)waX9i=0`4hQte4J;l}pa3X- zmcw;`NHLQ}rQd%zRH*N@8)Yqr7m4|CI;C;gZ}z#W-^gnI%{%H(fpHQViUTpp0VR*l z2vFgW@sJTE?nRo!X{?`t7`5rK>+Vbi-P+&DvwH(lreG)*<2m9Ig6m(ktdyTf4ZXYb zU1ytb$QG`YE8D6m7_2^-<;Wb-k)sc6CM^GmnE_Pigq(3iSlA~VH%C2uj?6y4147t2&4+W zvSlrQAba^fS|&BUcyB`N75|fF32Ruf@EhhB#it@n8-+mj$ zS@c_D@10b_)H`Gv)fhSUQ4x}9Hn!cN^92ho)YuwestGcHB64wvYGgsQC}&M6FuXx) z@i|)=$xZYx_WkQ$|MKSTtNR`{|EQ<83yZ~OGHc#RViH7ptv~eEz=$b(9D9$2vBI6u zJ*RPWC;J!^fsWu}2^i_pi1eLnLU~(%Z{D=b!oJIa)EuboR`C}N-n8}j-cNtB%>G}0 z`sq(P7#sx3NtQE9W&ypJXTGH^1Z_B|FX6;`EeOKy+%?JY7 zFFi{}s7Exv{%Z92mp}h(9>8%x&bzq(;Ng!y{q*KP|I^mqM-Q?8;MZS&b*2x3iFz$> zIH$P|z!2%){(PHag{?SX0`fi7ZMy&9$&WwUbNu%GyVvFp-kSUGDQtjPqwfrat~* zd1A5Br%%6s`Qx2c7ay+ltfb*`7Swf1+EkY6RkdU8wdO8MBNG)0@#M+I=%v_(5EDc6 z6G1}ivJ%Hn<^bcDcoaAet~4Gs!Ah&h?xv)25=I$9cJdHjc>x4KKV1Z0WfXBIS+Df# z_5dX;zu95xWoL~97);d` zNJMlV-*lqAu|%?XR@l6(poOpw2*C(W3WOvjqEC{-I|s|;&#gOo>og(+J(-k zAPAz+11BoV4$Qu~nj1nvbh~}-Of)fj43pee=7K?P0OL3!Vj~U^GF_z9nX4>P@+(1N zB(>Y9j|~9aiJYY+3&<{iK?B?0WJDz04s3t%3Vy^9-ExIFB;n@>8o1ahpz00ZA^fS@ z^D;POliC4nT4|s#zN=_B%$PMrRfO49D|~Zd`M$$o*j4p4J!_8fUes8%)A&-&&OSFCW}%sruVD`Z)=~az{aYtDP)Sl97+f z%OhBuDB!iB!kmb`3g$70i;f+Smh$B+P}SAx3BB-;h{2s=+hXVDVJt~9!q4Tp?SWbj z2Vd_ZSwKLK!eui1r$Ot26|%*~)R78P3k}JMB*v#aDvW{zqR%Al&5^FHOLgT%a0lgx>} z%AX8T_Vv0m$(V{o0R(4j67JJbV&7|iBPwEGf2DMtev1kU`k7_bto|Sfhr=-GG#LOq zo~puouWS&}F+UZ>uB*wqdFdUW7eh!U6_Y=sFK6wyKw&1DC{jT;E2S{vqSuXtMt1w| zs~p+03=tO_u99>Gx`Fh!%>V#E07*naRB#jGkS%kf64pQ%iBcz2@RoldN)2{+W!kTa zpm_;sBEI}xfm|b0X{4x8<@O_u9udeKb`}+{l}AB9)7~x`WL1TiuiIsJ%S#0;IaIPo zxxy8ql0vbOy)T*e3cFlG$pVu;3s%x9j_6f7%L%Al$)>EnBVRE(sB$Ir{Y1cpPH6BK zgx<>_0R@VZCsDAQNY)u(_Gu}K7)$UWO22JpYHT<^RXE!?LpCtB4&mTM+R&l3E28sC zTMNpNKe1Fr(6_&sNE~IiZH_s)PGkpm8XD=r#Zcso^FGpXi&@Gq(_zr(fgm{xY%U~e zNuL1D-H^Jc${)EV?f5-(ROO3Yn$wY~%x?6m(7p#PoDfGs9~R^gs2s_Y90LMkdYe}; z%qOy8jBZHth22}%zTPES1ekjD_=r#6OyPq^)S3E_x#y~u%uv$GQrRa1MOjFulA~=} zMvkL}sZ0UC@KM(yJ>$jmb(6G&LtRKVlVlRFWW~h#m(q2fB3JfpPaUBv*>d|^8oT5X}vn|sc zfu>bQS3|JqwS6wVrvn#q(U;~e(e92Cm960SoW~#}x}gNM1R80;lUzY#s&X%U@})%2q}`eK$r`U5aGy)AT6{C)j}>kKZLQ z^DFmBU)fy(#|HQvZJSp5il)te5jpGi7EKHPkKM@*{6Uyw(0FOUSP+{|PGOGYY(&0n z@KWM1E?2@H7lYMRiX5>C-G5?Z^W(G9N#lIRXULD}%2BB&!mW@pY-PjMevPS3Q;+bKDBiB9LCu72lf*0ON{Vr1(w zR8ke%?eS1etU6X(TWQ3;fNJoNE%FRn2ipd!q=j0@QIe119`sV2J>&rEHWzq%Xjl<2 zYO-MpF4Um&-1i11+85XIdSmIlhYbp?``;~I@4yGH^1`1l7$5_!zL!DGkA*f9E!fBb!06tY;I0lT6<{o~i4e)D58f7iTF*WVpw0mFPD_WR%T?`Cw0;n z>6`tnmA#oPqjn~p`N&E~vc*IMOOfOj6E#b)%ne;ub1TSh+qNK68N@mB+<&Bqf_>Vm z@LK+vGytJiTWbYC9nAlqs{3q{B*)V9oH`;>R(03R)+{gb|NkcLH*%Kj=`PWgI#Tj} zUU*b5%_GdsfPDE z+m(SD_KlNbEm2!&PyNUyQMcDeV`;Uwac-}+%`hWPF9f58hT|NH%_UrA3;sski zwo)s{TllCFIchc5H7ii%uPKew?lE<82ZI4waT36S8C^578UZE^b&@Cf#A{#xe8MYB z%{VHezp8m1hC7|tzG9=AOSe&^X45s$a{TQ;L`^+)&#$t^*qpt(&XCnPW?k4t*_x=Y zJtC!|)K{FST!dmZ?h+`T+;cy(j=o_i>i7$X5m{q2=Q4bX5*+fc zA2ovv2-ab6?P7$YJO?jfjAjqu?gFpl8@Sca-R7Nvz4Z7D zQGyS)@6`5=R?~s+*?c93Fbs~kISN^mZ7_eTO=M@Gf)b6lsBz0b z1w5mHG_>-E#n6PPN*4BumLu6+YH~{A8q&BKBTEZUYK$N=rP<|&D5!G-(JAG;<9Hvz zt3)dlwmvL%%Jp@J`IKRUY+0R|xv#eyVF>Zs~wS2|?~lV#IdJ?y@@`ug^Rb+?<^nVFLewQ!`mje=-_K4RJ|jXatzt2aRu_fStw(6jL9wlsG`v;Xajf3eXYP7v3WfHus2j zVD^B?7{c+8h<%Q=*H(lJywtXbx!ky`6dh1PnLEN5(F^H2w5M$X6(pSn6j;C8lfpFC zi4jM*OKG{(5QSMnWpnBczM!ea%R5IsnUx#|qwH!-7=J@lD+{mFG0R=vVLEoAb*LrG zo=ADNT=zw$iMIvSAWHJ(Pxf3eh-3DC0P|8axo*bD)Buz`#-Z>*HA2>i>UJ~)n0I%T|0QDc+i=Myxa4Ts2}owrEB>yus?d-U*`{ZicE zNL|yX=<Hk0202bLIMsri!zJ&Mz`HzBBvoy-x!YsrA#EFuiBB&wu><^Kbur{qFy@ zLFgy%Bfoj`{(t`MZ+hZv{_AhQy3h-?LntNu^AEoQW?8w9AN2kDw_ktxT$wev_M2be$xCknXB~GxLtIR3US*z{ zjgx1~s5l^o^?Xc08Hd406`S?f+i&{6v6rJ~AlowYi7f^to8WFCkRd>vlWJDTmCOLg zEY8)NF5Durh9m{&t<@5WY;w6*y1F_3%v?kPP}fenoHFxI`1+Gnm4mg)RW%%WuX9wZ zJBFRw+CufDQy5JoKYp`+0{Niw6(AD{ZNiA>6a4 z&uvL%bZ4N%$2_E<9ew3n?|c|(_m^V-c>DJ1w%53E^x@Vom)(E-xg~@6Z$(iHfL3p? zQ?q2C(zCOw9vLxO3tT9nA5SGZRV}J_)UnExYFIv?9Cj^0PFpwuBqkJSnPm!8Cbc7* zp&}5Ps~}$sI-Efh4LAo#&7^ZdFFL-jAO*5=r2g ztXxHBaQOv6+;SCEGKq{_KBxAT$xCf|M=`Sq-cwb_D9cJN0OGtMddfO31U5X;OT!KX zSUW2I5}Yf>G<>p(3+s5z67^St0~Vj`FbPvx`b&TW*?Bt8=K^)Cl}I@k$dg`sfZ z#F6x;M>EP%$1#u!jb=^Jxn5dK7PIISVbvo1X@lQTYz@i0pi_Fse(1m=0cvEEI1M$2 zy!?16d%2|{3C2NW_+`<0P^po2QHq6F3U5j{>OEQxH1?oSEJOh(T3-@kszfZu;nD9P zJB1{@fJlMeRO*y;mbReaa;Et+J7mxW5bn-cttLe+h4|vS9nk0HnkUg?(;fLey;0mm zjW!I2(Sz8{_=xV1#6d_?4Xe0HGj~AK&q%E+U(s0s6NRF3Cg}w0zNC)Ea5-RB8U<;| zR&p@qb?oMn1@P!@B7&=yI86)!bqzF4AbTfSFM6GlzRO>BU!hO;pbL8q*sQ~)HCLJt?X=GNCyQ)*^jSa|3o%F2;2SNp+&7U_Zc4LvG-WfQmEK9*CH9MY!TvR=qG z+Fh}cSxSb5AmmY8S$7f@EV;i*`;O-W7|A3;Pi*!*R_@Tm2MaWdgwLI(VrOkt4s9+d3lkfrpHlYK|tTRz?m)5d+D2=XDCQ0t9 z>4BHR#bzOJ=O{MJyJB zqAo%eN@9av8sKBf6c2pXNivWF*F{?QU*{r~=R$O*%cTc9`v!S=n0JZ{b2T>Yrio-q z9g-wx(&Ue@yLm)u06a$4!_ifQzZDwZ>(xgLmJPC&CG20UIam9hK70P-53PaGfs~!s z5~aA&Xa)cEL(xWA^0H)soj=itE2)TGu_f|2#>>t_wVTe&Dpf$ zt^I@Eyz_FzzyI`CFrGIspt<+O%vA5iQI89*ZVg-{>Y%17fw5^3^C+Y)EXSLed;+Fg zHuFtu#mUs9SOK~>rs@*+Zm)^|=Jy)}n270hfL@x~Lx3~=e{~ha1_h1NLZAO1?;<-j zW*458rUmep17$Vptv1d8BhTnH=gK-NNOj8bqH17CrRV>AB!BKSN0`IqBsEj%+oGkG z{5E>*Hf+@p-`Q#)bkt;UKI>f$E48A#CHSXGcURM)1MZQDO=%gFlxo%4tE&Ol2yfezUwU4nO?n@LG~2iS5-UE>Amn;k%a z7G+APIU}*ETkWMCFTa?1W6pU~x%e0Y{F*Z;Tot#9OpbJmziLw9I9$h=DSUk3SB8ap z3d|bGHO@lJO9A9I=RZpETFhdbS!zZ+mXwPN{-T2QSkN0xU*=}xXN}rgolqz9h4ta| z_Qi$YR2rLiCDE>*HP)APa;qyrL6W2+u?1<4i zrwiY1mfq=9gQ4ke%9S2N-6ot5p+dA8lnLmlKpg`UVtHdb1f_@AbvFl5cutY>W%$Tl zfZxS(Su%&JxI>BXXPtrMVKGvw!|j05!jkm`IqeW9e7cFj9gzeES2?Lkc9mi{+^znG zucK0x0CMc>QW3(G>-!XpFoe#0OFOsb;CN0jE%L8>T9eWkSND;NdnOFcMG4JA#67-F zPg#1rr5EXl7iq@36Uu3biZD1^Cm+D^G}c%+_Wh5Bo9U^%Aj5tx$&wA@ZSbK{b_J4d z;*QI^GGkz+Q_lz|tIOE&zbaOz!wuHuxJU)@b14F|OAygGgj0H~b)I}f6*n%{fQG1c z!kiKg&6US&206m?-mTg`)`b!s_ul$lkXBO2LI^#+ul4A$wO~^eXnE3OuTpxG#QrI~B#}#)aCbwFIlPlL?`pC7s24%12yFvb zJ6*@KJ?%(i0??l@ox)Rlo&F4Z2`6#VR9wmxw{oc&!bLM}NmWu|FH@uvt~_mKmvd3I z$sBh_+M}5myGyF9|H0GPF-PaxKv`MwWS$ZSVP>2UR2JB|ri2t8x>zV@BAGEgzB}0v zgts!CmjlbZC=!)vpp~GG)$D_D_i+_J(F(jTpRPM)F_|aNP0k@XZhX^QtoZQOZqofY zM)$;+JGr7BGqMAKg^Z&sxufjFbUud3(981_TT15amRXV&+zQ5)ck)l4dr{`z8@#tFWcAMeWJ<_6^ zWF#K@pt%WdHT)=?4+xGiLRX=2VX9Ptw2?E_^G!p~#kpWM8C|MA1^`wyQ!ljNtr{74!r_U%Mz)yqS8rdlP0o-4O`8^mr@yW?Eh|c75yAY6F_|0f3o21-{b-ox4fr zWb5dJ!d+O9$srS^$@3`N=!NK)nkg^Smtk&A4+CBb0PM*}8$NrmVfQ5$CJS*#N$!-9 z!iB9tGdQGWV}UQ<-ucODiv|qO{&@4|>9ZHtH_vXKz3?iBEr2-l=AiC*Kl}EJx7=F! zcl+hr`;VWkAfiH4jeB+aaYN_+xcSk5PM}FFl~Oe`FX@GWW+*?~pov15s;S`BMO@8~ z^K2nQ=QKsoOBR$keN3~IO8_hJLfaHqwYKYlQE)9&^&$Q6gkhv0Tq3>RQ4KekR8c`> zs+&8jfLK!M%*N6igIB$F&>$r*7>)K`9KOnug{we3eWJqzGIVmH?N4!PRwH(@x)Hn|*R~$*NTM`2>hb^ozcgJ4$mm)(Fq5~F4SN@T$68f4+ zbB*d(WzToWBQd=}e+;n1kV^>3q&eU5GVsw#ewf0xd#B;+n@9-|maZ$5BMbjiio$_Xvm_#$S@|%=;6gx)a zBiUfW&0#E>pd9Q`QhkDp1YN;0*vWKC<`(Qf$hk#&2Vj2ACj$+ASxujHn*R%e~Ne8<{@KoJ81UM){)7D~scqEE*iEI`M zokzCT(L&gxp_TgDkC=;>x}Gv_t{&MiuYqDo&?FsGJrs-8A@37=@ZFn4i~U-x+z$4d z6&*B(&7bvT)alcXna@(3t+U!O-Tcs{nU_{S^g2HYa}y9_CPaeuW3MaqX7D2~0D8zv z?bGm`YUnnVhMI>_q~{WDow5d|W*S8pHd`Nkp19^Zwc^>~3ZfEbi1|zjc~gm*FW?o~ z>Dc#A9J9jEArecY-xZMfi_fs{Dq@#g7imRKrDMI!lU-6b9Rp#F1^*R}^>UYpM(hTA zK^Wnok#^t@#e=JYL#*7aA^EtYWSDD}YIvhN{*t$hCqIUrbMwQ+k*zyNB0eogGtRXp zJ#Y1_drevH424r+XG=l|oAOdf^w%4Le2l9)XC`MC@ihKA9nJWf^XV*inW)F$og3<> zXs1!Aco@3%=OEc3D}e|FxilXRywI#}vx?9H6Fu-HhPSp1S`x%kN7{3WRuUW{)!&Pr zs|$LJ+FF6q#X6?Kg#(O^mpM}e?BwJ3ZgnLgvZw|Nf^THtta6JTx{Ot?xQ~C`w7Zb1 zjP>icA91bj$53*b{I~AHM5|w+C7EZ8b_@6X5wA@rriOYsWCy%~Py-S#etx#!{)0Df zO+o1T-aLD50=?bLvYj&dO+koa>_+-Y=dz!(wF$p2+QXN1kiTAE*WGR4@Y?W3x%E)m zEtKVn`PUzRe5*VE@%H&ktM~oN@29{0*WbyGpxtD|G_OPWrDXk4xgYwef|iu<8QXMP zL;3xeUriyud~rQpe0%@?XnK1R%{_9Md|dX05WEmraF${?tSU4 zs}$MHE29ghMh+|vSiJQrte@^RxxXd;&^HFybdS(zH0O}pmqAM@NcHe^ef>;OZ7Gl= zS~RR$*H&cxUf#$(bCW>$V3wHa8g`1R<$-e6S)WnoqKPn^^*59_P^E?*3k_@<4Jm^b z>8q*L@BMY7(YDc0l`gh`pNVbp+Hj9?Q($d#nlXN$K8oo#o3E~E*esC^ZNI;Lhn7CF zVViNboH=$8MJJdBICuGyQ_!mZ@JfQR zfR_sBSzXPUD;}WSz*m`9ElP!=F#s)Absb@#%rQWBiv}^ves!p(%#gTGAkJ1$NiGy3 zBDm#=3$cL(3lVFZdt2Dyg98j#UJ@H{928Z)_zYj%1Q|GY@gon`|4=p#1CVTm;lp12 z83VD4Tm@W=XD=k^ZR8*1O1#n|I7ySWSe(-;m^1LtBHzd_;Kc%X7`v)&rR>1Ft$)=; zpd|y08+uY|c8Da>A?z$OLLmgWyY72sA+Ewy7k!uU=ob2LlsiJC-h>CA}9C&Or3EuFN7n7 zMIoEWf*}$KHWm??aEDmL4@17Ev?y`U8E~xS^D&m@hcuVsUNweeCo;#@*`XfmI;q@j z54n26gZ^5XHN!#0V1%PK=AAKq#)nBiiMd@yTlpXzXVehF!!}PFa`=ZX34>|fR#twY;36=|mGnLNY zk|bGj_o_y0hpu~wviqqehXUj$yZq{hyJ{0zvqHr8)p^OmOU*gTQlk9whKlrpNS9SP z7XhcVE(46CinWJHQEtVsw_+PON|T!(V_Pmyl>EV-|AcL9jQr8_ky z7$BYEW`aAXLU-Ys04Z;RkIkWu90dZa{Di;Amh$>vk4je(`)|MgiW{ys4-Z1uFslu5+Q}$ku(%>7vr9S+3i=Bgh4^zIkVg83Kjto7C~q zkE-+CjcUef{(=NjJ<)*rD1Pyj^5)H@tSK*QsipO^*DtJkt^X&{BHgz3*D->@1i3zL z%p*4-sG%WL?K@jnRKH@PkvmS*hs$e%XS&@ypqXj`2WX3&+HNNWO;T3V)$`hKHD9;K@!1Er zxq-KN*<4s%|D5`2mog%Y<~%}pZb=&Zelmnv>9ima_dbKpr!h<~3-9&b?ph4w0AU$W z+X;D9%0q&B|8@J*VJ{e@R9?ThJs&|gzgc+X2tB(EKoT=nEXT>-4nBanR)U=BlsBY! zHJN+o!?0I{I7RSkR--sa9k|*#<4e2DWS-1x`Du=Mygwhusiw)}>ReQD4U%2XjYadV zW{qm$BWDyy z$_D@KD9ur*_FWHfJ{u-t(@#xSw^2XUwC9n9msFrKwne>1HLU{z+)NY!pFH|@o2gTQ zYOoP}I!cYTAv8bToGJ=c8cCLP6S4Y{4H=@MT9pZVvMxGI7o%UmoT}mIqSt%_?KY&c zXP`BDE4#Pk78tXhGHQRdH3chK=Oe04m+D)utoJc}?6p)~?L8J%()qB(a8=~wsx==^ z1dMnlCrGBQ4#EtXpJv$42&710z5k;BzwS&*Iw3LvriT!sbZj9I~vO(mey;XxOaG81MLjwEuGs4j=_oU!O4s}S?}!)g!}Nq@xD9iEMfRoW{pj9jt?eLF5#)`f^TBLB*JJ z#rj&tNglgp?8qLN2=(D51c75{CBvR&FRZjoNrwQ`#7$nGYQa^4?gq?+O_GS z%|@A8GJ%A?cErVe&8`c6iDvUI{-wZDbcnyiL3$-X}|66IMrUCmH%tgWlA?wHCHDMF>@bgCwMgzZ>&T!HFy zw{q{wRVM>I1atRt$KwI3@o|Rjrcwy?mvI`W3DS=dJIr`zwo)^#^64ISK3c1UK*r+fBaO!~5)=q6S>CP1`2z z`7i%mRnMFKbq;*^y10mZ@aJd3JPHx%STUVu(`kDAb$@KFQ~$qhO7M|l3`23P(^ct4 zyeX+rG461c*6~qtdZ7(fv|6Tm3Aat`IRR4lb&B+jP+rrQ72vNn?im3awEn*b5qiI# z9av_QM8wYP(C8Vyt=m8O_0>18UrTQ?8UbB;vKRZaPily%o~l@FVMV|9I^9OD;DJeZ zIfjBJdadYq_2Z9z`q#^ydbdRwlB{vA1%g&FEy&|1&!6?iLR;so4`boCH-a}?+z906 z=D9HlXDAz_Tj^=V5KvTCCf`ZQ#dbJV{ajeK3GkE*!b$< zjhJS7K0D+T{+jk)kJ9QAw&*?7CYd>BeRR=#Gy8iKx^6#tsb`<2{G0fH(h{Mb&CF!Q zy$bhD^E{yFJ2$x8&(L)Z^Ny4b$Un!z3-9Va))77pH>npV*~k9NC`^b|~C#qJqB*wIAI zb0NxxMLulPCOv1EC6?ab!WgdVMhnMDa7MvOWsXonrW!%G=C;m#UG$_{fLUWU15#@! z+DWell!KZDunw?|r+V;0MOj#Fvr@naASD@y)>k^!5S9^&_Xv(dJm@F*6?2QNk_PJ5 zEME4F;)4{HB;{crK$3@^v?$5vhk0e%IBJ<#75QI|J6-wCP5mE$poTJuV{By(tzE7$ zlb*3C>ZEngYU?HB)T5rhlu!P{NU(}gY*&)20Lz}`Ask6%@`j3*gpZYTx`-9_WS1Wc zJBIVZZ}N__gu8G!K}IpHPh92WIrPW-A|oLzxoV}O6Ow(oavatU=3uBn(j3ayoY8q? zCsq)Li<1`C=%2);1@EH212Y!j#2n-EInI`oHCa>*0Gb=kgM^kQ{&iA&d6aKezY}B4 zaY(1@62=d#iIv_HC7P~frtyx5A3r+6p2X7n^fzPo6}vkvv8w}9`h{w|#%ZK*chXmK zeo?~cmlSRZyR1G|v|-Ds%hKaLgF!F%!oR!XjI0sqD|_r~!`JYfW5$k=%r&T`rv-QM z9+T0DstA$n;cRpT35`)vW~)mYlz)NaP6Fi7^=q^8Uxjv#!1JIQ&(n{^NYC?i2{nx%>Ga z=0QrZ-k%M$V*LIpwkW*u*^;Jph(uQB?X-59_j&)AKySqrvhLz z>gtBF7Zoo!#bQ>^yAkfM>`Lz#qE5+_@cRJB6j-Tm(SwT(1M1fHMx%l47c~cf25M|L z!IAM!bsz4?AarQOE8hnev8k)Bnht2w9JsSLWj{I9_9r?FC4$r{TT9{((?;u-(BQ!& zx=tGsgwfTONLRLe2e0q&{{7oO>~{a=wLXA=qx@O9-fE+~$N0UtlK(@Qnln%OUDcd= zV;qDgVF5$Skb51~@sGHVyLl_IdW`wV`W#akn0jZMNa(z+^zruN)ki;JY|wD~%_RDh z=PzDaous?`>ctNqKD?vB_uDZ4hnFvOZC?9nw10Zd_ajd=QT^uqhu>fSp=WLBxS9Wd z+8ET1lFyzRi~Q~H|L_>$`Lk!g{`SY`9y*$hv@-mj-;2F^%sc|f*SGI&Df2%4Wv1N3 zw`R^>{<=R|W?{t){Ti2g!(4;;n&+N^6k@g7#wJiiSyIwxQ~sCzKlS(<0qFiW@vrw^ z?`*C=!_xQhr+f?b`NBLp#91R%E+?m)2?jImVp;C-=P?&B(PBJXNXv@bhBEIR&g0Zz zbSE=ZeMOl@fR0Y8AQ+xkBT*>Jjq6xyIWjqT64q+b0v4Q&H1+sbySQ8saON^zuvm@K zWv)R8vk$J1%et&I<;e`Psgf@hI;E7Ym0!dm)$!!(QPxycHFrjR+o&QQRZ^fGNuRATnBnNkkdG{C_GZko?;>mf=wHys1PatjJ~1ne zaQJK@DP@PXqwbTx_U`xmIcC>yr`N{jMXO3!1JnvP2>ql1Kro9?jd{~NC-fJg!keB;sz7(ULSy$+YFYj|g>F~jgeDug z58YIpe|C-7Qr(S#&Jy10oBRfv#9ukSD-Bbm`p@@hvYM+{_nL4ek^8h0y^y-N*n zw4xin;SO#$DS4J!eL38~E;kzCpSXu!)kE4ylPkq^gT+(elu9wj-;@n^ow-RFPplU1 zX^(SPQDZ&E!V~Tbx?wKZsXr;x{X_v1^AU$FH}X7)v*y7t9Ebq^`Q}&i~~|L?hqX_V@;?*SkSUO#$t+YW&)9$E>j-*5isjP8s${C^bmyS-X@2^ykKwP?Iz>0LS^z~04&k-TwVYa{R=RkC)SP6MW8CpIPZox^A|>OM zGo>C!uUQhv%+4Q4=K?u$SKY%`q-;*#m8vux<)63!kwzu+8w7Xa}}RGzP0$r>UOV8 zZpDAgIJVcM?xy*EWf3+&(4(fen^B?aMpV|pfBxhfZsJ)$|HmJ{|Mti4eoL6Vs;?yz zUvI8Ocn|X;SM*6ObrAGCzI@g#YaX2t`W0>d+=2I|4hk%F9|8q2%|<+(5?)Ha7uE&} zdjHJb4(@J<^s(JP)0OE1%TSsffAaXtr-l$l2#osrRG`)@AWDngmf@3ormWZ6+wEQLKs@ z#^YK;Bj=c)!VPlrvXWN(zCoCtL4K@YtwX=W2ahfVYeEzpKR=CtvWLgkRT>P`du>C` zdzby7_Xr809N4%}ZLmtpb6F2w5ZIW)$oLyepkjxg_qVU@^XVtRK7RV@SHSRk_4M}L zyASU^+)_!8CooM(=xTP>?zFN8Ssi<5hw(&cWx*HU(~Z%qA0j|R=VV>n1WKLL&qS}v zr9b}ZROCV~)hE6W772ZcKBtc3BF2yEHt7&|B`9h2MlxP{OHI8MSfEy=|Q!-YbqW*{`#;1 zz}IhP@jL(&Nxf;GX_G4-*gxjeRrPr)P|t}5uuoF}X+#c|Tuo{`x@neh=BPRs2q7J} zM5%~tC8o?gw8R4t9_ir&vzQn<$VV#XF2|vLhb-h{rHjk437DGsC8ar6j$u1IMI@W< zj|a0{y!uD*C|vQ{ve;VT!+lsHbAVQSNu8oNs00s_Iy17v;5P6QPAbWw8$whvb9quP z3=CS6788T$B-fE@B9`o>1~He^2F?R7C6pVL3yn&ioMEBogbCN^9x<-!!qJ^ z>Rq|Z=@htBWVCjFfu9$EOJg07%sOGK_&FcELG3EK8PeB0b%HxN)=i+29Rw&J!Sr&3 zoJa+U#&Stm{*a+IFf^rTO6NEi!v%XJmc7{4!j<9>cJU3Y7ngeKE&n9fR`o`mi8;cX zN3rv=v3jdFkw`QEa=lzY=6u(2 zU%?faDh8dWRUkSYF_}ucyMtF(fhjYlr`u75s>CD8p(K5OV@zf&5A0|+G=q>~|L6ef zzF!VXasZg5UE_d+O%)t6xr>gmnJ>lYU$zD(?ea(yTY6bFl?wX6J!Cml1!kz7B74}; z*D!O|`^<*t5sJ*CHc4?V^qG*?5Yd4$D%?z*4GQ-eS`STrQfaV6n+B0u@)N=9RxdCzI*ppf4x=sdAM$aKRJy6_G)kL^zPl8Uw`|}1AvdWA6h?n z_2mA|eLu2A9N1vYHO2%UJ@CMd{>FRj=s&kvxK-@>vbyR`HJW?hGIymEPZ2Rr_~|yy z)Zg~!D!pZi>{MX7f8TFJJ$MX0Q{q&=`TE+VvQ{y1beH>;u8+;*>QQ^S;P&$$_VoO3 zeg@(Fhqqt+KyV`>G({;t6JSZt_3h(lH`gCO>p>=nuG005MMu|P-?U(QL8}h4M>G0A zxRK^;x}RxP6b7jKtT(Bf^}x@Z-T&&VpAEpAxP>o#e%JBx8aCv{tSPNP#td0wXYsfv z0Idnq`}aD)#sFIjB=LzZEH^?-nf~(8P}D*9!PDw1r>Lzh)8G8Fnsn+{t~gHTw#Qdp z)-2C~;Mw1N7v3p-H_NC%kdliTgU!8*pVWkf=BmSozi>*drpf|D53wrTx}i_1X-Yqr zspc4IC!x--(Vdr4V zFoB+7NF9%SP8D>8n-;~GFA#SVM*#+Aj|Gf~60S_uaAGEaAv!s+UrQE_j#HR8yfq|* zjMequYe1&bB!<#a3|K63$_lWCf#RyDJsbK$rj(IIi)1m`%vXQ0+Gp|^RZy1ubT_wQ7s1#zQWMeC|9cT zWlEP^Z3?ymrJs4MuB(O>J*~2Q1fU6|aw#)-4DVRkBzITwLnPxCa6BL@M7ta}%3+|Q zi;3&>iv(tx*#&x}Ozby~HBo7Zn2_0B_U2^$$e+Is>P=cw=c@~&C9;QQ&T$I+T5zpg09qxY*?Dn;o5ZuMuW^oBZ5via9*;LfOQGB^x6qaewd?y&L?a6j^iP13Wqo~-&o>B>PffWSSGY{C>F~sgqtg;yWs9m=H&quZD7j+=^MWpiGZgNE&L7Ik%%kRf z(vHR$WYulZokpS4MtGChThlr6%WH zz)gzFC`9;aU4f-kqdTYxb z3)nxIKdk{5!&+-%_0C55AV1qO#qyxrTWz!q(Hdp-w5h{}46Op%@}SSQUHQlFzrK6( zhW>Or3{Lc2A*%zTSGu-$9*vH9`g`WmuiBvV-g)|P@4X+mz4lv9-r&5lXz>yGQcsog zva?g3VL1q{W=+!$z1SHiS*PA#Bm++U&gRGKH^57_hiozwa%I%@SWD}$VgDm*YsW1{vZ+ZS> z8~@t9kfX&xL6}zdGhX@?);?)iTD|u1mKvGKDApU;f4uwPg_aMWKN}>u<=ZB<<3n*e zsf*dm=6({ZSm4uNx=POv?2h-qxU?Obaw)`DJu6;yqdUeMkC>G@wqfrPp1&44GANuU zceRhS8|tKPpy7<41RM6CCGgcccf2$_x5%efPmHxg;1vRas(%hhn&u0B61!lhV(-U5Y z$Vfv679>Wdlg9QEw!d=~Sn)={XU-Mhhm>>%EoRNm^^HitSs-Id$@H~eh_f)a8d4Y7 z4Rd_;$h$75x~C>`XkAb(RuSRg1&X)Ch-Tc}eWaurw};%(u5tl~c+^+?h>6u9$e<|K z_A6gafa)l3+6<|rgmT$+fL#U}B+i$308Vb~-d#-xC(}7TlJ|VA7OCiK`;%^wP--e#N(N11t+VbR^13~8b^yD_tTvZwKR55 z+luT~$11{P=u%|8 zzv*U}ki+?_d273Iamh2pIx!~z0?rdmDOiNGo|$0hPtW4Y#LPOHA|UUs{ZOW6qXveT z!okfPJgq<@Z?zls_|HfwE}xZkxtNVp+vPuyuudSCL^yY) z_Z(+sXRGlw+@*nDv>LU;NwmgGeim;~cB=ez5Vp{SE=2otm4-&Vpg3ZH8YCQOa6v{c zssI1#Ek_yr>_e9oDr~tQ@$?lN5vp`N9VJPE@DM$@8y)VqpDJLYRsX*R{d3)@Ix?%O z^&$^J&g)o5wu#relH94)$8*l~>Ocq0@emtTu8x!AE6&eRZC||@oFan^L$h@AQ_-=a z<%>sNO!i0VH(2eQx{=Mh-p6TIvCG++VtL{H|81VSr zhihweZgfP{^@~&aZ7=;kH>rsh^#8rD*e08nH)kmQY%F;l*95RUR;I=G@86lfuz9HN zUK0@oGqaxT;FpXaKYRZC`O_C~zkj#I5E0ESko)8RVMy`y+poXBv$Z@kLS$w*7qK(X z=PS=NAJ`q#+LNyzKfHhP_|;E8{e@J1ZCLOjTK>Sc*0#{mw11pL(rYZ+bF7)|DyAwD?0(T@@Gqc?1^L{kZBg91kk4pZEw7ge(Mly z#M?S5j~qN8SsfT_$5HhTBfeM>&$E*(zICRRD}#-K@EjX!$F<`#CcAaLRxQb-P(CG7 zX@jon{i$m?E2dg@Qy<9LXp}vwJ*8wtKEk^o#0W`hBLHZH(wWKmqWqWFZ{9NVvuDo< z{`T#=H}BtnR;Rw*udnar4-3Fdcinz{_~g^s`G+XeXZ~M($RXpKV4EMql(C2;ofG@b z-T&+%k2!xYkrxKyIu<8IUw}*Um#y=>#JyDeyfjY3SVJGGURVGCKmbWZK~$7>&OZus z)Fa8SAR2KvStXe#ssJ`;?|zTt+r**)sq-llF* z>MxQQOsfK(T6UTEZ+($rKq=mM-uwql(8dZQa8I+2XxiVGI_)9y<-0|Do;${TGg7Cg z7ilwtJuVr0DHg;1x_N%V>?da`i~DHwgn(J@lu80QF?+N;za01UvBVzw2Bmb}kTB=D zo!z=Tr6;iaeINkO!fw;gNo8_Z^F~l(0%G(_42#289jZ!YP)=HASt$qqVN5KvOO`sd zEj_57zvT3;l~gAt-eKa9-Z^A(8?h<1K@XL?E^U=G>@nwVw5O?V@}SgSiLo3H=uJoC zuQD_>$Q7?_xK}|}GWsN`RIZrh(M4(TWZ9)m@apjx2|mDz(SZt_cZwgXj|z(e5(`DV zr07(Szp&(r;T>Iv2|LN;G89pb`*4d=@#h#oIT0LuH$mo6bA)akxN+KaN3g28BXJ$P zAc$SVGWwZ48&z_UoiavZVq$llb|nGlU1nH`=LW`Ehw_qT!pCKII7|^~$bDC!F^%F$ z+&F6zN>H<6F}FIRcdH}Qmu>(cV{?2gy z)BQ)^wmtU335#mr!4ju;t3+h-9RtmMm8BgZc zu@dHl)SzW!w=W!#YsNZS9JTHued&W&iUtse4qvD za*HZu4$|d;AK<}uMEH|Oi3g}};@^?rD3eEK$Xv(v;c1Myo?`yR;;06AF5u&2jOVMt zW*lW~l2n_UOhTlq;Y)FVX3CU&VTE-FHo5?7!sal|o`2Q?_8YN9emK&UUFhAeKoiEb zgRKeI!Zs4%%BNl^v_QC*aIYgBEiot-+UtGedc=Y`rcB4zhnurZ< zHdB84@%{VvhVH=4$LnD}z4p6TH_xsui?73{Ci>LzylN{>@7jt68^s$u)bB6C+bi?- zqe=94Z{O;B(VL$CQ@cfkdpUv0fK55b&nw}HZD~IIhfHImsw|SqOK`hNQk z>K=XibYq>8@r5Y%_3Qh$ub;kn@x#mKx}<(hHv5Wlcin!R}|6_geumNIZDwR7J6_>T~2QV2~Mp%pvq_ zVO*z_aXIC5Q)M`s!GD>RBtt5cF0lwHAt%|TuJ$fbz9?cG<^5rkr%de1?avWb_Cmdz zF_1UT7z2di^x#LolKX^XE|tv_zS2{f0;=7yuq4DQFa{12GFLw?)XP1@;fsSZhcuqp zI{%l9?{bg{tM4dRxI)Q*>!l}Qju}1^PP%wbDxqr2{vS3CP8O0*9?QaeVG=+^ds zvYTG7B$3ywo*oEunWh;}rE{DF+k0gT-X3P1qkt}?Mw-HPWWJFoK^im@a+ ze?Hv#N;oB(kQq!lJEXhb&E$-k?jcZyU!Q9?LtUA}vfJ`YU->Ayzkml9q_p8GISBs< z(0Pj(##H%7ioSF8(5~#kC}l<{Icz)Jtdb~$`m{mDgp)IjQ_%qiw8CRKc=s4`+RsG!n$&ccnC>3DhUtYsPC_$tXtiYEx(ZwXugj2}ul-)z!L5 zNXb{MKvg%yzuMFKam(^xMH?a-#GTNV&p6b9J@zn3YSP^y&7`~eR9#`y|9($OZ%IL; zCOFIpNP22z&XqNNRWB8}Za-b=9mQks7a0zqr@ODy>H&fca~%D0RdWx5=#g(6nMWk4 z3Og3@EfoCt@+C1S=ARaDsQ6zk?fClU>Vw@sZQU=DHFKR-fGEwdN6W-PzXTk`Od3fa zI)dN53vm1Ksl`iAu5gTJG z>YDtvelj}AeE;f+F_JeQpfU+HA8gpl7m21J2uaNde=7o6*y=rR?9e>vtMRG^{i{b+X&&41sf=WC-KqBW zc9lUecMWH`BeHTe#Bk%C^f{k7Dvm4pr`2l&|MyihddR`VpCDR(viP3MZsi< zJfvz6?&(y9>c8|751#e@dn3hje2I0@P)WV%orWn3wI#(IyecY(q*^rdOi5ik*?T+U ziW>mb^eWpYY?BU{o_z3T%Pe`|WkEXJ@0^{~7aP=F@DIDTTh}~gKJKp)mi7N{F z(qn*FpBkf=$g#^9&8t*=`Iv?U|HfQj(J`pmyZS0<#S( zNY(X%cEeeL*id@OdC}$s=+||Ip}@o?VJ88%3k~uog@lO;0t!4Ez{N8E6k-1 z8D(LIdJjAV=lYT(fAW+Fq>~Ti=_YQFA5*0cC@43VBlnBnPHB9_c;?VYTaKHuvQ%(g zIn7MIFD8ITnbHYx3Y0YDcScXexWJWg7rLW_B1a#`PJdnmolK(v0{_ul+RB5JAjuKN zc}+JLkReoQbBfwYhhSL6A?<_v+ViK^S1+I5+&+3>hHZ0I34qV{7EPJ}lvLNbmg%dV z>b~DI;dcM(F``kxkLed{@gF~gM>A-25M;jy?B9EcaRy2pG%+Rc8$}4aw3MqAk*0{) z>u^>L+as{O`X38Iy7&kwA(Bvz3g=v{)WxHk)?Zh4L=F+B-DO@0)@&5k#&xa zT53BhhqI!!X-BA03I3Fs+F`DaWvCq~ed15Lxp-pE7VCVwy93Sr&rzBDCdhIVXoy6K z@n>R)4q~4nWd(&{98%6HmKyTvaiBhhVyG&UC=;jZHSASK$$v6FV5dpfci7QU=ekoY zJg}Id@eI4zb@8{cQ^h_MCN1Uw-W`Y0-9a+Y_+Ih|0WY*1pE#0J{5Y*tlqn|Z-MH<0 z4M-C3jn(4GcNL2$PhS7;R@KB`}>MD7d6YZfP0FqT{NIPt-KR47JI8Oj{ zW}DIOR7b3(*afe-OPV_2ASB(5gCHHUjCrt{@7ahgw-+dydkVt4^8G96kMSVRdd-!SM?l*4 zH&yM&v!360Z}z&^31*FpN~SFV3WsfpsY>+Br0HAR{{H2MA7Fa()(;8mzI^kL@cD}u zuU@>cBnVEEwaFwjL7i=S3YyNdYq=Io-?w4U&691vMw>QCHEu8{00t_R;wyDF_pgKB zTNvy4`!)s$y!BMR%B%lc^XbTQl;Rt?XDhTu-1oHsIlJIHnWv~a#&kVdt_mo z3D1Q!WR&%!-}-gEO58#eWcu}i)MxcMA5TOh9W|_dFg1tC)0|ZCjS7Y1n5iqv*Qkm} z#4z*GqM_=s(FaGweLXMa7!#mV4gf-@k%QC>YU7qJFa0F_QO}~yFwdYVyKY{pK5BuO zZ<`I+{D9o5)X9x>Y6c2J#b&Kb+|~->Qe51(w^V3iWn*;q!9h$k@`*prO>pRX;GgZV ztpFVB+9;!;z9*>0{krJE`rqO=Omq#?9Ht&T#we{dQIVNLL@$_VXrAp=%7jFJRlkIiI}JVlhk9Xcjsoy(K0@-L4`Pg5c!((Mxx0(dk9vd}7D_EY0&?#z;;;4AU}9%k4bJd0B~~ug z-Z{}&%@Q@FAxm{&V(0h=8q0+tvI#os=o~?(znGu)k|^@w$_HX%EaytD;{Faax7okx zWUgC95UZ!F7%V)gL)De!Y3|>BGQmBYB_u9O>T>+->ezA2o=Oon5=l4ELJEy!`C-g2 zAUOZy6!Qsz?ua5DbSYDHGW>ZAEcnVtpsQ?mK)VS~*`XVf3&Cn^jKxRxJ<@ku8uk)y z!NFxh3}VDn+4)=kBqJ|70`4HKv4fYPW5)bI&d_4j)%rM2$1f7JKiW2&0e5TkoSL0;QT zVR+AZmt#1dGWtF%xRRKwd?kIwYI$AGLC*$xHH#u^#ZwEAwIr-Ce3)B{j+ z52=JIl0Itj-RAO6x4n6`_V}rpxT70!*VB;(yk{I973d*dUA=ns>MuY3`11KH8(a#{ z$Q`XTS(F*Smu|{=yg^_JKxaC}h*7-3me6s%%Kz#6?FTVYSM^5FvDwbCqGt{qtupCh zTA3mU5<){RF{u}5jYgPAka==-^TUsS`L~~b`tgS!J)O|)CjP=61)XAAvM|QtD*IT` zjcbPkKaMN9*pkRl;K#rG1P;^Acz4F}3EWSgJvR)3#LwtHd+`z@mK|DB^7Q)2Pd~o; z^!)kj*Kg?R%abSGDDZ6t&Bg{)j{ECJkKaH1Xo!$YFwZRPguNaI(06H^)?|m?e?P_e z?8#$qM?C8t3&R0nHI+1a2?}_vfQ2fSJ?>dbef^%sZnID49s(Gk7y#6H<-yP&&pLF- zL@<3?`L(z6@o?X_i>w)<6-1H$ty^JUUrVTON})MJa_|xY2;Mxc2MnC3=AgR5R$bKb zU-X(|m%K`^nkd_BI^-^jU;FFpctHgatoxY13(p}J-s)0Irw7BK#;Em8MiN*8*b=mj zYU+tsO{=rt?9kdGmJQk49}~kn3|J|5Kc!|TD*b;y1cqLf<}UL2Zrh(DShnlo6)k(A zqL*%Z28Bf!<1>X-wiCg@%DMzJxe01bA^m!&qW_;sTQF9KD$b!#7P6-pnyEgMFaIDN zh2|!qT>TGTv8Kwkx)}tGww5*iQBnGJL(}eRchxc2j*95erK@UvRLj1wj3fQ{g7FL? zd!-}p2oxEmpp7YYdK!m;BlLc5fmlv1eF5*@(L@y_Y+gT{n+@MwpH?Cte9EdxSP}RS z&2Y{c^j5hnW55XpFGrB1NRWxy&B8877t!*#)U)%%JP^x`F{ z?d}qQ36vb_B=!hza&`(M$VyM2k*I6vYZ(%UGzP2ZIAxnf2Iqoi#5Y(i_M}XIbg*l; z021_kr<>j6;^(T21Cv%zO%Z+da3=PYrAaW&AvjG@g&vbG^_~yOJ2vyKo=mAg5pd%_ z(`F)(Wr?rF~cLUlDx=s081vI+oGyEz(*%k|pqBrL zq}Ag4H;-+Hck{w?{x$>cmwxekIC0T}AFudkYFI0%lcdjd^YH3f-e62;DrL0%(!jm@ zdhT`yZS&{prrSWBfY>pW+O&Y;1P zF&qa?gjN$`IgYa|9?{S+oN8$=D;fN=2l}=3!+b74=mSDCB*hg zHT7zkz~rk9iDMO_h8i7muqQWumdO}rQ)=}6B}QhpDjj;1|}SP<4N)%)+o2&0+y zQ>xYxyhk9+VfKca>OWg)*`d2%=f8`tjj6o;`{c2oS`kp^k^I6{jzpEKM?+I?-{+cQl<@Nd2;hCIjzcBeMXN7ePdff6EJn* zGz(Y9e*q@k1{JQxy z(w=f#D{E!Dx6920RG>!z2_J)V+-aJ>^}vVp_5V_T8d=38i@;rz#5AjDGfahpTVN;% z6^S~+pDt34PsXTvI0Qo!E`fDXr})5wu^SwU2humrsj>SKYsEmFM}37IVwln~@KV@=5EmyrZTx1$BB4XNIW5GTBmYZh1fz1vAw|IvWKYd(&H1 zHOAKAPPsg3jzz*C&AConS1hDJgy)FYQ- zLpv<;Qt>=(>}R{li$2#WkT=WH@Vyu_kuz#qSN~P*OMuI834dgcMzY6cVWw8=pdD99 zU;e@8;)pe-?l?mwOT}iqbdFb_7{<-~>x4X`jZ&e!0F!TSo!OIg__ArMYXUsJ zq(Dq8x1`H;nnvASwu@cIPF;*A(y1Ydz)8EVstjEP*Cnfj@>GMRicL3>nB+&I9J$=x z$*Q9hXvV2AZ1C!#dQVO@!%giw17;)7ti5Juw`ntXv<*E0wK~tPp1gc^{oIdr*#gKm zq3z){=hr6a^jZbt+h}zTVmI>W^cKyUJf}xod<`}tBI`JPf9qTr!43bib&&9MC z^44VyG~`MQfn?w3GeRMlyGpEZ0#z!0 z4r3*s>*ZC~f^#Edz_Adrik${2&nzoJy4Wo&B{YBX@bj{QSgy@QR3W-hOxtJ5xIsX3 zj61l$g4iJiVGCmCJ~x4O=gUQ5Y!A%iO0hL_qZYCCz>S{rn$_5`xuVr!=GcB2^|;P) z`UIDAUZ?<;H@l>v)~xqB@WMxIJ^Z5gUzfa(*MqCI=y21`QKteizdC%_lT(oJ1jHlu z^$_ia^Zdn28{nAcZ!vTQR7j=N+qTd&nJEycW+>QN#h)p2=c)$DDDAWTH1AolLU&Y9 zv`=4ZImPGU^-ZuE618DRMoqHRny_{3d_L=UH*+hV`&ldV4wlMX2`|+;#A41TH=l^< z6o>MX*Er7;M3?%<4Z33T3auq_I@1ff2pBVUxt zGvcjbl-_p_MXh6Mf2AI>s^Xy-3yGG94FL?dS^?w_(4SXsdcab->|0%2g_uX@Xmm3* z)XxuwpwS-CnwD)9YG+n)!E}qWKx~cIOjS2#;f!KC?8GJ%Gnkp6dWKA?rlea43+~19 z8KiScjMkys%*9~E=PFMlfD3f2=NJkpqeu~@!tN=Py*%K^Oj@8Sqf@b>K;Q7Zp}8^G z?|-~jU5)hm+2>Iz#R%kraY*_uY}hY=I~#xG*}>!yn8x5Z77G-o}|$;l<)!q2&3 zi?gL?0}j?Vn|XY8g6B?&aCE!QU(q4&%>W5hjnC>E0ag232t{N1=u@Pjg)q=8-BLT}i;;j%pCorm|$KS?%Re_K!@qi@d;i?iaEgAqK>Om!%?x|M@ z;Tm6Zw0<#B;47mn2O*0f>96hyB)BRj!$v;$^m3x(MHRz2#i;$1n}kP_1~fy-A1a@T zA?j;4-O2B^#y8OTM>}$3s?*2@68Ss|`or=mnP=t|aUQ-fM!}!Y#7T@1nhut8I4a(C zi!`yr)SSWYF%lX|$UGS&*BtI7AxBh4Bc8{u=u$P#*Wx-UhqS_w?XIBZ|M=#ZO~jO* zhaKKWxKljQcO&)UM7{s>ei|!_u5Lrhz^TIX^lM!n%nGA7&p?C^lTqKSw zl5{U_{YzJ2@wfkY?V5I?8yfdM;{xR3XR!*%!eoVUG{+cVx)k6QNg+z~8O|7*5`v4l zj)?8lg(KbeKXzj>e$P2@+E3}wr1WaqaLhMk9F;J;xQuvQDc_*)BF_=Mn+nFmAiGT@ z&rO7388tbN?84a_y;iJhNW^fBqAfqnnP|E}gVb!BIn#;v0-&z8#ujt-*bsM9xCrVC zWMsOz1aHWjtg5|oSe)B9;INFPAy;K$QatW4cJ){bYjl{97`0Bf{aFNk-IrdclUU0I zU3FzmqcuX!diB$T0CX&5q>*$o#;TW1qVT=OtSMnJ7uhj9D+XO?Rbue24ig##MKD>& zNZ=uH3cmVP^_v;0{*9>ey7@FZiBb7Ha;Uotha3@@58)vU8rr^ro=A-K3Wb@fd?(SY zH+)IBj~qqhR&wOJR5mdBtNf+T0J;NAJCy(wz^=}nxE~!t7#T|cW|+Si}=x; zVI3mgHm;x=_OS8K6+9VywPG|zeHz0?0w`9@x+?vtFwxNN+eF)f`)A zv(7HbJUqj2QMZQ}n|M$8oFQ(ar!}lYa~Eb}QI6T_eA#TjzNo43t81%wT92O3su6}G zNzH#ruQ!e}wbV-M_aD4%*;*Z4=(Y{jIqKHly~P6cGZ|p6W75xBVF@`M^b;pee~mCzZdb)Bw&ngTKd`v!S?+;Ivd)0D1NQ8T$KoudV;q z%h~sANaZnxL8-qfuSa=sCp=!dECwTB6v2%U8owl|qk?Ij7B$F^a1*9&KlNdHbY^%> z4^=cZrD_P2PZGG^rCKp9M+W;fTOL+$cTDb)Gu9$K?aw^nN>CbT*yMtuD3p+^n2C%v zh|@;qygO3Pw6nP9XVGds3-nVxpQtNewU83MI^ozGk_P9Qv+jTPhyuy+;*v6OQ2+&W zTn|m0hxiy4!aSrj1r)-6mDgCDUoM$S$GmG|9ZuF4O41f+6`2#G2r|1@Gc!>La2Yc|<1^|y2$?scgF8G( zOG)vVP&G7FCv{}K|CV|BWZV|OXM7t3*sYM;zT_DCMCIX zk`4;`m2NCl!OQ=m+V}!dVkP6k#i2`xY;&?CcQufuoSXoTRg{&P8w@g4gwXdySbAhi zrogn>pQK4YEz-{t$8guIE+N|mc*j9kCoGHvjAXm>G5QELn2udB1f8agF;Lfd@Is7- zy<(UqSB}{!Kga92_I}lep@dOEq&^KyH7verZS8JtoB#~g7FUX9`S_Sf8p51&jWijm z`JN}ARr|%H-ZTKEGvf4TfCOn8sY!eP`3atCK{adH1=4=0rx@O|Rx&*3Rba;RqVnsv z@A%cd@6TU6yGhPQmU!c7F%(O6`MX&P?CF4Q$WBYOM4U%rjOSz^b|ecL%aNsDA1(lc zSnhz`s%n*$28QO93wK^X3gU!nz)n~xquhd6l_OYJ9=WZmjFST);S{WfWW!NE7j9r) z?F()A%Z;Cl9C?_((jwP|Q&i6XGA@~MANy`;C9wNvi?(Sdxi6ng>-K_ca|9VyN#W&4 zJ4r57Q3^V@TeS*s{HLVxS=N*?^+CMv7>;yw&h_~o7jZL^(h6z|Y|B6JrygY$3TTj= zaHkncPbK@$F2@>)laqkat{;eC{OP7NSsTmYp5j#Yx(@1AjAuj4oQRO!D_>#i$J3E= zCN3&gsY^eqzQBe!Bw*vR0GMQH=HN={IL70alx!A!lPT-{R}o1AKy6dO!lkPT6?Qf+ z3#{PVV++&&VTXCsMB{e~Ne1L(5>i(Z{W>vTe{2bml_u8f6$vUFt%O4ntrX**u3H)Z z?(Lg5A3n6%S8RE)6^?|ny4`R1K3UDg?02``8HTWB{WTe2k{v3{5hU~>$kgeTe%AML zW)Td@+N41}Uy(x)splHtiN_g5n3+G{Lj`ubW$NIX%V$(L$%l#uosVq)^Zo1p^`HN@ z--i16=l}k1|Nd`({qepPH6CO9`pYjr|Hseh(>h8L@nm}B<&UqFC!!Y*40Ns@UwwP< z<-+GNKV0W{r<{3T>G;X4}B+Ewwc~ec*))q_$0ooChXr1t$>K-G<7)=r1} z*lid_iE39>&}k;3eEkRQ08(nH{vPk}3fTJ(esfK`*zfr5VLo*_nX0j0%vN_6C6PX#HkG(W7XPCIOp;G0l-Ol@&1PU|5_`_BWyk5Llj+r8#vS zuFm-fZFC;O;RwWCGf=<_E$%5`gM^+yhI^t88!UaZDkebgak2$be8Hz(4_vzYGYeQ^q~zhB-pRQQ;1$nu76i(hrNv3EPEd`oV6H7ZD~1 zs8u3g5q2b#PrO0L|GHaA_KwS3?KoB{2UEVTV6>EzHYr@D>A1a|QP@qCpk-2p>iw%; zYl^sX)jN4N>Jxa*YV1nHP97Vl^igQJ<1%Ku^GSnQKvGsNB~9W#lS|W?M7*jL{5bWk zd<=zX3W(LIB&4Q1C#huISwn5iwtDF zW;r>VLh}td`L)x83AV$E_>e|g8Q_;Oa={klD-Z+o@D2~kqO{x`QcN&+?slkdH!}&< zV=?Y_de3r79fI0d&9s^(mE(0>N~i(L;*cLw#d#>ApEt}zu~2Iz zpkr34X<~Dwd6q`R<^A}#Iz~7?&ZD?U?fllQw;w;fe*Xb0mi?s08l~?oJ1`9-GUH_) zdcnh+vNH#k4qL~}DPpkMBnGz+F!+fyf2Chrbz`Y9;>dpfMkFLdoS0)n0pc}I%SzKo5WoL;_7hJ)UF9tf=zJvHe8M6mIdmQt|}#v!t4{`c?Sn@YDTqwZ<3qg91&n)vD606%&> z=*LeV^jM!h_UKQKX7O%8bDZ6b3pL@Ym{x%na$>_t6Q42HhK?ED&mR37mts`+5U;z8 zKdWZ+edEtJ;2a{4z+*kb&!5?=0QbNZ%tyV<>th!IL>2S&Zhp0-=iR%vpFVk4S1I0l z_J95Lufm}BN4!1*_c<7cpgUz$td+^vBr@vyXORiQh+8dV?T{pyn5aUR##S~&sXVg_t ziBKixc`{I)|9brXb#;+TZ#Ce^Y2qe=rGR&7s)G5Zi)RJN8aTzJ)Y28Z zYX2angK7s}aU!gHcodNr+4#7QzQ;g&KizkR?S7QcpN%dI4x4wneaAwed!HV-X$>*X z@7>LY0cubqcZ*J&gFH1fhc>|0*KZU{?WkF4cCI_S^l|&nXUkZ}F%XN# z4?!0Ux-LZa&5o)AnIX}=Z5O>j3PtPSUZHSeqsQ|B=rPa-Ef=~KZVX1SPZG_`Ep6zD55{LYdIW*Dh^myum zLOxqt27ffid_&0=lgHo6#qsF4XhbQPi4OQ~HgQNj7|8)*kFF{R+YY~Xl`vEQ`vV)3 zp>TTG9`vHI{AAtMK|oxF=DFt4%z*2oR9qnd;c+(jp)5U~Qzbu$Sf6w_usSZ0*>5Er zLtGLL`#)Tsyy&Ad<->XMs)^tfs9Ekr4jzwW={OYCJv(PULk)8NRmnp~nHVXXQO>RV z@i(o;eVh$+WBi+O&XO~;10|M{zU5Ko?_6iT5b( zeA0|owwx#-MX-}tkE_aE$ijLwASFheM>5ZLWS9G&v||q%6B&;rC#P69eLCHD$tk!J zV{Dpi#=V3h>OjhjI%G{>L=nlD0FQ<12W+LJ9s`t+diQT%0mqaXzy#NV)B>&XX_Zdz z@_h8@)0Yn>TfVw#uRXT1&Kt6XxL`)0;|%dBvrn8F4n?T@0|7s%LchQHe|zRTWRi_a>JqeKusi*> zPTM19&8SY!=;*R9H`{4R0dj0RPlw%<@&c~nQl);8FYuK|YiL>m{ zkgAPpcBX#1Q){Zf^p_6(2YEDK#Y$V{@>4f}g`QP=u)bdH?o-Kyh5(U_FhHE}s4&AQ zS2$2kK(8&!z*W>W&UYG9lN4KAjLDq(xuu#dJ&2kaJiAnlIL!oWdy|95)INOt1hMy@ zJ{sUPX4Q&4eqt@FHC;T7e)a3!H_`yWQ>0W79AigKNDWCFM>|nV(L&F|UJbqQv1jT) z@Z;Op4boIq`EF&zy6={kbwqvA70B(nD&?#}7pZAi#qpO(f^|M)H0YquW?fQFEF8th z)uQgWsbNW-sbP`73Sny}oGnkdvcO3sfi7O$e8<}y$+zEft*qi0NIbO7wve;W?CyZ( zyKP@#E)$K+Yp>wwRjj${I@1zrbZ8{9py7*Jtgq`o42R9sb69g;livVDrH2H9QN>^X zi|E6hHY*n$B7|tWQJN+q$}`tEjRo+O7)wE?c_>Rd%#|9G{ncdT_yPI(;(Pmy73PpP z?%?+dV(-FP-hq%jLV=#5K-`R-7Fi7c&ztpk&AYY*&Gi$Pb)M&nYUh=`fr63piJ<#)ofv|C7Jj=}Z(8mBZL5 z*MLnwixGK?a@yk7-0>z=x?#Xvm>|`-G|&Fq;>K0ORskIzSqzELAKOj5PA{VVL@l37 ztMT1Q5(n!)zT)9P;HUH7ek$tXjf7kiY2RO%Rj2K!B`bGH3 zmgw=!GEKZ_|15=l(B%afbC4|x;!Kq_GHi8hjeUaj;0t33Gmy7_?@1^TiE%>wHGG=R zo(bsIL`{PCbh>xBmKOFRTr7og7ogI8O36>VL{ayKDpC`2F)R3tXSF}|R{N5&kN<_^ zx=#}vNO`kM5b1-cA*sK7%0qlXI7qO8gVwIWUyEQnAWEc@0uOxQiLXxc&!x5ktSUeC z%)y$kPAS`wIC-fksjI*3Cw6!*R$)B~K2(qwJ}w?&7p^H{tSPoJXyQdARUECxxGn55 za=P^0QAH+LtcI+zvPztbSAoFh-(i!Pq?OG1VE_~^qCz|=N%jW;Bv-oO> z28um~z-+@9+bB-os;;wspTRz2Dyz+1wm)utso!3&IF50nly3G9Pb$#61DYA?WJ=VL z;}~5y*<*Z6zy9^NUNGkI$6sFk_M1ByKK=KP|M%bi z`|U@M+}7a31)kmNNuxyj?e%Z^f2R2#Kkc>0zrFhHFMs*-FTcEa`}Qq%zkIIpV$e&| z;^Nl#ckk}G*?rW`Q`weToao?jlCkG2gVb61Vz*Hf=D`om)D zdGucOcpC$pDerh)O@zFtP^UuCsE56t%;AtzZ44lB%eUYJ&Z}dpRsO|E(X26m3I6Uh z5e-_2Z!j>Zcxrg(aS?a#7zqfz+6<<4Bgcae4|`s}N6FoNKpLLZt>YlNE5Nv`R;^{M zkfIwp_?_Xs#uTNxN;7MB=T`84L3LV7f6|G83 zvjz4w2vxPCVt=^FS~uF8n&dC!9s+rFDPdpybb@v=gmd*jhg(sp9#<56AnaSHgA7IH zC}7#PJ0du9fE&|NM09mVInbQN^`}WGj6`iXF69(wbyW)PaZXzM#G;*I&RAPQ8akpU zDLUHUbXGd8E}~Euj$>zP0TAYA39TnC5rVXiLVsaY#NlZ=g^4xNFA0YoxX=X|iOj+GfBQ=emHGC=Fd zbo4Vk78A?KsxS*)iU_u{aTLP71+jU|0^R&c)vBJ+_a!>!9}R~cy` zyDCpAkr5@DE0YUk&he2C76GdEIl}Sr0gYJrhBNCbGwsLwiIgRsij1i-3k!3N0L&R} zAXSu|3_4Dt9|`ICcDz^4LHlc)7wEX>VPct?udV;NXIk*b_s=Hez6!hv!_+I+NWc%S z$-kz97cii!*gbcem?CVh(u==40eh&8SPi|d3?AyR7x{7u!;M=#&S9~(u0$UtjUiO_ zqG68H+Zi>JNvenAaU<4g-0S)+35BIDSv>aMu^%5ke|-1;y!)l#5f^Jf@S|^#<#2^| zswgI;(6oA^6w+ca&C+Ty+DW!qN?ueH%3RwWiCH%AqFSXU^|3-WhoJnMO$IJHbD*hN z)QixP>{Yh-T#+NLeMlzyEE!X&ar}fYl(|pBRlf*PJJzA!iA$72b-)rRM=#<`6-UY# zk&%-mZ`j9ETvqM*LOk7Y8cBA!?xg}CFRsQvS&r*o?80`!Tjv_{@S`_UY;?3AMFq!R z4908c7duSuI3YDD5`(<9NXMen6kfT=#Wu~=F2&C!&cp;tEkXFj{5d!x16%|0(kUvyj9Mokl z4cCJ75H#(t_ftREuVp+ILgrB_gjYT(K~Kd(v5rcNS>uP?@s9^3q1Sf=$!&KZ%d{5B!x79(%tXgZWj>r5ubi} z*0UUce0%!z#sB=D|HEv(_5GRe_N}+?-~8LZ{+mTJ|GkAn58l3e2TYZ6oA2YtkKcB) z!yEng2lpO4fA-u|vr&roB=b*Wp?B}z>3Di7spxD>bKiocU+$Wnwy1=KQg5$QYz*Mw zG<@^x^pt<#b%toxS?|{Pk1eUscTSb#N}Z$f&|M;m0Qg+Nxp3qXKw{PXS5flECWzR#ZZ*?_9_RDT ztEO~?ikNi(`>wN9A2O3=z8aZO6oh(;RF-RDn@2$Rh-&_(UXYzX;L&?3U7T@o+BIO# z3q4Edk{06+jqL_t(&x##5T`#mno7y$-@2+dHHCXY0p;+blZnlkS=RWa^6jkqIx z&ma^_YW~ir%nF!X&tN=jPzAK$mX;ceIXx$h>n%%+ft~GfniOO9xYR_?RJh zqwiuv%7RFJeH05;t0C}FcQNJeLj_79mWPIDWnQ27Phb4T&aeloRi03>SkqcdQ-AuWoaWZ z;ELb{&^emtI~n@CJ_eM-@U>xnnRcN+MBBrRKd4HMc8VSd0#oHM01Ry3>)_xEip*iP z$qLJn60{Q-<1#IW`w}~H=!9?Esp=uB%uq~2VO~)NU?atq;WEa%Rp$L5s@DY}>=pF~TJ56DrMLUH3NeT-~O`$~425)RFw z4m7!9LVEm{{rhowi8zSL$8Mi8j%5=1pcsXyjFn@yO6*jbBbHYGDRk!#GWh@>cuvEF^En(Kl7@fy~|5WvR?aRE=_&POAxTp_?ASQti7AXyQfk^hCdo>ma#$%T zOiE0q`odqEeZ00$N1YasgU(i&er{elzXDp|AP%ZwmlfHt`RJ!N;5liheJyO#nLV@_ ztOZsQ07*Ie5^$0%JHbDm8gBz(i6tKV<24h;03>nT;USo#2xEzmF49D8@&SM}lZUJV zlj4w?D^@myRq|=U9^bV_Wwj}n=}B-@R7wLvp#lWQ2t$sod`Ia$FR$4Ae%d~m#$5D5 zHeFx~m_jARgG~#zEuEzuw8~9S_{uyd`~0OhlF=H3_5WyFC-tt!=5xM$p|~Amx=g2U zj`LD7>zdD4p+DYUaRRg($xbHrQDA#p+KP*uXM$)aWpV1$kSW#Jwbedd1ZXbPqsJY5 z{cf)tqneQij!H+O>xB(gaKPQw6RKd0Z~QG6(4v-+SJ2@HMe=NUfrP$rBemcaw^Ynpus9 zdj@{qY7OS@L#@lo$oVhhLa>U|VPULrwq$N_d^CrcQ%&x9M_KzuJw1R4*bXlX1#~u( z(QyrUNbGX&27jQ(b82G= zgn(5=65@b|Ff4TsYe0P_rNf$xytLdDQb@9Ml;|Dxr8zRFq>V|mKoaTt`aM{>;C&M^#l<6a&vwOP0ex{hmQ#2F8a??evQCvLV{J-= z%DRxr9R8!rn5C5I3|MW+$3wibmC}S z0m%=lG;{owo5pjOo3bm}NpO+D+}Sb@RMM)iE3#6ZPnZi!e2kg&1A(1`V8%`uV@dT} zvQod3C>=01CG{79{dG`U>c|?$8!9?v<9}uVGfhQdTiO^0aT9W;oK`ybgTD(4gtKW# zJqK-%X3*Lc$PlP}YEF5ql|a&_z8!Uvd?AoCOkgUuoC@T&2PDmg1IrEwl~t;;t9uog zE#U&u&OVR}o7~W)CQ3?%TFlb^%XZFau(a^T+Rlmwt3vrjNOO;iqqujxLJpxFhcQuj zfu{cB@LdGtSLOmtM^k6#Lq<5<$6ulh^!)8O-BurBq~pn()COQGt)^>BYP(Xesbx70 zD)Jic*vJy7i4<~`ahhr|`c=^Ba%1%D0IT*OE70W~t#u(&|KlWT7e4C1cpXuzkBk;J zPa-gIG>jGnfGSd-!0e8T;5%Js)V!AG-2fTKQwIML~ov2C$5)9sVNC+;zM zus|V8YazK>)0A@XI#@}&41FC}sF{+%g*{y`)VP>!uQ$qRNRQb*m zI#f;ljF_MWgjHy1 z1Madwu{3^<#wG^xiCA0O00f~Hzs(9*Yc#z%@$WtJ#AXdoz1o;gmI2bp6qz`3v2|xv zjvTJ0*-SNmYpDLs-#z_weQm-i*6p*TG}uw%P76(-J(?h=1NEF`9%jqj|s`7X4_cTv(cKsx-?%aL!j`X-9uNC>{ zE3*vffBEM>|M~3cGh$fZk1-KVpAr*pC4T$v?WF3yoq;U^O{lb7Ru4Fd&s@rqOFkSN9Q zU%$P2`_?GVTn==v-@bE)Io1{U`tkGYH?LhBXx*5G=l0h-KUlJHX57bq`rzoCz%>k4 z;(DP)sgo_4Qu5d>we&!d+ARuj=2uldK2HuPjg$RZ0-HsZk4OR^si9ohW#=SslupLE zE4pc_y+5%*82BJp%$LwZebHtt-Bb37Djv5ai1NGzgrHGjD7B7n<+H8eR; z5{5=%6f3D1(j=9@XmL?YvEwM6T)QLOr9)BbiS8@3(Be0txugTy5L z#2FUoLaPAOGCS~Fy_jQ+=tZ)smt)2)EL{)g^H|Dk7ZLE|a2n3%VtoDtc&uG~=Q{g@ z1yLVxxCHFG%PK?b^b`ScB#WI2NRC~Gwj)zY&7MUD6$Qk6{=*2J z(_tt##gCEW zR>{LeW~#iniAn!$u-My0@^HAna*pH!qu#LzV(=?79+tABaTW?+6UsxlG$mL%l5z@G z)BI||At@(;4{$IMiOv)XUcob|*x>-i)dWjUGM7H)QvU20NYYh)IIrW*F$Zn{;XBll z5$O~$UOFCZ{|7h`RO*qWDu1 zY1h1T-D!ObwYR$f*e@p+&NtY^XkA1BvN2?REbJ)(=jpz9qQZ$d%@hd$MT8`>&U5$M z^=>l#F3m0Yk6Qar9>4v>^q-CbCJ>{&ngkcwdx=<=1*{8rSR?cqqq;54n`%06-o74e zg63=@r#+T)`{UCPj-M(mb z4Yun3TSD{AL%?gbJUW-(pXba`V3n7HuT*F<11N2!&Qb&D%p1)pUv_I{%7|r|*4Sjo z+7rcO1nl1lC9ll6f^$_nI5#-vr&;WL!6T3#QhZNG!J4kT8P90qTF2gVC-G{cMbvH4 z$sKRuz4!I&&qqI1_G&N!r{f&?uBlab1?cqs^Sf(tdDZkjZDBOsG=*Q^-#;Acst4n| zc=7V_<7ciW7RqxlrD_`~P7$;I?i8q;^+F%t>)OVSv4UA#_aPahxSZhqVa@n^=Pm`_ zw$%HBhq-iP|1@lE7L>YTb$5-KPCv4)iW&ATXGs)YLdb=$tl|4*MhB@qF297$=xj}(^zq*yp)81U}54}uSKV>Ray z*@P`@ym&z%L8?_lZ{Pj&yr4&Wak;u$34KuX>)tHu;CpMjrx$uUGkXNIeomkg|UyIq)K%8sc(!K064DFILlFu-<18~ zanF3VH7lSsKWgP&bw*nE3jg@<{)5rlqsI@0dRG7d^b*Ji_stc#W5Ynvn9)TPHIXCZ zDBe-(IYo)xYBHMJh7Z-CMAee0m})1%b^>oyH8F}76p&}gDuu8Gp8`jFtM*fF(q4x*VRMh~!ut;M9;!$ab z0H-<&eHFnDUjk%st_BoLf86a&LQvY}6UDHWo_|I#wb)}OwMU(43C$Lo{9TI zIJ9T$zm$vg%7!k*Qexa?Sk{o$kzJj+07G|(DIIG=6$ytKKndmAa;7jS6raRngzQK+ zdzgXMBh6aqFD5Nsql8dC3{53FP-a#+3^+Wq1!J*%+u%C5xIKcL)I`nsp}rvzGe8g1 zI~a9lg8bke3j;{n=!eB&qC{|?xQ_g?M!q84IYS9)IIl)cm>{X6a_KnD$XiwdmV@Y= zej=v~*=@sVkLxPpNf{nzhm|F!z`mi`?gsQAItnbU4IH%#%=!eaE=>=5()Uf^=`P=Y z|Dmg{mJm>uh-^}BooIz>{S!?DtXJK|R5k1V{rc3T;J4=IT5a4k{Inj50_#^(EqN77 z5cPZoQi5I`mo+KqYX4*B_MHa-d*Sf6kLCcseA8Wtt1n`XTNPRq3WzV6Jbd!_F_AFx z$lD5BNEA49`t>L*`#oKpSfPjlR8`U^ugf{7jx-NjmY6CQAIeE47R7Lp>>9KHm0sl6 zlEE}S!yk@xKdfg5MGHT(cyg;kLFbI4b;{H<9G!4LB?2Ea{ih01X5)ZD!n#zIX zXR#V$r*+(ICvZo*ta0a1V;c=strGo$mJr>s9CQwnF!);WV2<@FGW2C^JOB8q1Dxom z3MXJ&3Xe=@bXJkIZ)nWrmw)PA+RG-3bWEdAd>kIMV+4!MTA~smE_1Adln??L(W@VA zD}SUTTjdO}My(^?H_<3;M8U3qL{n;(7vGao0t;Pv6|` z+7*&Jk1fu7Yz_Q#57{)tP{z#b-uGC4SMT_X6+a((&gNM%WOa?W<7q&;(K`Q_6k_WA zdo{0nHTQa5zTN~lTB^Tzl0hvv>4`919<&kK4tsl(WV zIb^l0&`3n4ewHX@?RdkS+s|LTG@bnU+h<+&7n7DHpFDZWqD7TkcV54KMb=+ly!^{w z{$hEge!lMgtsn3H@y~zM=Wh%ns!-W?&!ndf{8*i$lL*hJcLqES23+321wr$@`qV;V zZ6BYesZ2~bcB11dhl6av3aZHb=lDTB;0QT?)s%UBC;6I_R0Nsx0pxe3@setDZW1J0 zf??28z5>$O35KeOvs^ge=-LH#%@jd-Sb`zgGHt}<()9TN+ZCO3Xstd$RL@4wkd)oT zz)R^RCqe+JL=jmDO?62F)B5+G=nS#e$zvB(v1aqlF7<$l`yv?pl6%nSgZpbBK7ac9 zrF$*zKj1yaZMvpiFzUWxal7kLWHg!OV2<4Ed6con&2_dCT65?xhEFA2DnD9JOkCha z`1DL6s5e279BBjJ^GbU09f8a=^=6jUqYP9g_^NG|u&?Nts)Fl(R9*tk_-7%Go=>r| zKo&xyP{C7F3dk##$Qx!LvpF;Hu!wXio=ROZp-+w^0|Y}IqL@J$D!{cMjLKS3TsM4t z&2dUWVqlOuD?E&D^3>@}KnW9dVmkJ;Dam#8R|k~m61>3a6^Ue57Ip|^CY7|XgsdB| zs0E*$;5HJ;TG}zf4k}!g=o1Qu+b@VojQKLA{4fkQw0%!4$qRwfvQ8+Q2vtnhg(b*TJ>oGij_Y`J1Yh7uweJdFX^{rT3n?(iAU=NLAv;Wt^c0Wv zvbjx~n<%h5!E@MU2ZN(+`wR0_9uIMnrX)$52M&MfvFPhOe5E)!Dv%hKc1qI%AFpRP z4ar;uzudwJnbjj4HEmoN9An|2)7?!J2MP--c~^Akm)soAh_q%c+6N*&H}otC~C-W9Kq>eo>9GqXQXn0%|GA6ANt} zs8!o=I)+bje0MapR-m(M7v%U@gAkI#c-YcQH9SG~W&VmezL(4FedY(Tg+*PK3%!DD#G)%VX9V(jMw9p`Xe!4|Hh>iEX1^CZvZ>WK&Nh zQ~wUIKXx$6D4AximaLMJ4h62+6jh5z%>Q#Vm#aL`!=HBXMMUAI_>X)nRtqZv>zI|z>P;;{HKD>G3hTZ11&HeBFVqONI!qSeJb)7l= z{&aMdmSyV=c5lFa>-sU6#j;W3c&q;JYZdgZ&CUPv?Ahaok6cMe7g?ky8DPv(s`V|p z?3`kntnkT1=)y(H`wt&KfBEvxyLW8!`0;Hs4e!0I;I=!99$JF)%P)VRtJeJ7x#K0l zI=p(KPoF;d^I!f#ng8$q{ucwGm%sedg;QX+KmFs?KOWtG;2r=kL@-%_nAZo0fx33< z=5%8Rz0qb@GGo|AApbebah<$Km}{6CkNHvA?VV+~&g!dnR83xZ2od0tSB?0@PNbijye6w5_moi7qYXm&qNiP)n@Y_N^0KeO`XJTdqHZ zs2p~N{TK=WT0o`0Qjnt-`--RawsV|QAyD{RY%Zea95gf15e#xg>>`9C2}LJk?Eoqp z`wkoh1-?=vEH|BwEb!(~2Lw2dJK4NyZ~~Fi1*#$;1PMq#tC)1Swq@gx>Py^2X+Da; zy=bH`>1*I7gS6@HhN`o0Pvdp*T2q~@ace7UIvQ)IYM@YolXWM#ff!V^UFnWPG8rVzh;einsz7xVu^pz9MO?|mi)Z3bfBy+UwJDb^)dP;r z>WVu~dv!FM_;N(7B-8!!6;P?F9ej~Fp{|l)j@z&*;5?2ypKWj;;ncY3#?}78I9us> z>?k=&ca`t20{u-DfhiwP5h-J(mV6bLvUEgiLyz-F3dTt!+r@^STGTT%WmHgQK8<_& zRwAwgPsTvx+U>MpHRN!FVhj;1fC`(vf^R1W+3!Nbh)f`MAv+A6a>q9DH~2Z56OeRI zCb2EY%N8EP*9&pHYRG+Y@(A1Pe^>0XY7&})jDJ5YLetdRmgMdd>So$l76h(^Z|$Xy zxuDA02*r^f0PSJD|H3>^&MTl=X98DEs2yEUf@VN;@3k!7J)iG^xgAG8zF84OCi6i! zyZ`0Oqc58OB#by*tu@k1!j)k_u9$V#?+2wVYrJ#BIJziu7rvcgag4&2gdv;b>n%cK z`sDazaXwu%s}W)93EVg4WegdMLzWGyQQhC!;_(5?lSLzSFS-}&KZ;H4j= z_Kqt*yT&^6m0Nqvc8>Fk{wrd$bh-ILc?PYZS~B8ElWrk&A)ww!bwxH=0^dC->v?$m zWIZ8UNaF!zP(?gVef6R9BDjN;s>XRn22YED#zJK~;r(sfW7Og3!5GgmcU053*${NI z;`2|aQH454tktF_R1DZD6)%nUn}ycNZ%Gj3vS^^rKZM|QajJ%g@cQ3Y=wLUlA?5Y3 z+wxJUb=N#&QRn}Wl`an+nV>ZvPCFoU$y^_3M&Iq4R{Vdofc~Q-m>!v@?e3li?Z7-A z^YxpzJx$*;@?AlA;DP#81L7k7b}0_nwiVc^`|-zd;K?@Q!PeTR;Qic3NOp2^v`mrnQ$Su8`_-oha#zgBj6tMs_ehP9WSE@3wgtBhN zXw@h1$u<0PvZO6(+YP;R(6qoxW{O7W<2BmVWLwo|7Oal|9hefzLT#g@sOCjbv=;0P zmKIr}dBbHcvTcw@(Eyr6>>2u56nMgpp6v!@_Z4SoKDtCorY6Rju*5eUKP`>q7a??PPCEWq22q^Wu3zo$O2-#B2Kgcj%L|Skp zeG;8LJEk3_qg64(YPW=HK@-*KAr^IX(2vE7e?{Ww;;x?6@TA(2jWxL_hx^DdGSqU! z3e%1xpVFO?>3|*SJ3Zj5_Q%rkJWu)ExQAry98$S-H6Sq#!9z{%CB;<&{sN>uDueD2 z{+-mHlp9mQL^ok73;gK6l&LY&7ehBtG^FSZ;eWJ2DLg>>Z?ff{paj)%jk@VB$M0r5 zCehIr)r3n7)ELdc>(%c?-kB*>+A#06!J%L+4&Lu>sAA(3TkGqq=bH;aB{kLOs?!s1 z@o8k{hh#ZoPxivNeoo^6AZA$A$NAXQ)Gp_s&KZFdb2Ml5j$rQAuW-uDO7L^T zNY9_DFO>|nw6TYWSzAitJC$R<($$(AIa+}&TIUrrU}hVx=`OSxe&2SEkcJ>aom`29 zz9Egbcm-3x=Loi>g9$%rp)GF5aJs0*%gIgQH+ZQD`<+7K4^;=mj`i^$s^VdMI$mwv zNmeRvNLIF!O1Cy(>TCQ~#4b?|nVi&@JwCBimW}dF5{%PG8qI|Jl(b*D5w$PiQY4k) zIBuORi&z01gUJHxqgiratS7l}Va}2FS+@}tu#oi4y;Y5ds5Y)X@g!@Z51FN(&J8!jR++2WtLRRM$G zgoUH4;vS^pi9kl|rk`!a0I=(g)gjtNdTI3wjrDcW!2xfp(%ei*3COCm+ixD+?=~DS z)_N4=R)Hr)EYXX3^G{ztdpxgOgg;uE6QbLX^od=7P#(zpk00Ex`A<*YdG)48@$7D3 z57N<-#;xt}Daat`IzX2@>dL8plxOnw&hzIl{`m6cvnQT9D!uNd8wuQOko@(}x(29$ z&TKU=CWG%&7sW`*Cw`cNa3`^QpB{U^00tdBoxCaL=TCPly!^wT1VtT98uG-S*RS7L z%yj=pul@SlU;mpH&SXDr5Lxu{Zs0dxz2D#moSB|MyubF#XQX-@f*vaN^cd$Z4r#LR zTQv}!=+69rC)w1$t})NtWw$%qNpoJ6Y6D+o)8+1=XqCXm`6_YBDHM3zfLy4F>fq2` zU65P#fT5~FF(NDH0SsVyAyJx=>v_{=tpdD5Q)9OgIt&2~azMkVeaB@Mic?=F>jJk5 zwbefevJO^|^VW*qE7w&YFz?>KfBo*Q~}qNs+z|^ zdUQnnk<2`tiB}ciet_lNz3n{|y&g`rA~E5}**C2%N|7vvd|P!UF`mf@$S7jL(8$6{ zvM)Vhi>}Zr0u)@Dz;Ouc)QK&U)8Dk5nnk(h^Ve=5QJlNN^bAXV6_05!c`%T$4qp`F zB=d}`5JWg(ky;=;a>yC_W8?Tw5_3rlM$KHTsUX@mjB7Y)7`Qf2|&WA8$+R2qTl`Vu9L=0`Ne#yos!( z8IC3WZZ>?$P2bU7%oeJ6smD5)?BR)BS` z?5Jk9!zZsl1+(&!+L?#EcwzcHrd?soU+C=oiVUX|Zlsc6D5^@zXO2R~Tp^>SVrirA zu;Y|`JR&x>U0kd$g95m9vKMmFS%cVwI+_}WNqHe0&T6ak99unINZl}w=Sq#GE@0Gh zs6a_JUK0l5^HpBytY%K9A{zUR$YV(1_!(WI5@fRMa8k~Ys~}TkuzrfE0S;;aDouui z#qq(gQ^Lu?RMl%GV>;^thZ+i|I1W%bsk2ju`Anp45a0OK@$1wMLlyoQehj42aA&g0 zUd9~lOglEy##KB(1%n+wr7Mo4qyghRToW?;j~l?rbn0RSqIFXQ4`Dlcn0a_z+o=Vc zn$PI~)B=dL_dM8NxSw?eZS$6e_U7Ogf_sWcH}KW@QO);kf$^zhP1w!JPmsC~LN67L zpK;K20i%vHEtk~0#>!}VCfKfo#=aTUUYY4_@4ez(x5KsZ6i^S`qn*7N)r_APkUsZ# zAP)$td!ylB*C-~DpN~P5al^SGK>J)XBch!e*Jf4Yid<>Aywp)qzxw7prn&)C+7Q?J z+6G%;M8@7!z51vq-(Ezo9jR(`!Y)O^;bF6C@z0(zv0htOl9w4D0r_3Z8)EdTN^y;~ zeGgy^C^5sAoIR22Fvjv}5OwQ8_M4a?nb+*EV)Bv+YHFKr71>WnJDRAh%ov$BQ&kl9 zJq5Q1bod+AOW~Nndnd>Bes{u{9AFGZ{16zo@kWwaXgOm5my{C?v@e?|>^Rl=l+kt$ z+IOBFj)G%9g&p7oj|_-)U+Ukx;O`oMzXs0JMX%H_o*!XFbuKsbYjSs3$YMzQ$7psAcMm-&o90satDB%|T zCW*UK1j@efbmfB(UhcRr8v zhj;J3cxIK25?&b=c6SMcd;_UQ9`}J*?`_`UuDcG@dsTkU!lI<<=Km^7muPzQzc)Y8 zw_YFPSTNXAIUS`U3QT$v%_&ZYreVfIq2Zwy%HFx>;Xit!7`Do26C&Smpx353E zf9Gyx&z^tr?8Rok+yKx`5xDdeg`CCfkMDYbfv&yFocvY+D zmu)lu^t5aMSU9XMU%@&{dh}f60%ITVkT@ z0i>UtIaVoUWpZ`tI=tAMyBizN^=cxkgB3mmD!a8gCi;Ej(S(rYWR4W=16{OPwH*g; zR1_=*EaPcZb`-8!H2`?~mc!ig+6HfJ`1BF;9#joW{cf6Vf!3`jp19f#>qcCmVb5Bv z-cpRalpL|m|8Qe9uF7esQ5}lK`E-&@+&TtzaJtr1#ftp`f-*HcHEtPWQ&#SP+Y^SY zwDWF{xGLU4-U?9%#AkIp(GN;kX$`3^6Q#L|SCi7QYmOi?M_$mCWu509+mmJRQvBb1 zUA$h(PUGou?DyA(!b$FAeBVUvGExO5ib|m^NOmr3ed|IvgmX1brS!7wkBaKTq}rsI zC+4(@SjyI?;4}R-1D9I;)8ho1Hv<9F5OV5|L0gm7uBWWXWJ^NkiWj;Av!kXYJi*sF z-fsjrQU`yTc`R4?mEwjNnFz@Ms0mLThQ6pv+rQf~u~*{`BoC?JKK_$GCD@0j#w#p0 zAb^#v$2;WFicIjqOwBrRb7t_$jcxeZn$bm#cgxr65Rq(-#;tJSJC!-Q;&bOcDF++_BFv~YNuhHAH)hOOfDMos)_ zq3Ri63%xz!^MSdxVx_);#!Jt`<29O8l)ZVy+Ydal$JE+45y8|RuzNlE6L8XXp0UEF zed~>5Pai+>h;D4Qrnpz{cw%^uhjU%P3SvS1=@UzV-4=M~b8ShV^fsmYXE%Pp>Y`in z@<~pauGz7R{#`;&^9(uf%wbmAF70&!*mMvL3F3ws896(qGPyy+gX}3=ZvCpXzV4cH zevdCe%B9rK`qVYh>w>kdlBYqe1?v=8<_c3bpOMGYVQXE}=Ck-AIjc>_#o{rh3TP9e za1p5xb!69(1LBnzf(Q|>HA~To!;F&E{R-fU<`@%n-iqR{-Pke;-Hg7 zTHI9xN?lp2LUavNU028Qa9;$9;Gy$HZ*G+~eYfs865L64?Y9avTO+FIN>w;^`^AF; zusk6F0bRRM&Qq8{kD=;!#~l8&<{?}L<3Buv7xHF+>XzVIA0);x(@1^Fl%7`C@aU7O zu9oa*Nrg0Y32MFc@+qwQXVwxF0+(2gZn@Ci`vSp|u%{??Obv{HX{6b=>m^9)vUP{> z2Y3By6Q~l>yAOA7eh#1NK7RiA_T8I*`}@ED``>^4=bN|hTqpS4O+Hw`E2M8eeE#M6 z$b=PHAk}{9>ltNDW3$(pP1($>!earTK*2 zKOh#7-xK1Qb3QRatK`2TXd3H8r5+jv5hh_e2R}F`lhqID$VE$*JZBCZ;nMq=m?j} z&0a$wLN<$$JJ#PPE7?#{TS>oapO37ckZ%G*G$eDhsYgo$zBY}g9Z*LV#n-@i{O7L~a<+_S8DnSv!iYO?v^TaS=$ zpph!NL%^eh#=sFVU#@+r`JY9)+jt6G3@b1!rlo?0(Ii)=b3KQ{dE3<)^@b;8FS~wS zRY0Xa89`Tv@^U;>ogkZvjvkyj=#$zrc?AUuT{=O=@;Sx>!fQg_QX@&}j>UpTfl&^6 zvL;4^%DN1hK%G=5KApGaVOV!Gz6Vo>bveVsMe~>3!k5WZv0Us5+HAHqO8LV)V5XqW zO4C$atxc>8;nO}fsPz$974_dNfuf1uq+snLzt+R~OX%PET+TRI zX}q0sk@c%-NwZN}@&0ce!tlDC%}g`bfMMMh!JrWShi&|-4vCgc;Ke}Q@1hZ2B#Bp z@ULQe;#uPO{AbZ9Cn~BDy%8jO5R@`bA|A+*HduN1l^M?^9By}PYNMi8p`3D(fURJn z_@%s(r`E(6(w#YUnzV^oU8^QKZmZBaKx4(?j78RATV$*Yz0WcxR5ZhtWiN(qjI1yW zGH-;rL@iZ;!`AXJiz%j$Z|RYpGG<;YNqcHR$z#GnK(N^$Jne%91$&S- zJ+rI7pFVq*!j}g>o3@i8w{UQ;``N^0T2mc&&wx&S_X@`^RTvB2+^UGe_*%1~Yh4qV zPum*|=z!=Hm^y5{M|VKCL4zeo?eW!3PsXgY9cX>&0c!lab_b)XC5e z0PyWN)uVL$V}?LtxzkIWs6n;Tv|KsOg65RS4JiC-Ce+EUU92guC)YvnrDgZHiC~t5 zJ<*_*6USM=!!&lKb7BJ=TwTLzKiM0N06V{UD0M~}s0@f#uCi0lz(*$1m5kIO3NjcO zgaz;Dpop;;?r&ivy}&rN=A>zhWWlZEkqgoISfNVCijAf~Rm(_nLB@QQzfTQg%CW;m zz0zunqWPQ>2TOPkHVPizXoVZV5HL?fbGo>0K2Jy5D4<8qcp}g^FwUjvctj%~YU8QB z8#J}7|D6&;33dL0#bRfq&Nhx>up~?Y<_k57M8uSE=|5wo*oJeXqRZbUvw9#BDsW`8 zl0GUJsQW-4Fry2b@}-sAG4PAo+n_n8f95ozyuNONH>3J8CZt1o2)kTX?7obT)JOzJZB|^<&cPcE7!oLkMK*f>^Sw-#m1m4m6ef^PhAm!5GPc$x;sp~m(P~d4HhloaS0M~4Zh4;uTV#kZ zD;aYDcv&o`@K6vEb|W_g8(ebvMulYOMdR4R`_slcY{P{3ecB$gZ= zXN#V|QgooY=S9Oj9Gj)c3MgB|L>VC|p{fAph(tU=&Eqw4)-r&O&0%#Cw#7SWpqMuv zFQGwR-=NNnFvzhM(7_b8d>H9ec&1jM#2_Mmt z{n8IC10=y1fd4X9|H-pDBo#B;0gLDOa_(rD%Ep61dPX@|6u#kK_EHaY=1Mn}p{-`W zoyvA=P2Ca_M*Rez5zE9aM0GEl|~QQOFd z)4^dTMX#>e7s-^1lO`6r)vQ!Ox&~EhS%Zh_s>ky0^aFbEDEXW@boGUlTqMb@A#oWIpEji>OU>m&?t>P$QGN%;S!M0eyU4ZU3 z?(v;H0Z_rs94-oH_nKzSEe7Y7x#k}2W47=9_iBPyvOjGXj_vCPr$E`}2b=VN!gpj21I% zP+2i_IY6OHz92?eIvz8m%2FT;6g|IM6wOMfM@`W&b)44)mt3Hl-)$PP1}X)g5dnp7 zl0N>U1|TQ4E5i5h3_l&XyCwkNlrhYiCG(IqLP{s*4zEfRR`9W3&Q51tl;J?4q+qgg zBttT=3uEO%eN;F!$_1=M4NxAl{Da6jV~&%Xj(evQJBLD{Zz0D)9#vADKva0^ppGy` zq}QpfJ$1OB0u{WWk{WX!3AjSF7&0sef0E0#+^imhs@)607VcEThI_WEbAc_0o(b>m zR17x%(Z$I``CjTmW2zr><0meq$8(qx13wKyJ%lHL3ZAXENM9=-s%Bpzhp*69@f1fn zJo!e^xQVx+TVgH%gX-OtSUp@QDpFrrK&c|n?L+*i)WCnQ^>2<q*IswrU1<8i<{xyn~A@E{q4=GfBf^+Z&r4{_a1>}j%biL zKR(~KV8{4Oa5lU9^~YQ5?w>t<_*nIJM^JO2^&s5~Y-y1Z#@+jm=6aGNVdBC@>U1Nj zv^J#qbP6MELhBd8uLnt3UGW}R)XY>oqR!Rodi3$C0w#U=@};FnE&{mr2u8>D)!SE| zJpac({n;~5ncgZO`gF$vqFxECRH4Np_9i8)RZn|!gPdV%;CI$dW{Y|r0wUezh;^7) zpVM`3L@5rnCyX~87vyS;j@g%n&BJZNcI{0nc`o;#|7RFdUArdlZibCHLAe>6X4>aA zI&Ktf1huV#X`_mbpEwF~6@hcq+EGijv#-a5wwd222Tf0fsc zoNM^&^HeQM0J*k`2Sw!u-8@4H=H%$vwR%@* zVYvB3xZC({EmoMhX+w(wS~@IeB7u*KyPo@`kh2wx7O1xFvx$E;QuOjo=ZBkPj%gTB z+MBpj8cak_uxX$my|ey)-{>7PBnY9SQzjq+BS`V!I7LSl_ic{6mc+O zq1h}rL@SvtQbt5?5dIf-mL^J6p+hqUhNLp-GY5#FeohbCFb>MDRJn0f+8l6b&-wI| zVf~<5fg=HEyd?q@@e9vb9OZDW0_QW?+SY*Yv^_J8aua=s2-u8P$LS^0hB^Zs=A0~H zye>*FZ;q36WhxpNKKvc^u2?Q7Rf8E@OB!iAUt(#w%0@SZWxA9SFp(AkpCaX7OcGlda>26bCA_7PlRbW6$?q2ciMol`J5z&Nb(y|a zVQOW?OquY6adnTS^~~3au->w17j&k8s%sRJHzhf3WafvnWXwYq)w3sue*Yt_;O4{N zqziV`?}>;D0JM|F50YN`h=kbPMA7Fj7BHg()R2p5+=6`f;r-`NA6qxXAV$hh~2IO z9A{4~P4D#4QDWteB?OupkJO~`K{pvIFj)z=2y@_%IKh)_wK)6BYi8zvDBO99xekU& zvwWeALYf0HC;euPDqBlv_2EDmC1F1)QxKsC*>e(?V>Z@EFQ>Yl&egP)2hd1d z)_GswXEd-TM(Oywc~T!JgyA&B&LB|C-03kP{gbbnD~H?G+kxdqhtz8aWVbm zKVB*9CBnsldynqj?+s}I7qU~TNsJ$+1k6V$(b z{NN^K1A}MJo__E$st+G6O`+bWPoG+;LIIX3rn`nHcRs3&4VbDRM|oB3J;hZ==V*|y zbCz8OAQ?`u(uR+0gWCbcQI(>W~|`ZhhZ$Hp$waQdpA#XLPFRgI}P-{UyFs=doMy`>Rd zO;?V^u=d5h+m=8<=3P;GLMMu1F5(I!Cepu zHa?<$3WrId1YE}>K0};U(?r--e*eNnW?G5t(5FG`Ap&K_A&e zFjLZJDf^}7p^au995%Pd>6cp@(3l^z$L)JCpP3-Ne`1resLjd(u(kO2TVe@kS$p^o zoaW;}W&T}HcyRZg*{{znrWGm+XVF5iF~^nwH5FNdl~(8AP(rW)!;Kz9gGWPWw<9kn_Ig2?6bYd$t`%n;hbDi5vDSkpq=1931$`32Y|zk#jpGu@i}bv$T^59-vfYVB zOUF$V9#YalGq6DYCR6>nyfERVbPVaD%#roK(hS9rAaxy7yDON&hM|fPO;n*t6CppB z^q`;cvOziK9+N+&V=y%(uQMzx3FO*;^tc`mOpv?0UxBtfsQ~~q=S~^(&B!So%D(w{unmBCdRYlc3+bNz&`XPnm@>1| zMem|LKj zcKC`bir(Y#K;FbH;fP;x~AUli4?TxZs() zlDR7>UJ|WIsz%dp99bK*l|PYPgQ&~quCLw@93J8~DigPL;+-W%S{NVYpX5{aB z;roY=ANf(WuKNn3G=A#4yC|fD;lT&DUwA6$33|oE7+!Nlx8W@cWXQBx1!>oKt9}o3 zcRf$0Ri>WAdbihP%VC!zsbX0uJrPY>I$d4!2UMt?M&R5?s4o8OHQ$L+n-G0ES)JhK zxGh4M34I2X(QMC-IvKBu9vk*&OSf|hAtT;HH7+^;ll7Cef0d3(A}7EmOWZ{oeE9Dh;-e3x$Rq`u znfRs_O*MJ;BNMx-GTSH;pSp3{S(lG7#x=_W*xS0Z_m z?(C2nbttDRsfi>p!SGRTN7d~3s@$-IEl9i0JKo7vY&aw{K(S+4iFw-P(E6ok2-t>- z@{)#CzT#wOw!_?*P+B;lg{o>tTB`mN)vtO5!E`gzK~#WHV#eRKlh6&$O>&xKCKmEF7`bnFA)4spm(Vjss;eQ^{KbK_Jxy4aS&lNr1=DMFM^bb` zM_!fBm$+B&Cn{s5q-ar^75etwL%+4?^&viaKKldbp_SKlyjno@fIRr2NX4HN)I*5D#q>j&{09HH6B>ahaz_gs1BcRzFM+_NF_>%gsUoHq4bF#*^w>> zRe*MaVy>ecj+4-dkDLIw=P+DY(nRU#c97pzAQEwaJo}G_>NUW!h9EpUD;eKWM&-B! z4m{yWThKU&+|`X9V3!^>9-$6D;MyNyr~YJ~20YH^R#RZDBn~8v)g+9Z0bEze6ME#nKd+buP&xO8-FKx9p|*}T(!>Ls}P3~ z%pAS?yJ%+9k#VIRGkIZ!rH=K1AL`>HJQt(R)t3g8Q|S0!_f7R#EhToSQHpBNY`S>I zgvd0fYH5Dm`Dz}ziGIT=4HU!n@*#-Gr3+hEpRtYuggAs`33G)hQ!AjeX71mb9eq2G z2D;x+$OF2})7D)qhN#FU%-_EM@ZKZ+jqz^X+x$@rU$iB38;~)^&)XIS!K!6J)R@zX zJ{J@W79QL;c+Mz=WLF^Xy?ONO-@%xAHy>}p-Y5dwWb{s9$TlMIupk!mPJ%BV-oJ0r zj_HXnpG|>bCl@x^a;;WYhRGe~}kStttE;a|#2;pJ<|84+~H2`-9yTjMp zqiaWu!F2Gs#QRV0^2xg`sWQGotoyhM`)1WX^8`MqlnVuV6~>UgGveb1LNP&=$}s)5 zm)P9zGO>U{*0ghizx)OHbd{0~Ozed}E^K9~E7D(kply|(+#oVkZS$a3x)~dFJg=|w z+)Pr)xmh>m_c|K`RP}B_NU@3*#MjMQET!HyN?J1z&j%`M1uE)ar$A~v*s8`(GKD>X z)bdLt1_71D#BrRR!<9nFSt1M~ugEW)N-KF8Xca{CX|mx`GqTDLz&$9ezFQ##Kcnze z923J8mHHWoX86M$RbXus!ThNMDbr_)i{uL7!2mzG5(R>@!JpY;2=Z|eH+m1fHLvki zhKU?+zNAc`HC-5PD_vUT^5Q;)316BEx%!th6{;%M%)}XsTqrMw#wf6att%BHg}$SK zXvnB@I4nD@WTY-c13qbGhYM0p=tQAnMEfx%ZTt*xHur#NJx^L-9tjv^OA#l zdQgHZz$xF4mP2z17khoyILnU8f@nUUVoIU+6~$2VSy(v+r>QXIZ5=VLvYABJzbd}N z5U?UFTUxDBd2XAQW6yo0a0rGzyYFo5NYx6>y4hh%Uu)VBh}-nx-!|)~6D+0ZI8#i$ z9$(UzIpCw`BRLBuT{|kLk@Vj@R;^z-wh+etI;;*4FzCDmRMAAeSMFG@f@BUZe&$U9 zX31>OKfbe7AhycQHe;g#%W4deO^;QirC@=lgw%B`pRh*?#Lh>csotx^XkbK{O6WkQ zsR}Im#GS+O5e?^EbO2Qv=wf>tj*FXF{PWk(n#fnLUuoem>8n>!IG>S{GN_e_&isH7 zBU9>BlbTfqR{ymJP$85n(0xc0Mo9cH|E3S$D`qkxON-(n*l9;S% z@I!dCTv9ON=ypJi8Wmjp3Yi+Mee->&la`e-mhA{|6rU`Ir%PvR!v-ea4$TAST_xJ*+s z@mL)w2HYsRVwD|+BEmVE6DB>RQj{H7as(o6`j;57&W9wR1nuue=0Hx7W$?#2@^QzI zsc11Jm+Pi%NqmJikIr_gisj_0%I;tV3V7_Hm5=XFx$sR;J(jCAkRUaNM--)?0_@Vk zicBz}GS$CfyVJK#|C zVHh;zw&I~~1R805+Q>Z?np3(a_}l||F54{TCCxUnLww{)j}R#_QYk5+A{%~eDyIK& zbVDg<9=_xTopcj01=bH+MAJXK{dpKtKz4@B6rCz#onN;LW81&VH4L($g?iygZzv`d z^|hlHHM_T0A*gRnP&5BV?{(wntNLn+>*Bxv^yy7Y|1IBc`ipFer}=Rf~R*m?5ZQ^I;3EGzb~ zqX!Q=?WsgY1TI0iI??6CFM9U(y;0Vx0s|ETfE4A{Wb61nu*g*f@_4(M1wS4@N&aWg zUKlpLegBp!JSP;B?>@Z3ZSvpx>A|3e4c-O-i|F(5<0srv8`4;E4P}0s@_Nxix?Zc7 za-q)6HxBFVaM5xZfYQ*gvWrQMrG_Jm zrQ;@8x$HbD1&A=KX^PWs0jVRzTi~NRmVf6$^yjx|udqtiy0qr@-MQ`IKaBty^C{lL zrtaRaqB5{sczaOl$Cmi_Y^b``oaX)ecLrvnnddn%oM-;fp*JgMCYbm^9mMVwHtEB) z-hKG+`t3VdbfP%ts?QH=jK1IhX=N$L)Dmbarl<*raFzeNnTEqZ?;&o@hPKD8Xbnbs z9MEcwI>TS<=%$+fVxvpu+iHTeSzO;zRv3>v?BeKh{Oo}h_{u&U9=zJ|twgkk(D|0SnuyKgf%!L691v!P%5pL`Bx8{d?@GHJ2Gsbe!@Pur< zUViL&rQrmH#Tv6i!#;BON#I6R9Qw%KKrJ*A)6O?p+bWwjZ0#b*+9jv)jt8G4P87XS zMo`tybnBC@5E$AHhZ8q92h$&U&D@;+=>dtJa{m0=9E|+B_ru@?DueM}hXb)u> zNfVw)l-LR}DNBp!s&j^>wnhM9F3?pWoBQ`1#^z<4JF3@(zeGg*kd5Dv7aY}#gbBGe z6FxpBVu6%4mX%+DgjA}n{%DtHI-Ud<)4`v^S-|nJb!l|l3Y5c4D6@}Wf}xqbSJBk9 z-hR^y_4^qSAmsCk9i>3w1e#W?yVQGXl4zB3*(#Rfum6l6a(EIV!ku@_2C>%Js^++e zlakZH%Ee`g5bHGwPbr7h)U+gkGWIKSC9>RyCz&74^# zi<0Q5Y(qTALN8!eFmdAR&>QQH^XGF-q z*O+|y>b1WgjR0EUd+X53_0e&=^A9Z=Q~UH~zbL?T|% z8|~^!F2`I#@g?LDH-Ydxoy7u*ZE3M6Z#ap1iVQp%8v>*lpBnHRY}2)qK}rv=HfG2R zLzqg^($m{W6<%=Eu_;EqaoEfDX_u_UjGZ_=ocT+0HK)_&v?Z&y4r}})>ghYkWB6J@ zgbv=Q#i$BRfK7IIKWo zCsfSU9@=B-|EKCs*zL%XJG~e8%cWQ(Yf*P=8I3(N_I&@3)uV^^Npsn2TpoQICG}vz%+)rv=cIJK`Nr)}z zp0tIA#3J~{x79`fJs0}sz7^sAK0W;Qq34BpZtH>!Vz_U!$fk}xF|60(C2SkBM^4Qr zzy9XhyIXgzxCm9<`Eq5N-;$mN2Dtt3(XPdwUj6d<)vI@Jz3BLdAATa7m#<%9ioG;%$dE(u^s%f#9p~}Y(=p;#^O0~ro{W(p?^#!!4LAP=yrC#(p-;VZw9RfDLLR`jJU^x$x%UADr@_L!dI+CaBRaORRhqpUf$OnKzw zN>6sSMu8pIfrTxVD@~bV`*!Kig!uamij`o58r!Ys!#g82hGd*zD#ttGGQfFc_d4Fb zqlY&}VyXAWEedmw<)uf#U%wXp-_w3Re)7)WT=+(+4f(azc&nf@%l9a zs(Kt~&{l}jqT+4@*If2kj#Bt_<*!blVOPF{!7<2g3iQ<+*8-;yedF4jx8i@ILF?(Fr)D@MAXMi_Ovy2!A|VB>PWd!A z7I_nA!Q@^d_5vK$B_``SEQCy#DGa<4v692@vra&Y&mxXc%0ljOxEC@~uY_DDek)6G zbp~34I+_t>HQN~5l+?Rag_~)|Kl|?N*@@K8DoiPpcU&R{62$^xlTeNU1YOf zG8loxpdCO=7JlQYhmh){f-gPb!@QJaDy1}|DR8?A`D6gD0r~9pA}|pB6@roM=`AJT zaOsnV;vs~3WL{ctiaez%TBXXxkCP7N`kkD2 zUH-be;oT5#RA06A3|Q0jjHm7HVnKOh09(E=9K4Mv_~KGEao^l+RK;st`Af=ST7at& zCT&;)h?emJPO2U%*lYqP&9xl?PYB?U)Tf_H`M{w4qp%p(NujD#Ad8*V5Qn9d9BF{; zyrZoTwbaOsj%YnFXot>J&n@IiSmQ2su6G@>?=0Oytl4_!GcVBei{;V&%spwhu-p-{RRk3e8sAj@J z;8&r!Z+)Cij4BljQ=k8uM@o`$aWPRRr!~TqKr_P9*3KMV4q{dbS(-Qpx^O{auBFW4 zQ?WYLVsF!S%R#o?{FR4R zbJ13PyO*{_+d1zVNRGc57i82=9Q-2oI9|Lf9Zat6_&e0N}_n*JqYexN=eL~G{ z-?DMwb+>0Y2Mc`OTRrmVyYIgJ_CdpdJGZXCdwcEOHS0#+ym?JSguAyOz?({5y?lxO z``(NOA(#Xm01{#!*<=nK(W8e$&m(Jrk@Ig)&aHANI|Tco*oTf;)75_Q-iDV+VTh}y``Z`d zhJ~JhZK*)a47ZeY79D*!~i;uvQ91u(RZ#oDjz@LzHoI~&7w!B zhHW)cO$h0zy7xaJYfuS4z?|w-Ks1<2hDQU{G$~;t)kCqrlm(T_q9Cl`B6ubVbSa}E z?sgLOz%!vSlBf^<5?S66gNsg*+Y;8-QEcXYWH>9!4&~RQo*qbn7}bOf&WrH)#Kz`p zodI`4B;U`28-g2B-)_G}Z9}t&j%g$o;YHA_v4c~o4M8O9U$~2qO%~0u1>%z($6rj< zfRKd=n7t=s=#`!XGKf_${#W#5E#hyupO1j)q;{PCq9}n?qR+I3k_b5ZORSkgPB01r zmdC@&xy532MEcU4Pqd}0R7dz#mIgyReic3DLS2}bbMRFG3dUGm4SL7GD@pd+YY2n^!MhzWe3riw|$! zYL&NpVUR|jN1yD{gVG6Q_|goofROGqhaBsKpn|%hBX`cTAOyuM#5AaHS{T=O<*hkKR^5YboQfnpT zH3TP-`V^EixRL5SnV~ofa}tF>4g;kGhJ2~je|%Jp!9EJk)hLfv5b_uXYxi ze$kjC8c6FO@^~J3Y#GWPo?R`>&+mFl16nO4GmgN_DT&3vk?@7dh`#)Fw(Pj>VlyHo zbslw>?8FUAycI=aT?<+ZixZ!C)!asTrUJ5reXAo1aAaadZb6o)YMKbhT8ZSS6=rSf zX$?h06#jNsO%sA?yfr-z@IC^BC53gWqwFIRVjjCKOG#Y~og4%K+snN=;}Xl~WW;w>R=fRSjttHk-c_cu%| z|Mu+p(`U~gKYIA&u?g9Crk|1LQ?F#({+wXX4cz0EzPa}5<#U~w`yH>|yum-tJf86K zdDHfm3@OfiS<3(5{NxkvI%#}t{LE&`IoG3hpw(7Ww%9- z0l>;Nj|NZb{^m2T4YfR6F?*|XVM4m358t#n3{rMy^8U>m%fH?fHGA>G$B!>xSkdzu z;xAwBiThu8n8Q22%^gkfe}>GQ=z# z5PoAZ_jInDwhxmRurXCXacCVB2mjbcq#*1TMdw1)3i$57ub`sx$qcYfvGGP!)+(i?3d5 zR8vm;v+#b?!t~;>&Knvp0n((?U9{k7%CIi4H`ZX!4uq$N$R1TuPmh*xL8b($IFHT5 zST$v1K3qwG8;{4y<`igLF%+mCUC^YKBq!l_LMG<@i~dP{LZM3%5*95z$Xl4<8dgCAd*W4 z8O<=s4t)YSADY8gw+BxWz(5buN(@=Y#1%|JOKXu19SNv{?H&oodpbaA;3<4-TGmzu zJSr+$MHM>yR8v+ep&Z`rkTJ3+s{;snq6v^J;zxnpL&TIgs#X~KO!|my6jU3<@@v|s zk^=}~KBphW^~Xr>?w*+Xu&Bqsx_|e{V~@D~j0v+^ckf*3S>L>~R3M#fq(C+kR~<^I zQ}jl9279%TNRIYn0;#mXzIj#-RIhJts%1UhW4vC!!Qc0?p2%!o&ze9SMJn&sc>$MNYYrJgXdEH$O&;=Maaw;A2BA4 z-IVe*zw%}J9PoJe_3xzemt#NC=y=MEc9)tIw!PM1-)M(=MpN zw<2lN@U!#09HA2W#UL}wtTB=4TWxw)jN1?>MI{lMUC-vc zh3dvW;UIrlDs03yeYVPfMY10oa*C=Nl@*Rbu4MK?c|Kpvx~^U*(z_A}vmb0If~DdK zxgJ~tl0Irq_)mB(( zGkf{Jerp+()%*=R-N5$i2lqVHx{tebUNBc${Os7RrIUPbnXMo z{&&E4*rK1YK$RaU;&G795c+STFCCw|9SfyF!mRj4RN6Ln^Gebqy-=UjYJK`}cRy<6 z)hO7(qe_E>VgyuOnMC65P&I|RwjD`9#v(T^r589{+m7Ag_K)8F8t3o!O6z?*#e(j< z2x3s9fd`S?-~wI=)ws&)sz=g}P7rW`iDY#j3dZodj&T-@p{;tHR5aB6jj76&ln7(} z!eY3<#$L5fR27z#Hw&fA3aw=&m;P7U**+uA+2naiQ_~a{A9ZzyDKx}?a#ko&LtGVT zN^%EZa#^(<*QZuvnN_{`H%{tWXCeovi;yB*xq}^^SgC1CJI~^i6D+5Ql8Fd&j_zi3D-EauhO!m&uN9=&604@iuOn{X$Xyl7P25zxVH(po@Q&Kk zh7f5i?A7iEiik<7{uK=6S3Rr}*m?0{%n28V$kxbj+_Zf5$&*J9?5ug`ivKhFsTz6R zDs`J@GKY|t8}W&2!9%^p-ZaDsPNE5RmN1vVq4u1R(c=ww^S|6~5~f{$Iavn}9rCB* zeBx;4a1$_x16_c-1a3p1kQBz6yOUxC8-RMwuQwGNQCz?N!AzfABo`LEGi}JVU%O?a zmFXgzO6pM!SB60-aaK;9SokM~Tw?MXA6kgiCNW8I3e3i8`jhq%aVSpSSGy>s3io>s zLz-4*3dg9*sRO6mK83hp2si0jXb3Gy zSZ(i`f4-wCa-%NQ8mEA<@9NXvo*69|Tcq9!)}H9yAF$i@q%pnt;l={OdG8qm zy0eOlnba!Ke(oV5uQ^B^kS$XM4-6DsxdhvH*16_ZvW$a8IULhSkmK@Xw)oj-+LPn+ z)y92TZE1RX5DSTda%JnPbJ81rC8K%M$tuazB@=Gd`&q@vD&|v7lJ6qvyC{PKN=I*n zqSWy@lOEQ^>7fkQ0#ww&RIQ?P`g8QMK!$! z79YuRQGcF5g+&4*W!0;)5n;9>+L=37emkHa0dOFP<~`JF0^sx40l?T|tVN6}yzcFz zECBV*PHyd59i!2f@@qNcC;B9m)PJPT>LtLDhd{~+zm^|#tocnyqn!FT!VOi-P$QnZ zI1r;Jw8?GwcNTd6q?d0R6>3|l{h+;}3^Ig9g+WZDcq6p?WS5syKNoulx2lhpDw1P} z(kPFrO9i1z1W^~b_I#j5(wK__8JSRK(9eKHqT$(RSQ`GO=DtazZ20{s(+7(h4cF65 z?fcKYhofc*fIHT(tgEqy8qaP7Y<|V9fOw8=DDboD{%1o$$%r{*08oj#RNKjzjGZGa zO#Du6VnE{p;gzZB3|R;&GyVq_c^3wV-D&I}k%tB7q{@s#fY1VkA(@;?4&iYR2Q7e0 zsm@kvzepMVG@h1R9GY>^okgDn(WrLK!&wcrBY(Z$AP&k@fR+Sz=wMx;^>^A<0MWK^ zF02t=CBgSo^^#*Xw(D9ic?luNiuafLxLzL_M^@Z7^_pt=@yX-v;NH0L+_cP#7m*pg zm(fC3=vZU<+6AyOey>3w#i6O5>X=`{18?>fKIPDJr`)g?F}h(}m*;xE20;^dWY+Zg zCw0un1cY{c`Ol(~k9UfBsPL9Yu&#UZ&XiqAcW-MIytda1si6h1y59B$M+#Yu|AGk? z4NqZb*eF63Dk-=swDFUmw67eWHr@sj!TIYyNr$bn7Yrk;LSF&O8%v!O#e`r`WPR`_ zp;a8n#1aXJDKVAkn#qx5@r(A#&%`>}H`EbukUN1`;Vzd0M;Cc;tYi%XK6a3e9HPP- zAm|G3lZ`BQz}X8mU5Gf^5Awjz3678`NVY3{DZip2>u+@)G->ftTeT*5vYthchj1MF zLS0p-=UA=bn~e=du!MGBG_DOE;&DhI$G8iWF60A>%6Y!d656PP1|bxX*qr+jdI>JY zvqHk+LduY30066O#&+y_k%5`pls6?-AGCcA0UK|!sq+2fE-@Ceg=j!&&*H-N}wUJGRB$<;zFDJyMmN7Be=@WU= zg$P>PEp%o;;`|TR=Jb|@Y5sRp;v$%E3k3U z_uqeycbnn3&v5_V72e%I71SP9l3nH&Q^*iA(o|m2dU&%K^FCO@R%Qe=<01Cqbw%u& ziTBChrfBY=n#n%~LQUCK;%ZRk_C+1&&0dI}J=;WwFzu2FUGH4TFtaGfWH0fI^Q(2H zqiR86?}4%Xa?w94xB91rnceQ_c@euZ0BVHS{7bLzb#xA!7ylxj5fm$|tT!sWc=^m5 z6%<#94d@6A8)o>Lz%}XarKo|m>vyd7WRCbYrfzUesoUsNJN?v}Pwq0#k`6o&0!YMN zpd8x$qGAoBuDDTU4nKiqOToE3lGq#;j0z-{259~h!hA)>sjg&e*ib~&o%9U?JnGRd ztEy)1;3reW6a*wVXu?<%31|Hnm>T)SkEj&^;Uq5|q>JBADhzf%%5^$?R9D$bQFg9C z3*%LYp{VSHVLje3PZL2K1XU}LEXn{FzsPZs9vA#!6eA-#2{`>aNU7ShD2VdaHt!}R zh=gbnO>8<@cduBYPauho$gu!jmt~eq#;k)bjo4`hO}VhOY~*dBngLZ_?xVFpc`qsq z>LRiu=mMbx99yNtEdCt)`5zh8h!NS-LE?<~(_R{C?KCupCnaZbYJ(~c&zbxQWcXl{2CYjo>g_hGnTh;+4M#RMp%#Aa zdEesxp8BmrN?OCM!U{I#fQzp)B^Nf&G9^Jaio3aCZpOZ?$R*DP&Y+O(90pMUKN< zb`TYsP-ak}t5mgR`$uKS7_LwczuaanRF;XOMe3QytqmAqw{kA%YK2e*IsL>R8`aPu z5jcr#7@S2E5KD0sv~d5x4hX;IpIGq2hu?~=qvZ4Fd-v`N2Hx7=%`#}zm?g;ddX!?ZnWyXq~t>1m9`ou)b^|$45I~v&+O9kC;E=lF(EFlGjf`AKE_M2Plw8#qm)4 zO~(rui;Jxo;r?XLo@P+u#1`0YCn}-DvgP_ z9Ig)PoSGtGB9qM$R!H8IjZQSGtB4XH9({ukZc5%P_xdLjQ5;}TyJX?Ll4@P;VgNdW z&^J?>Ib}=Xf+$Q*qObW`CMC|-5U9jy-52p`HtRdrTvvde=C)!+VQJNS#HCE|t^E*w z6wep=H&R(;sv30sOfwkHkjz9wqQMA`^iB;HY^~kMybV!U#ogP*<&VC7bY|#pcqcqBYiMlu2Ua0Z!zj5bnm~hu=Q+KBuQP zfNzg~t(jy@3i#afM6FHwJx+`2Fi=QshGd<8zZByR{t?4$F&Awm|dmb5%^&WljnfcE$gTr~!=V-TRirwHnN6Sy+I0Q$t zV$#OYL_}r)Y)au_tBHC(j4qJnFLA|tK`+ST1d^uK<#O7Z#oNjvr2y6^HnSdXeFwc& z-ggO?4pGaa%d~m25Plj*Rh5>;`dQ!~TdNt&DN_#pggIC4qZLRg{621?`tu@%{zd=X zLTg{iw-B%tncW4cqL8y3LqEu{kmN8!7M3qb7^yij;2?4ot1fq~cr+bMF+o(D>|RnH z;!rm+qN%|W52a(}5~yKM8w_KTYL>6qo|aj8@5ErnNvVOr0MM_urXBvl7_P=I*9T2? zB9Sy%`ss5da_FPOQpSdZYNU5{q^~fkgka(wQmB&t6k0BK2P2-s)Mw&>;~Kv!FOc2as}GSjYMbuuhbpH(Q%DMsti z%T#^vmr{UZjn@cMUWIFbK0VLN_`ywgc2SsoSppmH&ZCJM!?itq1B6P=E|}JB-v)EA z9{k;WN{mNTbWJWDpsU`}2rHG0@u)60a|4&JQAK}BS{HcHCPX$gZUxAZ(epV2k%J~D(Z4-`;8zrs?UYu(r*HtrY{)MXWY*@M(+_xxjqS^q!6qu98>WC1HT~a>;j4tL|81AH_ofi zPZPJ3a3Vmu;|d=eS)Njo2ei5ldTL7Ny1UTUz80d9PQsX(4)Bz7%tTLJr+Mu%imEgl z#|elL;6O+uqy3d|poSRiD9maPtzkX&Njl(1oMAo0=k8#s7GD@b5hg9V({ngP#N+Tq zS;YcKLqJnUMw+%R>Bl<^bp`wr>DP)}9rDq4Vh49%n4!Q*;A-d0kCykstlZ6}*=l9# zRN203MrzbV%V2#M+IKY$I&%_9lllc9ijO>tn^V%!Nz8#gop9B4#amogKZ#d^1nf63 zAv~c{Hc!EvMPwzu31mv{TsoQ&%^EZdJ6+zrb#?FR{=*0NzJK)O$0tv|fBfV-7lpK< z2!WI$A~~bLpw~>ly!8R5mUO@BHk|)#9-==-NP%s~

liz0%dCt!XD? zYn5Yh$ic3z@(0o3^6YZwl!#JEFd{Tv`3{stJ}GH zuijX_(bi$)J19qOdfIyVsQIlYx~VF4sY@gCnqr5dP==65i0_Hp zD)9x73V@xlEQaY|_l0VhLos7>+(4Oj)&6t~2^+7a=<1Ivb?MEUp3uqj8XA&K2+)Vm zSs=n%9Z&b5#`XauZgq)&dHmv0*H$@$ZcU8Nn}e}7HUCNz*G6TH-ED1MvRo^j{Qo^g zga+F@8&Mp4%mGQ7>29>$7~V` zs&&Q=ZWs>DCS5ka>&>9b03Vqr`QtYVOsR?XykgKiDRzmyFK9FHl`qa^ln3KUFxu*gI%RiW$I<0@TOa$giPmOYWsRP zPRQur>{$3D)DdcNUZhvG2^D4OpD7r}E)pn{X+Au4VwPNI_eJQ%&6jUuFYf1zJ;)bm zGybE`xQvGtf#Iy64Y=of5J9`yoUGR#Y?4#8c+^vI`<0p}md9q$_nX?Qapc*gt@}$p zU02Uwzo-58ee@aZqp4udoeyAhIFp?x?pJQQpVmI;o^82`^vEA zHcL-0wQB$P&5Xf}<*2|ooEwI{nFN#(P6qf76uRJE+phAt?kq9zZc5t?w=h}1Ww`5I z`u^H>H1#MscU~6sL|sg-l~<0Nq{*5NRa{?ho74MPZC9g@6fur;I{qN}sMsQ&RgHFl z(w4QcnOSOWw~{Hr2Us?CRy;SubPoz6XieZL=|Y4dsMfjL6h5KUj*G?~E+oyzQRT_( zH>2C%$lIL~RS(sC^i)8q(MS@V?ck?>P^E^8>nNl@MzsKsh-+J9iwArK=ThguT8ezr z%5CzeY(Gicw&7auPL*EiCnwHcx_s}RmRyM};vT+E4j~T#5G8!pi_Z2K6j8^~>+d)2vIb#YL%z@uoDSH7q9&f#83am6s}Q9p4r=C7I5%M}z?V1x60 z0ZBcs0Lpzhiw5F#xFE0|*Ycn|fCyT2Z%bn`sP9rUIuOzEr=6jxfd(by?ey=qgoiAG;t0R8iGSHal3 z(Tux^LZ3Zuy=;v41-hYFVTdpby*tvh2+^+Z8jUxLahkgWycsitw^|L zL8Hxrv>W*l6}`wVUluo@u6QNXKInEVvG1?Am@x~p$AX_zuUoO6(41<*6B*}={U|LHbv>1N1M4XWQz#ZouBZi$jewdC@}SW=ME%n|TY^BrPyM&m z8yQAAqa`2Sn_Hd_5ZG6O10W)I%oMbJ4mJRa0mdl0OiMkuohGVOne{$Qzv@-Tlzkp& zOWGTItNANCpGus)cZBa4I!fP>_L8|Je;a#}q;n&Q(1@VF#S>p4Nw)x`5%YPTq?Z(f zuZ@nQLMMyn0{e+uJ5eu+3GS%LjSxK_S62`3dWnlzso+`hhqn@XzWPV8;S)78siGe! z&h~B^{JM*H<|#decyeotV69W6k+ir%K{B(DbXubO8kph^hSg1jWvx2CQyRr7Xp!ad z$dY{M(+gqio`K?y#(OXLjaV2}PNWJYS1&U`PrJ>wqX;9RAXXk-%v*I!LcbB2lJXhJ$S9Lo>X2H`1yFa2B@5&vC ztl#R`IzUxQYq#elWww97?6#%Y&a9ZN1_=JLn(6maobH#TzOlS`@dN{p_@C!zp!#PZ zV7#2-{Jl#Rc)dy&`B#Z@$j#Vqzw*8Bm^vfw&&Xy~<{<6NEGoYQ*w%v4zqtWgfnbSp zmG`*P^~>}Ck89?sqyTu|& z#sikXb(zjTwIYo9ut>44i>vUZ&njs>mrU!UXkJ+Iv`$?7E{i}gOaF~fL9#v+**Z+U zLHd$iI61p&laW_yYdbYH)iz@KK>kW;aZGN_89oGp&j`XzCBfR?bz6|NUahnn>z2@D z#X{5CwSeXNg;@sG&$m|?D&@64(UzhwVCbnQnjI;il7iOBk>67Uj^gMAqU3od?X z$e;QVJom0%Jvj11ZRA4GhNpCW>dl_M@&qC8qVoX`vO>7UM118StoA@rw}k{w4ZIo) z_k$x}jm2jQW_=hn_F=y?+%8k-P43AH>(fl`YY%6Uk_HlD zy69DS;t=kSo@^Af?(_xPevLON`-P-)3ZUF~cvtG#@7IBj9w%RXHq*D0|cC5#|zBM+|UbZmL+?web?pMfR9eJ_r zf9{@iJ78F(x&wMA<9AaSdMLPF2>O20Bqack2?eEcJUG~Kh0~tle@tTzfddWq8*p08 z+B+4C_%lKmJw!PP4ZWkX1E3_*BM?0BXfT2*QV^m;8q0~xT?xj-6}XaFAdmay9>a5Uv&;IUyU|9c8n;K^m}_MKoHz%bCe=E2H$vaPrZMWH!D z7)g5`u2M;6Y3+@imCwh^gdC10x5eH%=`Vh4$>%rj3V6rzOQ4KW+V~qm(WbY}*C-a# za;X}SGnw%+Ibn5rWsW0MZvJMbU7*-WvCHAa7=PQ%MExM4i)lv>Hi@I|S`txIB&dVM zP&M4Nhh)>ALq8Z_fT#Qu>@8(*Sn2kFNOOVie-BQ);BlLXZn)E zIU#Aa`U^RX1`H6;_NBApDLNs3xR%O+KvB-4)>zLxEuc;~< zPt32SdLQN*gSA1>)rWc)sopLv|mc3nbb(Zq3IAv(7>SZNIt0Gu5Gw$;kz4MeC= zE_g+cP{bD&jbEQt(K{{n?x%c8tan)W9*r6@ynP9)(Ai>-KOviV4mStXfsaC3XF%1z z08D3Gi33-stw^}1_Z?b>w-mdia+rm#Gaq!wJLt$`;6`WrU%X2yOIxbSrhW*UjM3J~ zQIyZi$ZQz=0HOIbS%k+xkX(1jVA|LLUz&Zt!ZX5baO0s~f&9gw-g_dqG5R%e>1 za}s-#maagTc2jXvp%rzq61CYD{JNQ$h(3 zD_Kby7KE>(gr&=OfT0rvwEC~$S{wkxI&lg-z-H3;-v}fQR0A+6OKCHv zlbHXs+~&Frdg25CH2G@1HJwdwrrrQlrH^W`-UZ|rZYrqtfvPw~gg%BA3^9U;N?~B? zpwPk$*PGQDe;FB!+TM_$8XcDCKkTmVmy&m|%2GURxma9X>ld93-14E0T;I)RJ{;#N?NDFKEl(6gH0XSC?_`HkgTw6Zoq%Q|f#@!yj|7VKM=cz^$1Z zIA&^txMApsLx9J(1VMfFAf@9dVt(ATqb`iUr+2^-*V+}IZ8Q^X-&^_H7;5<3QzCXTl7->ar;o+O3<{qR z2eUTH7O?O+W4c*1d5un(3ZTEbU(EOm>{enkp7fXRclv3}Si>54Y`@yU3ttYV9R}A7 zjv}Uz6KYz&ze|jpmeYc?4iO!z4G{j6zi=nd+Lhn!H1A_MLuUw#jBq61)mN(q|?FGp;s(>Oe7R=@w+h*A=~@)mCT3 zt)rcjIX(_EN51w}uYP;~_7ZuCj_e29qZ`1tMC-sDJ!k<(N15xOpuT5FHjtq-@q5Xu z00XV(ymNbOnLq)Q^&l-<%N^n zgx_7?f@8d~J3!|KPcku`oL{m`tm8f$*iJ;Zp$-S8S^A|?ZjOb>Is^m-5-u{OT1zT) z$QAd>h8@Bj|9_f$�!_s7;t6qO^$g8Uz#-Y0{-cMLtrZX85rD5t;zn^}j!IZ2=PJj9u1P zd?$0y+C3@Nq|B=7h~lfm1q01@06I_x#iI}!$V5Z#p(brgvdASG$SUK-!SS;gD!;Zc zuy$-A?El7EM=db^P?kP{o zChc>qKJ2F}pEUO=S{^lV{0U}kSz})8;6h{x+D^_A~(27aTDcvYZjL#RPp(kW-b!$WE!^^4OS?1m6 z%Rb+vA3Muo)@2+qOXXy8154aDJkX*EJI`L;d**E@wZOk%l?@4KKdd=!V-(pga&-BZ z$Fn0pYdmd(@n6BVlnC>>HxOrOt3&VHug1p#Jsc40qdPpBoHDP2oF`{Gbb?>tdEo&$=T3lBkXA!#q=Q;BrD ze}-)H702tXw`m7a{@Hw`+*;Ou8KGN^TaQ z$51VDJJGP2GIR7TY*yo7VQlZlB2oJx2OT z;3ns9l`;S}9n4*XRS! zuNvIjXERg>d^AX|Z4~v;haKSks6+13veZgDUkxR!=$mFQ>j~j`{de*viQ)Hkyg!e9 zVttl7E1RjN%qB{}7PBYqgMgJIe2Jt_zOm}lU#JlQv@eJBs!jR(y&by>sf~mrlK-nf z^ew3M2t=CpjRhkLi1>-EWI%!IHGPJLt8@>Vj5NO%qhxcX=1#f1nc)4KiB(5VjiZjU zOa9s50_}V7Xrlh+J`Lz9s7qxM*}>GIi7X2hG7gP0{N=o$iQKr=S|@|cTi1K#EW*X7 zE~lORdW7ZR9lHBo`{ZfUSwMay*L8*fn!%(}s3v%%_PwWg?J&M0_mBs*H`H&v$$Dfb ziR#c+^h6fY$;avRn{vg!XJO%5vm7Ua-)jey$(*fcMq2 z&Xab8r_A8o{)ear^phD8h62^RJ+Tba*C`DY!Je)wRn2(WKhqNas>o)oCSO)sA9F`U z_lRW?NW~fYrgV^ZyqTvy0Bd;`+8{}{A)_FM^siYdJ0bwlqQyDsZ}azJbA;=OPI}7- z`ptSTf{}LS>z3QbH#MHK)vjG=VKYjR|Rv?2@T+?I9Jc4}zDcoc?5|#zlNm*RN!&(S%tq=}HdV=Cq z(Ja4iHC#bvj#S~idgF2~MDbd$h~DApc$(S~pU^CH_3TF$o0(^I`aUUI>S7$ z=RryA<6R8vOVYQae68CGn=TNatOQR zJHANYq7h=4Mpi_FUb<1q)h62nU~Ew+z*EJ{VF) zQA2%`g1yZ_Ns;fKDgHL!Fr^5VZliqbGLQ2DU9|d!b!VNPi87|Wj||(Sh%P7xeZPZ; zKSmszUbL@HWJ*N0+c59W5nU5)`uAn(-qNq_?9t24`>3D*6=7)%8TckCcdVBl)XzAP zzn9r=$OeZOhN_u`Kd6&3{I>7Z+E&7$C zw0i|0l}cxR-6BXI8~xKnyHfPV)=RwyQZy@dD6c4xi6tlAvu$r>_l`*Z-sS&Xlw}XJ zCgS3meZV3%oD9KE64~Q#iM_DeWI##b!hT2c+OHcAZ=gjmmK}#IJXJIre>PP&r>O9} zPruOVz$3*s^68{0*tf7mH-iy8r~`e-0o#MF+n+7{q13b(#)3}XM&r%Kr$)t2<{s~# zxW%Z)|Md6u{V>EfeIZoAlf-T_JHwFR+=V1O0FksgCquIfN|$lJ{8Np4 zzXnEZ?j5yDeL%*#49z0H9Bam-F~tr|L$!Ln9HbgCoG zOhG|S8b*sVr5lzKxiH*8?Oa}eO__wYTY^lxMvaicipL2q3 zs3q&a1wTk%!5ODs@&#!tU{qtVhZVXq!!i;h3!Ec~#v^5h9vdIxxsR-kJx+Bpye2Ug zrPlYmk8zG27;qv1hoVBpUZBf5x}qt_x|X}703vn-Aa>-&wu`TRjMSRm>S&Rm6^i&{ zsurzM3$i+E~1nNUBFUq(!MU$ zTixM;?X#p0w;bP-d_I0te}pCkS#kbRVH`#?dFx;Rh3x~o7JnjXvVEOc8QNB&MYip{ z&6%Ww9@43jk1_(@NgP^QIRC5uY9qUX+~2}15o#>~$SLFtO{6=)?>uJl0t|zprW-S! zBd5x|<(dgi&GOrsCtYt@)lWVb6Wf>M#=dz8IAJub3GCB0LX3*&G(gw`p5lL5MB%~n z11r!7W&M1%W$ZVlt2U_uy%L!>6eQ|MsZS4lGhN!aTBdgL_{4j*M{^u!0Ce$a?;YwM zbbtx+km}Wh7EySm%gMA2t6IG)#8A8Z{WFVeP>}!UhKunFp%PE;@;?nULL7%C&pb2trc3EJ zU8}9}>$()NZY)tolxxb+84Wxf^8RFr6pIxn+k5Os zXSLxbPCyeZJG5bN-Ku|6G~#xVtdpIizJP4=kBg);PtUsEyQcn0J_zXe7=t?zf+FG2 zz*I^3iDZN%;KGHTdr{pw#P@3)*S83SynTI3CJ3+3*jD}cl$vRC`g6?1h|mu_YSrtd z%p+-g$;7#xrV}7iM1p!P%_|msH`e4@It8_!1KBLr;<{D(xk@;cFWh=_}IL^_t2KjG6z#RltG;5v1b~iWlHajhIu@{ z6xZQk@IP3p+Y_ZG#Zu4dlQf0IqNIha-*4WC9g9`!5*3y|DrD98YR$|gJ&AR7S`SPS z^D&fYPQRlmzKJLvhF*ooFYCoC!}?cU+#cMwGb+IpK0EWv&h{=(EN4FcGM;49WvDjH zJOWo=8?D{DZZ)#`_1@csdi#85u zT_HI-->Da6FE(_xydqqFp)99xV+z<&Swuu)nkxLPF|4k{-RVa4vHZLJhwG>(L6 zjlqrG%$jrP9&F~ z2--EPkRnwcKG$;Yut2T0?x~PRX#t^mi1`Pmdo3;d;5h5r!40s4gFfhs$GmU0vFYg1|^bP;)Qu5|Vp6t;ZaxGzBM1rbBXYAQE$Lv}7$Tc=#wQOZJ?qs(g_8qzAgE zkY@HX9i5K`QW8nQzQ9OZ{S5iXJa*pZySPxeBYk?W6A1Mf%Frz*;ho?FSI)ySS)zBj z##t#Q3waL_lPWMcR}Ar0YB7@w@S8YC+yEr@eI2=oCgY0piREu|y`+AgDeQmWl}<5{ zb~w;~BZd^i$@0fg0?0xD2@1%F>?`ddp7DVe*flNRzvMHUvwjKiEer=A7=GlQ^uF_{ z_W85lc(Ef_&mslD-*yK)`DeNNAYart^4xc(JSGDXGl6c`PTb|}R0)?nG{>(SwLjL# zZ&ASXbfFq!Ndl7;`|T!Ckw_6!8JlFLl(yx<+91W*KGuw4B4TJhxbn9Q`MsPCy~yiU zg#KXf>kbXg|KP|Y6Kn3Wm4|xdZ!L!;S*QZ0*rO3H{(b9>pxgtTg+w|x>;WuzaSx}D zBB5i#;^_)ohqem@uKs7Yc*FWL7-*m0yZN z^@QoEj%u|Y`86=`1X5!ztUd-g+Z;1B`iS91Mv2+auN|m(`@J?i(nQZq?ub)<2vkLBzum?4&*W@Zd&VuLF(uZLYuksV}86I1ix&VN=$JttZoK8D?Z=tPsitdZ?|8^@dOiCwX~rpmJ2Tpa zq}n0Hy`=U2rgqH!QTyq4C5s}~ME=d?Gh%oj=?AeBlU6z07(u8~-c(SnTU>^GRS3+U zo<)vrJw)A~0o9nWzJmuDTo%I`_M$k2f#P&LXiBkzZj<>ykH0W7)l^Z>+xVUiCs$z( zEB+Y@kjPNYEFS^VT!;-ooJb+2i~~b{s7XP$s*+~Wx!K&L#Ph+$okAWscB=Qo6(io_ zl6gj21SANqi@H0Dh^hU`d1FY!$;)imyP^!g&1Kd%+EfC0|MXH)(KxGv{t(OyZp}{hB{6|m4rQ{O$>_a zn-6xHEwHVyFZ19DDqsnD5lwd@R5mojHc*Mj9pUFHb`?JNJTjKhr}AdM>|V6%GPL-} zTZBP%f>Crd)yoYWMxbynWGMV&5hU)Vj`lwm>Qw(0Vft1^B|l6BFWIK?^WVH_%JrV* zBgny!|0y^7@2S}jh-ylNk09G2SZ)_naD%$?p&ZlOIeikkSyMk!dwgUnuj*QbQkvjV zmeb1gz%;+STkN^@b9o4DI1cPu9)XQ1jmeyyxk82#Fxs)^hR4RTLr0ypjbEJAoc#%T z8>`PoJh*e5MYgc(u6=|eJWA2(XGj3zL(sUW4UVbc#U)Y{^>$MaKNVxt7v7ry@@%4F z(zT3p5OcT%gnww)f^l|4a|(o_V6W9fGiHI|k$zhu0)z=k*QqD>blB6p`iZ2p9M$2! zC@aUgy_{7Ir|VV&KEGkWkZY#-y1iAd)i8gbdWYS`qw-It#KKNDvOm2Dd_EAI*#l`C z9B~?$VgqKYy+w5-!n6Xcde}=9)9-pUyv^pC;yvGEH?8A*LiG6YHNE|g$k&hhMBxRn zX&(c-2@QE`cdJNR9IjW6zEJD5m}ByQmTSM)_U54u5lHu9sl=ssRP8p<>T43Yv*R1n zBzGqZ``Tx%rXscCid}Zby*swK);cWyPDSarkA^^;f3z7Q+)Un8sxLz*(LB1-S^DoD zH7n!R0A($IWO}H_tE??=nb35j{SY79XYJEGj50%eP@n=y@!VZMjCtooY<4k2aUkNV zo0}R=%H}c+n=18s4VPv@uu=UcNI!fbdEpJQA`Fo9f`(zdBjnS!bA-4Z`j>1Wy###u z-`*t9S%m&Nb*u$pa`Znh=-2b~TD^6-7E zw0n;E4Sh(TEh}nXWj2L(;SAe!jiFz)c1YE^-_M{Qjr8s8x)rgm(`QE4(kZL3rVE`y zT}{}uFzhBv;fSs-50si)id`vt!g}5&8UdyH<_13kPx%00`Ys$h@8@C}$mto-7Mub` zy#oAkBjBxgu=Elf3JEGuAOq~}QlPm!he9Z+I1db1*mvJ-pmB8mO8%Oei5&a$9J+lm zBP4bD{6Sfb8{R84=)A_&vmAo|>xKM+W(Y+yc~kaEn?O+E;DXC%RbsI`gCM-3C?ZcZ z5yEy0))VyXx0?CHhKyfJzhc*ZYp)-1Q*36JtLWk|*YNm*uJUDwzuUcoiKC}bgQPX6 z!nt5hwQ|AVaRGZfQ`AX2Akbr?vkt z%%PHu1kj>bgf9Gu>SZ~|uq%PxEYr^tsT-LiJHelle}#@L=vKJmgssd>&VOz{X+m}I z{h68B+6IUl*R~k^*>q{3oA3&J)D;96DP~-K@3BLOZ_?wP`c|NskXzktF!f{9mrMB9 zh`MeHcK=>>U4@@vKIktW|J!4xb~E|BfzRNJ2;XalynOk9T7mkTd8SV6?78T(LqWdF ziWa9AEwolO5)F4h6yR)UzA46df4&h>D4FsD0#)1gf3~~-!|cZXv)KUyZ8buX;)sV@ z)WNsxfVuCRGh(#H$%`V=#_v-4v4ZGYOOvP9ZC&m05KL1&RGs*C?PulzdkwxjHf|}K zCSdO2ERiCDoUCTX`!b!e5E8U*7WSTl;>H&uC8C5-q+DWEO4kcRWy5STUXbCDz-ws0 zHI1n~7$mCX(j0Rv$NjGR-B6239h}0gk|&wd%aZ7F&_rBA(NIj@OMA6VfUDPTAV?)v zOIo$RO!;SQuR|5hI>l@My8|fsUZKt(fu{AhmV@?5;=*!EQ3j-FFgmNfthsvzYBkjt zqM1@gV`@Nx>MRC_$~UBKPkhULZ&`Mk+QT*3k@4X0QuhnH9E&^-mi-DTa|7wAR?-mD z0V2JDs9yMG|99AlT>jeJxE9VqrX0q!V`q7~@phR#Cfa;3fN)*^w@B2=g0Aqz-skdc z#VoP9GS+(!QKXU8i3P-@1bL+hlv(Waf4S2X$a|inc0-z_#6uooPm`63%$f?L1{Tbm z&A2!uImCKyaiWH%fYfm@7MR+_vmI#BC{7JUpS1>YK}euehBD>F|aMiYh5%J+Ud zHBkn%wXRMV60UXgq*m6uk83LyoN~R8uyTh{@n1VcLr~a7U1oNP6Bz=a*qfntlbFOZ zj~)64((b`gtEPWU9c*$aE8!mwLrU~Z!x&uj;t%W!sVPCM|TZ|CoH&0UqOS)4bAJgyFW6D?bDy|sp% zjVwv12>2qb4f9it%*oxRzp!vW^jSImuQl@ z#0QGmwX*?Shd&#E1;%PDz5guolTQrrTmLYBOtgG(^YEwojG7Y~UdyJ@7rV|#oO3Vx zD0Y|MSwBWlssoZ$za^DlYD-Pk_*1eq*8d?+2JUdR& z3b*{fl#uqbUF`p;e?UYoRzNMpNvJTbFLols(vZ*dO@}}Y=zFao%YW!(1_G-yf?DMbj|0oIQZrO`tHJ3$<;YYL=3ukKEOo*O!)GwnUA$7C~bNTz$A?G~b znkFsr%AFJw+Lwk^BFJ{KpJRmvG8z2W>B0c27D!|sY$qXO3{MT!>{)hX=>U0O=MH4` za)N}Rj2=)915hW_$hZTDEkoq7k`{F)DGM)GBk9;n_h{?U;pgxrb9I06sFE!ZS@#MvQDsvM$hJP`40}E`XAUr z^{UsU5t1&8kGj0%(6BbeQWB%kUe^$pRI8)_Sr3q1o9YL~XB|eo>y=_1pL36XkSDnb zkw;P7wfHA!F`p)c+pO=QMSIQ1w>KS1-~CKG{`$(L$I3V3z4P4QXe2ogQNHARXQ(W{ z{?W)qAO5?^-6q#gBPhJ)Xx+u!eVOEM#rc4x`vxue)kJAzwq@PJ&o{h3vYbSHZhENG z4gGTNp%#!gXWAizrAsK7@0LyTiq@%b2^cWpdcU-^a!TM3G-0FqDI^xZij zEen{|zzxEU8dIz3-b03xm~S(Xbv^BEWmz}dnQVs`U!|=Rj_XI}Uv1$^SBTNth&|G(B)*jx9Y%O^tf{tvwuMtzzeTXNQ%DIlv^&uQSjC}zE~iT8&q}Y; zqA52}2bf5-*WfuizUuysXl%Q>dJMac&SFm7BU`2_yo`9Hk0h6aErE*sxQ@fEf_WDU zJ(zc5xBDWL!w_ILq?71D6kC|q8DfZ#BoedL$}NsIE+i$!%QZZEoP6<543C+p?hih_ zYs>*@KKOY)<3xRC2b6PSAlO^#-V^5u37y%9mF-KZhf||*kNtCtSRya;d5Lt)@Yf(X zrcC+pvtypT4orGzd3p48ga#GVhgx9bF#85A_IYSQhn)IE(fiOe)0&8U zM&9_=r(@tSd02A7cNWQXJbp3Dsd;ncYRsO65Aqq!{w=E1~{(VfDb(I=#A$m;cqw2n`KZU2j(G5b%SWmtsvEpzPZ zz`msJ_V8qTgTRh;w8j&T!t2haRc4}FY^r{OY9!n<)=i;+ozX7*K}_I{EYNN5u(v4s z)##le<|E{+KDy5uTKYwrOkrb4_TR@#n>MA%1)^RAd>WA2g-D=yQA5`X25Vh2r}135AI%w08)0b zE!LV;a^=;+;{I@V< zS>0SG->KnlbVMc^7kLnw9ArP$K0Y^^ZZdf{K_pn`+ktDwEcO-Z6i|5_$MOT3%I%TU zpqcOOT%NY{RN>hbLzkz4BTa#1Uf3?P%>ApL1CYN&i0$TF{VZNie`eW(cG4Hr3KcC~OIf&oC~af-X^W4U zw^sxYA?_Flq1c%cMnK9i=VhVn$sh7}9O~Hj?k=7FL)AyP3bKw=?{myyS+LMLkrp&1 zp@@I~Dq*|LL1J}+IGY<^6KQn4`5!7i=BH|8+IXErtj!51W8vUY+n>>oz*+#2VNmiT#o1YnG@GzP9IL)qth#j$r zko?Hel(x#9JmY^wtv-0OBKpFxh+}`*s*APl>JhUc>Acgi$Zt)i^mn@|4CR@a8Sh=x zIPX#v8YK-m&z$C2=@Z@g&Iz(X-j~!xW^@%q;^+n|2Br)xw|4vvUk^})5J57GgCJKb zlCe%r*<1I$xfvh9$(J5986n$vJMfi55|B{v{fOk(NO>}Wpywn8jfOaT8T}T`LA~@( zjb+)AD)nh{qaOy{7O{U>{KbfAHb%SV7(xfsB0;riCdS-fxi?hnRam`y`RdB9I6)#M zUp{6Z^|Ym<6=%NY7mt%&1G0@l*M8)-wPbgJASrxvY z;|2SVSGk_KK%!!r;7!3F#)vpKMxLK-udrzDt#orArXMo9ws zVwaj@vehKa+6{RJjopKkwO`Xq>(n2?*0mTLLf;6xv;g$gS$+2))B{!m>XT-A6h{C% ziUX3qZBz^T!e zpjY`X%i_lVrUJSlHK(9-!BB=qFGr@L5{|ie@k_}n+Y)vQm)XbFe^`se)F_$LyJ_&~ z=8NCe5LGMngoDIse|i_^tDQgai=f?t4@i^A$!-r{$*U%no#Kx#T#Y%a!@~E`iZdSU zk_fG6pb+-3soA>xFvE21Q!4MkfrFDZtl#eXX3mL!fH=L-LM8XGZGhPgiD#UKb+Ha(G}~k_S<~$;wVPkY}s^y*HhOyZK4ViQDyl2A}YGs};Aa ztFpXeaeA2k9P7Zf_m9#n>yER(hRtXW7H}tNEI3&^Kv69LBbiKJ<8kFP%!~eUD6+H{ z^7Z&xNA->U8Kd7*FdQvQdi{kzUQ}}PJ^G)^rxs_;vx8mxe>Xqz7R0ei-$ucUraRWf z&@!-K*{#Q5G*BaV=V0DJ6xomA1fZ=z(58N51R%)*rU;^QW=BDb2n(X%20EzLe?zf3 zvNX;o#QWLdK$==@t|5M4ila*Eql{7Pw<`f=^Bc?P%QkVqx%I&I&=^38=2V<#&myHT zB52sR2qtv#!?TuY3(K`npsa~=l1W;=_ft~i&gz-nf`L#l27Eh|E2Y&iJ1Wle*a!#9993 z=_>|rHEiHd-Pun|G->#gE%Qtx(`#*I8yaRbs|~cgAyQ*sl2Z_vC~w2A=@IWsx)nY- zn(dZHj51u9rV1l1{$MdfdCvI|?8cis^Jp{K=a(`h@1eXLRKLQ%7QlKZ@TsaUJ&(6< zsc(Fg83rCcREwZhU@sa+OVzXS6;IE!CQpr(JhbqmoFh)4&J%uQj1*>ep#D@jm4RAXsr`7tbecr?quZi+%63 zOYdG;oy7;?dH1S=c(vOngi;dA0K``XxbylXLk_ohWV-lq99XS2yPu=>Twm)ryEY|w zXNuvT-673c*EFR0l?}7ecgW5N65|f%$vu7~PHdl$LKIhYzq&;(dFA!4Cc}%W^!qiC zxCr~)S2Wf1MQp24*y5)@SOMHIg-7s9sM7gC*vPgn^e8aXe0*X3a6!FOwXg!0|ET*P z!z70xHyip$<|TAgx)NjBX=}nIQsczl>=gucYd5AU+-+2l)wC54?(o5_OYlvU)#I5Y>@|6pLL+S8$&RjxV1OOqqKY*h`=A(Tdz;^$ZQt5$~*_eZ*|FbyK^Ktw z{H*7=H*k~5D66Ut|ep%mCE1j}@c-CU5MMt7hB2U=XxmBwkh!sk1vOP3t zC%Fj2YUI>!*<;Uk(9!zPpOtfxL8UfpW7pPxrgm*#; z-n}hLZEq{HOD4Gd5J6D|&Wa)>TDZ{+)u9L#q1{FwN4;OP3rRqk`+|B% zo3q7`H}yg5BAyi%uphY3Fe{yM_B2ySjSN@FKAQc4UaC^{O3T12Wv>GIsSeV~uH(?g zQM`K|BIkA`t+=fZ*TOSUixx+FjfDNIdMS6@#Zh)cPC-{5d((>^zxVc30ktoo>2#1% z=)JXH|Li)Pe>3U7{lMp2$RB9pHEpqL=gR9{%Lqd|{PL`~(yv`-heSw&B|GK*?uO(m zuA#ku7?FfFD~$H#uOrQrmvXT>a=+4)NEwFN@e}Hk{>x`2b6ayKzO{o2n!CnH*^(E| zpe=bbGO{M7oKGB3c&-O(wTcfjyaOSi7~sI$grYNozpSduDnp-;Cj(wU#@U$uvd*VVqVJ^N|!w*q_+oP8uq9 zWABpAJ%Yvg3BBhDx9cvpmw1=y@xMZbHGSbXS8KzW8I<4;fu^UjGp5=O9f#%fty#3s zUkm1r@9S|Dp}o5eeV<*GZpRfrfimeUta z#6uCM{wzR89!)es6#DZL>^Gz^L`L;5|E#An|7lb58GulvJJ3iNPtG|537J=#ERn2^ z_%0|mgDBjBbao%jM7rQi%&h;o`^psTpNcv=cVO{*))+N2G3Z|wNfDAv8utT zM;7^oe?0CyLT2<8!wWQ_R}-hz3H{ftEEHP}&-aF&;uQGNa#0QG{4j+q*3$!qnIx#! z9jb>R$YWL5tZ1Qf#$f^u)3e}5pk1dZZ^2| z<5$^+-E4~7;FoR7KMnctRqB=*m9TshsAt>RZ=bmahw1P*e6zFj-g2#e^(xKsU74jg z*eXD@BNLx`7#^Yejzws`Gbwd_Ur`j)`y2RQk(aw)j{oEd7VVRvvJS#+eQi*+TU)aE zP3qk&{b`x_j(36j^TGbg^H$x^PCb@HO_?eA>DbMFG zqHi*`&wUa@#b9!m6?U~T3LnuN!4_`D1K7SupIfS5RE|Am8I;Wt`~0d@dTQ}IlP!V| zqIO4V7$BVogMJf+@j>;E2eY4`u^T47&tH`E@=*65XMeWWL{6LiC@qZ>(|=bu;&LOh zGeiw{SrTW02x|sC|3wPto=`8EA$*RhIR(aFyownX)W-m7aHH2`xK?<3Cl5NDO6Rzh z*Rb*J;vbyXpHlEyXr#M>$HNQW4g z7^Q2-qXhQGUkdtn1Fea!#PPuPX?@sruaaV+y2<8ksu=$oy!;S2ng zLDXFY%U>PLsg{(}j`p9bO=i4bEW64?M~#7mvyp~P@uDEP7PPbnHJhIgH+Jfm$|QBg zT$!z>s;c*SZWAclt@xNQbkwHTo=TTuVCndTnbI5HfP>CvLS{F)_Fu6Sm0a)E+~oLG z7#8!YZe6yAXU-)RVR8*pHKi#2-m__&K1}nCZBfvAO_n+~(%1HnIY&mm@ak#10WGCR zgTGwAg-6w}rciUqoNaqvAMNkumtks81!vbknCc2HxqbIex6HerA*2yPQj5SO+)aNy z5P@ZfZMUWBfd1@t&|a^`3U>Fuf}%SnIc zcbb27_|rro|GkrJ3ocfX?brrQ>!LpZ4A8j8l1%As;Kk z+mO}!3ucwVlpuPbxArY3 ziH;1H)Anuq{Lpy|Mc6qPT-+}oSbx-uoc}K3I&AP0PUy1W+p@1roBl02x%D*eT6QdX zEsZbaRq89rQ72;-j3q-=I~J53dKAmuJ(2&Y(f_wMoR|O4AA|nZ^8dxp`L$@k7VkY< z*!zQ|I6JnO$P{y&UZ}E*)1c$|)FpPA3Q>||lpf7-I%QV`zNy?f@*%h!ld&E(fOe`w zkm_~l@}u~xXQkzKFESK{Hbtl$DUa_ve=|_9I(jrps(-#9DEzKm|3#%f$Wi>=tjc&b zMWjQDw`R(P!qL2(gsbS+d}wuPP1jLcaT-KiIiT%`eiCN#t{8CCiTac&xhD1c()XdY zj>?^A6ce*DiCp{#CR5h{4IcNV0oyc1(G1^IUiGQydB~+S$wz%h{NYhMaP4QXU$JL~ zgOPjyg2J@5Uma{RfI>Nj07&XOUSnd)62rNY6krkF3i`XW*G99Po#vR|F@};RgKL#v z2%r1^RD1WywVYNAq&vV)?;=#-V2!Ym{gtK82Acz`yuUJ7IOP4Vh$u+9CWqY=WbI&J zV2eOnL95rB_AsYNtAMqiguy6VQiLh?PM)cc-LN z@3o)4wbd#EHFaiJu#J8fwAv5F>Z=sP>PCdYphnyKOWrp3{Y_LkKWr+zQ%LiemZMIk zB-S`2>s^CtW7j^ighrq#a?-U%QT9SW%k3-Y|I~%ut2v&BP~S^yl6sF3AT77# zmX7zjU^1`kCS3Mmc@^DV9_9)f$5}vj^Faq50m0=3)b7}zX0MCxH#0o$+5R-sjO&ZI#eB>6`GH21X+HXes-5iy`@*Atd z(2b@L+nddeUS?=Dq^Mpr-FbK8m&E*2^@W2%JUGl>2uN2ODtAm?(pEnOr zEL<}ymOisPFCX=y<%l?3XQwCQ{BokoWmt{Xo&xm;&)eqIxE zd3@}2cNiT`O();1%Y3I{vi&=a;+jl29X<2r?^} zuxso&B|F;ZkfC=zeZ4}9TsgWb{d^AKqEYk2D$ax&uLI@xnTnXGkr9q@N!3mb|E`ut z#$7CQq2Y*=Fh2#wf@v@?y#_~@V-kk%1Zh2xMJqj#)c&fTlu{HHBgonr;SLKGFzyc= ziG_Q;BJY{oPq#9w*!)wO1F!fX1nC@JKbL+P6unBroRnI73jJtq&nKWylI98H};Qp5-)?frRSA@w1q@#MK47C*iph3op9W&W~8&X@B z08hhe%uMxGyrBek;HjsGh<$8hQF&si=L3b`)1S{8W!}^2c78Sb9n8c?gDRt~MFxh7 zFieWmG0Qq9&j%t|5r_Xdhf(chk<;PvQ=imKeKR+$1((yd+OaYtO)3CTnSMTe%Sl|Qf<71hpTD!h4h_{lYv3U2yk>f0g` zK8hZOEqBvQ{`o*a-RO8bi{n#>-}$*Q%Pq#mQB6D_e0!xv>S%EDIe9B3@vFj{3j1R9 z?*`+MPIBE4S`_3{mN!Ns6v6%Z{hIti+AZXR-GlV#?f|o!qU^XUUNa?mT!u%9jevnG zJkZ5$FF(|m+l-g+^4Ip#{_`uXU12s)8OpF;1;d2)%1|4A%sG+2UDGUwY^;(pI!%`= z;Z)Ek$gutmx$JlbUlmG_Osjh}D!fx$-W25kxl0;L7w-B+nY4af!uDJQ_MR<@8yN7grg7qE^X*x-!@lNN9d9ERNBy<%&(P% zYOnb$3b_EC6)Ku$mvSxCAk6yte~>Hs0lJ=_v}XjrNfr2-a9bE=QRNiMK)yFnj5b`$pcjCvMu}9jfvyT?#4ajTkYB1z3TbSVB|QG zpEzd%2RaWMv-lCYn~0 z(1I(nS~Ny#7l&eQC0JH#7QMK3gC};b^_c%(T11lXBzA~~l7%8{TY!ljpXx!@|M zT~5yrwY8s|emeSySC3vvtJ{HT5GKFp;fMeLUzwYwn+!y#^ABn$nnJ4kWEczMda)yS zu2_COBDU@I16+;bUQv_8dkoAoaCe57;uy)@VUh5b5RjBu_kFbCP$myFf1 zPtuNNs02mcX#m)6zNH;jvH^&3C5Gk?V38OjQyjE#T|eLLvfeAUj#G^EM_}@g7lO z@tXXD)F`ePUJ;a@GW)sv;F%Sx)xw$Jmde|jf#8EQuwLba> znn|rva!1(B>2yKWDWSa|IF6PZNc3}}oO2vy=72|w7J$Rv+fJfCR-NjRT>N14?sIP6 z`aay_gh8;D9tsx)(@?X>Xjf`GhHu6!b`84NK{&?Lx^TmPT+!3W6@-zPyE?jO7RJ+*~-$-V+f{3#WkfCT1czg{=u>Zhna+Q=g4%9zHZu4pBLe zN+Qu5XPG?{Tb+B3o-_Or=Nf{~r`pf^zX??w;$5nr9R#j1teFlB(A+l)#SdY{9J)Cl zM|kkkvwhe)T}sIe`$-{P{g)yjTESUw0+#nX*v8xTa0?JXJ$jnU2(qcY3OIM8 zHv0rIyX4<_I~iFm0wYDITkls~X4-t#`fQ5quoPn=+nV}1TKH+EZMlz^N#2}o1DIF? z2g~&3tTG!3wZIdu>Bmj3K>L3jHWlr=v<0rMcVi~T+jUpZy}{?*o41vv?!vtfI8t6$ z@0~Cbaw0uyy@Sv(+(a%FQ%9|+(Bsw3a{s&NLzp)|r=6aN+)u4AyLo%;*UR|jMyb+a zkb)^zQ+-NuYp15`N7R##Zk)SMu-zL^50D}{^M)1uPc^D^&!fvgipuf-Coe&E&VR@M E8xtKW0RR91 delta 225686 zcmbrlXIK+$xGow#P?Ta6=~V#%K|oP@iH$B&q_?PuG!X&mfl-hmRX{+gDk7cGJ48B& zNC)X80g;|i!axdVzO~L;d+qC-AN#uYk4a{dY47{=`+nYS1v^1c2Ri}n(LtLYJ^g=t|GN(}KKq~h{~j2hgE2r~|M@%H|JwieJQ!WX zCm5_Ep@Mw_4&<=m33&O^ayC1MWXfe5d)Wz~9EI?FfeE%az2(;V{JH%O_G@B{YG#t5 zb-@=KnY9q9F?qi^^`8m9ZVvuapcW_M7;m5u{pEJF#HWA$-QM*yZ>~4h248mj0$Ml< z9K?RsH4`<6_pJ<@ex+P036ttqOos-+8c;C-GnfK4HaAjss?V%qMX)#V7SgBQ!n!31Dwa|)ik;F}VhnpuoX}vSn zT6beK>0fK=oM1P;dBVUWSiVruHn!=$ma6FuBStf9LUi5q-L<;=lSVHuHWrZ3M#KC5 zKK-Y*q-S1wu_U+>tT$TWV6?uv+s3_xhI;;g0)=QT`kG>aDeT53MrlM-I-j%`Sh|O4 z@8s!(0lHp_Q(8aaJ+Nsjc?`p&?H}0Z5ivAPec*_Uq|&YTyMDP_$0YHk;Fc!45#7I= zopw9?P~h`1tjvQ_oK}{LWu(Lp9m6ISX{LD<#_BHdqC$H4X#ZRTLg`mO-fy2BzRmPY z+I&rWP$IFT%oN3ZmihWw*qZM5;s*|Ddq7u+ENvcOSF>^>o1&~Hnhnsd=lmHvLaq{; zeTVu)gm6wZ^GtDk7{I`5wAK+Jl!Zdz)NL%O1;?<( z>D*x#%8!EPO4NZ28zZp{24M3HP1`5zS^#1uQ1)iP_D6m*oQ|Xq&7wZn{=Nj z>O&gpC`&?FgIJzGZ51N+{pv*RSw2SpR8E(LWq5{xkU7 zgQQ2~+Q7O5Uz+SXhMh&}4ejThZxhUSjqG;_71fQY+U(Yv&m>!nT#LQ%i3J&j*%4b4 z+V4M#CWc_Lf-E|bhioMF%2Vz%9bof+DX+&5A}CV3#Wo1>KwoE8RD5hS`mj@vi9AF5AFZhz{X_wIp@hHpr)xJF=a%j?IBYrePJ7)O}yJ~6^&Qil;}Y@Pi`ZE-XvF| zMLfm$QnbtTz|X)y%imYlVDEcw+^v76pvCST|DJB2aU*~*NlFD>j$x;MMGD10=97&5 zmF6{U==trpAqicDT4;JZ&_P#?)w;Flw~qAEJ~$B)0JL!;9wq>9k(DLZJ#W=4aGS0M zd|g+o%l*x_v@)HyMIwz``a~WtUu9J8`-8bNxixfB_P^z>>0h)*ahq-x&H=yd|Embl z4?76nf<)=|Xo;e;#+vM~e>d~UH^7qH$T4hvOP0!F9_;L|P19JHUFleOlP=pKW%pP>W~ML3cI?~ub5rVqhTZ2M zu`e`?yL?nlfh)r`!1Sk6_&75KH2=$P6|?Zwdvv$9@6SrmpT~()-Zn;3y=Y&WZC^qY zr6G)>v5qP1@6=Pe{0o#Qp0CZ$E2uzI^&j@~A2tI&C2|bwoy>z=(Jb0FFv9u8c@xm>Jc>g)9KG1Ru6LK4+_Mp=NSLXf<*|541>sk4db+wE3mwEV3 z-h}Ioq&KKG2}W_Bhi;yWkXZu<^@yCM9@SYEMyO-hX#eif#C~m4v0goGq}84>ivTBn zeYDLV|GrZ3i9h-8$+*hjUIfmmhH!@OE=buL3J9|o=Sl4FAiR>e$b&z&k)87mY`c({XWsOlv*W7@0 z_<1DYxSIWZ@t@w}%%e;TF@@q>Ei;pr#BV}y$(md}b*s??!S$O?hwczcaH`yTkf7;hI)re#3y_s$>c%(bIId*#ZHhS zp57`){JHc}&a;0A6;Q&uK)lv;2msUTduV2ABW4jTbP9OX^n0)6QHJeg-gh?vg;Psz z4`OMA3R}9@>$2&W@(}R+BUIbowr{A0+;Snnf(=EZ(GpY}*N;l0+7gKM=26;}d zBc*AO1ri>3(9;I2qeh;+BnJaFawS@)jLtSFd4O#Dw3<=icILX?*!Zt|nyxQ}wgn?| z&&Odl1BY{#(?OtPq0S-!;k?a@bS|4{yq~(z8htn1q5f@<>3<4Y?c{NRap|-y!Y#x{icRcr>>%V%HYS+u6j#@2UX)!*Vk|V)@WFCNVmnK`}c9J z(xCV;?1V2s{{GolPkK^DN_I*1(goXM^ zO;s@WQ%%Y1_o}x<-TquqKh3{SOIdcf>hofBsi<^ZaDra$bM&`-gY~f`eFwzGeyPx@ z|LF(Nw}^S+%^Do7iSC=@DRNAmj4#yrqy|y^-ncBtk$%~v{VvmR7;)3jopb&YofMju z$W8B-_KBO`Hf>Ry?kaFb5$``3{@*V3BjDP=DI1nHbqvcU0Ds$%1}G06!hbo~9ihzw zqT*dpmwaO_eIcqw%1&IjuAwGgzc@lz-sz(mSH6i3{krvol()!pIF)-1iTc6w?TxAi zcl?-zo;{fy6lA8>(0JS`inZ}}ocA(Muw??rFv)%3B+7{}82vDA(`P_I$JWvEey;rR zrp!xn$@U)klzWHj!}6ukd{(hPECo$L=^aSHRb#O;d5>Sm&uy!?i_D}4f%yjM(iICf zKP?|#P0^wGh%9SnEF1G~7~&Z!G{%4V=e`^&u(I&@7%Q94kz25C)F@nC^qu7&^v)!3 zy>NYaW-7(p%u!sV%=LX~gQA5Z?bJpX9*8jKI2Yb@Sh29|dgO<<&!;9VhAFR-3QTD- zCs4@*xNSV16iR4*)VlxYw0e9uIv?ZeVhw+jBp3OV?AXARdj;@Uu}I7KMyRsCH*?yf zq@6nanXua>tCS$sq*0nM(W3Qhe;;6#a|^OcKba`Nd2g$&$4J}r_vA``PY!rlHUi<8 z6JEzlu&QZ_+DbEwpAePgT>R`_c|Mj!%Ims49|27}gG!e@d&dDM8=0=2nmWQsHcO!- z{Vbk6@sQClkkZ4P^0;WeRQ=q(RfJpuqG|&*2x%uAN;P~a)`XQ!KHRDdu=RN&eTTTB zU-hElTCZQhg<3E6*Pz9K0-_Sr>5k}3v@G%PC>W6YD$UcMbf1nvaQw)DxJ_yUy~)qO z*6l7qj)@v~4cvA=m$eV}TtBxdbBmc}T=N+(xco=LAxaj9=5RjJ$}6>*D$9{GP$w(& ziG=IOOs%(vy3Nr>I*ox2s=e`;#Illg1*u4@Zu25~0SsyjR7)E*ke^8S_PfVNB4Wuds}K6=<=S{s)i*&KwG<<$PPHneW;D6P{) z*WDx}m*X(9*p<~2zy9rc^D)eNr{4Gdr!~o1bs%MuKY%fuQ|^A4)8z@{|YNRhP6w2 z$jeISkx#stq|<&-!&9-3!`$w7fBpbhG3?hecnP{HA!OQ!a%t!*p*G>Ag*`@iL4ujT ze3pwo(;*9>(s8f3?SAR`KIxk+kxqV)v}3 zePDUa_rQ>~E}OyQshqxp%i*hs+UiohArk%>ovj)~pM%JkCEF$_GA26m zS<%AYq`K=HQtZ@wn4+S=Of4~!Y_`o9GIl&o! zvsmP+0l`U1Vg=OfDxVFY9!`j76g*#d3`5sYLbTCup>nez z&^A4CBfP|>=UbfdG_Cj4^<$X9XnOsI;e*4q5x_}0kc1=g%?O-eE#gd4V*K$B;XZZQ z8?Y}XtdJx@8=Me>%fWP7^bIm|aY4#b`1aV?p>{pt^8~-(gsR#X7QfjI0w`P2};n zf;aB?T4jo4*5HPr2lmk1<0M5tLoW`fD(A~udgxTav9Fb|2yjVLQi&AhQQfMC`9DWC zze_A!ZJWtfD$PP-k72Pse3v2+u)jwy69+NEBZy4To;gzC7)Au|1R4@^?=5iT^hoRf zFy1Muun#(8@Cb~&1Fk-Lej3DPe$_$J87PAmP(*S1f#ABGR z(s~`YP>N@^0(cs?&AG-FT-9+T`Fk6L-Lo%XO8f`t`R!=ll4T>88^ zF@^guQXP&(CL+osH);R^5J}o62v#_KU2SyF8lTbj5VFdWbL zG~Hrio-{KAlz;Lyo?XmN8H|l7)Y21(B}|&J=MCp%B5L6Y6a@J$igQ8@L6kBXwK1qm z-t_g>F;|VsGtvUvl%@M@VP2=vkrcYGw4bdqb#{nO6%NW|+j~IQ`>V^&Oo@nE!)#pm zmhK|mK*-yW>4l1r&tzGf=>>g-NL-e}A41H_G3oI#ubB_azUQjFe|e9zo}spA2wPLm zf)9cNJU*?aYn9^kJu2FkzDHMHWYCd2d25<_b48x;`|{(*k7a} z1AZj(X%iT`?X1@UEGjTH3p=Bm&g@*n{Q5>6-F@yV2_w^TQ5Yllx@~?ZdYZXgP@~zW zzR}slx&8jod7d;hi=3A-e$7p`BU@&4FGDJ!&iTb!oEXzqLy~$Vjo!=$!|9S9d%f>W z_JkjcR;MmYbgo9}KyYVyMTS9-!yo+H_m3C9{=$&_!F5@nU0v@O_E`G{Z%UK(?8P6P z3}K1M2bGNKQYz{Sq;<>s);ODI@xGznulLakeo6Yv9*=b>Zfmkr;@{Va!1uHXVvp#7 zwav+7dR4P}q29?e2so3ck4eRFXsNQngO)+autT|A9;^Wbqa0RwT-glwv@>z4UT6Y? z(zxs&u>y1m{xeqQJ}GyDl}Bi!{K6!Dg~BGaI07=08is2wi}Y@oGaFGVK)q#=h%*8ck^n+Ms0ZqIAagLP&}M~V$&p`RlhrQq*o1dQL3b1xc- z0|lk55Ab(G&VmV0m7_a$W4xTpycV}&R9h7k6NJCUWTe-iITf3v+xvVxnQ#Vot;j#d zkf=Sioz@pBALi>-l34B)n62(MtRba~MMu(DrwZ(Eoq78>R9Ztnv0AHzIra(Rs&;8e zz=!1;N!;w};Qjo$7rv4yEYVq~+Z57pfWOCP5`m)p+Wm&*mPUtsNA>0EX3m*9HZ8{f zB;BQb`CkJaFOiXdYo|Sv+oLcDB-zl%8GWSfhMQVa4v~8%Bx@+$_fmWwK z)i1m)12c!0@8^y-)~*!M+7;6y!;WEYO#rJwqr}sg%>lOhM;O>Yo(nGMQN*dT1umER znXmw^`^PXPbOd5Cy_EyOp&{a*(Pc5QF3^=6Q|$KUR;8us6&rdVa*q-T-OwyJ(`=Tv z5<#}LiBJ*1wIWptif~U?h#`#U)-u&~N80oB&He<{up8U+?4`9``{xA|nxg&A(+vg9j(iOh^ofZ~zI)?DE+tVgg_ z3R3%Qw~2XPYLZVze<7QMLu=(>3_pAU{m6nlze(KYZ$h-1IQj0)*UkOGRw-AnUI?D* zK{nZ`=WkxhU-d1%KE37%bQacES5!&{Ca0j8M>RB@eJ4hA>Jp2}MFrl34VGQEN_Xm( z65?_#sheJM%aiO)I31OeCiUucx0`{Az}e^1Ihr%H-`4J>HC^qi2ibOw&v%92>x|I< z@+}wy$25ef4KI;A{*UAb5cema#_v2jt1JEFgfjgly(J&?6~apAHo&l|qS@7~XfEj1 zv8;K~VvYyT!1Aaf_94-HBSZ>xxI_4|T3l>JEw!4r`S9gh^{0aGJKv}4Mbu2mT`obh zli(|H>N@abW83C_c^c@+g>Tx5tzHZ z@=DG|y@z zu8Wl|xffHbpy_)K_^$EScoiwz$s;4nWs+s9Zf~P*@iK7g`6I39Oew8{-$%?Ob@1>T zh$poYSZiu(EBNeOl~|?U7uBD`m{`Dq-S$Id`FjXErelskUDA$>h>!V9y&xUqj54}g z78DbmhKB+!+jOfFG>OJZv}}#n{1b9zXV1k8^xvU!;}$exX=;He^&RHPfPVMTYQ91Zxc>0(lhuTafow+9^MPi%fSHGfZ9jqj$8{o%x9n3Xu@kg7;L2(Lu4E}xv5s!b8UK$&S_ZObW-5q|?P@O1kFSFHPqbf6_Sb*CirZ{IKxZ(_(>JciiLrbTF469R#m$4}y|Fhirt#8PMj_G0efz#RYw3 z(}R@3d98aWZpJH@?!&9EN!K%@K`m*G$&Iywk`is{oKcQ7ljnZgIZ>Y$N8cvr*?-(` z0lu!pI>;YEx!uVF6rvhT3?`~{uX6pYtEqaPMX#271JP=ZI3~Otk zF-}m#vFNC5nfh}7y%uXeca7=Q2u`Q)uTPEl*e(yY1O#-o7G4Uvo-3V3zOB&(UL(u< zhhg(Kk=TrN?%_SN);Xz?=#rR<#`qS%ZBHMYe3@wbEaai__gNJplAWRsehe-}1rs)l zQMH3XwYMV+n8rzU9RdS5(C^WQ+`=9LvCES%M=M*?&K+AD^w{Lcj_g_>7$c|9#vp>o zX~^%aa13)ES$aK<`z8Me%TuQTV8pg=3a|?*4WK^bJWj73!`Pbz9(Ugj1~jlCQ;aEc zee#Lw`gFa2ufMwm#MpO9S9fggD-n7srjoLyRDr*))YS@;u>ya(IESXTW-={Z%eaV4<7@D~JidMktrD&J^VfA9SzyJ*k-`vYj+5 z_itl8XmZF42?Ihp7YGKM;v_d<)7{!|(>)-8O|R*@$95*M{^%uxr_E#UD;jJud2fQY zd>>XeYY7)+Uv$RM=TYAu!<=k@DKg2;=2BJsuRl2U#6@$Vze|hD!HU5+_=SqHwDy}& zF>aN&@~3OoQXIV?5WGG_am`Q@@^9m!+AAN!{6lJ1MD5S5bTvK_>he1W#gs0TU$uLk z^b~0i%Z+woz4C9gvJ$xXKkKLth${3rqCiyPF-(Ps-~-{>VB#m;-1X_;vf_Nbzk?oL z4RO)6TERO?2k2)YHV3@BNj!!j+-9he=wC=s<`@49v)|DZnkUk^}urYbgU+RvLDz{2VjWF%U z!kL>tK*igAd!d;0U~y6tn5?ms+%vDZdp<1d`AYIJEO@--4k-ytF0h-fq}(OW3ORXo zvbwIY!O2_A;n`w`0WZ~im*646hO2rrq=#UzE8vF};5fZbOL!o=fnPLgKr;Vr!o*O8 zN&!VS+8Bh*o&CvzA(X4VD2;vM=7*%%-N_3XN1m-FU#d&Sam9U2((ucAz7Bzh3B%Ny zGxpDKW6D=3$FSE_5kvAdipG0okJHYjP08eVmywWxY&ywdNJ5u#2{7n_XGG+H!S<#+ zk=+g9CUg$AYh#q2sN{GlymM1Ti$k&Pii|gb?P8j@BDP zC2G2#XLXYGANgiKkrfu`y{}rxgKkHHml(Tq1pPBeTcDF$vHxV(bX?|9?bO4@abQa< z+UT)>-?SC$&FHWBOcz{EMuZuh2*Vg1^ma2RRqYf8At>V#yWpdKXYeIdknXcCe;S zJLALR@|%#|hU-9^81Qub7?!TQ_^pi*@kBhk6;t{W0^k_dKoL>mHP8v`n9BDByQAo(=gW@pxE=t!l9|E7`aEJ=Q%5_9 z#dnx0wdZA+6O=QhYqj}^Ul6g0fuZ;alWnz7L4OR3+{+TVQ;Cz(xj3Cz&;xUt#xNrB z$OK^_**G=3dOwu`2z}(TI>_N|z0fZPV3mlW`{KM^gOFJ7l@AxP7d?~zcpsiFzWpbI z5Z1wKUUd@4bNgc15x^3mlXs5tBY=lv+~p{C+HNq*L!{#Z-<_3hr=g=s>;dIGfQyWi zN6!Rd2U_zW=J5Q2RezCt5_6V$0K`2!e_{MD_dqtrvv%AKZc6h!aq97BU5z;e)2{aw zGc^q3mD~AW9}N4b^E0JtREBF8|6X0_ZmiXCzYXBM_WBOf)}fsEO^^ALo#`FChw?wS zVguoNhe$JGYOUIx;4f>FyLPK8cma(dRTmbg`zM%7zW8*0RMieqbjU*qkj-7wBik|n z_-Y_By4j91SN>CdOR9m_Ds9Ln{JV^2uNu=~CW6PL^v;ZmyHCyFh!>KDs6?@DR~e3W z1rA&=V^mL~M_BcsU~fXHWm|z7;gKQ9*eOww|@4{78S zqIs{J#_%}PwixGNAS(-sZ|5yU*D&|m^LSkH&*2+RIQfvR+(MH5@4gZ{-*<`obt*R{o_^U`Q-TPNJr|& zuv{)}j;%glHKq?Grg>c92e~pUKNQpLO4d{2cr0INvA#GUNwkVi@w?{sKiFHEEc5T_ zm_8GUvmqnStmzlK)HX5wFvGi?MrP2x`m4)tk+y!*1{~2LZJNbhTOXV3|Ksm;KX=(r zyAXKYxvQO0tkE-$BA;c${R@U z^=uQs=L-=~L1}DA{kl(vvVao;k%QFt`;P0hWcOoO?ukHtg`CP1a<(D6W}`=d@^+NgDAT1r()RU#E?!c1(o- zqZ(cHwUdO`>ZULMw9r+H8r_#Xy_d3c3>&8ge36^zBrDXaEP&z`k*)n?R}`S0{Ft_F z`pfeec5fjsq>46%;D6~cg~zJ*yp^8`%7|0otC!B~`1$TI{;V=gpv^l8*7sLx=YVvg z+7k0D;h)X8G@6(oW^Z793k^#3(UZg`EWiwt%467Ydvh%ATH&kW`@tXgXGLE7$Ih^7 zHhJrcq!Sh6Z^-h_%dfoIZU%Z$j*g_7*0@t|v*ikF1_`xvhq{c>I|*EhLD(I+1G8lkI5ow2!H_j2AJQ2;gfB?D42GsWj_A&SkiK*EYs1;PBv2l`rN zfhko-^<9kS=Dzpmo!)`DBw>Pm#g!q;zbb+spOTjAZap^#)aoy#yzN$iSavj=nnGjr zH|zii#$>ybZhToL{db~qf5z++K0b|l9)0IKi`lCSuVW{HsmWTSg#qT?&Kt4~3L13J za}sz+=!?XJPT5EdJtTnN6vLvsSN5Z@q1_}x)#K^YQ;Gbs_=FEhbZUNOwm;+@HM`$D z+wjI+8#&6;(YueDb|z^w5~j$8QPNYjz2+t%X5IJUh>;eNc6yrRfb7<;-#I02A(Tb? zJOgA&NtPW%2T~EiDGbHswDdPH;>!?ne)h}ZfpdlKtK4@5I_34eI~f!q-jl7^DcN zg2%8Ns308@#Ha&{bx}gJ)2ld5kE2sA3r8j?M;ydvr}or;DG$sM{p3mlg^LsGj#R}{MX%k<2wMLy*$wL$F~*V|J31edYpSZwWh z&o27Q`-QvyV4&&B*N-HjX!jas;M5wI(QPqj_t_FBjL?tP2av3SP1X#m#CD8me=K_O$>FLuyMj{)0GhM_$`|Q!lrC=!_lRop{WQH;sp7LY;J>hem2C$g zRF_Rd!asV@x?*&XYGWPbt}?fD@k^c7PX__zdFR-cnJQP1*S0^^S|Gd3Mo*THy0rMG zTvkn!MUFB8&kK;<6e4?oPWL9ZMMjr`YWtMm)|0l88taIDG%uC zhe(o5m6fr6k6HaGfTr3(Ur+J7{hD6q; zzXqSzhq@2-Kg!@MBxonKoqZmFsYJ< zR?pn0wQ?@Jv@M`(IpeRsPROjg{!_~8SD5}5@-3B_R)t^hNkX>7W@UYuArJ2`zJ-vB zIU6hRwUby)k?mTd%XOK;FLR$X97lu?^`F_~WYI192&>^*qWejoh#V0mBWs+EXG+U% zbI!XBnDl*X0d&&7KG|Y$5U1yV+HeDG%5z(Q%j?Ho^uP~2zjF+G8eGrNbUSierrkH^ z{VVTYwH>#Q;CZsv8gTYyekz};Dwc`oX;6mgNe$&QYs_rd#+D{mx%tgBwapHm1u`k4 zc=Mxk{a4*qFNH*wp>(O`@;Yi9J{}M=%Hr>3cyXdcm? zAK7cmo#RQLKApNdAZR0_DDqIydjzB3d_kLX$=dvS-KG&HfZ~#Alni2pOqdA0bjhdepy-{KPg8g=>jtKAI(r_~)^2 z4^hya_k2}aRj&6Sv!tww1y?oMEvALC!jmf=uK#DlYGn7_HF6f=W{6 zv@Xc(9sJ9%_qkmIA5ymP97h0ANZuAwIrMrrC~9 zr?Ad^BqF$L$)6%AIWyJ&p;3`kQRhuig!<)Y?h)_k{MZ&9PBFa9{3J;Q*V;tQu3=RK zu-_68(KV%`$Xh%R-F{;ELHuVe-F15(GH*yX?J}`0{36k@Hq58MdiHvcyYS}Dck_!0 zwp8pYQtNL{B|QF*k4rUQfob94{n?y+`9qpRPJ0YV3MsO8rA{iwH8(JQ{vap+n4XH!juH^*sLBA0Gxxw9JqUsksk{h^RfGbA33eh>D?Y8~vslE@34~ zH`8a!Y}n&P?}=6yaDc$rCB)Lo>BjqYnia2`KmD=w#Z{Fv|3E7t4#i0v)Lf6&Q>aI+ z!~4?Wrjyk^i&R%%vPyq|=r(uXba)J>dC%t_j@|iU zT$XotV6{SHcWIGp%sDYSZ{X$bsf>EQw=*+&!IyM`T%lzGs4Q-(dDWYj&yEggrzUZQ z@`j*@2kFs=h$x*Tj;Ap0u_q4gQXYA?c1=_w-^)oXSXEL023Mf!pAebRSE2=a6)_;Fj z2Rx+#O@QzqAu4^LeS7M<={QBTaQ<)ldyCq4J)Y0H74OQu>V+TZC<$XSWx&}m!mJc;APucg<(g&2PC|il|APWiRVMO{n1a$lj%s-$@>7xkT zPLT4}zW!I!+PC*!zi*hPbK;pBa^jz?`R$T@8U&;W&xPQtjoGiig30#RMQmM?FU>#PtTsig!EuW5Hck9qnFw?K9< zOtTzI=ko!oyOT8GsXGeSLXB%*=|b=onGz;oHfsEy?NAWT_o9@^lKb(5u0=o*$o*ihhKDScwp8?1!I$)`L*4 z*A2^u8?rll&2WFya1QucQD%o<#}v#|zXHqH`1Bbj;!?vdl}RM(08b8mbe9Yx^rfUeRF(jDH;<+GgmdOx;A^W z59ZUwvnA~1ek*|VH=seoCc1t3-Os{-KJ_aH=0TYU-({zeU5Kij%@x9|YkKe`Nu#i! zr7LAa&syF*!la_NBmA8#-+j1%)P8YW)Kg+ec3@6*H`F!zI(kTQsvq@K_r&Xj zpEqe%JuGkHU^Fk~E|KES}3utDe-~AU#v92oRo}NyBngcS; ze@eBRmHHRk`RuEQL`$F>G#HYJLLhKz_S(8V7uh7Fuays6Su7racV-KYQjD<~t4TQg zy_T4P*IC?hwe#Qwq4t)MP&tXw0=FR$4hG{uuU_GyOBn=!*;t{SA#0`L@Q!nf{CO>6(_sp&1tl*nH}pQh;dW zNf!iIge^?PKHZjd+fp%CWlOvA6gA^XkZLO-PknKHEIBq| z;-_IwL;xGfgu(7qgyVSg`A-YhbsDy*-*;o5(3&k^h(*X>R=L>*ED-W2j2LCn)$d6` z>EwJGhdGoUYX}_*8qy>aKr?GN1q7A?px2*><8HH_<}oc)b_+I)Z-&?%1{x?$sOwe% zqSRCgqwRqj4PX2ISRrku!qdmgwM;5d9#tRb&z3 z97E+p`Y!@YZz#%h&(sfLuHoJat?Vol!ue@CJG|1*r@FINU%^-#{C zNmh(B9m5U`g;QGMKW_U2Pjmjy6$H3A(wE)T!lDLts-^fsPHaYB8WE~3|+ zRh}jO%(D}`Qe3!LC`gt1iw4bj8UIVL@b)a&F|0q5rVUMKbQydjlPyQ(8~U%RpZ2vb z@ep5B3}Qo;5VDF1ChT^|c`Rdw8BdnE02K@Eb05@|yBQZjG z!N~uZzn49yZ_EABSu4zT`19V$tSKPpvOk)n0XpD=Vgh_T=s`b|q;Wm1UF!%(6R(@3 z0f%E)2OWy1<1*-h4KC@5Ji4$*KAQX;iM{^{SpF|O3`_U)KgPty1Er?r^#S@H zJEH^sIG);(9I12r>pvMzIYG^rZmHGFc~UG;qRXHpbVcVAjBW$Q-4%ogMO_klvzHeo zc{uPhHkU_Ze!S2z-)41qGn5-=pCeINg6y{0-aEAb`z_s@TwN!qKf ze{BDcKR%WIa}GfN$^oTUHjrd1+8D$PQWyisN))Y$X86}tUY{|QF?0Lt%|o19ekm-^ zO#WebBk?G8H4~x%77EsmVRa`zl0#%Sro=ayi4SO;50E|I!jW$S3(I%yr=DwXH_vC= zuw=FPe}b>27p+^5=S1wqWZf}pswpq8di$-up&{jQ1aNB1F>5Z0>64eP5(lLhinupv zoFNMU!9eIAXANQSBl#9Z-$1d`Z}S`Eh8-z}Gih+qs4 z11e-=)h>Rfm=IgD7RiJBe6RD=m%O-zdUq%tSEBMBA?FL=7tO>)lsOKWp^l6}5RJ>< za3F`Etf-kN1MEQHZpYa{Pb%8S-AObR&=4AzzIAR+S7xrF!-vV8rPD)rHvYQ32=Uj^ zI_(TQG$_&?>?S8JC?_IN`unJGV|e8|{~$xJNkzTyMAAp=JA6JT*UknSG48#^=A3tF zy+K%xsK{NIT5G&t>>)6hviY@NzGYTU;z^RCiCN+MmCx6c7{*FT5CG%**eJAI~4NDQ-67&aBRud%@;m3EOZj|RT2PoF`bYN!}{Nq*l@ z(;T(uzjWK|J?TOsa^T?KwHh-R8grgo9+-@SN7t}&VV{@E4Z0VANt-W&>HDSMktWWa zxc-i@7bnN({4+Nd9Sps7I>iWtg2KrIS`V@c=zxbp%eGi=Pg*8-rtxOyd4G0EGoED+ zR=FF^kj(=$7nta0q%yZ;Tvzz>@T>&m);I@chBkzN(q>SSaif%g)+J&(W!c>9xd`7H zyjim%^8B!fGu&lQQ+dSWxZDesopFf~+M4oYr~qYH1uJmycqoH*YTM@ZM;p<`iHgz9 zP|Q!yTS?>>a&1VR6V#sKEt0WJeq)qd&QABw##r*XCU(H$Cluez;h;ngMU|dpf;f-C zLX7qsJa1Q7GBtx*O#Mi!*Fg;`_GN4x0)9@)YuBs*f;I&D4N}zoNG_8a!a(~usF{?D zB>OfW>JtY@J*51sW$S6%fO(I^Gqcp<=X3dd*Xvhc^lLvPzPw_R`U8Mcj1+c5n!s!F zWkBfd0wEF#Bc5@OCv|B9v~e2rLbjqBP8mxB2PjNaRpvf-2?i87ps=sd1e?e=)^pAw zu+W;RH_T+N&QWOL3WDrFe(-Cj)LlkxTd_CtdB}L(^U=7Ev)-@h_8tFSlg_b=pm2oI zn8c4^XTDS4CveY32m{nuAf&v^YcCi^@Y|kNBHq*UKN}`mDg-!OIH1 zM8hP|*$|fb4VqkIS;!l-$pMa~`Zz>XDNyU725#K2J15gRAX5YQA4)OxRgfe!{hWT! zn86tq1!ZZch>-mlKoooh>I6|d4tm!U(0CbX=Qe|@5fy+t;7ZMPZ$5w6>Xdk9VSmPV z6XB-}&F^~jzw`Saa}4>@XA3jASYYD>;d8SCQNMG7CN#2?+*|cZ#4w<6;25S>x_HxG zYYAYxIh~s-*&u(YNGzto#Vaa_(&Olk!B9osh-ALU3r3UsM7JAwk<{7IB+8(!$mv@zXCQtnYQGx6U;hXadQLWCFF z?Eok-xTzjQsJlbhj&8nIY{1&}E8M#%K$^3Low@Z^)b9^@I~_V_w@w(hKD=o{cd-B8 z8nV)fYwxMugojFzY3+1=6>9f40+Uq;=ox}Y%oN{4aXJ{dbW z?M7o7#%;qb*Zv>gzC0Yt@cVmYU&k)QC~F}lOClpelBQH-9VL5+h-%CuLiVi6go6by|rbJF@waDVy2y%A`RTLvQkY@$^wJpUgiOp;wqg+!J`7sw0Z~z9IbV5V&Gt&oQ6j2T=H^7zQ5op17hD+{JAC)1 zM?-CF(_w9=vbzCUh1i}*z_uKv)a3A#z5A`uRFfOltE0`Un06<2kH|)uEIpa|mL-M!Z9H8Y<-q8{NAAw5yp5Pc zC*>Jt&z-Xov#zT8@{Y7)&CIx)6L;h@%kEZ|h-PWenm%U^f>cp%3J|!Dcy-IyIBr;z zbYs51)NgBQS?y)1w}R{V8_J)Q?L6O_M?A>FU=vR#GGNwYj~?O3@yUYd$S!E*ZsxmYxKU4YbEZV zIvOAITz*ZGa6sMK3wLULcf(Pu)AgXj zGX;2D-+a~AFf74ouqXEc=c`((8@n5)2oKFJ57^tlfme&p(IlHV9?L9b26ypXFOzt} z+&k~`iNCo1^7kxaF#Gl-u$P}1u5dJ}1ABi)4Fo*3rFT)WA&D&xlrg#vna9=kUd0V3 z&m+-E6LWtN42w%7XnXI>0bs(Th<_2u=eU1)QEYZ@;Px})H(KO($VJIIM6M~nwU1o+ zn)AW$zcpWR7`^TNEwb-+uOkI7#;~WZ_M&6A^%EJY#V1O`1G^;qZx>-Z-c^1#{Vrz) zA#Oypl`r_Q_`|@OG=70d?^@ZZLaBu?_hst!JV8QJ^JD7)3&Vn7l@uMmT`wp-pVsqg zHUd`whQoapwxpXe(~RYD+AXjd`MB?L`!Stk&UIh>&UMO-sNh4*qz?$)5e2<677`dK zFgW+PLl_6(LYnrpB~8C`AgrwsQgoR4{%u*`vDtn<_asH7{_?mos2e%bRq-N<_x3sa z`%t?!f%F<(6UPu=W$}TE?(odE#$GzAgVD`wD)2olJ0NEEWiCv_vO+>-+Z22KP2Z~G zVRVCO0^81zxdc2rhlY)0+wlUl*wVs&DyCb^>g9}$=h{I6!a*S5TspK<7AO52`E{E{ zn3`kp@l*Zi;zRzc#s@(9)QpW}n0{uFU5A+|)vb7oLs2I8v78h6&WUdh&WG zM~Oi62LtYO7f#3kqMk;<26sS(+F!(u+!C6v*#!j9jYUFQV*)15SyHn8IdS-}jkPt< zoZ#ujo;+#VLixeMH<&PuC-Nv-W>Y}M$?NQXx9h&BmyPv(l!=6_jl6*L9AWIG5?8RZ z=%jID5n5iQ2U#!}BS>5l=}B|`xkZ~!j5AYunsV*y#kk}LhOwr#Kh7bCO8p$Gs58l7 zZ+=M~a=3iibW2}f?C2>{ym(d9VbTHUjZ)?pzCY!33{dC}948Zm+}^o&XY)G)rHl#X z-g>AhV`x&I|G0(jjSPOTOu_w0?-^I8N%f9fHm!|&CB(s7aKc%o|RtIkd!|E{zCK6LG6F?4X54vfDw?*N*n!?wfe>6LOe@qFJpWT#;voWgwv!91E7wCg6sz81*h8w6s=&uoCaplM4-YHu6ieyaj4 zth4_v|Nc^4S!`V0_9xpj)RHqV$Rd@stro}1&9Eui_7v3}Gd}=0+re4xq+Hu|dTgrW zV`j5HckIDyjZPm*9X!m6)Q))Uwe)oMFs<5y@Y}+&_|f21-$o%)CP5mfIO_zVV;{R7 zP3@LZ_Z_&YS3h;TR^70>LM_UM7-sNcd6*ywJVM}yw-DRV@B#e?1Rsi7`ysgK@_3Dn z*QIF796N8@4jZ!pmlA{cukX&Ep_NGl*Y6QhV+Z-2s4CVpR70d#QR_Myy(rSu2}deA zZe!rm@9LaHBFD^6^_wAAAdhRJXAx;DN|Zgc?Cmg;<%;Ug8L0gRICb;?B9M&z={?=Kh_dLMbbm;m4x75Iy&%t*>%E$8)A&7c0aS&FPVu)OzpQqL2 zM^PAost=4*XLAMLEsRcMgxL!;X550rfhJr?u1DQ9nB!ny<7e>!6oz9TZGs`%MEZPh z4kI2u-GNuBqH$F>D%ovcwCC5b-K~;y_jUr;H&e?!^9h@&lSf#BH{kpoXTTWdhqGQX zPE+NdyoRBPGl>`Ls+*!#1_M%uoQows(2tg(SbV?(;rtD@sf4hLVL-|mjMa1HN$uQT zTTToUYMhEbYu^0jkB5NB9P?J#HRd<>XQ@~JiQlgXEd2!Yu@OJD2Njha(1MK8d0PEy z6xEFkn7DUj{b_fJ|K~daXh#GSfMrpD$=41YV%UJGlrf(%>`4T1IQ0ZgckzMIsqJs} zGPb8HyQORD;@o+~ zrWKI8qCkYv_YRFfFZXvq+h)qZyhlH>Y#Rb|8-_#wBKW3R5@?1Jftu|@@B=}F-wv8W z^kVg$84dfB@AZd4IMV+){Ck}dxg2W00R6B-MWQx;%%YilGa?TIxq3ldD@M0LTh&PB zu7y@h@V*l8K81f$il(#Eu&hT*o149vp*Ur=NVExo^b_;Pbg><^BU`P-JnVZ3;E$q5 zSm(pQs(p!ygnO98DtNy>onFGI#bFss-tNuXfg2#+nje@qrW~(`a`({7*##~W1a@H z9sZk2{n_R-(~f1pX38d}UMh+X~IkZ`Lued3XW-4XdyiVLz(?6--JXsSPuQ>{X&-vGe0Io zmU6+iK4B0u;y>TmSf4hl@b3&lnLN+jC|>|1kw>JGL0DsgAvJOtepNnc6*L>6=#bN^ zKQtgcDN$PT($S?*WG!R2W=WQT$q$G$Nc>~*Bn7?l8hD6HC05Uo zW7%LC(K=|-3i6M+I`nn!6`=?Plo zAqSAKAv^2*X(IjT60|MYjh6(K)`UQeFnIC^`Z|&icd9P&TxO?!*Ah#i(tN7%^W)!= z9%o2Ldis=1dAicxsB+$8jsPX8K>rXg1+`ZxAgFr&&AuH4P;*fVM0x>H1C<(|#IA4< z#f|sCO%7u?zSaRuf`FCb+8F6jv)qS_WS9H z%U^IOJR}Q>uujHs0O=8X@4h_JTdtb2gNoJnemojKI%oU)_pM^Kk4p4i1Yw9YN`wzc z=S?ls-Dy3~KZbdGjv9$*j>g@5^IduTnSPkF&WF~#8*{!15WO5ehGg+2p*@I`LwLS$ zwCx)1F3og^P@^1mVqr;U|J4jJ=0wBZ0xX@QH%j~q_YWK!T}Uvwii8YO@;pF;{eGgD2eWm!QbzDJ(nf6ee@ zJIH1nJCHq79u1LxVB_luf}l*90l_S32IXfTwPrO z;>O|4%pqJuayk0olY~FcEyUzp9Ia`>AYXp6P>h{H9)DzczM<@5ZMdPBoFIPq9Cr_JhgKQFIrN*% zYCP(fq$cowJMo;flj#PPG!RaGt)4FN>6^WlwU_6~0U_;-=(Iz8&qUyRJAABRtT2H@ zEQb#Sl#ozz&fu8h$%cMey?`703{Wd!wZZ;}eAD)~eS7m!yHYmkI_;U{QC7k79}o~k z_ZkZ?vE-O9@yKxUjI`pY(WTj)?_WjKEJ5ly|bk=~^8#J&7yUG_8O=&2JC;!tM6h1foI$=%1p$<694 zUQU|cDo6)1#P~@8EfJy0n;R^V_citPnT87)-;FWZ}XRLuM~KfkE@r3XOIVu?fg0lmb^YtJ z==M~Jsh`HsA-I6L+HvQ{GH2aLO}@zF*YBUte%WQNk+5?1$4~TwI|-{g$yZ0ZWL#P{ zL9f5fl~**v5Qrc)}8no9%aT@7%7Z$gtM$s6!=L>&Zl15jTeAB$bMMlva3pT2kYRMap>5menX{mzQP$M0;JcdFKnesZMk|#l19;p0Sok(o#d;Z%Wq8*`ck&zOv0OHSZnwJg zD18vF>^(BS`^%{l7uMf!Z^qLw^by#Adc=j*PXnb>v3Q};&(M(Wp!nsbb%@G_RvIq3 zA$=DT-Ynbc%u^Y`xxbkHSO$t*q;hC~cc0VMSLeK;#ci7e}`2%9(XyAtZ zP|KcRJ~)=jdh`{<<|Y3cojMpF2SA~$BWU;nV(twm`vHJGpDTwoP#s8E#XbS2U&uIP ztmZ_Ja2&XOHKbI>rQlTV`8v_%>K}0u>y`-+>DTI{(`D-CS3djy_5GVjCzhZ{|r^NYuCfZDirsLLMk7@lmgl%QE70^?4k@sUi! z0%fRtQI;iy0%DuT0RHEHGAe#0j3a@Q-F`+6#?C^!!}ZNa%X4fSYu=Al+g3U2_(-Qc$x69p5iQ-;dlov2zsziDwgjCc ze3@@`@?-Pmb>(h$Va0X!yA5mTdR2O^xW8P$>;+{Ge7HQlKtk$T)E5= zL9kimU3j8nV@j{(7Y(n`^555vI)(2|ewjY_zBm)MUsHYwwVx~kr=&H0n59JR1I7R? zWleq)Vx2>*QuFZigVdUbWGRyKC#vrotRdzM5UAkr3!r7L#7NtWAqmxnNERs?1~=d!E$} z%6qh(3)IJA_%FgoQ;~jw#sSrc`PtCenjM0tZjb)fRL$8rvU1B*L)LSD-Jq0_Sd#%I zVT}kYax=-M9-wx}^c*(uGzp93(tqEj~{qm|s8CZZDS{ zKEcpMGsL!7v4fLdtZfi_bu{nLmw{UN8;gRBc8z^Pt7dpU@CZbzBW`x*U)!;H{X{oo z%8Y*ZsHT`(#&_bwdsh9Zr_bEnkPsIPREz65>l2!*S^6$nOA6-=C(h<_+n6bj7Zm&6 zaVf3l`nbJsjl6a`F= z=uh_CdMudy%}aMvbwzH+o7F|A0!?I^X3JjDgmy@+Z^;zyj@m`LVaR>tovxO!vx`8q zRC;yah^WaQDp&BQVqR3*Kk|JX^d8mTA771(oH+>B(T+IR0f}@Pov*6?@&b|8p)A>d zyz`p7f~7oes47Yh#h&VJKX%n9_5Jjrz7pgg zKfSSbPP-G`h})1BC2;3_FHp}gW>s~Bq2&l(3KXJ40;T@1%x{g~(T#oL-~Rab_Ut92 zKVQ`hrH;0p8qEa>!v-StEIJ8Wsl^b(&~H{!k|vAm)F2(lBszFG-6jUY|}NbEgol5#v}Q<4Rd8 ztTFlQ5_d8iife6*KTA}5(pEOPZ`vUa^C>kFTUiv-${P_!f5>dCNrraN-S(osEP1L? z!EfAon&|B5rokI!mYF_J&Sbd= zM3AHRXA9IJ2s!oibq6Tg&&WWVzAF+4t@U_4_H;0{r*PBGE~wqowzmVEmTl-*6cVp_ zs);N9rZ*(`y1LCHyT?;8=ojuWlZ2QHw;kJV+Hq#wp+YO&pcBSOudDHnA&lv4bgq-4 zul2m=|DotR%@O!$%FZMCBGo!Fd;rhQ`a+~yV`8vWThuN;lpOkspy?l7&0VK{H=9wX z{K6_*hP|$TD7|JiRlZg!qWE5}sPBeJQY8Mf50p=}fF`jPqou2wcuw5$y2hGEp0O84 zqN)R4O1Rw|6&AdOqMk!d7Xo;ndX_ku=n7}e@fs~h=$@3hm9yuCVY2V$`VYsh8+cEJ zjIVn|Tt0MUGk^__0v=#{+srWiyu-1wt(8Wn&JE{iV@!h%|5^mOg1($Rldw;$DeQRQH^yE;ijDb|~ zhZq-(Li7Lvx4d%{{}rlw!jQ73yNxv)S1h&et8A^kMB|R$tT{Ql5r@8I6FKi6;9h^9 zbZEDSo5}BxKv)^urjDE}2UFsO0$XK&9s{&XhLEGn!Tw8$6y*Svv44M^?CUhZ8XK+b z;X*KM!F(P-?YylzQ2PP=k;@D>W_D<*#PW<(6~VGH=d{MuR_x0#zRI%?4&Myuia99$ z*4FMK`vO!6k>2Om3{nbq3|U<$t_^<_W_X@f*HtRBh@o|_-rw-cfs(CHw_YaSc>FHo zSa;)oCA;)cLj!-^VLWF%ZmWcqFWhyCPH|Zmck|ar`JIq|T|M z_m#tnhNJu5UABb=)50IM*|r85qN*&xkFX!rknR^}3vZHgxqH!~Ms|uV&F@DGcgzW~ z=E`inQ!CBU4mFoE4G_TuxghR$Mc4Z7xT)n(- zGX;N@yF!O;{!URZDu(dBh#@iU3P;pRotPs zhNJI_*-VTokWy`Ih) zO1rF~2HT>4ysNyM4$lFb0CK~?O;7}%&hX9NFnMhlxPvlqNsp8st@oy`t;-cp^725`XfDH`P&G zu&}Enir}4Lwy*Edt&fIYA#VDBph5LnTp?0d=}GVqEp~z|71=B`*Y9%VF65ZtAS|^UYli`TS6Oi(@`RLX~a} zjJA>k<6uuat0PzNeU9+eM=&OV{ z8t(WgYY)~pe$IPp*64VUhjK?Joxjcc0k3~f!sgu3)r9R4Py;r%8Yu(V`i7*J@=D{a zLetAb;jUwWg`?##_q6HRPT7i+J1f4cYRHmO$6~I>Hik7)Sf9PI|Qw#0ucC*FO)zM61M0 zHpdwUw*`l2>-EQY6rm=|Kl<>W z1k)}x&Ae062Ibu#mkUj>Eb}LKSk1abrt}r;^?!Z$NpW}L7@|(#6s>6*d_cz=V5NQN ztkrTr4#Kt$zEpT zo@4%3>k`O6UfhwGiA$N~+Bc85H3xmIm!;v6U5 z8EMlc%|R2Ikcy?>!;NnieRnmuw`mj_cyN~7bE_+;q0BQVpL45 zE+`N`YnWqM-`I4wzBE7~ML##8ySi6rpxR>hBMxElXu3T^w4B9T1ZbmtsXOWb!(bK_ z&^+ooP(qh~f)qc04gc*G|7)?sy6Gx)G42=kCfxPBuaof>-DJU0@f>*%@&UY$16wHs zZ%{g4)Con_%FEOa;bn2Ubt4pAK@&yD+1a--%2NNKt8IaTEl2l0Y*zE@t6jTFC=k`5 zNv{HEI0y;ajoiH*MQ4P^=cnug%#!y;yEES{9$(MfT!yH@H2|thjdlzK2hyYzDK%yz zS8Iol>NoW`=as1prq)??a^*$(pW6qO98_*;i_Tl|e*AVI5I$y4>SKMzLht_~RxniN zH8H@8o6U7H^)W%U`uVvCN#2a)x?FJ8WIn^2{U-)KCdN9~K(%D`Q20T`zvH8b9U#L= zAij2C4HD@JAfvEtC5P7r@fz9@?8VPudHx~-G2BGId|sS?5!{IxUXa{+p{80>4^`dFZM`n`@}>f4tCxlFFXAA5 z;Vv8cuNxxROAN7ztbYvfMWPtO>9}9Ybo^AhN!iYVm}O zG&b5Lx!CELasqL)xl=r;rAo65?|PVNVn3J9kel2Q(@KTBR=F7=!~XU=)@}NVrV}nw z3sC0S#j~+_98Z$b=Wqf7<=CCTr=Cz2itxyl%j9d8n@$4o{yis%avRguc>lD zc|J0QPu3%R{h4Rzva>xyrep_bQ|EDf+XPyq#u}sAh233Fs20gP{@(o6^lHM;u{^ot zuC^o(&$CB4SVn#*6b=IR|0P_wa{veNWd#_Ik(dWj-25+CAsGSBm__HW_Nscsv{W56 zP@4L1`x_^83Gmt2UNY2RP~L<+QI&}K!q6{jf3-xfRJ&MP7p0q`g9wT-XQw_#0+ROs zk?#m5P`y?eh1R_&`h`FM^NhWm00%y|8`QVoiPbBo^%p{%s4rIox_q`39WFMDWr&sZ zWJ3u?ezPdSUt&1R*7_9C(L!@&Av`7P)teg`$1+|$iZ$uEn9>u8zVfCVEK=;pfAmOm z`&j}@d>qRNE)Vyk7s6h1$8Swo53%;aQg)-O*#5festJl$-mT>tk!u>g(QV>vRC|Wp zy}y~4pu^BZsJ#e_YUTw!&3VV>Y|!2Ey42LzP2N^TqO+xcv7aRUrjg~D;nAa3a#pr| zoeJ&;EZt2|6{Ge!VBN1V_JWFr5-JgR^?q6`1N&ys?T2xMme=I>3aEt$j=#h=to+CH zl*2N;YS1ur>cM+S_jVE=0#;`U0etMCEmX%$Ifi9L2U#*?z|N?>!tU$|#W1;MQvc7z zL{Wh?VZ3+*tptssH)QW1U~ZO>GBdJf5LQ*CRL%mro#m`az8N^!6GI=4N%|@CwAR+WI?|1=D8c232Hx(`GE(OP`ga4(t+aQQhEY{9f716`chZnBfYFMN67Zf)M1d6 zYwT+oWC`5>tEZ0xCw4hg_Alb>R;LsC zVn*g>Nj7r-MJU^ENSh<@!nj>9_>J>gCA>^}+l-BvtEE0Wwwbtm6}$Uqu*G%rF3-8( zS06p)?HDl}>0dlsL5)##wz1qd7~qh+Y_wf|)&^QCR0=(-B<+Fp3n*D@Uy9XnlgmGX zPt6~^>)E=_sn_ zR(B$pe`LP25&xq9=teGn#AWS44;0Zb8_2W9raf9=@br%k&kxt%_i=BC!WP?E~&L7S8=XzP`d#vU$Xq02A*yCVtG4A?0UTpUCul>o?o~wjLFJH1RLu zMEo6)C$--wzz(c3_5k7E|A|`8jK%@6euXqa4SA(P^A_WTNOFuZ1-|R`F~Ph_ zAlBy_>l#I&TV)U1?w+9;LJ=S98u0j*34cy+{6)~!mY~mI zD;_o91@|ZE0SCqv!BF`_#DhymEvHM`NBYhb9u&-UtBVlq9Bl`Jg(gg#N{of8Bm}e&wV7*?z zJD?I+e7ZDM@(aeLFIq+?UWB91ROnTnc=PS@AA2e7fmluc0<>~dl2}TS#P%2{W$I-C ztaZbtCXjKU2E#%-psCj_0=J@z?-0Q$}aw~%kF=!P9pN4pKoYpW)48F76Nq1El4h*Q_$hpwadc8d`cZF51yjr@u&hUgrr zjv-cfjwUzJffbku#|KJU21>s68Y?Sr-i-WkwJpzbI$E9i&C6(USL^*3&H5}pi5)x) z8tZiK=>N(WZuY-fZ=U0?XdU0yIH3id&XlV*BXU%w-#7Vm$1M_v1oOQNwOiN&F4{y9 zLy-EpXb=7X{46#i=t|n{amNFCd=Y z!BfiBXochBv%VkH3FpBFC$JJG`?s)-3wt`bA7 zGT(esU;Cb4KGN+|#C%k!cC}xayT2o5QNe#voNb+(A+k&#htWXyESTA)p>G&+q~rQz zRNS03R?_?M(yiH2NVSKv>$H7PZ4PJJdi8Hp>%FSvK-gfA#m7zEn$3@zXXYC3{+TCb ztT+BLp5YQvw3NoHbN8&A#-pL&Ix)GVw%|$uk*pKgesmI1Dg^a4jy_M5Z`e^`oTG(P z2QvEu3zuh{99(on%arR3BafU!^8HS1@~v%aliAi{2~wjWdKgWg@(P^-YRDf5Pr?1x ztuB_GxAK&BkA%9M_*U86(2|}|y|h>8?ZGe3ao@V!5cozQhmHBvGty``K8+EAnM>#_rMaEF%=29PQJK=|>@>5^Z0Pp2>Ws%Q}vSxsPsb`pZ zh9UZqX0%S=+Ji5Qh8`Z(Z4$M2Wsthw4oT`lM0p=qUTSx0a`TyGmAB>QMJIJ-hb!Ot z%HRU0B*gtWgpMR-VAi{VRNjF4&g)=zghN)D>aEX_rg5O)9_|i2@DU6#_%O@UHU6dZ(Nh~H$T0BlG5R>NOJk2ff+5%~)t+AYyP z$>?=ezkG&gq)`OQ^ zS$uDbQ?Q>w|6g7hzHVY*Mgxr42FFiZrMxGiY6dwcf6W-_rI~PkKweqwOnXwwrPuPo zCcb6s%2iXumB!Ze7m>#H;1Lklw(CP71bBb>O6a?Aozj}Rq>$iHET6A7!uTUTAxKu= z`?YfqGxi5gG!dJgPc^~9>Uu@M$^8xw7UMttMaa!HQ72*obmRRg0ihOagVCX;;0}fM zdMAfx0u_et>2=JHauf7SIylgxo`P1EQ5yyaTU8h)Lm^JHB98^-rFLAQ(y%XVa_{cG zRLJ03VB-qDj5x&#`VW&nW)NTt^rhi|Rw0H|YvqJ;HESTs&|Lr6;Dhr#>c>fD^F8~V zBRdU+8Efy0IS@dd_$ffq&Bh2MFve<&=p8)! zY>2#N=n!p{`v6MCzDHB70E$osl8_vu%p`^(b$jANBTb-l=R((1k~_;(=3#NG4>$Ya zo?YGYCWiZL@mIV4A`A&s8|2h);CS)`?F9L5A-QRvkv%G7-(&m`W=LT^n&+J`o8TN)oosqB4 zwO%n1uRO7{bVHd~Pcch#2PeLa0<_+YNaxC&Oy zpj}xX5z)RpPT+w^&u?lmB%ZK@tcO{0(;p=k<8&hK47T4dA7n+pUacQ(4Bgw!G7Q<$ z-N^!bqsM%{Q9g-g@ov(vJ@#wUQ@yVs*z@-UCi#JR-dcE)WW~Mct=Ad4q&>UN9#Ugp zt78dJX}=rhuzPTKWj@!G_;$?no;=Jy-Eh$8-N%UBrjI>0j-=E}4C4F$B4impZ=m)` znf#bCBP=A~~O_?JLgK1S}Cds)85R@wSw_5FdOQ)vlW@t-YM-4O+dRUGv{BlJPr4*aOKzh2sNj7+$VCZ)UbjJL}C zdljDdr?KJ?w246i8yi#VzZyb0J#6jNWME=owJ(j^Lar0x_p=X(f*I>yBauu27$o`J zH?Z3QE#+0f@l7DCC7>b+bhRl+5o;Nq-=2+5E>)}>ug=xCuZv-HrO?L?ipX*Vu(cyd z>G^kRUM@*ybLD`7=hBu(xs59cJc=T{qTA4cHwfev6G`>O2qcU5$d2AmNOY61yRJQd z*RMQPnq+xU8PWGxPoA#BwU)fFJS%&O$_eRpxu-lG#Z++w0(BCsMJ7ZI)4S&mh`hjC zjC>a|t!`ud85w_zU#_KEUEqA`N~u$1F#;9ijl0Nf!mEJE1Gj`UL%ipz1w-znw>a~+ zF5#$C?;9uNPLAh^T2wvI{PtxrBumwI&uw4$;HkZ+P1}Oekm(3_O!n~FjBJV1!^1WL zACyuqDsnntI7HgHOA&bBVP=`m%{NP5qqz&fqS^j<(i$ zp~;T((ZveyM-|;|IL?biB*c)9`HRq-{)@PYX3Z{dX&3uu3#K&uVAl2xulIc(G?vKp z40?7g=ozvv(qJ$Wy=H0z;tDre6jL>&@z-_l%6-Mf#c$dWZfs|tBRPR2WwW|~N^%M@ z6O4SQIXgkk20>&s%8%+%qjCd<`nviQvesd3ev3h2yY>$Pr{*D0*}~#E0qpO^35fL2 zuGMI)`3jk$+Lfn5tEjCO_Fh)2d|zAV<8XHAlJxvdRkQ++6r$Fu{L8QZx_S5?rL;_o zo(xO8V2u#ia9617UFZK~Yb90ZB&+y^p4$|T%-bSXanyg35*FOa(r zrg-NUAoO4-xmglri(N4Yj8%_-1k3>PTsVoCmuwop*`G|Hky0Rm|nj|7w`>~A6)bSQC z(!j}?{wskmF#=B%x@Z)8F9lS-QOekk`v0TxK>u?<|KG-A$q-p(@j}bR|LW(ipO_K(SZR2b z4K1SgfzipSDagga@%Vew*Hb*W_hns~Ppv*f39WtqT|^BS>PCuma3Dn8z#Rae={T*_ zPPCYgUZbC+woi6T<~y0ZG)i(0Z$d4*TY!Z?^1Vk&yU=@Eb%8GCuntv^hw>WlVO*uo z6cklSR@Tu}a&@2e9lDWWjd*0m%^uvNZ7>BLvtNx7XM@j!0*Ur#QbcOH(;)4_oJ0}* zw7{Uptj+P(p~G`A^$oHY2r81>_A8i`)EzmTAv4ZcXfj+1TxJAl9|cqS+O~w_-Hfkn zMR^z2t<~K;aw@=0_3R^Wp6?=nfqP80$Lr#2Iqq_Vh8VomJS(w(1?uF z2rjY>rUS5@$o3v1k-D5{(3REdJhM}dl$ z_W?u%1dsp50`JeKM5}gvS0&rsn|0D3D%GiU4|_Ebr1CTD$CmoNXaeXX{HsQs4b$c5 z?uSRUR@q&iTAMj#CercP_Q>|c0t$53VF|9&0Na0^UeyYxzQdjRf^+b@Jv&`0?BKAJ zF@363V|RmA-qeXFC)KWrxvr_<1Qx+k@ zIL@Q;Q(eWhJ9FPD>Y=PZVS(xjwsC8+EFQ4f95ysn$}G)-NKb&_#>WMuEUoEFl{MpX z3r)0vg*Br0(}r*3FV0$B|I_9FLtl)Eym;$z+7dk-a+>bmbI3I6=v=btOJA;Ckw+0i zht$}gMn>vlLN0ky|;WCAT=x-_E`)w ze#(-SZswW^HVBRe8w4DhsN;ZbUf0WzknQkye`j5e)1v9{b7kR&9#y~*{-S_SS(B|1 z=`y`l-1a;7smetQJr)rhc?oE$EpaN7?8OjPYzM#(+KV_@*TzDPfUE6W_Lr3!T{xck zEBkXzvHFAG5AL&8zIi+k%vE-I|DyQKIqpwbL2yuE>j0^c(5vUF0Z*2~;DP|TcG1^G zbBRUkrE^!-`E4{$d8hUjU4c#s&+pkRU*cP|sA?JY4^*kC&0gy|T9-?<%OHlc(GN?z zv>}2f_KamC;8Ygxcd*g||LS--HhW6FUVfZgm9>(3q(UH{Dy;Wr+kM5Ki+GUJKv6;G zsgjgp0>Hr4ycjaQ)I@q#<0!+5n)jzw6z-r+&AuZiQshyMX(TAkzek3sB!9(FN_ETY z75?=QaDH+D!*38)3u6cq2SF!!?dZ0~yYYfO8U17of3^F;Pgg>Nd`lk-4DLA?fnO@0 zT7~zaDVa1ZeS`Vn9bOq{Hw37-Grm*F-K}Zc=FQ{FZzelVdj6P8C;I91A7B*uh|IG+ zv!58_%6aVsd5<5Snw)O;^XKUbGTWV^BNu+Y?K$uApJH%c)T03nSPRgA!&r}q_8TP^ zLs9%o+1;Vj@$!jc$7-oVok0Rmzwdb?=fth3KpSO0Gol0PtW;7yk53Kj8D?FHcjwqn z<3xDi&NtH}bEy;9dTMf6ZUi|o!uWOwl<2f|t)`*z^4IP??+Dt3M=xm@AQo#N{r|+i zz_hoUfq50Vsy0)J_f~Koh0|FtE1O0b+|LJIIeqPG5Xy9|^m(rMJ6*XWGW^$V){|d` zhH4?OQAxOa)FEq*m*05uh_uI{32T{CG$zbQQKjdk3N*v)S;~YdhvM zxrN^Zog5oIe=w$tK8Z9eF396;@SpGio}glg_W|F1HI6b%jA<2lPi5%u*01|y(^K%a zg0=rQ>L6o2=7afXiHkw51LlbERZ9BKE`|!T6~_TCVzh4Gh8`5$*Z;bDiHti3u8^gB(6wUz~iiLu$81e#zWV)%aZb zp?KlEM00{Ainey#0$?panZ?G2VD~z)wVSE!ST-2bB`-!r$LeaMD?|CiVyZKeu0FXi z21Cs07gcIW%z+ul!1RxSOEMJ_>{mH~`2{{Ww>5q+U(ASqCxT%B(`L*0e>oCkLvs1rxe7Rw>g*aLn8_RK zf5MEh-z*%idT7`~s|W*AzXjA(Ik^6Z2A-E;1j0_Ux#-!_(wy)B^4w8OZ3N%=f}f#T zIDLUk{^g}n`9euV)au->yaoxT<#`i{YX8F_F8jtm>v^UOi^n)LrWGSp*GhCUD?REU ziA8LU7R4Q;FX`&AS?Jtgd^V6KMOU%+^|9lAs}ESM;_HvyD9 zid%MLcWn~bhz$LV5sKs;ws9@RV=>>Eh4qa!4art1`)`#nNm1DqBgW&6Y&UzD#8cF;N&`|J7ad#GH`$!6yp z{>fWbdhS18oXRiignD z*_Qci<6?u@fkg8LbuT?+ECokBjEe%XO2V}YYGtt>UvEOi_1B#`kB<0_?VrLQmlyCr zfKN}xd^Z3ogM%}~NNsGRXo{g9gNIRk{c8>DO@4}@r6B*z`!JV|nG-T;3>Pr=G$5AF zLd|2$T~i^Fh^QD77$^e?gcBuouHXqvv7a z{b-6Z86Bw&B@xNap1R7tZ%JfI1~4FJycQi~e!|s) z9ck@x`>X3Jsl+5)g>r(j30{8i@OACbWSNEQ>4we0Ex)aKmG87Ew-CLsum%R%!=Rhd z)>{Q&EQXp4H3{Sy10C%OH3`#AD>v`}F`vFZ@{C1ENV<@U@I3>sI0rk?Yp88Cw4Ank zUmA+ZA($%zP`&UQ{Na$JpYOerR5oh);$K!M%jZj!jh^pfkTg+TU$?68$AFefqJU9y zo~9PG_dM(iwtzhfHL?o`<3?l#_X>HSSUR|_&{~vfuz%GsZOF(;{47cLx3U_-1I_^F z)n4Kh7+NLf;Tl1)^`1bJD)(4pfsIYK7mfuLN}1Q{zP!6vsQheztkTOTZ|(5DUB*sy zHDsqLSOiB_Mkqk9!Gx;8Wz86_V68lSTyq2ez#5RMPFFsu9M`$uvg*NT?qeCdH*M3V>TK%}fhlwc z;jy9$)4$QuGpHxYToIvicv}n??9}ri$+o}9YkBHJ`4PJ!k8cxQV6WnzbX_CIocOs; z!K*I!gAsm$xfs&Sa~Z8Uyokg?_Ac7)A#?YYAooKQIlDSJ9em)PYvniZxy^~JTt!_5 zRY_Lq$QZifQVc@~>B>G1!THmCgaY`SNl;#de}B;J76Y7j0i*9Ttxe#@2Em-rL4;4P z(_=F}p+69sS0d@9h!K6PvDKx&}av;E65?0EB64 z?<>-xDB`rU3d6wD~Lrg;FAhI5h%| zEMe{gMw}il524fVm#~5>1RYbn|D(_HsT~pWzt2=vJ&eLVn<+6};6p(eY=ywGICzmh zNG%ZO#OzXXjSR5`N9;gZ8{--YIxNIERt<0 zZLNwav_8dZZ=rZP(!>%pyp+5g|CkvL5ng{JW-Hg#{3tBFmX62JGF(nuRq5jJ$*9bCyI?rLx5#Gpk5Xt+%H83!xu&k{BW$p* zD7j;2>HCgr_rjGofE^irZ&HZ}(wzwZ@ujoQe1zQqT#t z{<)utAQ6B2U&41T2R?a9@fmdp`!#M;?(-cZOqn~`$x zsnf&U3|=v|#<2lm)=sD~0GfJR2@p52Bl{QcdR9X{t!U|1&8K8fzw1A2o>ym>UAy!9 z^d6CplX*9X_l#R@G_xXujcAM}OvTq!C#vlq`@T?B{3yG-^A~y1^#GcKeum8-+>uy? z1oJa;N5=C6TeM`+ud;!Fj|d@7y+CxRb3c+wg75aP70a1NHwHL`(8O%2D{JG|jO6e; zOLL&JI|!A;sb?7K1wsPmgCbLiXC|PrJHhF61^j%*j)gU&Qjmv*UmuMc`Q(p`oo<_I zmu%hl#k*aJzqdRho2v;cc9#Si^gbm$oRp;%#IE&-E;@=+iyw@>ej5my3(!%D43%<2 zM}rZm&yMc&voy) z^XPogg&+H8#Jl+c$d(AJfH;+msak2mjSROT`)Tv730JOC9q+$#!2V3f(IgO>chO8t zrt^`SG1&t@f)6GF-(71b_@y(N2Zw#2`Z zwLZK&wJURD7}=2pwm~1T_?SXbQ}gllszt%NLpn}cu^NvD<+6Ls69>HpuUS_LcCvb3Txwp#}w0###K(pVpz#waOfs$TGNO-kzb{GQ8cFgS52Hy>Bj* zK6!5^=$Pt8f6qerW2iZlw+susGTA$an%ZJ3G9SN^5FfBEF`#DPao(E|d+1u#4t+Lh z&o%A;z$Ih?bsX`T@Jk*BMBRhAKA!wfi;X4IH*7}jWT*I{7LJSAjHvw3RjcU?ri=~s z)gN=%fvVbmq>Xs3*U!P~Y+Z10n_YGBe3;uem9+Nx(`euUQVscZrPUB_fNavfjiOK8 zpLP(H`sY;XNxwB$-N|QGy@NC&Q;>anFaxLySl+F%uu#ise_Vd&iwXVqACYz^V`Lj= z*RC3%m!aC#1>KGp+Zwj~ngqxzl_3Qz2#tCZ&({;3Uekn_&%p}$d%nuYp6E>&|Fq^% z*}X4PDk>dQ95RpnqRNzd%DA+It#V_Es2afrQ){V~m2i-{qd*_!m2BKck6+fULzAKF znb)dhXMR)yEE-Vg&d14$ZI496@qcF8id~_?z-c4ks*6#Xi4ne|XDy9uuKq>&8`ZyI z-0@)fD$_ITd+2JcOm*V-8M-AE-OPvX_QY(zNnNBIJ4~whx_xn8j{QfvYs$G#b6@Vc zphTE0VElTYvqix7JUhI<|3-3&7ov#2@oTry&a%(?&lULwgyxej4zE(MpDoidisVXY zYQZkBT*IbVa{Bk=LFdUT(M+8t8#{3ZfiLb+&~`3q07U5bFJU!LfDZuF5>)^py}O{v zQM?b?=iYw-**!Qlc!;2a?W+DgbY4sJ;pT=~TfP*ppnr9W^&QsK?p=WC2T?G9XNqDMz-EA0#0&5zn6!rE>%2Rwuvgf9^!>>nbd+3GL|Hu2OU zy98ik1ZoCEIZ5z2V!3W7-IFKCbdBU47rIim!iY9$< z4HxNMQli6M*uMM7(jz1JA}@J1i*wEZ$Nx_=!e`COM9@e->4R1)A9`9foT)JrvcI{F z>S$-{X&W5yzVZ0`t3pa0JNPQTJKu?v`v_eIovo_S(k$^08d~ono&{BK|KyYJ5}QoL z@OFss3T%z0rg<#BV-+$47|P3a*RAV6=G@Hj_n-GSuy^_hs(Pqbh`^{_{z(Nr55u&& z78x`wM{U*{R>yLBWTcDHL!~Ptr@!8_IU%y{9;G@X6*X*eoPsayBU93LQlcUN0GgTFiaR*nQEwD=!drCh%nj7 z6%}4ko36-!dWepF_uN^xY_0^Y2L8eEW<{=TTRcV)fy61`D++;M3|oB&xhE!h!RQSf zhmA8YL%JIn_O6gZmG#$|irZt4X^D^7R)?FD;Qdw03@X$v0a`tksZJm|9egggd)|7< z7K)-2|0+gv2FS|1^=$^3S#D5foqpZ4#R2$s)(x3kX+5_;QAJ6upr(#U{ghm3V zVQ0$b;Db)V)njk)cgDWJvc3x<2+`=WuOHQHC+#myDp8qtM0xv^ ztX0Va15+5Q}Gxvq8njs3%k8MXF2tigNNI0jd{Zu2lrIGO^G3T(Sy7M^6ph}ku{}D6$K6yV{+P_+73Y1* z*3i_QdXEq`#fnfqsF6S{ZOCVULaIhXzn=I|B_WS=EZ!9Z@3_mXpX6=Zs*tsgGQZw%%`j5C5X9)?MFee_^+*|Jq*ASF~_hch3UiKr_}T%D(+M3USp5q9FyKF+ zWwoQrMJuGDjK3ATP=}J_yK4xl^S|e6pB$ZEH+y(CC@Z(8%d?L-RYoe!p@5^qd6!ic ztV*3ZN0s?FMY^vmHW1q5MeuM{FROkMepti(h7j(WurKYt2+k90t#1rMMb zK;Jfl-m_hV(j3&6n(cFN7!zmKx+2X{*2^3?4B7f_ORRk zZ~vK6WIF^2e8s_sL2tp{DH1II>nV#5#l5~%FBnN)DnRM5wviIdR|8d7-E@6tC zHzaM^LS4`rp?hEe?){!st>Z9FIcMIFD7ov3mDNA#j{W$nJe%_{f5OR$+>{EDkGIVKz$9dy+n~;3ZUb)MT zcTS@rPy}_ESmnkNQ)QOvU;u!GI`nTBG^ik81%eq&JxPW?)C-dT-D==eFe429%M~mN z1pNQINB*C9=KqgjLjN}Mf5!PD6JkoMFz(urn$>e8$aRSaMMk-UC0|7UK+P8JF12i2 z@PB_?M=+4^Jqg~2W=eCu0w?dBP7qhKYP^JL6oO`6XxP*HYjq>fM6N|O^@)?w(`@dr zwq1E4u>28>3Ut`-B^kSWNtcVJZ=O|rb3H@Czc_b7FT^E)-79?v|4hes?{Zhyj#JSu z1XfC8fB!{&Pn~gC0eFStOa(aRt!<;z8~mTfT7zisV)BjNMwN-jS85M>Ux<7ZW_ny? zDdWBdo3sVnXQngUNlqGBCx=$en2P_j)Rm#OM|s))F&j{o-_?whSd-75f5Dz&$Syzp z2~86*`?Nqz($DoF=EgUc2cu-E! z3iklaIxixUVSw$k_$Rlzm0#W69T&Y!nay+;I`9erRgGEf_BB#`Ayd9NgSok~p_L;8 zeXn~1Evi3eqgl1|z}m`mH5l~vT7w)XP0qZ&;oGn5uhU*9o7Xea3Hzk9GSqz@C;RRg z&x>$OW(Xbu1M`Pm0YI}=YmANnb-3Mk693XGVL($m{i)CEPJG<@16|)Y$&Y<4d{6%% z#%pK9>##|lmoZ_uBI9UalU);t(2_X?zn!*&_~1}+^GVGE{q_!B?c0uD2T<0(&PIfB z>sYk`#NL0Kk>J*hQw`h1p=FFSG{!?FQF^GxCh$XQdy8d*Jhkm@jrq-i_CrFde7iqX z%A63Zeb3=+_1bC%!xYy7%_D!{e84{-gk@=dA6{<#XR1WVBw*(4#*qb>!U5!uIkCEj z@$WGFi<@cR%1a}U%Pjq-Un=MdKjP4oXnp^owCA|z21r~(5ip?fFDzV2KSrplMRN z$}?+hEMxRd?yEOTU)vvEPH73_38Dw;9E{!ie&DTryMLJhSeOYMk=kU|g7Ewwq z*Z#yp8>AUGrqmb+&v3vW2zV`9I5{Y=5bwQMM3C<*Kl$-&&sTfg)ol5jB=@eAys$0< zkF6^D2x;PMF=(i#@94#>`ptz3nQ_tW7+Xa|Kq|Qq$C_tc2+aVSK({)p)%JmaM%wGQ z`S$Lw;MZzWrpMmZS1W|GS3kVLB{HavOagDg1{wi{2XChn(C6R}GnRc7UwlPtB23@E z&oZk`+#4pJw%77#q3}bMgzv`}v0a#qZCz~pAcn(B@V=p(50-VsBi{)v4!$DexFz%Q!PXN50#?ZD8wxadKm48wF^uAGw=5dsmt=bZC zRHgRy#jjIy7Ho=y3y7%adJy0l0PP`lp)JG}82cA>xo#B2m`Cd;RcElUHbg&6)r!|J5uKUVEe`a_n=p2lr|U|6tgU#O zdn5lbmK*!{|C!MrRF}RdtlD#Br z=>O2w;*b9|QSOu%;dsLIWCH@$&H>JdHKGwC ztmCJW5vx!s=43vLsJR@qog+X;gZFR5>xQr|wm zoJrmTx(y$lkrFgGA);^_1Iqg{SDJAQk@6mvd~Dqk>NjyrjsBU+wJb*E0y~o?XB%Ze zMUv3$drMSb%Pv>l*X~r#-j?+C6UXo$hye&Hrby-OZAs!)OeHLAfbhYtL;jx55f>|m z{7!asKq#K4IJp#=7w-j|#T{BiZ?h3B!F-HB?x%vTvfk?MX71&^PHl?;Wn;!8Li%83 z>g#H2DmaIKAxkGUcNkz6h1@DaWxsKeK%w1k9AyM_!n~eJyghF z6ga;6$VAR6WKaD48H54;qGWsj?`!{gY4{VU75!g)@_#<%e|j+VzcA+iZ%_U&4-Nq` ziUZW+MzkVFg0*O`vtA)vxMl_|IPOUY&N>=rVvcB5vHP3l7sNF0LN6VkFw^vX315vS zzw=<@R)HWZrrw!BM?yEzkrJx$Ovz$vII3B9Uxo1(j9J+uBfgef$7K^|UUEG;!^L4` zf{g}3#F<1nN(er~dJ=kssWOm7kx*@Ra5Eje8{??&c3(rU?vIz?fN+JA%^};%_fyu7 zrH&H4Q}DDKV<@Pn3!w(Ki&C%=0@m$ zemA^=>>LDNw){_f$z_597Ldgf4u@YwTXKMY)~=N@+MwVNH2l@|@E z-qVRRc+R2W!@-`aOd)o9^!U56VBt`G{o7}6-(-hBY*p_tfNvQN!Ydi{18fJOZ2(Mk zTdeX1H`q5%FgF1_-9_-vpO{yB{d$BwhDQzV|04IPOUfPYP_M(5ojaJQZqHV0B>pt| zP)mI3{V;)_qp=7Nz$WscMK$80@2SS=C{LR>ou#3pN@g1R2^4P|*NUh77WcoNiax;s zn0Jz~^&~1DO*uznH)JLJg;1^--tTaM`TK97`$w@JlLyPomAx}3TJt3A(-EK3+zIlv znWhJJi9;$~>9NPc&>tULQPDslivmT28O~YXAhr=-5&?7yrH*z3gfFhbgVU?Dw(`jX zu-)OeFJLQ+v=qCK5^c(!+avePx*R%>b$_gHU8Y|rva`tNT^JO))QkO8ydJONb&2ko z^&9}dR#*F{+ql#O1Ue)dA61l3?NlrdB|?o)f%PDka0>vufo7n44f2DMjkYTfyw~C; zWQ=sW-Ek^EDh&(RN-sBESh}#QT2OMY6$;(&BEqzT8gFgU#!Cf*Kf;TdLDe+VSPicm zrhM>>4HX)CT>7Q*)Jc;M#O%qTQF(i%efkkEIG;cNURGBYsJQIK(1(B016XhCxn=}7 z0q!;sWznk%uH$2lV_(wEe#s?^+bPlNZ*?q86&47r80YgYbn8sD#ySF zNXxhxOa!uTK(#zhK;+D>#La#gm;4rCg%+V}5VfncGthU(Uj4@9+wGa( zQU&G@0!mz(8p0bx(;&4--@xu*dQ&Mmk@9r`{jvkAfphiY+k9h`b0500ho2U24dDa_ zJ|65^^Adn|4AOXjL-I~LT6h&CYE7o7>k=xFN!Jx#vEKH+QdKjw^X2ewDmiA@IZb?K z8bgv~UZ>?ui-<9&MwXgHrA-FGmYXWP{>EXNQQ!| z{$Xq22nR;v)O&PUXet>88|6`RVwj3=q>axH$jE8D2)HcuR>z6A%k1`5D13*^Yog-6 zk+R)XJz_J6eo5Rf3g}r76a&d6|=%bY-ilNF4i@Qt4w&T+qG) z`}CUL#g_?fw|V+B##KHHavI#&(w&?@M}|uQBec;ec-K@XNZWw`c9Y}`hOJxMeK_EV zMr-`g@P0WRU;RAIY&ntYVnq>{abs2LDg+-(ZOMFwuoJ0z?J;w` zuBl(D94_#%v`0_ojdLe_))JDtr6j-Y6uJHl9R(R(-EuIu0!?dRK)~V&4_WvJ?z|sr zA#(c_+x|I+4BR9f(%|u_Th4g&oLWbqwb!kvazr$K2?crUf7gGZkHNHj1HDMh>s!^c zvP}1Pc-0n(sK_-Htoi$1?~>&w@+)q44jN5cwarcv54!AqZ}^p)FpEQkRL&s~0kiT4 zN9kKmn-;~_eQn}nQ_axgF zC1o|=lS)~1hMEXq9m=CXc-kEuXZshGSno4k?9}3Th^9A&^{{>T`Fr5(Eq;;P(XlTr zB=TdX!bs(&iMk+bZ`Z(0h^#j)bia%*YpD#C8oH6_Mf129^0_0` zt}7w!?k#2B_`+A(X?HErKu^AG>Ol&d(=o zd0?b--%2vipZ)O2Eu|9wHz@YeRSbNLryUxzghsK_^$M)#^97>~UtR(-FiSGgb$+bQ z+b3Hses~GMQe%)E_sN*CD+qVO6lrAGh$fm(u5J)(z80+T&K}AOykax@F%Ro*#+EE0 zE~Pmx^Td5enJcz4K# z=|SEk>cyVe23dF>oEHi#J%bThI_4>}pJsKxJjBYP0pF)tUz7Rupu(efoMtvpo*iJ% ze$2LonlE^oX$UYVR;H4eyBr{z-zEee0Q~512|R>w*Dyrz400$`Gwn$b^MFL1U%u_pa?^i~?4?fMDS7Y(EQw{Nz#sc_lGaJnoEV&824< zoAQ*q>5Qawg#%>chiw;V&!!tdKq?%7z7+REjT(T%D+DuxP6$=7!FO9wD~ekH$5P)R z=d#Fyzr21_K$$k5AI9D1{ravj>1mbl?=a{Z07!3JftA;p5^sILy790|{RnZBT<~)z zG7mw_V0Z0YBH_BiwB(Z+LrX)6V|7>CC0y)u;v>EUFH+|b-e{U$Rg!v)js>l`Im1yh zaY=LFW&d6GnfgqW{#wlQuEJf9xH)#h-3WgWPz{bank2%*M?2ZPIUG}AHzDxJ*Vg?W z=lzWCsy@nO*JOZo@9krK9m`*RzkHH^Xr?O!`_OV*i92+Ha^c)&8Mz^KjNpQp56OL< zo?$9Al&TLvsOR6vE!>j=b4IOCm0#Wm`C`h}V^!;MRIx_wp|_aP3Kgf*??2XM-6+UX zPp8#Y4zlUn#C$^WQaSww*q9Aq*DeD8C{2|elrX9E6IXv04$rEDeuVzJ%P4;|s(}Mv zKz|N*lSwF8(KySNW)+JQ?Y>kNALioAEuU)s$-?Zj(ZV}Z4^|YlfGI9VbF!c|Wcj^g zx=|0OzO%P&Rhf%*x6;G23U(nQO<61o7X^v?tQcn?8|7Jlp9b_Fdy=iHZ`tE`TA@0a;0%#?v2FwlOos7 ztc&P3v;h+#T+SLq+%e@s@7@uZ@*h#3Kyw@3b0!nTLjEeCBKG`6{a7IW0y)Oj0tOgV_n* zqEDRT%?5F!4;eOpQGKy2%@;cegqQt?=BjHlm4ooR>r*-!%{8P_jHcL@Jm-OV1uF+E zm*r!>c=|E)nF3WcmC3Kt59)XoD7P)$?mCkt8@@w!7-TM~_=2_-;b6I=;7C}xYh$C< z4z7-2W$=Cdr-y>m53Flqo$G6E25j45nKF8$Irs3KY`Evf^%nw^_SBP1cj(U)U|d^7 z1H?kN;+xv9bBf(19^CM>8UZ#0 zunPI9da>QbvgA0GXX~yM#gWwCg(2B$v2=h<19%*>n46fmN1;7`QEWOsGu?$JDwJ^?3u2qh%H-pT#(5Q0 zDgFu8vj?w>qq5mFn63CSL_mO5tbrBru(G|tJkfWzF000+?1LA_xW=y zI<#!_$p*m=bI?`^s)~S^mjU_+AlWuv3EhYP*#rRk+-U4gL`bk?-{9((b4{6gUkOqN zWMf4mr_PBV`CKf1()wEX&h&XSu-MrF>^>=;RE42zykd&OHTifq%W~R8K8b$mW$p;% z)|oeMYisX#B01nl`U$!FsZt_d51hS47&WKSt!_e?zAh@G702vXoN9cnWR|} zApMHu84CC=`|tid)xO>4mqNV!IsA7+4*vwVVSN(JT#S=Lp2gekP!qs`&NSzQ*5~04$aRrg6l^U>+){argesU?)jy% z+_ytj*Z$0H5Q049bmHj7o6F z(nL=_?M=@>jO$7QMV`O+LtJC#gAXbE{KM9=e%*P<`FLCVZEU70af*r^ zMS@21!e_`I>klvYwgNEcA0YB(M85Om&i%Ha;e{loWGdKTJA*+SD?BZ3_Z3|05o z)M_Ni;AfhV<23)b(_iGQs3kFeh2MMJp+Jq#H~aL&f6vDol^Tw3w>+yq>zvGNrUMf# zTsm}+BuaEAax553;aVW>X1~HS@|*23OiWws{If zn~S;`n_lQe2x{y?Qyrja67|AglrYQ=WFgPp=eq!kLg7J;x?-$l1m$+?trVSo98t=D z)c9IEr%l?u%uTQW2iqw*s)%26T&Ust$q;NBftt z--$rc>q$h>bLiNtI`QAZE0Yw)n6CYNCezalIIy2Ew@fz>7;KUU9e+_XHtMx&aTG2~vaJ|geWKgRJJhqU=^EabbnL^pl*}2~M9$EF z&{V%w4d^oVD*)7+NBBRoUg)ZaPC~p)cG@`;DquZVrU_$9sfDjIUITRK3G1GqJ0kU3 zN{lDR+J5ouMITFVrwhzYZG1=x+->=tM8O2hZmc zk2}RFm*F0NwobWpT6%$<9v+Kay@e^OU&FFNp-x~~LE34m#!-P42~9nNxOEQ%Yf~pO z;jEdE;{3_<(YIoP3zxgyMP>q{kIw6MZsQeHOI^f1f@piVC@~R zd}?^&G0N(98nAnYz-G}5?dt=>GDDQ{xy{5K!9Fu2K0L!FtL2{MTe6A7PwR>!mu92$ zZ#wB-T{vTz@JVO8Lr2#amMv)TX(cp5fT$5jva;!t2(mF026l{&*8V#G^a2YI;ZeQ& zQTfu9Zj(Dcht4=9?1n^-cT0Kc1t=9T^gi4#E_#{pp;q1E_~N6Z=EHLUD(EE`5rGEa zwgVag0d2WyWbLJ7typC5sfmt34a0_B+;!Z|Yy9C-2|atCpOz<|t-2D6Vx1*Cfu_o^ zrGz-hivaIeZS9HF<%n#XX`S5PM@BwZ)z^sjIv8_~i|Lv|PxJ=^MTKJR-{D-wJ8G^o zSN|I!goI(H0MqVeK7hLz!eGPWrX%ZKjV;_`IN0CZ%(Gm{3X03}efLLS`(4A7|FfxC zH-Lsd#|tJ<7V;Q84v@RN;6<=)b@Ks$hUeNa!nYgDu^rQw^%TUiz}iPcq74doE(aJe zoWkc{QxozD(aS&jD^6Yb71S+RCa>8idxyhk^if~sH^-}2|Lp9ZLs-54DyxTCU=J!n zjUmW>s8Ll{4KGPfYD$nh6JFk z6+0--$BcIC2=cg2JE+?>nsQDSif) zdPwgAST-(XM<5vVP1b8PvhzMD%yX25F*AUH1o!D4r3wT<>{{<>3?)S|SgV5bQOer_;oG?~yy#v_8pt`+9UEY*K_UX ze|H)1NErig32-|R8c!~S7W@ZU7FNFtfrEyJy0FAwhJ(b_lOD)B=naaX2#+!F%w4R^ zU}OF`n(*{WGj`k3pVoN;7Xm(|$F%zHVzqp!@~)NG)g%#8XFiNIuYEwtoE)+-OYTkM zMuXtV{ERBlq=HNcaKWP`@nz=KgY^c}aT|WpXMedJ>g$#hJ>DsQfNXJ3DU1sR5}O%- z`!9s`n7A+tHE@DbKp!ByX-gRL3b8bUC6)1s$ULl0LQY{@{C2akssUIc335^0JfV6Y zg6-V_jYbk2^bfg_fdW+(D7XYIVDVegq(L~z8$(?ujTj?|##2~COak{8B}34McQLSl zsP)az8h{DCKvhj@%RD;)%$8Fnb{9Rcu8PUTPXDo-dx((ok^sDq)V3nT+z!_C4U=n7 zNA<1g>j@7;qpzH2BL1v{jpIbCmk-Yy*<gJN)lKqLp{!BH3;8}i+x0R~A^CgQn|JwfLMT)!_C7`wnkok# z$OQ*u(EaSKig5wN4Z{Hv#n3?N&-4cNJ`RcQr%x)zL;Y6GmmklyKb68oN+w^riVEHf zHqr-ov3nUzF|DbAA$paV;3Cy?Sj^l&+@&&pA%GU;yItgBivH8%lFtpyIUz^Cjjc0w z8q8`o(gbFvof6cDE(1kQuzAC&p(0U>ia{NWQ(n(>^z}dTDUfoYN{OP7x)N`K@UahP z9V~`U@Ir5a-kXA$Xyv>K6Eblmdqc%rqHT1(rSgycj?UPF@(u@c^}X*uG1!ZGmModP zehCw-JVSz2FqC43Ceof$I)JA3WlC2i>YSX8gUS6mGMTp|(&%+MhZFnGar8)Czs6hg zyFU7zxRp;JBxis^j(V=gA>2-66Z4-4A1H<17iTg6eB~jHx68d=;}rv*Uat@0+jRZ- zq{5OP?L4yVV}+nWFoOde&0GzT&lb!S1DVW1dM?6U#kdeG#(EFkAadj5rh>IYh|Mmg z5*H;TZ1M~7A-Q@&?!%%>7upY94|Cy6j!-Uy3=>N}p_^HuWl&>gro~_nQzXc?K5=x~ z*cgjiI;j3r{Ag3p9k=&tcR4F7*?yMKyd;q9Km%^na_Za3FL!(xCz%J*gQDE!{lD$) zE|W^Z|1!BpSe{^r2P?9Y$|)E6$+%_`H@uT%o$T6RB4(O6Ou3Z16MJ)>{q;@u{sXTd z5Ss7?n{WM;(jw%U6d~3&I%Q3Ka9GwZ>ClFWEP(9VzF57DUwbNIlDO+9V<%YFA6sPy zr2)&S)Ul>fRm#yR7*VXO3Z#FLYf7&L9!ls-ku*YZUCfQdkb-=5*>W|ic z&Y9-_GiPrgc1zQp9yv4)4u25gsEr!c9j31+FRu`n!EVhEuU~f3#ispPS*-f;^#q|& z30VRpudyT^;w{XGL36`i_}->!w5}Mw(CGJ}Yq2Ai17oNgK~@iv9<^D#!avzo6rZS6osA$Uuw8G_%7kGJF>f}A4x?{ix$RC5%h>#W5J}s!*E=hk zW9G!C{Y1%7twk$au{KC_gk%TmRcwE1`F!D^J(+#j@^xqFGeLuwgkM%5tCQ9siFE1; zRs%8ItjYn@Qmx8dXMJ{${s&|0*H6OozaN-;kLp|{v}4E05FQ}NIGUsArL;VmV=6Ga z9}Q}?40ttmmF5jv;FRdymf7l8??|nSJ?cDgw!tRVb?9hH(UQe?cG{H;EIXSew8;e? z0PL1+Uq6T1|_^eD~&=!gUD0H49XHPUR8Ndtpy9 zJ~$A>Qbe$y`WA1)l}&wpr>+kOefvaTzY=l4%+86@6;b8C1BDJpQ}0eD2LO%Lc&9<_ zi;t%dj{hZp1mxo>pPXj*r0rorn*;YA7a@av&d%QY(q}|AsIh%VEI6MgqS%X}yO{Ef zWpSz#y?Ic*ZY37W8!SFE_C-iByeY)FTk z_U!1TPDC=Fkn>l1^V;{qdqB-%oJ(6B7#y|_NzsbDXlKC-9r0#-bI*IP^4rO?Z-A{PkO^LL=`X4mct603G=X^j zqSCLjt%C5ONxp+M7`L{ryuGQH53d+W&^2Y!ApV{Gf&S%3rNYgqt}erctBJ3O&dQ(@ zX#ZanrH7%-^kUstL1dZeECZ@(6L$N^R}E?H=}FM?Ah=d#s);X|)^~o82h|@Ee_Xho zx9bcco_P+~(=U-O+rz{VH!p~cPR#uHF~8+eYl!=c#X{V@*cNUlp`~42yFQI`L??x7 zDLo5g+nm!2Rs%bCOGiIwK!H(O=pOXQ-a*x3?$;Hi`6Up>yvm89Hg+~&iq0N$@w~#> zvhYKe4P>uI$HGes(`GTa9Qec^x+nPfV5C<&qggtXG%b*zV+6f?m0-hEC>WU-T@2uO zHrlPwIw1b^G-nz+=lJgTx4T}TUsW(+@IqRqML$fS-)XS(j4X}qcf9|L!>!9md|BMA z>U4{;p!TCYy2e_2BM#O%NjHNVX?ABoJ7)&Xsqx6q3{N*!&Gacn)2IvmPphq}QzRE1 zk9S`;-f*$>_UdBL5V$%+hiednj z-&?~3tJTPXAv<={*nzm@hprPxni!75hsz->im3qyz22TSq^i_^u_B)1yx}(?BZbRy zdWV*~-ibfi#-vgDV^DZ=M1l!fKyFi zoK}oFMi}qF`DHkYY2wNGI$z`I0phKP*E4U%XiuD>4>$F8`Ri;*K)z;d~ym~htNR`-i_8ktVtRmF*kb{rwQ;Ct>&<8AjYVp2$o-s=IKZ+ zF3F%D)5M}hQjIcNYyF-VXBXT_<8hZtU%xXQ!10)0#QLRK>&UK#R#qPII@n?8O7Xt2 z>5OKG3xe0mO{NSKz~>hZ@GJum*#!73rmA7U?Bj ze;C(GnXqj~4l;nky0@b%c)9S8(~oP~kCfNC!O0F+4egwi7%Ex=oH;9GlVW;~w*A=W zOpcQ;L5173Sh~5DUG;6Cq2v&QYp*iZB;ZZsCE#t;#=hy*FS99AMsx1)(^== z=UpYf7xY{BeNLl`pvM*=XzL0Zf3oKrYx~APr;wfYcH?KsGuc{fX@gOm7q)5wJz7za zDv^2*V&)rv0&oe&0BclxX>EfCOThj&S5KqgVX1CW%zOqin_vg3uPX^l?}%wwSgjc` zXMyyIe(_(_%Q34pi__vxx;lWD5e@u!g7`nYj8ru&Nh=96YXHEa+aMOeist>s_Rrja z`EdkM7LJd=J(bx? zN>>>tqEK+dIWIvMe(k^#jDVJ z9Z@5ffaPUgFR@+BG;J~6 zwv?InGrE|fqQS74f8L{a-iGHWeOm(%%i&-=jS@OQxpT(CXUG)cmdv5IWL05U^OxOo z_7>60tb%2u6VVRm6w0*odb{4Yf3@hHK3iOM&ybDjG`t3s#&##DOR?SYqw2YYdQx%-^jg@OEjl_-D>Z zn*kTYKF?H{3r+bq5-jD(<{BJzzmh-cKG*`s+bVq9CCM+5nsuyt$lwz zisMKd=}Qm%V4fZVytyfA0m8OOWCCWEULvdkvV%UY%)O>R^A@0ND_7OXx)+_^1`3o15a!#08t?1oaq zegKA9F^nYML)Mk45p-}aBg~0ze0S(6G}qh$q(2ke6bDF)DBWZeLz{`7+@}`K&VTmT zCt9AhF2y-Cil1?7`^^Tog8c+M&&RKkFBrN-Fz_Y{f@~3zGR+F%P6eybhV~s=63hGDHmK@ouO`wZZB-6_)xNa=aDJZDsqL_I{Z}S1Gk%yMhp`*$*Xu29G+W;L* zue&DaTV>?G1#98%{y5+==(nS&&y8K-YDd?enP)puldtN3jnO~CCP8mU@H6?5KftR>cFVj_CyyB8@& zD3s9)9*wrpUea&emOS|oUmy?#s<#QL(sFAIA;un!oE7b}Cs@umcd#`+)G5U|1o117LOrgW14r&;Er_o9FIe zi?G6&sSZ+9gFJk{$U{aPCQqC4!lAz@Ki>DbQfC8J3eLAHb3Yq@nTk8zdPbIa_IC&X zY6qucdl3_y-|#!31DkHG3C7c!HK2}~I_^CxQeJzl??wMF2OEii5%oxOIajlSomxWY ze8L0#FfU3h7uX`Q7iK2|6l)-59h1($G1G2TNxV6<>~YU3R%gz-++FgPueeik z-$6OQCh}9mlZMwjFez9Xch558FUp{iY|3Qshc*CsC^$d&LD50cI^_$%-48wZ7d7B2 z!Q4w&2KucVY`!S?BtsUALmd}|I=cP}8>EtWc1zwzZbRyB6zZnUoD zNnv^6ihC#lpq4XW0d@uGG?scEkYb@5(cigdh6uEC&6qvK4Vrj@Nf~2%&Z)RY&W5nE z!lEMD*tqczqkY>7e+a%~CmUbSkVJOsfhBy()MnKaJyryREqX)udlLC^I`#lMXQ+I( zPrsP&%OBw%_WC_KUVEU&PmpVpKIXe0nhDI65{V{?$QeuG8q|q-L}=-Wz*3!inz5YB zE8o^|dxBj6#)FPp!9>w!x z`I3O}iNzE&L(GG>i13^m{S_-KSeOwI`dN2BjGN)ixJtE3QlE7VU9q}Hr7 z>uq2?YaXyyNlW>l(rL_tr3|wUHW7$WV9MMxPku&Y=z9V)?1QE%a*9xu6`IHOqX zuc}uhkY2O^BI%is4`6^C`M+r>=sga$}l@&?iWvUG2zr)%r*80V} zdS(^AP2c17&DUlk+T!wf>iyGV2mT=}YKMVctaoLLu&5Fc7S$#3;LD6@U+?1zZ)T?F z6uAe4m5Zfoar@wobG%hrxKO!6E@EKUX^Gud?8V3+#KTl8a&RBbXh}jXZa$1J=KNCJ z14XO+U&Or!P*ZKYE*wNex+q0J2uhbGU1dCT|q&JNL5Pc zARv)$LwXA>2`JJeLB%C0?|JyX{m=f-+25YoXZGwl&WMGThqc!8-2J|<>sao*=}FH! zJ}mZo4;X`is`Y^aVXbo<%Uh zEcg=ok9BkZAdPbxTB-ZofyAu}Gr7Qf-}fz&gHeyV`G=|Ci#$@v^t`&ZM#ztwbO~(MZQ5mMpUwa7rB41HSRDMr{|mQ61<7~7^UkXI z4+x!q!05Ds(jD|g27s*M8IKlt0Hx3bpj)p1&nBetNI1sQ3HI1<_dcbE` zcKMPzJF#0&@dPd~-@|B*>(If~p@_u6Kj#)U*g%TMLzOwxjTXzfsKxRW0-s=bK%GVC ztvDM{nVkAq34VwvCePuk4P8_V^x}9@1{n`q2bqr?RvHl)U9k8Xb}NN19#CcZqkfLR za`+*Jo(9}5An*L39DLUvu(5MtS>SuNjiZrQNepKsKOD6wo;h-_r6^m9yTtgYyTX?X zQ+kdSR~7~7&>|XqrXFhg_4wxy?(N*^FIO|tt%dN#e{M$yy$*9@eK6+9`e5h+dzXO2 z9Gc1_w*d^=86@~2olH=be18Y((Zn{hGY&+#0QI$F4k$l{*t;73%hy8eU5&J1aDqTY z@QzBegeKGC@jtyv&FKQV5U_BB4uo0gqEM&R(HzLuNkXNPs zVR112Y(vw*!`Hl#iszhttOWm<8SPz^X8lRto&S&d3Vo1-Zs2GKfxq;VbEmH367(Pb z7r|8g2DArr9!1QAwvra6am@FDanR3kfAndGL5O0B`ePcnKKVDov$-3!bF&FoD9`#H z@W!EqRxvD$I73Zeg7UA4oXT$NJN;Twf>J&5wfAu^%@dZ2Q8(yEOFm|Y!|?$IvcDstwe^A0KO23w)p>I^KZ|QEqUK z&Bm8T<^JLSkoe+<3O~J>_4cNzAOB-8)PDy_xd{dKxZUeQ?UG*+409@P>SD6UZPqtG zI@QO7T7;auHoO_&k?P6LTcN3Dp)s7!?+Ve6tPn{O=p&5+&bXiQid<}=9b+D<9t;&T4TRHuJ8#i z8PESxSql1TO&;I5{>70$-4qIkAl62g{4iqKgH4=l=IYjuGj#r(hdsdgY-bWAPQkH7-*%e2BJ|}-Z|eB4P1HhFD4z5sFhjy#~% zNwbI)rQo-lp6dg7*gw!$#&{wZSeI86N`3k;qK}az9IA0Y33$IN_n5pdz|qx;1YHU- zvEj=ke2?mb9WDGr6@8u3_s`MaEJJuo2#TM6RDEdwo>6!Eay9YohU5u3WCrxV2{H6f zDQ01S7P~t1fsnyOEX5S{pfx8U#NWhuOiHvWX?Fhn=vm3;aTQ{W!6!>XhFL;UUAcJ0 z=FLB7=Kc!WAvlZ40UluQ7^rO*^-g9B&&v#5Yik{I{wTZudRu$JdRHapch0ik+}mq_ z^0hY#Opt*5QA2+ZPcXR*+TdD59|+H~AJZBLzC2M~+Sc(VzqEivpAhl726`*k5xC8R z_{;@DJ(CB+sEL%JnZjLzUAwB7WC&)4eTGJEHrqD%#G z5u3egg4@e<2{bt5AYEPt@EE>p`JWEtTanB8cK68$!kuY@VVfu?S zSQ;Qo`_t^htpM8;C8rmEpnV@`RFk0q9kvsetlDT`a=EVUP*v=#&anykjpd9_nNPa* z7}>Es$$}abqwIj@1ik*e7(>hq7=7W3$0$W2DNBaBwXGDoDBE@p+s2#x`P}U-Ysp7> ziTn~ehdjRM-FN@%NHpJasz|~7^-S#>OAq7hFU8sF9`OAQvJRY`)E|9>9J=Urn%-)T z#PP|Jv34Km#Y*n4QFl~+1%Z)sA-!AATm7gn6mDt%PF8F1EPbN}M5126C0?8Pb~W&R zYi!-XvBF$8$a0IkcQM$%RgU%LTRGOkUF2VRES+DE>68O;@eUzIHp~s&8@rz&mBq+(Yk=u0L^~aBy7KHW4(h^Ho6L4+}7I z-pT;!ga-~_xhzN!AiALU8Op7pg_ZpE4RDd?-i%Hxbg$`$>F9b2A^k2&kMs-(tym8q zPZ(&tXnE>(Im=G{ZjxyW0&>Rg7yMl10v*fyT|cI?;~PP?REGb zPd@@o;y$n&!wU_+0hmzFf-nGLKChZkAPy(`cqr8n(1dAd_D1mZl=HNvZSqqOUg-lI zf}q>|2!{0ZZe?O_l0 zGxKAOmD>}IGJ=T9$dm$=aqw1>v9+{WILqm2dq}CzWXCaOkQkxr=Fubwf$EFcd7USq zxT=kTykzk#iaWrR$54S1I*gHLP9P~(gZJkVu(vFhj1!9Ftgoxfv6*cOB(tUViRpxk zI_!VQeQ&!BaXF(+3fr6ZFSZ83K!Ux2FuL$ZuvHKI|3>)wo6Yp>Ea@!pZRnqk)VkFi z`1Qs;&Hdk@OY6(0()>P+@U(QRY0(+(?$+h=RzX2ZS+nQATFPW3>pwhibudD@z*380 z4Q}uVM0|)mgz`qh$8!48K>p}}s^$U@JhNC;TAO^ubOgRNzmj}a^z(47lSnAXVY4Z* zrkJ2yy4@srBh$#4;4_nfek52f`5*ViYGCXNe0Kij}S&ycL-a{j@hfi@%)SeCqbbTRvM@Fet`hL zIqoNq5gP_LJe?Hu0j&Yu-MU}5_D)kFUworQV+C#Cl-%{E)&ikA$KpGmMKZ3ZD;Vft zof*ts*hbwPCWj#P9w>s4(6Jrzut}*ifMJ6%lM@bi`3TnX7us9snp$OtCE2@kSnZYq=iLf;_Lu;D#!HU$L?rPN8qf zEu_2Llf*%~i6!k8G|WDZr$EKfmK$V&q!^`5A^QLW9TJw^jO&nqw|s@DcD=tQUPt-g zoEbgun_0DejYyi?LwbsFUE;o<$+Gby9(SK&)(y#!CL5N|yb z`Bba&#S9$raze65J7c-!_SkwF>Eync^QGP}mQ}BseYXi5t9Y=q;h&j%w zlts{?(p2)DkPMv-{R2Vi=fiQ}$Lx;CJ@7*LCZhb(;=|@O;ZQ>)?0tGtSu2_0eR13` zpeRnyC-Yjec>AW5-a4TO;kg8mzVDXNC3>k`Zmo;X?2rF0gQAc-pRIb+*>t8ON!5`;>-P{nGmwexdE4g~hd$eRq#TEQF3JTpo~) z3}kIb8~79xoZ=w53gEu+H?bMlHzA2}rEQ3;>qI@V2?{fq=@;=6a{NToN!Rh4i~Y<# z@=rHr*OhTtLenJzcZCK`Kg89gt!q|Q?$cBtO=Xnkzv}hZGi!~lsln&Hf5iGj?x}!H z1NG4*0v>Wx{4vSo1tu^iAkqkYeS&$bZ72qNHHCF3QUVi?QVlwC^pm-K6~!9|+Fbgj z4kR8G)_e4t^Q^y2a0>!D+DQ8av}_X?2k?3d?*1VIm$}g6++#(I(q5%?uQpq!h=z_- ze^ECc{66(wRNLW5xz?YBe}8k(zrDKM5kMmwSW2!}b(#xw@KC!u@rMZAT^+~aFFwH@ zym@Z|rH3v|&pJzIHJtYd_0VRN_j9rjWS)+ORGmN9N01HNcT=6!s)4u?pYJMi>F<@c z)MWCXY&m9H%DvRu5dk;(D$;mTUk;?!8)3WRmE5Mnt0ge>V#c{{d_yunQ~d1VpZC@h z79SgTzZ&4SMch?qb?ndtNAW7&AB6t+%cFiw-T|Dm&+pqa-7g*22-jxNFVyUE)nvLX z_}_BtylUG2jy;Ly&Ex^PE(XCILDuup1)94SgbkI#%~0_jVsL~nq;48k4sgYMFS$** zIVoM_f+9Tk9~xBGc#RCX%-Zr80VgwgZ^HMh!v~pMCe);Udmv|KgWb*A_M)!BhZ-_g zw!d}}fFFCF|6Hctm9T5RAOeT;E>gS`{6hR?ch;=OHK#|uj(jIvIx?|U$mVwVcIAG6 z>C%Lxpa)?cJm3X>4$fMMgeF`Ba{je+$b>T3BNd1d{dlpVKF=~!@rFNv!~W}y--o^g zazEXx#=bXrOBIYVYyI9Hp_Zj-rpVdC=5leGjMLvws)YD5f~_-W&Zi{sXeJAGzxz-m zx4#$*QWdo*uZ@5oJ@|Tjbcph4vhT+YlkVK8flpgaD{_xFio8P&-h+{Z(zdh9G8q2w zV>WRhPp(RO9U%1GhHphX*Y|Tpqt}V9j2zY)z2Ma8>ZF`GAr@a~)NZtB{AXL9WmrVm zMNQSzmMg~05PB(-fTrAZ*f62oTm1UEjIK(#3>|Fe(i*G3L3o^v^9fY4?@dfq{OfhL z_Q6-_9Bv;%m!(c3kU9VL82R7gY3RQgd^di_QmhvKSVHTYpbDeY4ef+9;X*+(vkxkO zZ%1;cnaTGvjcJ9e-gOcHi>O0CgLRuoLG`GnV8oJ13x8q6*0kp(T_PIw?p16!uTD?+ zdM3EMKgL&i?fU8Ub>0Kzm)ay*cN}sugjwdG{yNpqgCz; zgJrI*X1|*zRliD4EC%@qM%cgGdmvnDFSF<0>w^DZeGm=O%bmffF9%Q!4va)$z!J|E z<%C8+o$s+t=FtFA5eLhZ^=$IMIrgShxt@HXK)!Ffmw8jdLLeX^s(H0fZL$u3)cX-# z#*)f{U>cSG-v;pUj8dk`dm0Ke0DSj*JV`gR!lk;Mvpcc_6%qpo6BCv_rf+3T4sS2B zi+tvRol8=tPB-KVT z`|6Piz1*Bx>}t(P#a~6I=t?AC4FUa1gVDb}iis&YKCGX&QNik(3*0#6?RDijucf2| zrN7+gx2QFJqWQwdzz}~aQ%&1K^TaXIXDLU9}Me>1pwG{iS;wM=BdR8l6NBfAvjtU*+J$&in6wf>1pr zmj-3yiQ{~TwOu^im>Ig__?-c{i}+D}RiCQ))!!Ff8q!zNEsN%7%%wl=ld{jfQN&i8 zonf>zc@((a(y*{IC6F!@xMPT8=26|S?NN$Nk22DJ0(0C1P-@vgZI(Dq-%Z|?3offctUpd~8T(~Eznby-@PS(q z-ynB9@yl*eq!*exwCw~-FL0o3fD?ex=nX2L_02R9|4C5N&Y1cz)G6?8SmW&E{d1WL zgp3-l*WG15SV>s!RYOgxU}{FO$zVYj@t1%$UMme=*>_#o=63( z<Bk#=xZ+h&PYFrte96m|v=G)a z&D*6Def;{S(F-D(A{S1)lhqbaL0~F||1A%(n}*oxrwd`X=c&T4?#ZxQgvw7vn}59gY3e$DeP&QF`qOXvnH5?vG#m=thb|Y8v|c*S zd;lVmv^{I+13<|np@i2p#T741s21wl{g|14J8Dq_NK^(iOD-R3I+-z-X7tLE=s;xv z#%mN^bO4+{EIbUp3W1*dG!QQyZgfEjyN7$*B)iv4cJ#8Pj1+ySX{_%L6g~3};+U-z zI3aUiY5&ZhJQ{E#I!6|o0F}=T{9(6nbFvJ+8&ptK%9KPy?_VwSX-;3E^hGvGgbdVw zloD@y`n1L=k25*;AA{i;9c&Zv8%Qd)2S4o`ze{9Bc{t;LQbgf24GGZa$1zx%wd&=Crw5|c&>fzCmY^>~T zAU!sAzaUewHZ_u+r9LMujLD}0j#4SqdEqDWfUA%5U%En7#AGfPWn&@;MY2V{Kddc& zLiiMO29jpYXumte{$L7_pW~Zl@_%YrQagN$<4i;2&|Ehp^ql?)>0OxYm?35@pZzH=?nwU-QI^Gopt~ZyJSc7Nw1f&y&ZblkCns1T&M#1%Mv@2Cf>eY@$7Nppb?;MPuTqg*v8NJ z#y2?|Ra7ZU3No#VDFHil>;C#0f3;}pTW-kKE9cLY2%PeGd!}ZlfKmjfI)YV13*aas zcRMv-2hNC*zG4@#NEiBPO7)v6!nd8cl&BXz6ADUU8q$W>2K)n$B(30YZ(Yua*{?+b zrSddD9NvwafoO}`c{m^ExRi3J%csIMDtBvN9QwG+d(V{Z8g}0zMb@FXeSp}{Fp{e( z(v0yM!@uIZ?{T@W?xOP-CbcnH)br zdgU#OlgZ;NgyHhH>Rutgv2qvE{{1`WnpIxg@|eku3(SKAkW)G8Fpj3&6TYuifu8w6f)bZTLiSQNes{xn+M>?AUAUh6UEGZ6 zbj3-n;a$ccm*D5qAIb^^;cg@3xOqPxtisph&&T9>Pg&ToF8OAN9A!#}x5|uY%dRNh zp+WUgLDzm~^!<_cVm;oo6ssaoA1i}D_%CkZ8-PlosP<)JDPF4G`9f$;IK}{d?sc+D zTb(ZlTW@3XnnMoP?Uj^|{Be%O+-BPE$_jCBMG?$GJaNJXR?_L?gB1)jsCr${VJ$Ay zp;acUi^1tp2<>9w_g&>$t543mOnLref4EOM*zJw$-1`tad6O9l>`4FSo;7=nJWL~7 zHdLK9*t0WeQPVu;yicfp)?8-v{lWJsm8;c%Mc-!Sb7npO{CaOe?=E=#Gqls74%h4n z<~RA^%*nU9HgP|bU1%TSJ2Td&*FZ)_O_EooFaz?ic>8ugi*&70IO}f${?Q<;H}kf2 z;sFrK@Nun*ImHW&!@o9pR=m%@$=>I3Z}T`$W<-FQm`6+*`-5IYL@6*-?5CNt&Ss6? zVLE*eUYhvop54wd6dvx;Ia|q*LT|eqRk@G1&%X2Im0RY&+-UrCsFhKH^1#O~98zt7 z??8EjNZ92(b?^3tcb=)k-~BVaQXHcl6>hUg-ocTa=jW~35=e16*-nnfj;VYN)km=3 zj#8xE1tMb`u4fNNfZ+E71N&3_p7opIUsXsFGtoQBBZA8$$Vc7j4DLw6)bS5*WB4SK z5Wjc+e?XeMV4A3$1=s+Uvn0f#2fahl#T006;fmWQ!OE54fv(i=&k1IUWNX5jPRPOqfE!ShkS!WV5 zI8Ib{`$k;PdA4yio!98kM9vd#mRyJy&*T~f@2Q%~PCtYK6Xj>LTe&A7&UTNNgD+wA zox_7uBLuS_JPCfa4$ZS3EU915ID8MH{6gFe*Xbinc4H3`*4!n=e_69T)++gLHmsF3 zzHU}35%WwiQL%e^o=-AkXfg`;8y~6l%w)b^Rr+yYWZ;bme*uN&~1U2 z1iU_gtOJ~!|LofTiAy2M?0;j~Nnxrm8JA?Jx)!vGQKrhiA*Y-@%+?Tgx1&(`C$&f9 ziJ64G7pKTnEW60Hd%2yvbfdU%utmU-0?FFRmN5#-UxQ?Fi@!@?TV06|rE7d(<3OrQo|Ue3Xr6&Y|~7ZYne8qyEP} zfAsYY&z3K8>v?wk*!@f!ao^|9xuJvt)QF)#Bx+Ls5}B)av;ji$fQYKPBy_kTW`eMv z;N*Ecy58T^BX;kzTO4H2@hJ^%bploQ{Uue{y)}7TXE@SYyss$exUVo}C1`H7u;iMG zmwiL1ll13pw%It3?0~ZE944nZDAI4he?UQaxd*l5*McL}MWbF0!pG9+M_6fw%o(Vm zccBZ*LBA|q4~vlp4>^ma$UDxs=u1fXI)2S{^t^XNYKnTg?p?u^~OqzG|FmLY2+$$$@vBc~+$1>}~$t{nZ=e#TekUZ;Wx+?8h zq#VUOL0QXRm=xdr25DE5Zsi|wo%B)fUUHbYh_pqw!tcGJfACDMV6YqZuoV!69!9~> zc=#!apL%SwKgZv{INWrjD3iy|zd%GT*ZZl2bSj^^kl5oVZGD14uSYk|(m~|%;48G) zzkT&@M0J4z)7bzKv;a{Bh1Y?nZKgxAp)FhPMgR;f*n20;0-6s+ELy0=$73WTwMIPw zKAA5-;J13iw$X#4q|VLsO3-8oFSJwm$@0U|B_b!yW|Qe=|GJYj8s)Ov*F#TcyaCqW zJ1!QqCt)`n$b+%hcI3Lz{E?@#I(6!NM5`-!OLP4c%1xxqg9Ak^4{0RipA|XD{b9!V zLiH?eeHtk0e7Yk1o;uj=E=YCg!X64gbY>DK^ua%*G5nYG=u@WS95Gce)cT8wpme!( z>@{s{7hlfb(4M1!S$NY6U@Y>4eMEFYTEQ zj#r!QR$84rEjs<=7qTVy_)_r9V?~T7(6>H1aB(A7Q1n=I5sDIGLw~+Wo@8Kdpm!qb za7*W-Q8*lY6hy_UAmJ{6>oz=2S?I>a-=a%R>MKH$V*zxnz`*@Y1@qSBDkB!hQq_Ck zn%Ne{3DCRsFn!eKbsPmxMkfYn;SNUXP>=~)$u%$TF|4|`bHX(IFC@2GN-0eN*BMw3 zWAtDeT?FV?xnwGq*)sz%R8sN9uMp%83FZKnw}XEz?NzwVTSHFJDkUvDSN$`emvHVw zXhQl~;S|B9%vS(*?BWWV1&B;AL~OybqKF0VD9T_rUYsu91do!FE1->>k${a2=3tRa z$5LvbI$1=ZJ@yEjak{`#Dhop`C!n8I7-9isY=G=t$7rhck;3$Jh}AlNHPbmz`e?+3 z6#uzs-9zb((YvJ(aWI`p#P%WCnA7-f*RLnZAUt!f2?E^kOyB3#8r!J=Z9Iay z%t!|FCs~$8$TU!sB5q^j429tbl|CIuU^O!C&<%T?K%Ugfoh5toi;3rj0%Z!l9_m~u zN$k^j@OFC>mSv}l8$c9*386*1xPs>l)Mq90#ot{{C<+-`emy2^``}`WsmhQ}&cVZ} zyIsIk7)Mm2nM$BTejTCEZZlD5wB|{$1dAE$=;h4%>uob`eDwLil=xp#1;Sp?n~hB zWUO6c^4$YZ=m~>VoQEkWVP?ueCmUE84NxcW?ulg`Knuj+FrR|xW)!!A!eUGB@E!NY zV)v9$ADveTUsX>a>Rwt(-CFD9RTo&|D^Uaoj5`nX<3{Wnzq&+KW}ct#_O)nEIu-&8 zftT4ih2@5>;0YCI*r9~3N9F6W+T_4b{mO&21V*nkcI^F={jToJhV0-YVWN;Ucl2Rq zjRVyMJSzxrAH87?qqp@}kvk`b*f)ebv9LiH$syHH;hQrm*^nPD>cohTRZq;*+?dm8 z7u*!Y(oQ|Mu$vQDymhGsKn-ylU81=OROp{w4O5JMyYm43RvE(N+R$UFptVfssF9H3 zAarw+0Ck~iq4f>&V3LGh+v_??8Nkt*j*@xPW~1?C0uULq&xd6U3rnlG3SD=t$yx7b zIqX1x6A0_3ZC_D>7TNWWN31$nF8_^?uK20J+;0PJy=oWqmhI>m-HnKfU*O(g@`M)| zC#-Ff)?98F4!m;udRV&MNtv9AQj^X+Jm+R|m3-VafP4}!8aG(}nFBHPMa(DEPY;a1--wPoSmtRQ zNYFl*022|i3%vOCCr+TdZPE_ce;2`AM_tDeT;BeTXvVI0l5mL2pLK+&ul~d{c^-pPa3pd+ z7)Egehr-6;*Y8+_M>rp8FrFzC9-*1WeM46#Uh~OZv6ZsQ2R*$IuTx(9eXpKpjVM*qHGR< znCi-7A8I5Im`f)8z{AHK=mLYCNXps3>Ci>8LyB?;kHd!W%aR)e>+~6IT;#Vc>0_rk zd)kg=OwUA@-x8@Q`?f;|hvC5_{);Gpu~{#51{;UF^qdbw+6g4O2#{N|-{ds<@hiaC z-(`SEHO2y9G5%w6Bn0Lt5AW`}8&E=;bLsva)9p62nKzJSozbjm>cx5Ra)z*E^!}sw zSrN@tG9At#LM z&K3VMdu!olVs&>oA@ZD&Ph^yfLX0;~d1G^%5h3(oHED-aoNOi>RC zDP5hYb$31|P@XGtKv(GmX(24#VQtEE5K5P$%KM12vK+TU!_3VP1zDzxn7o?93 zvAAaxVy2P8?aD8lR&u?2Z4yNT^2*+3Pc&tB$i9i=x{-0Z_(rm%o!te*1%PdIW zfOd`hvU{FQn(>1e}R zipPcKhKAG*kD<-GX*D-nAg zzQ(5jCGiQGw&}$v;m7U(@0WgqGLXMiYH{a{o@-s^#Cw}vb{xMiPXIfPd7!QDuU6pX zf`mr`a1^hDT9`ZY~JPf=EcdlJwgnkFT|c`$@$6uovvtpZqqt?hPb_38TPQ8s*6R5!jR% zCD*wVHOk(WDgMK4;e_XYEJR;jz1Eg*??FY13AvemV2#Ofy(ID?Riv5rY-{0i zYf=;^GC@}|%{1fsm5JnKXG@i-1hq7^ISN)o?LDe~fpTHaea(mw-Ej)-IOpoP;7B)| ztXBT6yrlMVZ(^CNdaF9ILIL1u8;jsbkDs*(U2}S}lm*QLz%S6C1Ty~>+hUA-L)2P< zJ_E~CFsC;6JBufEcU0_LTao%nuPKqs__wLJH2qsI^hv>$2i({*6YvVI( z|I#*dk->K*B^gOjHH-SG)qY82jm`qa2|bNE_a>LHxSa$W@btDd*!D5y8@L~q?d8Xp zO9`$b>-#q#3U0dzU}z7<0btLgi)(JOc5&NoT$hvn_`Ai~p+9;wWV{=IIJ9;^!f2n* znoc&JV)rJXPqE_3I!>Rh{JlZzhmQj5IokI`a!y`r%Y*j*oDCWqVqbIEXa*ms#0;RC z**5??PvTBNAyuqXcRA^+PVO6AY$KQAJ+YZSFQ0%clhmZ?!86fS++HKR&9@s)t(l`e3lEbC(y!TuRT9;UB+fL z1oQ)GgIeNS%otS92m15;aMANpurHO%JlTzslcd}+c<<)AUEE<^i-)Hyr9vFck5nAb z7{RN6IWOELFru`Cg=O$`eqRv%9yB+n?upb%>k;i}CdXfdaN%)Z#quqQt^!T=aq>lu zcK7{`?{WN+d^PA-BN#G91DCV&B8nJ;-UAeFWxm_$#fcy|kjZa0Tb zOv_xs4#s%iG4r~vXQJTMeI*?;0dk;{ekkGx05?7z@Oek1m^#_-TTae!E6i~k={m>q z-d?dR4$y3+M-U$jV5%aKun7qtjpk}VvcV|J#-T`6Qr^jR%{jyAAi+xc6R-5Jw&yLS zvod{)K^iRQ$vKuR!+&6e{{gZj38cg;5gVj^Xv4#kk zn_}_pn~sXz<6%(i_uPX5s_~!irW-}XA}Kbiz#}@BHa$N{8tm5Ns)&+R$lcyyGVO8! z+HLCo%^OD^%FdjLUm$U=SdMg=8#`*qYsKpt_Hf=-(`!=LwSsPRT%?kCH(t3hH13!9oW zzWIVslXDFy`Z)0Pm`GVlg9pV)1T~9;=>&zwQ*)|f#78X0svw@=L7M{G>FO~ zz^!%S8O8cYWIX=Hw4su3a=B8rJO7xzf04WM8kG55spdV;o66fC+5^-oqGxV(KB~jc zJu*!cyv$~I7!fK+-{&LxW+DEUuM3TtHQ+$8jJP=9^}`svOF8=8mfF_WbuGGf&tx5CZp`+}miQb=d@Q-) z>18j@S<4=CgLMlWK4KyW0n(S58hQ@yccHFM%5UT!DfA0G*hYdaSxrqL)nm+;sk2 zqIVdWDUja23?>jTwsb#mnDl^;f_{h;{Ix0Iv>>bLz9XXfB+?G@ z4)QLYT@q}w*x0x#>xuYF#p!Yrv2EN5GA{=zfr^CXk@Q1gYJ)C--wx@3eW-dit*yh~ zWnDh%%_!K5F(fuvkGr{8eE3jzybbrMvar%16$^~i=KUYxNK#5c@zAQU+3(Baznn6oY#j5_Q}h!Oo|x(hUKBJM6!DV# zcK4ri^ji>+C2;QZ84o)2F{7a!+VBQ71j7OU?$JJ4ZQI|o6Ph_M=|~lwwGx~?_&wyp zp}YR?5aKgthC#b2UWgbEQLOT<76W6Sk3cuM@Cl;NavvR=><+`|4|_du$+|V;x58F@ zyY4mdm2T$A0gOL@tZg0mS_KYEExeTD;Gkpa0eJ-a)qVJ^Oco8P*n1B2E@&L#Rqy$& zMH$auB1FwP2yMM*>*1A7wV0~td%JW(D&2UA-w3%7nq1@zg$J8AjQ*sB`>p#OwYujU zenI+{S(4yIBcl8T1nYCbn=OIUG28xtoC^*M3es#A7M2hh^~`oUT>q|b;+Rw7Qs#L} z1K-!r_YH8)o*sTzA$@yKWu}br{u8nX0Uf3a7?1r{WA!V&yp9p1>r!uhF%+!(}|k3 ztZ-is%3jnQn0|Q= ziB}f5-X3_*18Jz}oIt28i%4Tz{=v@zK1(dbQd~hk6XYkG%6eY((qvd!j%y{lFJIobKKCd6qn@^BqIK&d zZXK0vv8D|>1f%!`J{batwpfb04PjwXr&l?+SDhXLx*wY+z7v*w+$APe?#!v4ztq(o zxX1FZl*j-re6JLfx+Aix08Z=+%9; zQ_>O_Dw2Iv>Z?L}wB1-LG0ygb5$e?|#Xn#&@xcqgzXLTua-C_gg3LnKCISH3BD)7U zU{%Yy+<6rDf_3R&xm8%;kuA#*rNCBO4$cR$rtZSQdOFMI@!*g&p&I++8rQ_n@)1GbAR#PxknlwY85rqpGO>)_qw)E-NyOXS;Qis?g{+s0&@t3 zhzus`C9h~s>HCvtj6GKG77rS{m~*jYNmr0myC_TK5b0A_{1Z*}ri=WIpt<`<93QLF zY(pW|M6XxPf|(^CCa2S*ngnkjL@Vdq4!j`d3jto{cevFKkhKs4uaXt&1msJoYhVO} zH<6l0-uNB&QxVBRa2Hp#8QILanB&;eR^L*8Wxa*p{77*9(o(%2!XxO&CS3>sZzjW3 zniJG%1Sp+cNCuN}`o=H98H-~o&&L|sMoUM&3rYXg!|G{KYdV{cA0qzD7N7qv$%5IJDiCv6<6(?2 z3P7y`A~)Q#bT=7MeVdWH`)Qu3yO^xNdnn?>On zS|9(h&HbKE2-HbDnV)Y)kr}<~33t6vWy<0`pevj3QSSspkXz{F(GQ_%z!kj{_DmJ{ zRZea}S%eRJzHi~&Y`0&`+slfg+>8{|Ie1h-+`s z4*Ij68_{g5joL1O3M@P`BV-P9G{3jwiaAdXe>=FL4B}Aqdr(GYXL9+|E-ZkRtS}ni zT9JK7(QvVDCOd%WCmG0DP&z7tN~TSHuHMsX2v`wgZbj?~%rukEzP4}dvgr@xG~j}1 zq&Y?^GD_%zRhHDY{&}V5=J6{@QOKUwfzM?ntNa#SmrhCLvS)E~zV2jmIxphnxDSDx zW)s|<8fFiHI&BQKG4GKFI{{ zKblPz?bG7N9}!#I^vAF&0Bg))R024&B|}hqA-UcA2P6i~>jfV4s}hINu);zgp0G$( z?NyCO3F!O33a42PHI%m-mo2Piy^nb1{ON}@-+`7;rK`|!6Td5=H6I>-+(&=8yCmQQ zXw2yiBv|$3EAYG3F zb-P&W61o<;e*f^vyvY|)=8U#Bn9b-|DdscmJ#tLix6wp1O|BEe4;0oMb5eqv>i)FS z1;CgHsVt9FliO3dKpSzcg?Vs!Stx9YdYE=qy?UZYDNiI!h4i3dOXC50yaf1@AlTpc6Y6}87QtV!nF(YH z{qFs^!269F9(=9^)Vb%>e}%A*Ewv_+L&IG6PiUR`?wGFjFk@*qMSzC2nzy4Fz^Xl$ zFod%xQ1xM*s~eJRbVcN?m~@^Zr31+dj<0P-1UxaJoU3JcrUL}_C>L;PR}x(rV5@TUcGwG z>Hu;*B8#+(a2dHEz6>6*VS^pI!gOx7Le&-il_ftX>=|xFu}12rSUJ`AS-fYJ#lN(2 zH2mm$Bx3GMx$6$`8vY>)0Q0z`2@AyjUIV_~i;;KWc(VP__lczy_hS>^8I+Vapz^0%~4f%clDs>CJ>fH3Q=mRM76^WVL zl!`kwem_-Khuc-2dqyB&=Cvd1b|yeIE%sGXBCy(UHv)MU;)n6it}eG$#Gm zfOz|`jbP8cu_No=v33bdiU(OFS?R#$cY`XPFKXu$UkVC#a0er~9~kqDCnzEga3O~v zVQsNKYy;c&%i>kEpgzPaGvue0+@`AeS!iowMDO)b({zBK3Ri%Xhpxh%$d)U7pR$1$ zVgj;k&!HuD;vvv(#<2@)wAi=_b=uQG`V^d7MUMR5%z0NSi(hhD)^j7Iok=iT%1WW9|;HY56K6|+8fqpIFi^+4(7 z2}KyU$E+H)N}p^_z7>Rs`3n?R9HF8U)il^`soDX&>OSXymt-yjS(*h^8yEb_XEOA+ zMzQ+7BuhhzgN&O6NVymoBbfNzksx_oJc>w0&!GCDnjHGUwO*z+90Uax!!K5M2G7J_ z2ycJyC)JwXFI}^>=aaCWBNQX>Y*CV*-U0o`T|+RZ!3ZY=YXYCc0mL1jYT#xfl;jTL zju2mB%et1e>ZJ*pdsVg?cYYM7zv)&}J+O7T{L0zvD`Eixm@$AEDM()gTQiEJ*l!Zp z0n)jE4O1|%8$g*z9D^fBXpV19P*{d)9L{UBI)tP8#X*bT>V2eQP@Zi+et$H`OA9r;z2>uTIiKhzwP?G20K?snY=}${yLqqV;Nav;7dFRx(VJWo@?{ zyPLnn?1pN=DB?6a72Cv47dX_57V0?U7VeE6rbixaf7A3&C$0Kej3T5I@Ge8-nmzXk zV`=_Tlw#y5a8L~)UmmDG*EA>L1t<&r6su@i9Mh8jHP(7hp5xDwRKC&@QOzCr^t!SFqCn8j z=;99{KzcR^KomVDw^xR)8yQN4~DLW)jaqzxHr|-Vy4jc6nir$ zaeOj79DdYLgsw^1j9YB+ZK0z|U3@C^WG9|LCI|bDq~1C|yQuHgtoRgGShT_=BKNgR zkrlsfklp|eB?B{(q;0KqmQm&jBg>=49?_O3zx$Q~4qgf#R%qThMCO}TWOC4`VPuC` zq0V+o;vo0j&AWj!S2bKJzu^_`u!SMc2R%D@>Kz%MVPM7NvZlG*Y)$DM!!`u=$uB{# zGfJ+<_v+m0D@l|-t{7{#9D4zPWwRk5KVbgAQP~HIK8_uRm?3O)KqCc5*6FdnM;4C# zh2x#`J=$&_p;8!8HtV8!7VrIZwoZy;zvrn-(O}@5UI^4yG7#li=9p;CZr~-GSV%`x zbMEWw5p4ulF$YI9(>PaiAW^qR_SwQ&VC=Ry@TpwZ)d=a^mflB0&1(CikrV!cbRXs? zh~myi7$os+umYbC>Y`BpMkE!DhR8o`MFH}oe4DdlMZsU%b%Zu$#YIHoj2$k;nVH*N z-5K9?j=*WMW@<9uxvRnsl;0#MMazc8r$cJ=f3fxE;ZXkl+xQ5HNXQ;TNM$Q!$u^`c zskGR)sSpy9L>Y5gvSeREQDo0LWKA-5qR6hX%pfJ}j54mw@_F9w`~E%8_xRo4e>yrk zj^?_q_wst3ukAe7)n<&Z4#}rLgBFeH$D0dc*Y>rgALNuf#Pek|34uU1T3Ay6H2!ax zqp+Rdfg)QE#`A!+!7Za}W0#-k;Dygj`Q~<&=_V}R(fV#@((K-FPveLv_sgDiSHz3H z#eEzrTY|en$^Ui9p}t*O*zgN@(p#iBDRCTZWZkxlVQN8M0Qfhop(&@SZz;1S9NmMGfkqRw+w+8} z|KO%a2it!f8$NdZ688!28vf4@K6j>gegM03Q4cJTH~y}Y43kTP9Aq|*Jry|BBgPx2 zV_gJ0TMX-5E1kZZewsjZ?C_C^{tl(M@P3rN&h;)NK-%Rd?}~RuvIwhZ21%afAjI1W z!MYtJpV(#t5OkXZCWp|sPJtR7<2wndCCgPsu%fcJY=b+$!{H)|_}c9~DW_f~@&2YQ zHf-B(;#L-z`*-toL%Y~50V_iGs>leR>TWqr`S3+8{?+!;;XZL2F3y4!ODnH$$$Q>9 zJuqz*{0kf!UwhyRGn(V+KM;ak*a+OxA(kDG5Zl{~IY0?$MeX)Kg`F7`b^n*Vsr@zl zctwTJRpZkHhyEW1$1g1!7K!X-sWRx^hx^+17c`-9J_9d-_zIDVa;keGu}bb*u0LWW zD_v_pR=xk5Ij}DDO;rIB+e@qHHqsRQ!;TQ7>p*R0E$lEJqGA}$Eee4F-Py=djCMIv z08K2c8@}1^y#=pu_@|-cW?z=e`KVk0@SPTkQWNYdb9}ovjf_0qvRI zmdEBGe}$huyZQS=+J;~s(adfT{oyJ-tTPvEK1oDo6;Hj!-}{(#KSBaziNkbC6end_ zTIB&hn(<>V#yGuf2Yh4rF(zwZhdvqEgO@-DLRV|Z8L@dIXFdwwc+GZyIY6lB&$Obg zR8`_&oG`2RytQ^($Q(4UX)XZ)URL18xa<>|r_hB&vLOmqc6td;M>rg?;Gv=;BOgY; zMP{0~j49r1)UbL0**iRLIiGy-T($G}UeSwjg6sR<@P02|FmF>4!VcsJ)@`uD3kIV) zYJdT6rY^m(Xx6yi)5H2#QeSJN^>xd!wC8Cz&BusjAC=ZYVY(am+z%Mhtac`HR4ohQ zSQ%$ZjsFNm`4>*=FMMuH8xXV*vJ^dg#}yF#u#U&5<4v)4gf5&IP};f_J6SGWfoIv$bs< znRYUlVM{o+aMgC^MTF&5Q&|Kcc=kWUvVbCAK}Xtcw3);56Rm@L;e7L8`OX#r^4^I2 z<#F>PCgTB*b})RG`CJssEr&8rm}ruczd%81yDZteMcQ64)G(|e+%Ah7dTAj{E2c%x}=J7^&@4jr) zksJ#KK^ot%2-vY=yWzPjMRu;wZE>1SbKuk~-%MI8bDtM*oRUD)(hjQ@7(Uk%-GVF* zZMHA|g)9F>y!p{{MZ4dIa3&+@U72jma1m&S-s(NW0)(1&JqdPi;0Ka5T`*u_1{d3m zdh328FttZ)@YzFLt^B9AzZwV;oz#YFntjba;s-2VmsiHu4UJ(^sj9H>{67eS2K6l& z8IIwdV;Fa~939)r^7XuR=VfxcW{RpZMx(bS)*$L z5@8;MFh)`OqescQ^-=1fdeve+^$jf={Fkas4EjzRr#rUpB}|!~XvGTr+cMC0K$(kY zkr9Pe0(!I0uwo#6#SYv_d#eq8?uee`rZF}89iWnqUbd&dTK}#0i8iPp=|f&6ZchBp z`nSv4SOyG!fe`5vI$&x-CW>#E>dZ;v9y*$)$sBdkBKsuhpx(lsodor2t(i%uqu*o>{4km3OADOUFgCt! zIW+b>$pAGIf#qNV82_YV0zk4v$zxyXtYq{R3dhqwK|gD%M(cv_K=#@nhLC;3XH3P} z*gFKppH0P)zF*}4K{uU`mt-BMrwz4Px9qWljXQQCKWaqMee(meEY*K)dJw!4a(#5i zMh8qR;$@u5jefXXm|l+|5+bnDU`w?i5?K`GiJTG#ZM58&$K-^;_9s4xb=tvuI&KX6 z+o}|VUw!{lx(?zuq7<)Z8gqT-RA@YWkS7RF7nWfj&;#uuVP^DuUjgAtz#JWWmf}i% zmD!!=vtidgFfeZP@Tp8`^<&X|OdJBSlB$nw z{kz>+zrM|@q1(@UqS*tjZ;boI#kv*{%!2M;3pkl@MtM?V;1Fe((e(EM zOu*7GGUwkAw%+W3+xslIaM>J__%M1{uCar)5Fz5=qGX(1Js8Hr9ICNg- zK@&&lANwys*VQS1Q}=s9HclRoR4D45ni5}t6$)qahEZ;>OmvvR>~lYdyE=26J@I^> z8+Z;ZRP_q+=TaA>_)ov$KgPO)#4Wfn4+z4d^BZ9xLuHMQ=Lag=V6>Y9AX&e82_cV> zZM(v!>zAzgH02ykxr;eY^!Bx?CZT*GQVf#p=Yk?zBoZD$Kj+M2n1?$^&_T3~S#a)P z5%J?z-Pcet(}fpJ(s55&o`)E;ac~IkfJxalpEmO_No7VpR$)nRFIg}2Eu7Vree{F9 z(&@`Ji4Pl88+6HnmIikoNnf*k8B>~q0%nGa43#`d;Rt84!kaxEI}tyhV&sL23smeAZbkiZN++c+_r=NI`CPeI%miBZhM*FGf-Sy@meL~Mxr=(tpURNY{$P@(rmK8r&Cagbu<9~qj|6p@L z+68cL>cF{-06B#l8Hu=OU=eZPlQgeE;k{0Kos zk4KD>%JyhVC8X`LFYLXB;h;neu2oYM=I?^*gGNjS9{Je;@vnG?DqSg7V`e=_e`~te zludn8n~kG?uPzNU32Lqv0Tmx*cR8+sod&%)0PJ&#Q^E}?8BtrjyvrGY)3nY;y6IAI z-tdoQ+c(+jY-Qb)M(O9tqyB7Mc=9D&C{~5>29U5OK4k)?p{Ok+pXdJhAaE;!EIvgN zW}zQvi(XFdda1as7a^x6WUaS3|C7=P1Q&29hZXc8Z7?}m792I&nhxa7>lzx5maCPO z*~|VunlL~(_q$I(`^6DV=P`N2tD9oX^YnV=3f7Xr0IeXQD6&gHD^WP`2{ImkP~{KL zxxu!UBjJ~aYd>1KeA}Dr=Ofn-T}gIXq;g&rVs)4Iey|VOGHU&s28Kj=eVc|_6jsKu zEb#H7m293m01ICmT8|gjG`B?DXM1-p`DCK+r~Qpd1#SQ%{9#A;3)3a{12i%Vicn#N zrVH_A;I!eMVqD+Y806pW(y$|%d2#jV+$AN+#>Xs*^F&&8kFzlrK!ud82ON_3_SCP8 zcg?^I^q@Ic=}6Cotb*Z(A3=>(w#9v8(ci|(X7)_yCDZGzDFr3nr=9Y_mehj(_aYZO z&IUXD2*m|b?8_>L15;+(bq(#ZuHl2k?JvAE)K5NSta{2JT^6CrArAmYox7gIsE%c& zt;Z9`_#cH($=w@;6@`W6JZH9VXtSFvTws6sEt*ANHU)uOqXESNaswJhf|p!|#Ysd# zk9Y3Oqhx&O!(bJQ>g2K6fgj?UDJGM-CoaBAx^J>jfjtIJC`LHe-mXS0qZN6e@|Bx; zomxZm507-Ej>r06Ba|96*+O5rOe}*Wl>!*R*B}umi+_9I9J+cyI%jFOS#1y>14;j$8DFN*p zu%dbA8Whwii5xxiMYt!0u$+x-NjvRRI#rm~e1Os@%b(KS*LT0aTc5jtE=``8ON_>z zqU#UQ73LhpuTB6_&i##`Em5maK&k8d=1Gh5*G5yY;U)9zLM!ZxrW zX)uDyJoAzK%GbDZb~!V3(pchiazX!LX}*5wZNhoEgtM9wJeZB$$%-fA8<@uhhV!`4 zCu)h2iyz|E6kcni(}NnNWH|zd<}L5ACUJ`P*dZ8mj2xZQ!6aNrHpft%iC%w+hZcK8 z+O*sl{bhbbyu^col8)n|Y~cxk3#_?`MM^;(Uml!#vc0q~8ufxq-?& zeqXcc?yXJ*WNt0I!`^Be0)bKO*6EqvIo=j#k565S9`Qv~_ps_@bEwhHmNRvKM3qe2 zIuWLAXKnkwnzKXxKeX@_>#rngw7N}J{jYHG~4UK@#nJ~N?fNouTK8l)A9p+ z+A7rcl|t_V163-TsI*cc?`pjYv|m+6ct@T*ZmHd4(~-p=Z-c+`+fMV9zDFT`bRc& zNczDcc!O@txq#b|k`w^gD18C6oLH~*&;`P&OJ7l>0%SFyO;ZW00<%wf1yRa>t3@IMxICz;Q=G|MJ_|pHLgXr>q!9;T~cv#P=P<>cXKp9c6QqTQ3D% zYY9<=H?O9;P5YZVKV-Q}L##*VHHtQtK+6+T81l~9^>weo^l>znpMnRjjt7o4pHMEy z>{~lICzo6hb3x_=rP;f(vYT&||6`IJP$583#zR5w{7#m2|g|~m-EfRSW*)STEpX~}rE)OTI zQPS56HC=hDlpXt^m6z8QZc>^(Xhq5GHLPjV2ljyl@;!$846>zw&&Jgg{7xba<^rQ2 zERxIxuXKymXEx2iEgwC_wH3}v+T;!|LPy_e3??$d=U0CezDIb zlj)z~a-*>~LG5IpZTAC#5HqgpbO4H?=XCtR$Q!7qqv*%IQ7}-}x?o}54()c%Dq%Cg z(k8=~7xsVBIW8ZLaV0a?2}+(<68r!m*v#J_<2HAs&Af#+BdxAlrZ(O}KiZZ4eE|Ii z;TJ~-Akn|Q+rzrbQvof}F4E<3s~ODwp&&4zo&&A88+o(Xi2#Ck>|B=PHRqu%gtxVY zeL7q1Bha~O+~|5jb?(qxR;7^B=rlaJ1W$^_|G9>-{s)1j1BloAkYzjPzf}YR2UlMZ z*_aRqwo6ZZv_<%?lfV#`b7@DI`Z}S4efs{dYMb&qrpiUHd-q3h59<^NN;iK54cOmLL^{Br)ji% z!X;577%$Jc4+XVjGCmvy&Nx{#=ixd`O=0BR#Fch1Ep+j>OL^6{0mkB!^*S3 zG#BqvpylJ9;$3mD`Zm>^k%U!W0xg-C<0Qw63!P3#Kc#R_vF05p{$uUOLOzp=e`CjN z5_n}YY4uLmlrYJ^&nDOs){I#-Pln-8ov;WP9RP`BUkFGo7R2X zGWy0-a8WZ&1k-=)-s2$E#l8r(qh4R>-vAd&ogo-Uub2p$iK4>Wq z#xWUi@jr+pXk`mXjHK>vvhE3b&=Kys-30Hp_YA(*DtE;xOL#ke)6l@^N9y-e$^8p$upDd zYU0srw}bEpJk=j9t+yGZcvJ`-OX64ERLzS6tO@R{>&psZj66_n<^>!3&brO(#=Lx| z&!Jx}LS&~zv3|dj>GX#kNqpCwxi>$g>?yeWxif?XmxTZClX}f5fTqM8M;*X>p+Po3 zktqbI8_g+EPq*Y#j`>trne_F>*Da(R7j%u$e^L-r5mqcBm2)1%whBK`Q}Vm*WnjJA zP)r#+Kr1)Vby=f*_Gojh@TX$qBI4Z}J)E)%CQ4EneKX;;rnOaUA;xD|{wxYo%)Odan6WLM2~v2!eUYa|>c}Pl03Cu#Bdf zZ<#ZWSL2|G-5#1Y{u}nqwtCWmp`c8lMMxBA<_b6*-Df9we90oo=%fkmD;ky-?l7O! z&~K!47t*Ab$DN=f zB-2hXJ5eI*OeFm>{Glii>K4QBIXHP#d1xgR%VqXy9O)ZR-z<0_*zwo9joPF>Q(5pG zJD5YhFT5H`yN-GG9JViKC;`{_X4J`mo+O2?=Rb4134fX@Jeh&-|16%kAXuLp68dA)E!js>f(N&FL#gSN}%Gr=>orvf_TjN|9%BY6DKA%ZyA2S}ldq zKstt6w<|aRVus^>)xe5!02ewOOJIOFAj||K1^%>ZSgU^Xg{Bvnj^fAI)zoMIG_08b zBy^(+d6AF3(wY0_gP!Yu&gD$LYC^gtnz8Y(=>k3?NuGX)c%JL#`Upgy$Zpi z{NmY9w*qhb{B_y%GBOBk`>x=^+Pmz|3g_e2_A~c^W|3oljBIH2TEjnxui4Fdpy%Tx zBeq;6H-~u;FZktq@4I^$UBW)%q@Uax4|sj2t(O`?>Ad-%c#h4Q10~sC^-uJo^6d)? zh-J6042J$eT&~|*9s2>*W~`yeJyaf8 zP$RBICNS2Rj12uF&6D{>R%x;4mF!gmwiCl9lZTD@PjMA2aGCryW*!M;M1bzUCPR=S zPxuH>fCbAsM79R9HP+A?*kBiYXU>jiBX0f#{sR<>m(&wu&LC?T#j!d9749v%aPU0M z_Ll9iDs|iUaF3#D*O5poq^i#KV>cg&8efFN!HEqwN8JVv)w7{t4bV<}tNXL*AfkjZ z3*bKJb9+igf6_v<_1ag&F;T%;G>E0 zYCilJMRIUg4&bWsuT8x+y77Hf*=wFH-NfnKrCXaCUfpYNBu@;kf%GhU1tS9x{Sjlp zo`hp8;A$Yf)D`O?vD$P`%aK{ERD0dcEsJU*XY(>-n*Us-WmsVsXiW6!iM44!=-g^8+26zJ;TudDnWP-+`!+s#hmjW&N= z(keS3YS4i=4WyIETj;SKy4ZeD2R-&+80@e27_i63%5E^Fq9kd zeup1KPjl=)`X$ae*^ocs5bN|YTrvc3n2PXf22FzQJPHi2A>38u-l9BoE*Nj#Ftnv) z@~uXhlw#`Xqv&~bpWJ?Lhq%z>OTem;KBC^4uLV;QFOqk`ZQPSP)$?oj-~FNrYw3v; z7BQ;F%tVIG$^!E}_HpbJ_j3-pj0g^D@O;K1!vX~>07QHr>T_!bp~vkX@&6z|e?K6B zNWgQVNExQIJ2unU0ZG}!PP~xfpOx776BAC%ajST1j}-e0qtXc)jTWCeT?=gJu~8MaM3pd|%Mb7UH0{*Tllzb^T3_W=8Bx{e8^(Lp1Xl>+ zzZ@=r&5V~|J3)|1zx$kqlm5w0FrTQf}(`P!%k{4a%WnwX3WDtCpwJ==z0%6V>^ zGkL&^JvRZDkGcmmM~tX#A^#tO2L|BZ5Z-@h8@QSn*o&sJM66b;z^}GH)b*S5uJ-kl z80#xfbwCfMyenL4w9NU;b#C>g!Din<^(t9bxEhDYlP1uAuQap{Rst*VmRX2yc2WZu z;RfDWMI7$17{5bSiB`NbR=M~g$di}P(Voyc&=qq*te$W0{X^RK_p%3ZtP1YpOvWpS zl!X`AXm%s_v^7{Uk2TlP`0r{JC#sw3tf>Bp@Z}#@7?G~q6<5!)$yP#YN4!y(^7*uzW$0YpxV900*@z? zCDo0&u9m`10Q_@kVms`N3Vl5_dOX$B$E}_bko#Bsx-X`1+VX$S#~oHpDR{tn_$$Y( zB^{u;He386Fx<5bj5Korz{5gV#^Z>SfO_s3Ss3s|GaB)Gu}8q%~>S))%%Qu6FZpDW%g&Gs^4rt`Zxr1(4~BRxM%5r5$Nf{s{NwKipBo$i0`+0%u2HMGb3<9u%SDH$)|Yk3A0FZ z-j;+usvqH9q^)4ahS0!#Pm1Z_**Cl<8TbU2ggx^a8AHOr0IY>Gxq!)L3xUChvY&1I zTG^rqU%704gS_NYGkQ-&wyK#=!^X*`*DVos(yEsuIWF^W%LtQ4i4j|&oS(rwNE9+% zYz%%En75Wz3{*)#VSWVo+BdXPfZ@-K2KeI9AmOmRw*L+;DH|nML%ao69OPDXMvY~q7pP; zmm=UECb#oHh^)MS5btmNgYdJYF`Q&*`=B;$EI&9!>P&&@K#e(-NawE)`&a5ewq)Dd z{n)pgSb>QxfKEBUuW{9Z z0;zBmW`7y1(Jkf^spCaD?d!>=1sWb!%cjgCjWYZ$i1($GosTf#F7RgKjznf4GgvZM z4qc5Pp9I4Q0_a1iRV@fG?X8gC;c*)gD6#>AqYd1#^Iz<#`lHQ$Wg2wZ^a{VXfe8eH zI!(Q3;+j)bj1a5;I|oCNQ#Blc9lWaQH$VNA%cxIcc?jY9bAT8v^Xkf5XmYLqI_tS5 z)^ZD+t`_jkXK#YP6f!$zkHFVwrLsD-(mKh@O_yNfqGY|BmC{093Xj1qBK>hp5e&4|t4Z_Mqy14uT$TSD6`6IhFBAxWe1FVg7YZi_*uqD(osKZGBo-Ixam9DH;ts+PU`qqAfgkds*}%8={y ze5_m0l-AXGGF^ODlQ6E9-2OenO}`UiYtW1MqXMgZ0zJWZQMaHSGP!}7&u=})1EG;* z3l&&8IB^YRMA$ySL4c6bxLf#%WG>lz;&N#3!?_cM{4YisTPS3f*>+6Gw9& z46R~bf;xejs)ZPb2E)*>cqz0ghzPa@z!})GPnEme($z&h{~%1o@b?wwAXp0UJ?sE> z7FL?7CsqQmb^6Tbhy?aucf4Vj(sl7yeY}{+d<^l4Vx0Pe0g`LNp0}btO@Q8a!J2sh z?8b9wgbuuA)gRQ(1uyhBODx#3HfKC(bWjoy>^E>c(^JN%fO$X(bz+aC;}2XhT)2KD zNV_)v2+GYfP?ksF$nlQ3=RecgZnFOE!p)#y9l-9?PX+Nql~5f3AWaf}XfP!9DJNcl zer_nE8xqtLCcsx`imA0tiVYKDVbIjipPT&l7ID4CtU(CQPXi~R%P3M4IL_w)E98ls z)fw`jA=xcXnmibXJ@`1CAUghzaHr7Ye)^DuT>k}elaz?Mh~e|s-&-3cZiktKppprU z5pZf9knneHYAJY+^&0RVnU^1L{Oc5g6Zt=n;R=LW1xA_e$S?#{v1Bp;ICjPJ1x{QW zRcVpC;2$*a=yT4mLy>XZ`+EH??iwZp1tNmUSLsE17^6HVt-gTNYBRWFY?zEXo=*zNBZ{b={q!9F3 zvNtkjmuaPzBN|X)8d1hW=Xv=ZeyO_KS>?=ioSrOf9z2|Craj2*?J~p9Zh>HFO!>y5 zkIMNMr{)o876l)71Mrw4LrrGXGo>*mFq)#!Cd51pN0rY6W(J`See&m7i`e?UY3|3B z739Hh!j{mju5F{hcC$;T2V6ndL0ENR9v2A`A4tw+h#$O#USgY;Q?A61w>qblz7xtn z@y*ISb+9YAIM7$>_-vm@{e;Xd1T%JbC;-|e6HV6fS8)}2JE5ZJn*LG$&pN-dCp2H`{oE(9@4zATQKf2q zlp>y#ir;)r{8r5D+odV}v*C%JTEwzLZD%0TSD$Cd8lScIx(cFvAzU4vG-LP= zLcA0P@Z}b`_!*k%EYlFuW7lp#n~A6xH1!&|RwG{V*w5BL+RG3L%W>zBO>!nT7h0P# zo~3r_9itpA-(i06yH|Km`n`MJ(+hV+tqjF_xG=+*&r@0Z_Fba}gXUY15pMIvFRDO5 z(5VooFl$4&pX=$m?$F`q94sTC zdVZX`sn8nes#KTZd*g#Q6~PQxq5T<#z6*N(gK$Xw%f-CU1Q?(#6)k)eF3DBtM8)JB z@M;#I)_J+owbfR4N*WHon<}T4989jRJ)I)(Xbsok44J5t_j?8|IIP8=i`#0Q4?&{A zZvavPdFGjCloe(hA%OY%UfKFx${>6Ux$0X89-t2r%RWeq#JF0#VFy)Dv&tU5rl2`& zd#^CT{b%Im_>Q;fvePfGe|m2yEj`K#-@#36z^cW}>sbpMG&#FO=IcLCMnRXrKZrGn zY`vp&fgYbfGdDm6xa^M^lLzE~!vmn^3*ZW;!VL>5&(H^(0e(vCwxcG^ijI`)=(u{R z6535G)?G8*vX3o3CfK{DS)R)~^8Toy6dwXoM>RyzbA-EnXp(h!hDvrRvkSi&NaVz? zJ{STguStxN$M^tNeE{{0C<1w+4hMG3C3bGG;GWVQO9v}1aKy-ch=Q)xX8tfrcTfDb zmaTCidCGH7K_!br+>6~Q-qW7*QSEp_ahu)=IE$j)3X!8>v_-geV^jzK%$E{JQ|}eU zYw`r86g4aVtDnsdzHd(nL~QcHx<1V7p5ztgr-&TpYkxa>0B$*-d4Lam?qdu%f$ZM! zYZ5-1$lL4#(4tSb6LPpGN@Z2Y@JHoK|C{dyeGd^fdA=Qo(T6WfHA_W7!W;H0b{mo_||&P70kK_L!_r-V9<*6j9|VRUI_oRii7Wr zc|Y9Xo3Zc4dyJ0%uy6l;?#){LW9UcH$1fFO0Bmag*H*n{ z9mbP-eEupxwHl1DGv*A5$LObN48UdqVk`2wFV4(;f9G-}?#`Z~L}txH;(?TkwMywB zU{zgh?d(Oxl;RaV6#Bj>q|2TNtwr3oVfxTPCujK47R3&Vn-dZ`pmzeJ-AGn&tZ}cH zH^oVhrA(7b9io+-WRAF$Je1O#p~=hu!*Uk1H9?yhydWRdw>R}4MB#BXVG#>5j%_Ev z77rt%bKd!^b&wKckcV1i1C=)nVJJxQy|e3_t@52FJU(LVO@dKlL%dA*Bip|92f{b^ z<3F3vo%w}H`X3#y-4?uh63;xO1azSz(zAss#=s_xSI9AZ>x7v49-euf2)8!2SWK;G;-SM}U$F$`u)j6(C#3$&iV$dc3?f z?kW|FwQCp0Z9o?7wQ{D&=V!;v+-GFtFRoh~O3LyIFS`9N{{AXQF4QJ9NPl2Uy9rGO zfG3R@CGS27gKYymKDzQ$JAfVZ1N*7BDZ=q;64z<-nNG8CzU@fM)oDy3>4t~%>h3x5b2P^ zon94m1qS#lBZ@9e#tu3(QDg!ioh@!2y%X;&tYIyn^ZUhD-gIf%I6j+nuDv#|G7&rc z|98Ab8p6kdl>1LQ7~FR20y+h$^N}TGm`gz5&>l{pT?rHCu?kY~3ub45JyDe9nF^oGubpb`SnJyUSJ*xBF5SNn@V z!PoA-2bHR-`_8g)AwUem1_JT0rN|-(_1OyaMi3IHNk=mgCAjWmiDbF(d zvbVXJ8ZlO`XN@&Ad!_h#ajQG{cra0qr1SSvpTdc>V7h!H>Zz07kWy`;z^|PzTg>;l zjvnb&_2Uhv1{KbX7?r(#tAEw>wm$0!O$L<2j=kAaw%owuQ8?ZAO2_wf3AByKtiF0T zeJ!BK;atuWo0r0R1$;o{Go#xaXa)E1CL%#WQY0{TJ|jVPyxH*4M269XO=DfC%^#cj z_;#Zh#MM#+f@>$0xsQj2q2DG$bip7JGZpB33oeV9$NTr{pvrCUK6&0SzWpe*YPL}B z4ofJPA@@Ojsfj_9D2fCDOIRDwx)Edd^Qpg27+}~`@#opQR&n(GaI^N~w>Hzx1xbU4 z?xh&DhV9ekl{y^yXPXWfdN>m)D>h&SO8~bz4l4*xbv!|aZZ+x{mMv>B8{dHV^$0ZAsKYU@y}^x7z}QMgl{rk} z)BKc*zbU(&DhU6P=-2-8gn^7qF78JFQ*2?rfpKUf8Vz5Yg(!W@EMT_RP?JCB<<|nh zhEm8IbOjvGEL{4#wdF*j(`dQk)h~h69M{IsOp)h8X=z30EFVlg(AVsO)QIE_bRzMO zVarJfniB^Gh?0q5M&n6?DukYS-Zdmm7<7VReCNH4JI7~PDQrS^XBsv|E4@gW0t(%( z_dZn8lN(>Tgraele`UaBiNtmz`5{-{$&nsHF;G}vmITi4DGeP-GkrIj=NYuXcJjr` z#fXr=&2I#eJ^BNu;+PHpndyLHP~^uwYoQb*6$`78t=qCWDOM$O!NT+G>o~v4*Xqtb zenC{;RcgKY+50OZo-aj8lSE1dUgzOG%P623SOT>AA7hT@>}}zOJLbS1nv47bb+iqC z&tUHf)6jJsNf@O4ZRE@zvVV_Lt0_CgR`O8RYubYK?{_fK(FkW06|?X*gyymsu|T`I z3)cCZ-Nb$hsMzO#MU8cZVA_sH@ciojyIdmd1)9$ zIn{()y!U+X=VViiqrDQzag7y=Zp6P`be<``_?Be6 zO#hPiO*ogrb++QWt+NkoQaLbwBk8_N&o%TnK#r=tQ2Y$SU+gT1@070Wh`VF=(w$B_m{b>S?7O?~!wZsY zrWE35i4+&85o?lCa}XFaniABe(ZWO5r9_mg>m4KGBG*>86aowR%LN{EX&m4buk!6S zc*C3fuc>el|1qr_l8ab+Ss~t^K>O?jqYir(cKO8q6lK22GP5BS5N z+vtBeM?*A2sLizHG~mZBa^ivUVCFyVJH(H%tm)WE$BUhC9Q0w7_`3`?@}}R3WaE+Q z5K{d1NsUei1FjAj5R~P;-fgw!(jH`6yzh4b`%%IO4Z;<|J<979$>EE>H0--rTa-o#v86^1E6J!K^v15`tyM;MYrr0qc zIe}|F?YodDVh;ycHofyvUR=%W+YE@8MV<-Do;_)6A{W3-%4A%`eddL=WvIGv1Px%A zHuZ;r*u(In-(6Z!p>#(d(130J?uOj=bPu=3-fP_YLTO`_2cM%~yuHmL14;}}|L?Uw z0AcW?Z@8(Ed0ZTdwCe}z#y`yWZya>O9Ik3=A=h=k#ZIWDzSdh_@zh={FgSCu%^^v( zBNx|>hXZgu8#~fO5fnLo6oS8CHvw@Go_qz0%>fZ1DGgl`bnk$oV zwHx>~Y2#?Oktr`eXQ=pQPlqhD$2X3j1(sPbpUPj*=tPR#&7@XPZp^cHmz2-+2L+s1 z*}nD+=$OCfss!>8Bm_G)6l$x;0mKIYDTI9tsa!histThoLgFT`P_$@?60gP}D$h;~A%@xNs%Y)nesOj}=1vJld zB3Q|sP}|?H>HPFkjB8TAR*JB;rA>+^8!M>5g{OnQdfr$<23@dY-uUS)Fy0X_-;vFd zGI43>-kFckIeQVWyp)02xleqm8;??qjETR z@uLIpPzMO=j}T{4O~ne_PlI{eJt3$%pO9I_kCcs8BKt~HYI$ZtU9JsrMoa2~hiilJ z0|aT$=RgUC@=9xx^-~o+B!-Xd9sYJ-cW&;|BfH?<_y(!;WBeL~p!mM~o~64`2zs*z zpO6D98PgAmFBo^`0IK~S5g)8WWuclYy>f=^I8Y5Gn16<(5&%54t zWc*H_7Dz=;)(ev-mbBb03RktcmERLIauhlGmqD~RSM%wMj!GB{j{iQs>XlJ zo)`FGD9)awuogJQU0xifUX6hMZc{-36&8cJ5A)K_!Kf~<^tlU`W3^saj!WBraSG~x z%qo$PVkq;@BrdMcGL-EUZgLRT8KL&VFUe6o_V zmgc&~6d$$yfOF!ntATi{6km9PDhq7(U)B^dUsVueCuMhA5B(q;US%+u(vyZ~sEGIB%{ z>0;bet>T)b>YBd*B$tl*Rj?Pf9qw5jOT=Q%--Y z_$rIT{H4Ch=y>S=p|<~kHU_9ThHIg2CdTIMX;G)&fRh&zSyyIx$TQTimczBoY2e`3 z*H_jKPOJCIS^A0PZGLC(N2vOR@DlGH%LB3n07Jq$hSUkTm;_DX+VDINdBr_vC3ZV< z#2vBPx+m6F;~FxAHSRa3=(wDolhS|lO4*(@6`ib}Q!m_ACjRzE)O=K@Y`X9PaNRit ziG8i#ky~jrP)+5)Ky7+@r|5Q!MvMeSyKR<*A|KZ*|2FW-RL87$_$}ppU8rK_W7qWO zcU}B7dGD?iI>Z-5igJk3Q^D9sm_PHY@WizaWGgXt)@6V%AP{x+TFQOZOE$(tfck9i{&RD6UhPC=|;ZlaDGPn2q?8$+~+xvh2 z7GnauTdAi24PAjF{sBTD5Z1ty^i&BYY(U0Hahpgs(F|3R%CkMHVA#xsX*XdM4}R(UABE}3)p&^`%f{Y<7 zi!a3c1qo}0GxrzI`@pgXLP@&ffY|OU{lbE|q816 z?=QdEE3b2$jsyM5W6)h_=O7kjRuZj2W6^66LL+&_q4rruHqrAQvuE6B)i+0LB!({c zhIcKN$;k`60}mS5pH-DPC|1>-$6)TY+0~R_)QxIeo!;x~GtKYqgg*Hk&r0L@jkxa@ zms3>sTLBOP#pi>KRt=9#y3)K9kh9eW5~Bk8$S;W?K-CRes3UeMJur_V@?iOY zF}Y~=JYvX$lXrzP-!vRna4=>6b4^rSJo-FKfLH(8?CcCUV<7$uuW3=hPM{i$!&-#s zRxm`S>I(Esc;?&823{H799V9rp3^Cj71n_K6qCr7Y3Hh@MFc?hS$EUuYcJsP?YnOv zA~i9VLpIz;K*I{4a=KAxTMrXDUvZN-z5^X2kca@;Qg?8|5o{BTs*E{~b{s3LasA`h zKT9h9Gv;0Q9cC`v-#%4P)#D{m*C}^S`fs+8*XK#DTn$tg771Q(9JEr$dO=URFI&$I zwto1ctQ}3*oE^C|=H)H_^yF-E-^8Y(x3$#S`1TY|D^`vkh_pKS4QT&hweDD11`Av` zVL~G=@f}e3?G%D)_W4$}kdQ;Wf*r`-11A@JoAua1ck&*X3``gp-|!(Q<$kD7nB%`( zP`PD`(3EvKEZ|s&p@CP=fZ}t(0kL6ziC*sOCx7TLcpi5UdRI}lbON!6JxjN(hB?S) z-FjMMO6wbEi0n%NO*rr!h$|Wv&9G`_30)rwswQ0ndy5kR3 z0Geh~JY%Z7=%q&hO_XEG>lJJLsr+WD=+1Y&CAFu^eIuokVh%t<3_wU}yx;*s8KfjQ zC|Pq8(fSCNRK`nRbcgMH2&y0lmX4xJ@KMB>Tu7SlFNmF~7%0$Y#Sae*|7u2+240-~ zF?MfV$;<6SnsWa8_gR<+hWjG|`=0w>wd{<@2Ouo0AKdpb;vv*6ko?K-LhfT8f;n2P z)%DmnvT?3;x_AzbL}-8w~_^;|st~EMxdil(z{x z-H{3&VuY4(=X8d`6Cf_=zw|((yEQpH?mIRcYI_WkK8Q8Y`8%lh zU~~n{+qMHm>$p-v%>47~nd}EP`&&AaQ5cmQ1^e^_{dh4NcL?iTV*SIwxP16kg~@pJ0T(x zKC)&H7fMBxC2O)}$yk#;ON5w3QW$1Z{^=1awkf4s9FJivPz0kGDdL&IK0!vB# z$7QrrO32Ewo^L*NiVTFg5hTPeKH`iz%a}hd{_ch8$V9jD{IR3%3gnepl?~9~{*WHM z4s|OMUk1Wz=P2d-G-WT{cJN@b9MK;v4Ms`*v0~3d1M@&OvE(?3sWSqr1*jGAP18!E~E~WOgOA1thGF_#~L^<)raPcmV@PcBJ?qVFU@LyIOZqd z-?lWII!iWD08;U~seS3>0AWzwoUw~r$7(Gkc@PI~wX0kM?qd9e(W4k30VoFby3)L(YFZe)>p|_}Uc^`ZkQqP1?kkceei&Gw1Pq4r zZd4-}!h7}du!dXO&z4O}&4&7$_w#YQP=2WlzQ07gTRQLWGGCm@Z zQPOQDg2;8{(bALuT}1(qhS)79#wh5x8AI%Vl|htrady6??>kW;Qs;&5ZRWsotn)eD z`7~;C$9^g4pb`o4~v0p6u+WjH7CjyHqDWUZz-cc=@S%1d-vT?X*Q!%Y{_xp)o{rlz>9C?5>;GND7=zwb|cHE zG5MJ4TN#^AowvoI56>V&B+2dkO*Qnq52YEJDe6UNXh|#8WbI~-#h{}0#?Alvg9k3U zD5C({=%$_t-yvAkc-^h=iAIoqg8*Jj^w*%5e3Lg1dp4=pr$qUeA4HESKS1GrW9DzH zx^(84!?#IFDDRrmExcK_d*%w;K)fqyFx9QBhrjX4T`<53?pi|=s$JUfpb!n@8p&qYyyghAvstK z?<4!U@03p01~0%tu1%jDWuevg>A z!EX6WM0+*{_`F~@Zw#)ksL^)CV{tj%Fwt0r>!~P$x(w(dCEt-}rT|-{98jmj9BATU zq1PBU@Fd+-=hU-WC={vrTDWT;pZcuZc|YXe9{i-G?g!W@QwAx{7+HwMhwLiisiqyk zO9=J3sKF0rH3eHle-6v$U-$F9zTGmwRc|g`?Pc&>yW-EIoR}I~zKPhSW0h5^r@KJd z*jw$nH*yIig{Y0O1t;&x-=E~fnK2g2zn=4eq5S{IyP)Vzs9OT8L$L#HVk##bMrayH z0*xJ!Pqj+2lTCidziun`uXKLO#YyM=w0}5+{C15fQx9fM{7rOcsJr-3BSbP$RNhOk z0&0y-E_dpCGymq;Ll3P&l%L@I`lD4MrRg=aQ2bt9bC4m@i%0HXn-!H-2`?Mk-G;2& z^5%^b9UN-5j;3FJcHk<9={1jJEnWs##t8>Z#(~M$Ca+jsWikq=iHOjUgmhbO$rYNi zN0iqqC$peq{o)m~LOI4mwL=aZv36Rn5ex&yQdT3XMftslhesf65!$&Qq*IxH*2*`o zh?!H}BV}xCJwfMq*c8WA1k+g)V^D3o`Q%KXQu~deLbGUqH=X$7OY+9S$kx$t*&ZiuPrz@*RmY8P|dqKNM4*}^dsr^Uj zTgM2kWdr6COt%jAKJ!*5i{|MNqQ&M<&ftMzv$Bj`>1>0yT z1zcBNFajQQ{}Vc9kAuV=*H*-wQgzCbNa*8fL%mi%ngc&Xi2|ZY@BFNqeTO0WZU|S7 zZV>=bafl)a-zOKWpL5vw>!Kz9w)wGOX#R+UCljft|~y5C^dj_r@*d15G9d?x-Xz^ zf^uHw0%J~6If*dP+e}{VGk3_tUYk+l~587v;n?q@` z(R0wxFtBZc*$4v3)~{w?1S1xN-ba{bFB)GDw4mD4jAds?YQ}U*qjl!LM>M~43leCv zSoThrF8FrUtVqX-$S%ipUklY_LsN?GM*fUKv0^sveFX1?XN5vS_Ocp;wNACS!vmYk z*=9Hn$ErH$!@+IFTNESKo0(Ka>7%zmxGe2*qT%wrOktYxTQXFX^7WInc4zBzpV>?0 zzG2r6cqA9MWoQn2DOE5xe;f7+q5uxSqJIWo6J&N`oZm~vcp)1aSjyqweg|qo7Dj`I z!W~Jcohg(_%b=6hW6yC)Gag*epAu2S^#8d78Nx*OkNymFQ1&3pEi+XBm`HPh{fc2w z3H6fA0R;b=V6If-C4o9S0T(f(NH$C4Obi1L~Kwo?>fYdiP3M4pA8)eIEF5&e1d z`%4S~!@=l|A`v2dP%$--^Tx4CKV(uT<Z{^oDQkbb8C zFwb+qP%pJK(Z)Wq@5t<-b`2Opz)_hn4wx*DClqW^nzQBnoZ9^!(4G)AK&7Funk4<= zP@?$5uCIs3bi?|KNP*}LkL5uAHOHA^l2A|0kqX}(>!mM^LUzp)BJROk&Bs^mm!13Y zhitCi0`pc_Ij?P>0Cb-*0KUCBCITf`IH?C}Qmjp-nuedyKN5LivhiyQ!C7?7XLYJX zF67p=K^Af(tP|V$%_g9WfN_+EfqBLyLo2|AgCcc958`2DIa#s)nB5qq0yGWs7a+s! z-iB}E^;Q#p!@o_PEIu{8Q`z{j8~T20aai+l*fCby?qMmWZerFhxSOm`>BT4yC;RI1rs@oXsay4|{8IHVTR+%)n+07!Fq#VG| ztvFKoq0gKEGcoCWFop*P#E(+SblxIC2-ji-fbVPcEP5Ks8#~w1ha_EEMD}C30wd*0 z7~{K=Nz!~yL&@B=$pOhvQk_0L&2Hdc9dXc9OPAdk{KghY2*{t%+$ZGPRLhu0l8lqp z+u!Hl!X9i&`+R9^-gVyXKl&y31M7DtODlx1FepksqTr))L}e-$%>qMiPNE)%0~We= z$%mZ%ME&X2c|f{tAL&hTfrE?bOudi51b98{@ouo|hx;);Cr8PE zM?p5q{DX0XYzIhp_^zeuI4we^g4z)mx;7kSw}nkwUq8=kQBW}$M2-MBMjXTXG1VNJ zvmu)>o^3)^9~huW8@xuUqk~QSU%|i|Hvmt}{w#V1dx#-UlcPV0^r)p>Yn||q7I*xS zUHgt+nJQCF8PiZxNTOfQxn*QY_lg^f`EWc}`lgQsPx)YjD(qSKa5p1xrdgI1N1rRCw4KT_^I{ zot}uS;PilUm@!mEv&jCtq`v(^M3K}ZGyvd{| zg`nw2KUE;Wae?CKU&rP3uj87*z{fBHfpcNYnj3$S7(;EK7mQ`|o`1S*{n3^=u;Rx~ za6P2&F|X5+23RVky>5ND&&&gw$-?S;-I6q*G|T1xQ*%hy1_9#i9SME8+dwIy#{dNK74n@p zsse1ygMcR~LnoB|o6ODpe{(y2;yf)#RGVx#Cazin?2}o*wG~~7@vVcUl{Xfs0g)Hc zB48YF3BM%(KX)w6@wGM&@>i>uy>ozz@5sXnEv~NDRzg|`o^RWGdy&w8S&ZD|Ov#pL3nkfVioFja zAC%j$1on5H2LRnvr=dT`It$v`z>XP5^k|KA&2yR#>_*x$|4I(@D4lhu~^& zhYoA%C+xvxyNA#}<58Jt%E2kjuXPUtNfkgunECk^^7nr}<<2r-lY!tj-59Lk-zx(T zj=*+xSdDu8vo|i^ci&aKnTDdr77SWVTQuOXQ)kyKX1ZM4bHd11&~;dnhb664a{x=Z z0Cuf(zW-vPwo;9uZUKx3W`2tBzIeIl0h(P`lfWbB+s(RJ;?>ra{r&m8NS{hcgDb*5 zeSle-@R+It{wzPrqiD{A@czj%00oGVmhKbYA%p6tS$6C!y+z6fe@-7U zwOT+24q?f{CL8zuM(B0QL3lM3{5$$4XhcCja{`mll#2==gIcegU4DmvZcCm4hY}Ns zgdr-jC=(LuJTu_|JIFq5d)C|%sQg&sr^Zpcu@astBk|?9Wp69t`zh9pj~TLbM=Iq*wtrk_g_uj0URE*kYLd>TDzWGoNza47)d*1$*{&#&p>kq zlhChH`Fy@1CO(MD367+D30wcIp~hjZxtPq?AFyhAX>>eEqc=b#W@*Jk`sAu~j0F?! zS6ws8HxNGV)noa2dk-X;Fu1HNU&!0u|hy%4SA}JB5h?mRp5J!^f|rKd(P( zC9;#G|8{5X9|^!c#MD1Ob@u_E!4`Z5 zMRB4hcA|W-+>uuZNM1h|*^m3mD!^==`z+S-X-?a`S!BOtuxp^%yK)%%|o> z5(zI9^i#lZra|2f`kdh5Dt5@105y!J^{RVCyccgw{i;#<;^M1$5)*x05L)TGaEmjW z&6HVAiy1XZgWCu| zrKM=9z=yvsYz2%n<^_mfJaE!s3dku-*ntFr$Qp{4g*J@-Q1m*#NBOgA_ikPsDYdH& z6{|Mt`P~I8^EexfE;t0thvjwC_`Q7d=Gz~EQY6YZ1@Xsn8B06FOu0fY?^IZqJVPGn zhPeHrPtJZl?5jz*S7qD&@HqPPq{wB?hfI*?Lf#qaY_l4kxhIVMQ=w>+?yJ)dyMS&68?=hEY-C%u?fddc!O4?FDS*z8%3yRGWtQCpWv%|pyW$4RW{Srpb_M zep6m~mHf#U3foEA$=|sxv3;rKVan*!BY0yr$WofBNt4w!p|h%CX$LIHbq_&xgh!`E zABPVBuY3z$BxJJr6(l8g7AVA1TEF473a!C9u)IA87SB$!&>K3BN1~j=E`#RW$?fhd z;fa@ohufy8XXV#9L)xyQ?pVrtowXCo_rAGt2{Y-Bh?b7-DIO(=LZmLuu-8lx%AS)FW^mrxVEA`{IDceEQj_bJi`{Un{7S z!u|vUCmjRVw@Xwop$mjBsuk^W07e47qGd-L+CF)H*CA8a;Qd7O+$u!l)gkO8cj^FK zJcAv6@m?E*kp~?{6+S|ToP9Ol+|3s{CbsZMN(E(mUEH)Z!=d?WwwW~Z`mRSGXK0|b z<&bHq{XI!;eIQ>a41>uP+c_V78Mw68S8_j0mP???-MusAQ(#<-x1Fw4`~D=0^BU&| z+Q;%ErwtYi2Ho=XAu#1(0VWh@`U~~>3gsPXygyKocCCnT7*@Gl;l1lo7!wj=z9KbT z;GjQsI>hqPA6F%(05ilZdRj@0{F-QgS+)?-&X%&|dFRg?mc@*>gO%j)2J=Fj^V5wx zVMlFj@ds{10QU8vEt&+L9M>-i27j?l>yn43e~+Hs?$w8QCe$WMeq_q+YCH{4ypeb3 zym&y?m_vp=-uPA}`wAne)&=%*Y?y9b7_LEGo>~PAL{&bHA~mJPTwtq)tGb$rYyXiL zut(v&H|~IC$`kf_`q(%3Njwia3`6lcL`iz+v%`+by5E1TH<@HczRtU>;CV*kLf(~a z_ftniAsVpD2eP0rcu@^pJYZX>nGVY+ zdTGsirqCSzU9O>N-zoMWzLP=e8)k!Y&~v22CBi$UCC`jY=9U?zgSNJB4Temt0Qe_~ z!TpF{))7?(b@M~MKcvXU72~A3i_^6MMxX%1 zRikv-lFwK4eSOs3O5gdWk^KCYmfLExLbpHkf6X(w;qdHHo&u;SFCn1Fw&(`Pt15u1 z_^D|&KYi(HF=e&vR#bX4fM_~GxEUk|1B-6@R!{aag!?dDzmb(vM?zG6X&`lW4~>ZkiS z>0&cgO+}dClET@3TF*>32LN#5BSAG3ccKI3MwU8X1bUc_xkV3XDDE%8zM?MA@L3+A z6=x4R9{&cekVjj<=RPi+SJti~aIqP{0L9>Q42}V+9*l%~iby%=aG^dl*W;XC?Y6E! zTuV!?YX;!}aU}U$)j*)2uy$LAZ}8PN%ui0zDna8NDRm&iG6e`dp}k#>d<@9mv7A;| zIc@2qB3@N>Fdo^J!6QIliWHdH>@X(<_BiJL8V!o7@SX|N=6mg-6y2^x^z`vRa6 z>En)o%IV@wnG9abzY)V6bR9bAgWw)uK}_&O!7`n#ROLnAg4qXZDA9#4W`t+x5sg{V z^xu6!(k;Bt6^Y?p3STjHr_LfG~gByPdoZT%N3QTxe(qIgM=YeB>pM2ee&ePMsD3c0uspi@?Wko(x zQbT3R+E=7ACdxs66(i00VtMiGGeII(8XFJ`cPf7(8&EWn%OyY%fsD_D_`+A0(>hO+ z-X}%ow?rrv;S(pzrgbrorKpzkE2m%GYJdGCB@yxJXwqgbKNC0QMPS*vZ`a1TJMJ#D z+3j!rHCPFe9{oW6;-7*@G61YpSOn{}EVHK^fmcZPx`NgV;-h(<^ltkF|2QDK;8SNb zl67FwWGP5^UiI1Yr*3j6C;>EHN20Nqddoe}Q3Aa|rFI?)ZzJrE8<9Oa+=zFq{R<|; zOU-bP?`aIWr8gJl#&qLm2^gBeY$Y@?Y!9Ykpp!*=T{o!=YxIJ@tPkp>Zjnyjb;hY# zb)G|h_d@mb>gwFLm826FtlL`dJB%y75q{x3ntk?bcCa6`S&5+S0+R%Bv75OAq#P`_ z1BLvo!{^rtKw{8Y)PA%gmAPzw!9hyC`b6Nj;e3^SBP(MDR{OS#K01P37K|!{Yeta* z_9*6XwJ706o(B%G?y5Q`##jg ziswAg>QXO*%fG|?yW=|Lucquz?pIWdaJFV5s@tq?(FW39EU`=*5q3Z7h~RiwF2O$3 zivY-=hMvpTbFwGRCeb+k12q%X^YV#@ee#F_|Kr^b(NlH9 zXjo@`Stv!H59*e2i|f_?R$M%=%p+nCLy!YiywC~#hPUJy{<#8hF5Yu zt(nMZnxQ6=F5v~f8wq5_Tj|ly!7=foaA^!+fU){C0kTq2X?WZ6K<5ea;uky7Z|z^B zvS&h?RTZmGIVC+buCJcll-W~HKYI2UBrZS)ID;&ti0%~vHV!R86RPxs+7gTLXm+%B zON8VT-A;GytNC%4HC-(38f86CPKj%|<%v*jb>&&yOWNdpMMn#BT0pp)>D&ur0S z+^N&2;41Uh$8MR{ud~he+t|E4J7Q_T=J!VvT(P=N6}SpSSCRwvL7~u0=;Apm(pC&H zt22t zxa)S$NA+BgkM2#$Gj5?rb3etnf{h>+P(3>XuLh9SvFG4G1}76$`$s1);4{37NAiaI z!fV=s4O~dylu27A83+HJ=eZ(mr&~>r@J7V_@d7khkosEDuizOJYzUlzf;?r5f+UaC zkaZ{uupz-Y;^!svYh<3-5P!?d?OA7=K8?(r8dQisju5=#*2P?yyQaoO4g@Q~jQ!6* z@TBmA4-p18P@>;{YO)me0gy=ltrz6PvX$zEtM`U4`)KBfHbgx;&ve<~hVd5=h3ZD= z5Fq8dl^qrF`xj>Ztd6r{twc4xjbv6xIBqVHDxDgUvMGdkZNw6Lwn7W}Q2-E3gAdK( z0gl%F7ZR57A|4GTU%3Y7-X@FnQJPikzo!PilsGnP;gFjhmZkeAZREq*+6$Gj16&t! z=@p|Ffur;gcU67R}u>b8ZLNp3ghscJolHOu9H5 ztHl>y!y~&uJ%;wD2tHkBJ&tT_%0O+L3uM^aEdu!-F!9n%0t0E2I!UDC?r3md=K^57Xi^JUme`>a_E~@-OS33V*Q-)b`|-1se#9x2URDrYl40jVU?_*o?d;G(Cz{SJUofv%2 zvOmtxmku9H%Sao#8oOVO34E09+rW`vF&tnV+u?*qit#)Mrm2!!5{Ya{ zvc-D&l=O_^@}vUy_%6@Xo3GT~sC=j!YEQKN!x700%;^#^b`uBCi>Dm}8!^k*!f2BH zO;y})f0i>|h zQ}7G$>CNElAT@)fiAfu|eieSZP<)7vx~NO#_Mh30SU@M4+R8rkCg5$JiFeo4-dwBa zN+@N3-uiEa(LYY36D8bn8LY_f|3Nb8msA0+nJ+wL+X8ym+&W#ivK8>ULZ1KLc^r^g`MR*cM7emHRMcKFu2GEl{G_Dz;R*Yo zCJOS)9OVcWNMT!OJnhMvxg`TcP#3@#p9Usw1XA)O3WikIj~1f26cP5pviMLvxeIST z9?W=Qm7WXcDIg~g{}{Vu&)FKg(wu1f3*=8bG_fA7@w;wV^RJrd-QIh_N?0z-6 z*ZAbY8o!DGj)ePJy6{Dds)wHygdu+%uNmFABnDNq$iv_YERgUWZBm{$)=59(buLt( zC6-IoToC{6^8BAq!nFoBs+qb^f@qLWq=L_A0eDpcpl&T)MXL6~T3dPiFHDk^1J8cn z6vv;Jl>Br&n-a>5J*a0NIZyocTJFv{c`$VbQ~ar+%+G4YX&vfjF=3WwI25 zQwAd2narw>p1U#`DWYEXq)n2|d)Q1iuU-serv6f5&BGr6pusKtIK z5hHmT9n4jKbN#r@F({^x^?_Ob#L#QK{c3A8aK+>FqCu1)tkU?xteVIg?G=6xVdVkM zPRR-VIk`vk(M1x8&vfrp3=}C!5}cpsG}#~TsjQ%11*rp=&_cKpG|7t+N4?IN1;MHq zhe=bVp_<7nwE3l1FVW$V>rA2Y$u3T>ZsZeBnTzu1~6g{h)SK70Al#~Ap{|nxA$}A1^ zh8g=$;~^}p;9njeYPf8UL4;3ruTk>m^FLdZ`sqDQe}}&ZX(&I57P+P+1=?4qcbWE5 zT!_r<@(OqvMK{n2I1Gk8Js5zrD#X49O;O;vo?_l;#sfZBi?v;<8L7`&b;+gkN1c*P zR#~B^givJ02LG++FVuSy&&*w?*EQ+LtDY<}YdKALF0$qFW6)+XH;LC$QHW;o@VxeuN%Jdn##dx<-FMj( z#5Q&gMe!nGHv*|LS}+O^#T%{CqzXBvaxC8XwRZgYYGp2TXLDo(fMrFx5UzH`IfvTA zA-k_TH1QuG*C>Tc!BuUW1Z$N}cwI-^Cj=fz#ie5XclQD%@C~tZC0AH(nU)@SICS97 z*ghQB)65T+a4I`x9&{Je_dv&}ovmm_&=j$Cd+PD#ydi&FUv3P1O6xa$a$2%5t{W6b<_&T;ow5p)Q=^FzbGg*qx6- z9e{9Cmaufvfb!>cFiB1Z`)_q|Te{3I7vTU)4*MVTuYWd){rC#CHMIF;AANc@m2Gmv z_*>+09dzL4r#)4;IEZuyY=p`$mlNZtS0Yj1o78pBcIr>yN!9v%@1}kwY=?3lz_)j{ z9oF>{NtpS5+(V9Agp2L#B6Sd0wBS1U>__RZ=Rg-TF&zF&B`62XpMJ0J0*pc|iZQq( zOHTxb4Lv~~UsO`nYr>l&Z=*PEw&rz|IP$y+ob zOCUl+P=;kKX@`gV(0md{XGwXwuDb8YGxQ2&3^+yO8`L=oHE>;3Kw7#JY86UVD=OoIasb`^Xqs`8fks73fugILe`YMw* ziPPRee|ALOn;)7mm-dMJFUAb@9=$Jh#+rRKDG40(eW+RPLFl@K9^?gDLxMF?K@x8Z zH%Fjzfd}qzwwZYL92qm01bYpXdq#+8U~}Dm;6+(T!A?!6`=CAxO^&4ZEHhQtpCmhc zUNko$wVCd=5n*nBiDJ+dsIw4mJAWY#yAQ(!;rpOwBwY;CH%=3Tey>waK@bL=5I{;r zGR34~8n86Jd&@hq(MNtzI=RUv(14Q|RZ-qzTBL)1+^l@mGuyc`?zK?PoljpDjDtR~ zlTrN`l7aHV;T|A3)K~_yFOM2F4!8Pqe@cJ;wyE`WD_Z3QKYsb9hy5)>_IHriUL*#e zoNX5BK8*1mk1Zly<>3I!Aju@EEr2CjCJ7eA@9@+a`+xf+-7&Sf0li3^V>5r#$Z$KH zU}ItN&FXF-taX^N|1O~YHwBhj7-;|};LwTl*ghl&?QEffzs6%28aF*)6CwN`M^;>POb~IRd}}E9Adeo;@5`he z09-1B+0V_gV&;sS%DbbFVlji;Zgvap%vy_SI4=oS#La)f>i@}8U|?YE-ca{(hI;fo z=8(QH%?Q4N*Qt|m_^y4e6A0zJJ9W1pp}lcjdvxV#z=PPU56xJ(?q?MZpkOEnNd&Jp z-r7?SW-X#eFeX6&&W3P_sQQ6ajB61rv)ZeisITpGoM!W)|K44QwYt;Sv1xH*=LX;B z?(iAcG-KlxEos-;nd!?0tRT?*^Ain5b=|{&d&shn2R?2>J8TiiMN_R$vL|cC(~SHG z#ic?EL6yC%P0+N8Vc=X@f0u`~F)_hV;VSZ@?{Q`r^pBGcF+M}T0s!z0MRNEXF;E5em~<~G3QFBJ38$N9{rAK2oX6}4k zxR|5wl)hwtva@Twk)km4)Fzm`Hyncuz_6t=?N#pQN5 z-t*ck_Zu&35A5FQya$fvDr4Rb1OO{6EVtTgndVrV?1f*ndvB|%PE2??s@QR5VeyiW zl(Ew{=umstSY>|-I|pEdLY)QHBS0&q-2{i!j`9l>r`gRm1+uTZh>=hUbpdm~-m!-0 z=$-%3Ok+u}m3`1rcR%lTjD5Y0QBMrd+GGkN5Ud*lk%t}P3346ia00OKPQT(g1M)zE z{?WD9aImS=q<>>;B=f|5DZC@{RsJ`kpSro)i3tW%uW(MP46e%#D zm_X(nC@@aMGDrFq*H(o|H28{^*3E5H$MJGCm|6~{oRRxzM{x%63dE?9B9a*<4cU~m zVMaQLN{l!eJ`35y$^p9pbuWMPH4)do7`h<}=Y)AqA0j4Sf7tEL9V<`|gM{~>6dfel z-xnFO5U$~GL>+n_R9=5T^2Rn8-G$}_NjTVp&nIp5EeMA|hr@^ac3c!0Lp*KE7h><3 z>xARqE^}(`p15OWF>&k_fbY={)=ANU{rI$EAUEXa0fJq>xxcL5o#n0w?y_9Faopqj()cbaxh^+^^_EHu8UhdXLP zR??LW*E3ok^3!qmx52`RALHO(^s;{fjY0JBa3mgd2|=EA?M_ znkoy!ccmNNoBB9&?w@8o{~4pomeqwSb(H_(NC#7791%M|+*>3WDIdD*@7G&+hE%Y2 zXrR*D>s-TBMlmxjttyDikPr$jw%WV1*?g&Oy|U{DeBWLRIY3_?{CiqGGsy)A2219= zSa(rseB#hPTc`M<)4Mpu^0<=;ONE=*qfMpYa0Juf5oB%d&rF)&4n>dbMFe>|OB-B8 zYU#pW4-|l7=Z97;QZ~^ib@Cgg73(hQygu8Q8t~yIK1}=T{cIHU4Eb;JzD5QB{KkF@ zf3wcup6{gBej`O^%4Mi67?w+D5|x?{qo3L89d|y|aw~J>p~ZD}>3#R#hsE%$HZa6Z zsMi4ivYmDjd;{RiT`;Z~+!PNi4OTYBKVt~9edEa}fexv=I421=WAs6LKc9lHS0p?N?1&D^QCH$AG-&+ItJVk&jLwYMtm)SvrbIEV;(VouOLsO8x< zlME9yj&91QNi0zH7$5*PvM1{@`{M(V;Xi1nLuND`n^Pq!=k$6c-n>urx%|X!-%Dcm z5nKU+?o@&~3E(YTunt6tqE|!S5TI%SdfK`q#etMs+n(VoGs{hW%1viD`4z6`&RtA9 z8*8j$Anl2<0lcpxpcP)!+m-~@pfVuGWqbQ2v{Dfv>g3Pw3q6yZ-Zo#k<%#UR`-V8h z5%#x7qo3KJ4#Bqp$9~dQ<1@#e@vDUdd=NhfnTlx$H+wU#3B)#y65^ z+F=wBv|sqpi-5~r@r=i=#c5g}P7`G388WAc}T61T>X=Iz$G z>j^f@r@uZ_u>>nvan6tdhE(_tXw&9tMg)|XeKGU9>lMm*POo~R-Y(<3tB8a>@6{MX z7SY`n#(q`&;yEyyx3`>wD0{g`ziUi6&txvBnl7Sm2rJeJa6Uji1eJ#RDd2x zGX^FY0+^hIBmFAE^ZT*;qPgMnKF5Y+kz`2Z;hR_U7Ant|?;A)y zGm3_QwO)a6%RI3d(uMJ~HmDcqqWsW5NiItDY!qrs$ha?lW@7nP4bRglZKWUlc^oev zK0ln^i&77V7(9;jqEP3E_cPo0^8R$CF#9M(ob6ywu`%)lF8-@fDunxunp-y{+m^bO z)6T#(HTH9h!>rB4EVh@8qC{J59$TAz`s8fjD$WLaKQY0_nJH&68OpHMHZV$%Ug8>y z#OE3BfA)uR+b(TRe>ppm`p9t%@um!?dGuu0sSh%ix}uCY@O$}KQm|Jy5n9~>e z?r52+dx)@}YSKQbsB7@hf2Pm$r=_?eSBCC~<#R`EyN?T}5pUk*p$Y0RIGw11(ctPG zfnaa~xPVOZ>ia^0M&6}sqj+a|8%CGP4MV?g*kIwrsdr*MI1C!r+=lf*B6l9`cDhM8obJ(&h`Yt(iTPdw;e%#b>XRf(I*rY^O8!W{lh38Lruev9S*r2 z+?UpDCdV8HG=BR0QB2VHoJeO{;}*m8InTVP{3>X%#a7Zp(6C%SO>KuJT}1Z%8`04D zMTwTTJyIB$ML{dq6#+`YFB^=%5t6_E%>9pV6NB>huFC+_M4%(Jw>Ckt^H=T5f+c62 zNm~y(6l~2GIUP-Bwl+p%@=v;X8K>3D_qy|{uo*>Vg5tvN!`?TsgcSloX(c_-Q_*B8 zs5XyHRe$i3(L-9yxqrJtJ3|EeOy)oOTI7S)hA`OxdF?X#Lm)91JR@Dk47RDT>eab2 zn!#Jx4u7v1EoDmb<<|ELwUx14=%QW^-p^%w)p%h4iw z0>qywj3XdqY@nMo>EbO!CXoE{fVA>!3hKZ9AM~&PL@=<&>j+@?^S`~(i~sRI|LeE@ z-@Wty)2II5z4ia*JfQ!6Ape)QuE8XsNsh|%3;3`!h2Nu^pkp=|dhf=4`;ouA!DC?VGkkPF-bFt{Bo*&r%){#JzgMZNArzVgmzvPq>)bs1{yHce1p{QW2E zA$_jm+3xYjGT8Ij95IolVc~9;mA%dFI9STo0+({YSpVDDL(A z_Xsw-H(bQ06+sgl!=-*#L0M`tuv7e;mw&{^Q!Q@`B`mG(mYAJxDpW6I^I$(IoA!`d z2@+l3{RQsDcktS)7$6=8g_)C*Vu1v>_7p>Pt>Ehm$9mG!UGBU(%KodDIMx(eidr)j zzxBmS5FZ=-IcK+uqMZkc2n!a3OkUC!Lu{P$Bm0k4nwSR&YoIelIv*I*-1aQ}+q2uU z0AE&ZXQig1rs~_$aPEjLVk7$mpHVX%4 zvST|FDFl^a(Jm?_C68tE1jCLBTosgKGWgDlfR3R_=b(dYlCt2iI-c(FcJi?m2$=i(k>fXkO5HIoK3> zbLI*bzTMNTZ8QB*RsK?G~V zva;ZkW9s@M$}mG>{;C4ggAovR-9?UG&AFeGmV%&erqH}Uf=Kjxd~U~jrkc3P5Q%h{ zstJ%Y{nnnXBf|HI6EP`ZHhk)GP6Er*TL>dL3(Q7!&Y6g7N zRup@{&FVm9#b4p-3+;cd-+yTi9k)o1+Mj35v_H|IBu&*dFjwe|E6?u(ge_RsYXb|a zEB-a@Z||u3v_mt$-`)6TK1>Ha$F4T#(NDjAp7xQ+vx{kHKP}zBXC|HFY}d6lcNmY_ z=nt(J1%k>@FD9wMk5d#qh%Y+m@Ix`8KJU1U2D$iF>7$sr_sbb&F?xJZ3XK3B@n7IR zj21CNI8d}S+6?q^JDnpZW$0R#c4hnicy1E7;-jKn^-@J`Qr7+W5PS`HYE0Edeos*O z9Q_&13(%yQP19J(s9x>p(lCjR-I@*CV5cA7`e~|?8gpU?749SmCE4UyLO=OX$H-6T1-|4s_UIy@6I3Qu6e1bHYM@6rzF-A zQONGgI8j9fkA1J_V(^c|!bObdC1pZV?~^JhIpoQp%X_fOtCt_4w{}vkIj7>oHos|l zu8f1TFrTnZl;s|O#Hn@1-2Ro-jb)PL=Fd&rg1g^HY&p_7rhR#L+}IqFrL*r!e`MbV zRWVy|c$n2R3Rm4ykozwE>u&TDT#b>^=8J>^u`AMfgws;@6BKM>zorqj#^jfIUekG%60%g(8QQaGQ$*_XJ+mS_?ib@ndqFL}fTGp?);=@Z#TuBJzdHl}b% z8@;4iipNx4HA9T1OVaFJ(6crw_{qcVtf4F2y%YxNACit5ThI)5@iVl`d;RYO!e)&&FXk8vcvjEV6p2@9Ow}T zk&-}anA#~1dtI2k8K{2giuEUd{~872QJ-3+(z`a+S`&tzo;MIy2c&U!L=&3iHh$3{ zdIH7#JMi@qWH$i? z_FSAHtX#e1qW-KrZK_*aBA+5QD%Rgx);L0bk#t&Bj>*FxtF%Uc2@jCOy0DFQ@%bP{ z=ZzG?qa?iLwHLWH`?QTZUwO+|ogG}UNPE(CgV{|=4sf8w?$RkFOm7`8K%Vi`eXdQY z78{fiZCgk^p=&+!1})GXJZpI8YmefmGL@(a(fqNAg1wPTl4bNT>X$zXJXMC+sMOzx zto233U!0tmIP{dn#Izbe-~?ZJKdkJ^Vx5;dkY2ox+#);x!V-IaBI~~B{1p;&BqaX~ zE+xnHpqV3OWHDg7H9V|)m)}}i`ITS9U)oV6I!{kN_Vh3q31>c|{tZRihUQur8HaHY zLns0bR!pDs6rpgd$Ula41A)6c6JQxIQW(PiqIk?NC;el<4>ga|jx&D^gjF&b z)@U-4F_?>GJPT=~b<#m|@#_{t9{Rjo1!%x;exiE<(>w}Ttg&D-iux8HCD=;S zbt615iZ7U{<~yONplNcnDMO1CqgG$lAYOkmE~bOFN(Iy|MG%$+^2wY(+_$7V*%7vk zVj3W%Hcu<5Md|NX@-Ke-FfVtabU0?y++RAztRDfL!M--36*-_F-<9&+e1NtiDRobz9pNhF3+G)H_PU_P)MMa!TdL=TLS;r1AtM5jU zq2=-W7_V`NXJKtgpIj7A<0*Ci!fC1;X;`)Jqj~$H&C~O5F0NItTwd@#(^LPR>G{Do zDP$10Vhb3fW8HR?Sop}u0D?!t2Xx23Mo^q@UwcD(pM3X~uU#J4MlA2z(X_Ku0T8qE zDGp!5ZV1?1<+Fc?U@1T}8HGbMT`a{PqzCZ;T?%H>OZi~!X?IMNA-^lj}wN8n+F zpY$@6J_blEr&>>h3Wsu@aX2_@lPBvgW6x`#dnnF8Vy`ui9JHiKfES;rA+OT!gN4Z> zDVYDm*PF*f`M&+bV=39mmR;7emR*)fLXwn{EvB-Qtx_Y)C9-5+KKMjTWhX?oY-5+m zzGM(HwAg15S7!PAPT%i+Klk(ceedV_BdJ%J^E$8dJeK!y9PhWL_rWSyWffp6UjTK> zGDt)*Ov)=ynk4ru1OOJUrSqO=`_8SdoUnfWDQ#In$K-XTjo^t-OL9MuAZqDI!yc{O zc-jjL{t{ESiVL9D2Xl8(tW%MNYp4bB80uF7nZ zKBx_t?RzWej4sYNuE>;1NX;KumjAHmS_5VR3$aA+s*VNt6d1FGNS9Ik=<;?0iD^mH z#^1XoZyRH75plImXdw|<6^0P**3<<{c<2J50YPWW;Q7KWCRncyFNmVY_S z-$p}r^%I2pl*A6v*N<7cA4VAVsndP^SF}EpMT!Em-u*y~0ffm$e$hyx%`B2_9$jud z)gnj0uxv(*JB&q`Dd5g3P7RA-PCEYd_kI1dL@xxOpdlX2OvFQQz*kt%* z-(eQZm;9o#<#d_1lLE;!uD&CvV)czwh7WJFgG&-JMVV;(%H`E#zP$>B&7_8TD9@F~ zmX>SH3rb`>-c9Juu*mr|g(-_uEq@W^UWk z5ynGjN)=gg*uUeb;KoME%LRhX$ful7P&+n@cY8hf@sF8MTWW-9Nk8Qa0bccAcnkHlfx-v6>rO*Jgp1`*1sc^YCOf6Ez=ZDWB#Ij7dz&#U zW<3@zNU`r1d3k}~PH-~%gh0S2UjxPe?M2i;i!-P{hJEeH2frt=yT2h6+9roKg!-`c z40+db24hI%e112ktBrxk@ru;xr(9D{qTjL58~fO62jR>dhmP7MGdu(7;s0Wd^gH8z zh6#kAh?qWcX45KIp+`)V6dkFa%;IG)TgdUoV>5>1t*u>3P2RB~;{b^39|S0Df~`5$ z;v9@$;16*H^;(F_v+&l|mB|)|yRJcAc6<|NY-#01*E6Eef4XD&l>59`1*#uQ5$yoB zJC9^TjnCGirNz-+)3|8UpmzxJKyuA_iqXOO#I-)5y29#~yn*v1nY_XPHo6S93K3N` z`J;~kC9X41CgkJ(qlyHsMW@BAE7BC@h-B zx7c|J9k71!v#fOteap)A@n%cvRck)3zy6?aFaG}1w}_+&+mYGUP=YOL-hN~hSVd9J z^;o&+3O$RrwK)77=geP5r~k@Yd4A+0AzVl0f0w3|1M&tyYI*$dEvJ%oVfS_?86VTG z~1p_>$!_237H=={cI+ZdNI!E0gDtv>hRBfiXjic*W%s5spkhUQxxI}umS%m zO34ZiH^@>!6EP=^Ig+frf?=hJ_XQ~>sI}#nu4*MWt=U1?RBoB3baD9)iq}i5>8zcv z4>6w;%A)XQ?FEk33CHUWCu5HI_-?p(xJUR2ucY>~mY&S`>r2KRLq;(}Ub>C{FHOqY zRQde7AfKjVD3a<`Zq~2}Dbm1oJi$%-de=bO5cn{r@O5tGU`vbh%k!#?_ zgaR9jiAUBWWA%KNfJHQcxe0zE$s6$HzTjs z?KsFd@3^^I^m%svlw$%VM=u5Nl=vS23iP9j95zo-ly|W-q|@lpD(}F8LW<3;ZQFjX zM)Rl?btOUJ$mk)KjGsEaIat%mtC2D z@5^^^S!d@qVpWv2N3m>G(DuxyWiHdB#jqvWEkqoGe-oldD6D|BxGGVj@L6S!V%Z!(>;7DI|yi9#$_f8{tM)62jyq)rKa~HdWAe@W!lWW~B zNAJGkK}GzgUV=JCkwg=}cogx5k2qj5H{k3lMn^gkPc)BLw&(@~vL{UYJE~jm#a7Ct zN)Eb7u$;9rayf1?4sQS-SVNlwDS$EfmRJLgVE?FwBOijg8ivm+4-}*#)X?gy*%`Eg zM^r!S>N+hOPOX`WrAFP87Tp_(5B&21(-f-U_crKc9^a!QsZe-qXw1WS@o1+r6SY*M z9`3o@s|RT_Y#q)T>B*qWgjLN1dP z=wh;cGnwDGohQuhw5=N-U;H6)E>bA7(PPXVan{)FQde%cIP}>ui~@Be<;hPfYl!0% zE~Z;y(~WE6FSJH$=dGJxdg_^xG_kZgs1U3FK)q6S$jW09HZg~Y+xpU zqxv(*3IH=`0U z>BjpeP*i;xy@TTQ^GMtO=3}IQT5042IFKGF;#E}4l;*h%YHm{9(pR6~tY~F1?W%lp z;g8lb*UqQjx@b!Gj4svk8as4$k{S$ke1eEqIK`lj>)5*$T3uLnUv&@#Ew+Xsx$RbL z(iB-Mo-vMSG!3nnvgNobzXNXV^wHYCnEWYRS=@Z|yif_2=u#C|X94t2;hw^IFPr^y zzUhSW8_u3eT`l}>=2l}tm#$8HJFisBmZFwTeP8uu0oIJ9F<&R~(3k_svgo2|LPB;Z zCO%BUo4L&5t^JiO)Ys}R!57EPW9ReJOa<>r$=u`<{yR-zH}2o#(-aT~MlS2wmCc}~ z_8{L8{6yWcM~;`t-8R!tpWZ%)o4?9t9oq z|5g2&j#4zel}tHaMJSF_Ux3vg?xi$A4J6rCzlZo%bBF3Nx8!RXE@Klk|FxujCw6!_{~V-K1x;6<*u3q}x-kyHA0&1i2~~(B-mju?p_UrfnaDN(o0{(R z(Xt@k_Wz}Z4z?J*aZRpnVzZ7XKw4G5BMoPhbZQ(h)yn++5q}|~UA%GkzYAN}jkAOj z=_MeLQAbompB1s5p!?YA6p#Qjo`ksm&C&MnR=Gzgb})ihz9b&I@d#pu{ce+#OG8y> zXK|jHs*V=wiKBeij@tstJYD0^(!(j)S?ULxrtLsL0YD(#@={)@uI?WTHu@bUbR?a(Zy!l z0X4Hv5kT%CNLm!$DjM@BVK_RB9oS@K4|{;mJI~UmXOB>(DqZ+{w)LviHe+;H0#XxF zDa%l|BDMiZX4FmOOk-19V`_ZAdX0|a zw(jx4K`=0ECW|dmfi-4n$3_8<6?5_%$`ul-_t>Ka&b4p){Mn%mP>j#!o3`vB)>|1Z ze|#lU34_u1=uh6Gr@tiULBYUyVCVj(eZ_JbD6Ji+JP+Y147osT!#o~c36rcNaby*n zmOBuwZumWXl-5bU#3@YA*L5U_S22NZXMw^s&|_Z=wA8AQ9|mz(3!%avwPUACTB_fO=&q)fZKq*(b2t#F zhdg~}Ad|iA=+$*I>v|f_7`;x28zY!-N;xZ&uX8)r4DxHyGCckta*;z>CU_f|oGglU z89ED;ut}Ao=2qB=lVi&3_upClz<+m8fC7qVFP)4%ds2#1s(!M4ZapB)8=t6R-8AxB zH+WV!K3;b~=az?DgptVmO9gy^cO%{%W90rMiG2tb*@`9l%5BJzEr6^*3kt+G;xW6} zSOZ}=TB0pOz@}zwQ{9fx5I6ri)wej|(us?rPO4#~BR5Nwwr#o~Sc7$Mf&@XywV^8Z zFyN%IG{rV23D3I$K^p5pAs3nOplpTUtc=ct+>)qd%clrhDbg+1e$9#cDolIbv|ktGMSen&!dX|o+)Gv& zAe*MM^kIFk&d)wfc;(J0sP7%t;4DUONe_|#;QfH3wuDu+oF=lmv$t2d&x~6A@;d&f z|IRo0h|vm$%}?R8Y8RY`x_Eed$BaYx{;&AvFucL~wCI9Mq($m!pa>_2lBI?Ttw&X2 z?K56a(B_w$Jx3N2Z|UE(%VY{j4!Osm?L?<~Y*zRf({>aJs>cvrV&dv9VegA4*bA-N zti7E>pHo=5|L@RjkJl_)Z$nkyOfp(ZLKoq05mNZSeY`DpaTT?Xbw38voVd0jSv0*xTp5J87!weC4;AY2}H*h7hrkL^_`&nRht9r zWeoQtVUxTe{9@`G%(ZC2w~7U#puU&dwCnQN>X@_*h&k(#6rnd{Cr?!Eej*lsl<1p9 z;^};n{cay>@@@60T50n7W~?T7-aEMbuS^|*$5U1;n>Xrm{rhTx3AP7zs_KwE7YO83 zMFDmA47{TXemauEL=YC|b@IC5ogG%Pf3)|9?(@1hVTYK9((j%<(VQ?=>`p67qBEs& zo(BsC_~w!G#H3_{LP#^Uh=Q#7SX&(&mc2RTNiK^c|7yQ^H%>sk{_jxXrDwc#dAMcizFJUC|(zWyb?m;I5ouR#J zZPaM&U4Tt72U^^lcA9A6%$JFpr@m9-<@sz^04dpKeii&2;ct6NTKRD@;7AOkdkBFp z>^>0Nl4&d*r1(9$YS?CR;d#?O<>bDYR?@x|cB&%Zrp>qxef>@{TE48;%63W4gAWEV zEH?`vSKReCQ2PggeT=1eVJ`zqu#bsqlbrZn^~~u^!)P6AGV6+Fc2Q1nX&;y*cX&}y zkIsKA9+>O=KyBJ{3-yebjtXT$9dwYdCBXYGU>9G~xZtRzt?B_nMpN_XmU6}T9js;+==pzkZgGbkLbK=4%H!Qlns46Bdp)f$EVJ(l;*K+`aLajQ z6{WWkxa?l?18V{r*OvQeEFfcLIzgLy+P9HG;umM5T%XiqehqmXYAkA8bJ%Dse5$x}fjdN$`jMvS(ROMyke%rkiqA_6>DCh4cQVf5 zrF5KOUfHM5?MYw)bq}y|KA~}(0(%bmszlpEacy#|d3vDVgktFW2DKv8eT|xDTe&F} z-gx$H&8M7w=rmIan8=^U5(A4V{H9CA>*i!jG!q`#iQ+X|bYR7>b`fT&w>s6m*H+w` zTk9MixZJXS!9N{_7*K!p>Qq%Xz|~|q4|f9vb|Wf8a4bY6-%IF6p+Lf;@;?$EVn2<} zb_DT$xV$ib(b!Fzh^QL#gY|>)s255|zpCgqNITtlLZI_pOc&XQgB|r*c5>_WMBwS~ z^$R1Hs&8K)UuCxWG{k{0Py(F&4tpcFm}o)_y{1gsXK_1+D$4r;d#}Ogj#8|FdZMuR z;ug--EYf(~Ko-4na@4IUT58$$^1ha4)e|I!gjP*V+a;x#|!kXuAdPlZ3aeg{wX+^-j!@Ak?X zmsz?U9qkydATJzJ+OGY8!(ZVK^t|eqIF0QY>_|`&>Rb95U%BgZVYchs>MvYaZmBu9 zL-5Q7?sxH}uT(;W$G&s8EyqV*I`Y~fzvfO4I8dtKWBq`DC=jD$5e^_?@P3KDhf^RO z2iApM1BE@px4mlSJ46P@)#K)5FO-!a9t6o^4fFuE&BRBWIXDWnI){YmqTj-Hw@r2G zn2@buIKyrqc`e-rF_qRpx)5Gx>S!&|@tLum@XWtXQlB}EB z?JZb8k2p;kX$#_xKfWpVS}sqg>Stv*vV03z!*Uvn9ZB~vsg{HXX#7Qm0yty6(tyuK?b-uIDI9S&4@B`=u-V%+Af2z&kMm^L=F zsbyQ##+KqE2i@RU%$Bj_`MA%a9e{7)x|0Cd2sLWaZ_L;6y(17@K{D#9ueWLX8L`V; z$9lrIP#m1=E<#NwZyHC&v6Vz69SS){rJGv0h$sx9*B8q@X$@TyFlG@QWnIH={Y?(~fO85;iH!&lZPcGW-s^CCyM&B_8r zXc#O41OFyNcLCeB41Ar0>KRAxo3#)(fI2u-(iMIjS;=!^~nCJ zpWlAq%oSdq%+@)X&423Gs&(qK4gicY)rGY}9WPO=Eym@b0u*9FAcqQH%jrDOir9Lz zHh=vd(zW4ZUhu8B<5aDh-sMw2SuvLpG|Zs_B}BOnP7|oKM;%qcgiVV;o zmeDgRGOGd{QLntF$N63^I{4pKeHX&7Ehr~KLM3BKTvZ$34w*76A<6sRpf$XFmzBop z3p)@_8z3p7q_@9tk!UL<$8do`Sv8Nh=ucch)+hI0mDi#XOH$fH97r|~hJ);Mh{lQ} z0aB9cHS%?8ju8M>UQaE|h&@RQ`vRbr`+$@&^uxfA)&_NNxEY(20XzSX;{_1wza8(5 zWonjzB-AZ%7=jsv7y^|34r*hmAT+h@t0IJ#)uKJg*stD0eOnTooz=71=|3+%7DEvN zaOjaoPVgl833`p--n9_#B;_Ym74zt89hv>!wuZy6_`#IAkEi(T4UE_lBJ+Z(aGz;` z$i;q6;Ns(#hI=gq>*Gu0qFji-+fv2KM!CIZ7I0QK=&>gLkpT>^dx7HGsAHB`BE87} zZ_;dYo#?!XiHAHeFYHl~E$sM>8$AZ6rV6*;)y|(|P~WLnMT_zx%$&nPM>T_nB|fXm zB6vxzTU5BSO<-(Gqlyb1v8gL+&^x6j-Jqt^S;FvJAHkS+1aiUxw+Z?mWnDhAS2Vi|W7XFj$bT9ek~Y6xa!ZtnGd}fe z?c;5+V^WP8ln(IeUogZAy5e7*d$GiiZQQ>e+EWxr#-ol7m+Rs+Y5Xk@Q=}NEj@(%fg`O`LkZ3=ubkRBpTR2;n!NqS~n&eQw(5hVtV)1qJzQ$ zdmo|I=vH;25N*%Qw{qjw+)KWUyN13fgzL-RtGp}}fc(TpK|Y)Z+Tw!$AQqZYGz}D} z6n3nmSSV&b31vmG?Nj{{I$S}|u3FoX zMp!61|FZ%i=-&${!PMHUH;@D@!G3Ge#9=GwC^|xg$K#=Cm6y;B`!g*cs(tQl|GG*F z{f?6KSeAeiqI}PH^zAI17as?=D-yga6x;>ABE}8m(Yl1GR&Xyv`<{&rSiPUBbiUb^ z>YL~4;qsNfrZh(0pL5Vu52*h14#PUmd!2AK3FSoM&mwu-5}k6&9c!nW_>+0*`^@9R zQ~G|G00Je|K{My{1-=7za9A%;l)=A+%0eEKqK4v29AK19-B|6K($!0Gv-5LCG6Uv9 zXURSM7v)5?=%%pD2B#^CB=HXM*rwsl?TLl;#um~245=o#(1y(3-4FPM?E67?*YqHU~Qj$0I7+-tDm zRG0W3vaow6{JjL^CnrdcHTK>H*_{Fi=Tu@b02XFl@UYi>Q*Mv8?0ekQRxR_+#IV>t zG~$3q@9$+3>1-LL5kJO7mk}oX7UiM>*#jDFhsSAD9GKH7qXGjQX$wYlAYSAlwbpP_ z1xELP72L=m>%UJ+C6K-@GVHY->dO2mr*VKc8bJyh z!yGvbQQEW*Wk33+F`OYPMB?lU(Wsz@JJcAkrQnNR7MRFs znh-EKEoIjPj}kb!W<9e4=FE${t~s<89w^Y?`%Cd|{rknm$S;5;c?bhy%02aW<-GoN&gVx} zg|`5+r>mu*+wX$>oMC)D)% z!R_ zCLmrVg8Q}rV&Y*Ms|AU?*F>$?sEQKjYq^UA*w~HpAEzp+G)@2o%o(3-nOR|zG8XOh zl=j6ENo-j8pNIcIh)B)Dpx6OCeD3NurMl5TW{TPyw8_EHKZqN)$ukx1xr^KR?klbJ zXBjGavuXJtFYdv@=XPi;^Q8Pec9K??I0A$|JZKf#@p!@C&D48i8o@ue3%4_mC=1?F zOY2KWPeNDWF4LZXqq(mF!hnx(D-qRu{TJ?e9v@UUs=WKM&N#NNR?8>ghoa&>5R>FRjrn zu5HSk9a5ku63%pU&^ae5z^zs53^ew55NN492h~VaR1+#r-O%5JLAy-`Yy-~3DdeB)R!|2K(6`wB zgK)=^8N5*k24h&}UkqfBNJaRE&F67|-;-#c@&w@0#bDhpDrS>VYK7al@7o#KtU%@& zjr#K}s|jaB!*1Oc;W)p4nbHpqstNWxTORO$)mdOq#^R`&faQh_v~WPMZaUt*lM|QT zlgC5cUte%0UG3`!Z=-W8NU(Vb^nR{j;H(UKV6AM3q8{>@@LWUm#5xNx_~Kw zOzrD<^JC%zK`f<#4{Rhdq@=@e=;Tg&n}qvg3pV5GRI6`VJI56UTypu|n=GE-;$Rls z*Pv+rFA6qvpOc^EF+)tWi(4#*R*jP;*#bH+oXH`u7LB=zEP+lZBq#2YzqngevpRbg zI+%X^9o!$OsM!;%Af&_eu<_*0vt1$z!QaDf1BbA%upQ%QMC)Uh`ZI7|9PD$qs~9tA zBU2z&(fF@H+i$U!$i-;TXM&{zOrULiaIT)#qMgAB2+nGs=KcrajrNgsXukBOQ8h?_ zYp&qTy9i79^Pin>-7-qKS*}G|K(zu5Il!zcUYkWl!@+>G!0RR*mF2JvEQ^3*_|>Lv zYhydQ+iL5Oc26KPb@;ox;B2FaJA?MxGrM}|2t~Abz9+yBk7Vo6VI%D0Ol%W=sps&P*SqnNHvYw?d6)EcIh@p)a_t$R zHY4T1#fG1LK+XM+sBBkHQ-KiY`gCZShyVP5SZ+}0Qe90`a|6dks{#6z?RToGK5mJw zDW|2G{I$ebY3wC{ms=)m9=k_B0qXo+fvBjz=&07<5haj|zZSrWyU}qS>2(n{tqeIk zVITnK+t+3;b@^uFr&iMO|F)6hmh*4bQ5GlNdi(&{R1intJ2n|yS(pCh@@v+1Uq z=AYw3GZHqwFHfHD->;DW7ezb6&NR>i2Bzv03^H8Cyzvkz{Jb}QAjFoW`6OPg#Nn6P z9EfKsviTF{TDv;VZS5FXCpt?fF8@XMgy#RBhx8*hoixo0%>Wp!!!3-f3ecs{gvQqo zeo_=Z6#a?3LKZlHaJ11qmG}3_Ic?KqjGBQBX>(@BX1R)Da$^y5pgIih65bVTti1dL zW;?$CXr29om>-2;!gj6C35oj>jdFg!j1LuB__ON$>D3_4ITct9Oc3mNk`$Iy;%P`Yy4%8g z@miBsOvtaK@le%t^^W~Fqb(qH{lryGGfr^si_H+O?2U1^S9%oU2oFM=Z&$-+thU*Q^GZWaO+)n5(2z#- zv6JUhJLArjm1!do2!=x2*PI<#SZ{5w83??mk@3i`Emk<$j<_F5u_*rLb*^k>+|1YS zXVT4!NoHJ$pYwkl3*NDXh;B9;Ie@pxuLt?!5a}bZJ9ncvcTt>>a-B0ZU$Fr>qZPfu zGZ1GYE&2Jw<#kNAGov#Tvi^_+!4X{WB{)z#Doh>?^hc8785cYo;z%51v&R`bQh_H^ z-v^6`sd6X`>>^oc14yFL)(rVtQ#_3eilA|}y|M_@8~6D*;eAsmoaG~T%y3K_nrJo_gOL374>ia*QlTkj(q!VCD} zSbmME7S_sd9yUdi_F9=Z_ZPX$h#JlL&tC*ZkUD<5lxDCs)P^)rj$V-deviekc7gy4QoD(&ArQ4n!-wbgkRiJ7Wl|2fcYD!Id)tT zrx*v`yapZ7XX?rZWWe7)jIAr~&KlqAWVPi#(V>5J65%rC^nsYu*(Nnn*gB!7h9|G} z$EsQe6dNg~-9e}h7Q8|*=|?@{y>?V3#y|uNcuXe;6nLCd&I*tW@=?l7xTY&>h`U^( z{fSAev%Un-|HoL3`~1#fuJBXBH`O&-rFE&_y96!Ym{83-W)NNUHe3eFES z7CsGpRBUCW>F9lmeo>EmG0y($O#87UR&?g6CB|j^O@xj#Y3)|Yzqm&@KcQenEsDr2 zaeR-RTA^aVGfUx&T#{|YU*ZmI{$hF~`ErbT-c2*BZ#*)&T^8g96E|UA{%cyIvtjCw zU}JFZ(nHjs7Y8hB3pKu3cu~Qsdk*v7S4h$V)7iI;!#$TG$(rJ4Qy$h{e!di$P5G5o z&C9^62Uri4GM4GV3%yuH~pr>1^M#+^!$4v;v(0xSXtLR29avzQgJ z=@ukSOpaPkGf^DG-8}Mlo|&AV{pzXLYzyqLnkmucxM}_h=s6M@6uS7K?h^k6ZN?}D zxi$NnfaEAF4C;6U1ku_Mz))R92X{VIZU%sx0{TNQ+jM3&<21zT?9N_rF@LV#pLpd> zWJ+1Q@Y#XmHxUlI9N1Ul|F;TbT11IN-Om5nl)GsSzTVOwZ-+RaBK0cuqGRp8ilpCW!C2|B|Ebqkb! zaq3zX7Fv9PA)>07p$_ND5QjH%5rLdB20Fx7-jHm{LGoT*#kah{)cEI3T%8Or9K0`% zM$nnu3Td=lKAx33!f+z}5q${6Kv02|1=e%Or)Z-Xx>|jsy@Kic6rRFg+2Qe~Bt@H2 zS0Iuqx6lx0Boeiha!N7kM97;)2_XuwKrxxyM+39^=SX3>-By^3K1hi5)zgjM0Ix#ckJu4#Q7foZCT{%n9itbQW z&xvPd>@QV}3}^~6J(zJps_87JC!In3{!9S|F7ncrYzufjs%V>`OtX+GZg1|QJ*8^n zJ{GkR&dXF)5XA#H0|?+~p!Te(X7aBtD}?V|SrufJo6Y*`BjoUv;?mo8n}Ph8C`tjO6ipyx$a} z#qjsfvobBgd78pm0rDppBF;vOI91KejX`M zC}qR~5i(2yDrwea`4BcZ`5oG{2O|=~nGdI*Z`90=O331r!d*egYMzRE!nPO=p>>!T zxgF)!L3DfMD$vTikBL=9-@qsLR57Ewx>Iqj`0;necXcm5Kk8~@J)aP9VHph8(tE>E zK=7TmYW6eDtmx%nnwb>8=ZRNoX3J?o{Z-}AVwBAXJE-Fh(mgub?xd%lr>ijUz2=OZ zBUg2~J4Amm94t}G0oa3NgrK`?RIzEoaCa3OEVSj6WOiR-!d=qguL}&?#%vvi5-#D) zp9DMlvS5JV_mjooe5dMhMaY8_@g=xq&eYpCF+lq6@?)0QA0({%r7|TrmRV>({e45Y zzY@7vOVdg+b4k+;Yfo>sa zgYRL{3dFIyk^OAi?W2QtB1gb>VNY$3Z2|V|HTrtu_sj3^aW2_Vbj@kU{sI5ZR3C7> zpOyr|DxqFb2lP9B00V0_?43+AFMla^(AW9p!50M{e+Vy>TJsaBHAweUFJXGI2LXAC z6ECaGi0`?=kRh2^^}?s?#;f<@UwmU;>`2D@2c|8wP&NH zX*Ve#4~}q#IwaM%;+$j}TBvX8syOC5%H~ZCGK%51I!fw&$KQLS?cOcBC8_;oEa|@S$+@X9VaB8BRpxgWsZV~;xxi~jb*erQ0X55V z>|zY^z277f*w#AOD32!6g7PxACG;B76+Db*K3o$;0Dk5pPzGb8DFAcZzuD$EH z+Wx7gssA7s^~sbXG#ac?MG-Jsd>3FL9Wd8dWvW`vcf4dI@F=dUajwiixJ3Va)!Mgk zmc@SnkhFz+f(JU8FNmE>y%-)nyL?!sb8o!I>ma&ND@!ob1eaPO=;qvyd;Lm5Ebh{l z0#Sv=nbrCd(937eyQTVk$8>M&87B3)I!Y^WkEDo(>ovq%Zq#dAG+^YbEvv-eE`&=eKufR1f=?@|N20 z!hYN64;p!R_~j4U&f`t*MF@^G+B5$L@%m@4ALSGnocY6nwFB93CU!AfoM@UH0cdOV znVcG66zIA4YC9>DB5HJdYE!Ct;zv|K&D`bc*S6NreB$yIAN0I$4l8^cm-Xv5y*6~P z4Gxh5O@0M?93&6%7&gV-e-LV4h#?0_F~N<{K)@53j=uVVzyCqlQ(3BhTGH4)gO!Kd z!@>l=M5Ox`mzRMBHXbqEqM3f>yTSDFjmBsVq-0jI_d#_+U=L*c@8~P1w+b<*Q;Srx$(N8CVz1s)2D-3+x zS6oWByoB#ZHpRNclhQ%oi{vr)y~8gowfc1z{jgz zkZ7BX5vdsny9aLd4#(L9^!iYAHAc_LzBtB4n8~0oFEf)m14T=u9G$zAKIk0G(o*tow=*_Uc3Y5n!%_`60FO=&H8r^RdOMoc5EISFv@f z6fLq`^5VGkZCa(HSTQ(cTbL zBC~xnh!8=t2(H-u$|xp2@i)YlS)$qA?JvYto>yX~h|Gp?1AA9U`|HM}SqJzbNe(*S zMBI;4Kdsr%7Wv5L?5p)W0dd8YMnM}VPEbM-pD#QlmS+MsxkG9f+`!@W;1VZK&ByOU zZW7sBnzEv|x`hjytR2U`e&Eb;d4m?gcjgw1Fl{p!Ta(1%su{QtryQ^(04nPLw^{s* z#4*^mfI;sp{QcY19V@?WSS^$pVd)33_Jk0!rm5H9qO=#-2G;S z32JCld{)o!F}SO{bIN?Q*lwuv$%|v(K8xBso^Mb2sNLU7C-qB+b{cH<3#|{pvvvU3 z)7=CFEbgSu3=pssW>oU3zIrT?V(JfhlWkwqe}odng(z}l-4NBu7T2X@F3&arVp|j^ z1}a4jeDCmmiE!DMPC*EQU$RYc6kuN>C%0tlfo1Zb=qBm7BX3I1nY@i z`Sv6uL=e-Pqu9u&)LHO(j&q=Noju?CjQtQJL@L% zGRaJTZI_A0?%R?LP`!MjnN;SbF?`_H@7DJi%c+qi^;4p~-)}_D8-gtWL(lfuuVB>U zCrNEDx0;09U?*{foQ4xCJPGDe(CBZEb+;+P1#V?!XU!egc45Znr(LgcEar=Af(VXy z1#ojG6_N%zA0Ht5a?N|~YnuvK-QRe9lTP@$I<_S&@i(jX3lnX|p!ioydec7uNBPSj zjRgw-@KViyVw&Q)jD&!gd*492s~b)(_P^{M>p;_2Yi`CX^TVdESVDvboi(=!eF*}=gkY(@Ex!IicaAm)ak!*rn@JPf z{vI5BFlbR=Cw(v-^gpD!hmDm-2C1naallD|8LL1)Tvqq3Him{VKiZdkblSH+ih*Zt z^9SF2C=v}_%At$<>*OVnKK$JLObdV(_j6{j_~JIE**`U*7#er*j9x{b+g9DPW>K{y zwevzq;)V(=^$&t7vL6RJISb+eiv62>gSfVL0%jrrxKi#LhG^48<0F2nBdXjUJ^l^X zOD|hLA(4K(Vk9HX?N$Ga=Zk+Lu8EwvbBrdj1|56{hwDa_EU2zWNK?*w7N&t2mZ^NJ zb=$Ja;s<;t!2p+g|C;m6{}#xs4a2feDBKN;S(M}P<|KNG%Ghs*T<7i~tlxCtZ0weX z*0X^6Nta7YH^a{?)ZO%T&Qz+?qylgzWR%oLV-*E(BlHEst6_NrURtt=py(44`KENK z=YnH`d_+oos;X^|b_fc7(i0Jr-&$V-o9ac)fxInW)YI63?}+K3$&)N5RBsy=;fB`q zR!$78bCfqe8#;QGPP<*3rUJ^2&HZ(pIM%)!x6P$@%m}P|LDKgIe{LLqV75G+4|OPR z$yNg_8n8YLZ5t`3d69fG-ZQo1;TM=uOt~1yQE)n zTtQRTKua5^c1t!2mYB<-!mvv~?V*CixrBU9 z;|YTxH`TKwB1G&lP1wX>zs{fVJT(oqvY&b~+i zxgrEk;8NfpQ?O+xr&MIX8#VwpVA{Jp(>vGwa-5rLwd!l3mn?JJM_nctN!!VWsOOL9 zs0)W^W1T?P0*;J-WuOYX6E=i<^8~Ylly3clu*exqbW6Z5-8t`ix1$T;|M@AGWiTon zOLSIRA9g2OI@FG7v{>Rjk4yI5Q)9oQMmKVt+faWG{AldWth2oI0nncq;K_vnOS;}& zurB|RHt{tq2gUU5O4u2qIPU&bmwhU}8VIs@%*=jC6dk+hk;UN2cwBpN`nw9n@D14> z8hN-lOyd>6P-0LMb-Z+-v5tG1#=L=~@Nn+26RSFNHUdJ7%K?c*VX0%E*ry|tU4PO{ zxmKNTY?u085`?sem?jP(p?`5-+w5$B)r5j>w>C}MFjNQhU80bj1}D*3Ai|33doDq8 z9n2G+a39;++LCL<)^$&FH8o#I#g*E29J0W(UAT{+MR-)$Vv(ULx$7$H!ku>YRpf*%i@YUd9Tr9 zV`A$X{f7^d#RIhrd-w8N>ILc_Ug$0xjA9n8Ig9E7!Gpfm9%%E)Pdp^VkD^r^vWo1Z zXl%d1`Hcv?rCRyCQ9yI z&b-4hcM1V-Ey+;?A+pg5lDBREqJy#cybp`NHfPC_JGK*>_qTpD=vC$NJ|2pf7#gpO zm3E=a&7T7vH5EzpGVS%7ao8gfk=_3w;=U;o^(!NZwBH)#4iLw;FQE@h9J@`kScBf1 zRQEnGKcnd1h;a9((jWkifJP83lIUa81IWg)8-HU3XgqL!r?*zD`Y*-03axG%)tQQ% z*^8+ciF((~S{<1N-_hN&GCcVn%MD-xwmxcUmB!kMKoy=4@%Q6yjfu*Ycc*+er5j@3 zbF2y0Ij)zd2|U=85)}LbU4L|q>8Ch_&w+N#GnfoguZ~ktm|zG4(K24@!*m-9&NCGb z+q&hLdC+WUE6mlPB$O@&&*VHfjr+Qoq_8bOlNg~^Ab;r)-2rzs>oCozYSI!0&1N73 z%pe~k$NChcN|(xUC$#xZ(sZ2jh)SIUlSd${H>zgt>Vo^rOCFBdd{%2aN0cIp86hn! zad!d2J)tFOE%ng^4nPh7*nL)>(2h+5JV@jl<{w>MU)aMTb6jzU~v)y_4$B?i9s|oDAeGiB}kH!;{pFhn^W}mD6IzP9% z|M-LBd6C#H;V=`<5k&eyP)9TH7dM6zP<+_M6iluAixzi+XNxyM{MkX+ zn_&FBxud6w@{iL+!|2rcE5mJp$6tdPaopw2_7gw>%8$Y~yjMgjj;m9;x3l*Go@Ae@ zc%rmCI)fg*R#R`9I`BU3n5*sEgyR?bgvw5}UqjG`BfYVUfOq9Ss>kvx>)s|b7N$nA zC)qr=4&m{9q;8?rm1Xq}@|bhPS-PLh{T_`})+XeBJ;L}QIaf_DLXm6W56A$Sf36|Nb)24Sm zU+(Rv>WW|A(+Q z4~O!7`^U$gEh0;{Axc@ZrA0=`mLx6K7=?t8BqPHuvTq?2Wl~AmCcBWat0c*mv5gT4 z!%T`hGjqScm-pxUJ&xyip67S`{=ji4bKm!MpVxW4&hvbouTyt|f6vL#w3AHz>-W^T z)?T9zc(E0L&E^oc0`Howz`KpW+NX^HarZ3}B@Q&rmYX8{7R+sqc*9YHB}f#Pe{py( zL!n-?g!*~p7tTGkoT9>un{^ZobQi3fhuWieMkJb{w)zBjI{X42ng{sn=y#D{3zWdm zeEju6{ppnH9Y*LvSi4(>7<(&Ky&X|%9w_r)Ue@-mOL+J++w|kd`~ve{J)Q6SPGYs$ z0R2&_(X~?T+GOvxBfqh7$Ua+f?1(k$oe3hXV;L1`U_go=fV z9C=abu{jCY6!^brFdqZai~O`zWRd`Su|c!_qBuU&5&cB`aVU6Z>1WNP`O1tpZimdg zl#@~RS=*c!oIRRNg7qK|sgTbEx$A*$77ipf{y~8wV)i!&yWI|(2IhS@wCIEQXJ09p zz|n5%2PhvtPa&k^$d56pPd)2DzqSn5udtuL|GVK4_-M_)rx$w{19r%N-ykbuN&iaT zgR_hBY`x62|8@j9&`Bv;ez}4ILJ1@k{RVp6% zNbB|9xEW=&G=$Bh4}Sxbgg9P!GKKJ&eFX@|!3UAFSaySFVISET!G{skYe0=4YL9wIwJ5d^)8m(TVg$s*f~>GL2x_ON`y7 z3Z$}Y2OS!tI}a4#?<=?P@*Y-A3HsXPi&AFjzp}ibW!mjS zb_S#b<(x|eQQVCQh&~6?Yi3#mbWL}xA32lj$CH--(xu18$6w%j+LZLr?%j7=FP=ro zN0mOXGeLv65AZZ#{U)k54Z?yY;%A6-VmWZk_BuB~IKK3WgQHACc3_P~cKl3tEOB6(vx0>@&^*vXJrKNm}b{(DqP#y0v^!*96w@w|21E*}-+{O{h^xAQJ< z=egsxljG4Wmfr~mj=8Ze=CyhYq1?&l@~v0nPS3Js3{mSx9oMi>e_T4q3%S8dHtcHu z@7P2InPf^|-OJokT(Y`f^yDSX*gYQnJNPg6%M(X*OgH>CcmHQvHzpMWA3C}oI0$(W zV0B_QTG)qk>e#_GyJXk8=xxmnv~!!=R_>iTCf+x`ngq9y%t(~QrG-FXv~|?dji2&M)r%)%mlpcR*;8-MN9&4=Z;{ytb2A)({KAqqYX_)WxRya}#w7Rj zB!V!ah$|Ukc23JYZKxSZOJ?@kC`i`j4RecGCVe&j;MA*o_p#6oE4EsN$ofk->lI#x z{SZ;cby&H0gTZ?<5w?z1F88zrZ3BHY2I9{iSAWZEqL@)&wk28voO+B7#)pRK+AL~` zp%U*k7C`aB6R(RWCjs!NmpZ_^`>;xab3^LbjZ^XN{K~535h1`7_JTjV5~&f{N~nLC z$jGPmB;gg_zzQR)OxZJZ+C+W%t4prVwpWpG&;oQRsO#t(0sgD%A7hnBl6YXOJYnqD zK&39k{09HL+&HX(Hw!a(kQedkY!8HTOgZ~rB{g$T zV7?FEvE$NzV`lnUyJCR?znjrlO~6_1k9D?3?W7lUg0^fwZ_*18mpc!FsIy!6#FBYa+R9M?v^V1lJ#L(hsqzaex;B^c zTkYd5*)!_CZ=t)0P7|C+ zcW6Ttha@FhMsHbbp7?&E$V1GA({lW{O6)`rmnxLB2>4yiQs4ubCb;BvfCHk#@R#A_ znf;oV8HO<|ozPc5t1fl?Fq2y#sOx$Lv>k4n-AzjGvql7(m}eKssa-?UHQqgYa^~%` zJKYWs&7|nbJ$aSIW7jY66mDn8KwNvx?D9BIj%l@Hds~g6-@$AO9lLVjQ9Eadpd<6 z+ALcLD|eB>=*m>p_9?~x2es-G^A7E?Q~4|&{_*eCtBMk4C(nv36_7MYl-WR7Nh*~@ zDcdOEV0d>xV%MCZz+@xYK&@=8Cl#mG*A4BfNDn=oJW|Iz?NR<}rd&fi`YEc*9pDb0 zuwNSsAz$2fFbqgN*d$}c2<>x;`KK9DiK{@z88I|=={i<3sB4*$((%+wMYpQ4RAdwj z8-a!0!$@Dh%qVV$;z`1+5F#vS!Qjmgs_>O^L%++1&J0z&ObgsEI?>syn6O{<%6;j_ zk0~e)Ad>mtgXS-WEHUi5An0c&EbpQa?&5L zk8i(6>GeTM(EosC#Km4j_&10*{h8n}YTFNiIe=?ag(8J&YExXxU;wex&1I^*XrhkM zt0=c?m-4w!2X6LSbL_M5JWPchJrL$u(KQkOXK4}2)ke#XK$cP$E^!veJuLGZAah_K@{{u-Q0 zrJ=hDGzX3JN_-$URffcT<>g?Xh%n)1Qe%O=;O^2DD=DF}=b1&>QJ6~*g^Gm@A*w1% z0=*tfCv^eh_g;EmMZNd49?-01&EWp!Jf5JGyt1buH{FRF5%WXozGOx1}28yHec_zyqaq} z=)W8}q{y%S%qSB>e&N06f=Gr5r8+XhVHRit|6oEaXEqTe+q~)NovYHmQ-|u|;LsPB z7RWi(!F%RkmA-yi>BdGH^YAgC@ zg`M8+ynEvHA?VN{_RR<6(}r$J zc@(UxHP^$UjGSGZYPt60=PE*;^HG?m0$b_lHPYBos$nAd5KJ5pOs^qsTRKu79ndTu zI1}{Ru{FS6(_D~dK1azouW_#gP$-$(%|(VubSw!z6mPe+*I4#+sDr=* z%i1!$Q)+2@@D>q4S1u!8tJoxio5{PMsH%W{ih+ydq6o;mpzmr<8{mN1< zb3IP&y+-?;$~%8jSn|0FRMl1$w(}pIs;Z$WEN*Hm5$9|CI)bVWcF=5%bx5*S zDx9b88wn!H{_r#qnuBH>*vNlwEE(7c4_LL?OCY&(1ur}s9w>@49&e94Nar5uk8BLC z$@8DA9e7|e@JRZ76&Ls2q3<_dp!0pBBIEy+TEYShPrn4_Y}SMNShQ#4$PTaEf&ZXR1SXN_=YzJWoquY< zN*aLs{k}kWv%wNqZN=2^?TcfnM^I(clSdkFwTKVa%9+Y(hF=Zz@as!Eb?@FmdtEE{ zoub=k{#xbk@CA?=wFzohRqTYw{XQhRIffi=OQPo#7Gd$8gCVk*ZEde4Y9s8%ts832 zCvMXzeAFHvQ4)5zRV9)SOt-%bXHbBAg|h;f#vtwJOFlLMBKEI8#S=AuB!0A#6@GW| zLClK!W-GWuD?mN`4fHQ;mqi%j)YT;X{@|G>=P4{m%E5Sl^&kcc4BC0{t80ACnw-vF z>Gxww(w8^h;8-mn*`_eV@C5_-YzpzRaPano(VI_#H@=v zt@##n_}&L48_G>CAvuqfdD1g1A6ziqJtg+_PpLt+zd^Vq(OvA71OIM5*H$r(4k0j> zYvBLUbl`rl@PkMj{;AIlj_Vcqo&I)DeV1P9Q^@Wac>!1CH`ord!mw} zd;dLU$Kz6Ub<{_bU(mPJbv2mCSXwWnYsccXDd?TpCFg?5fXj)?D$xcPf>hFeSxW5K zA@S%@0&2g?=*A8Z78wIUT+A{N%;3XH8}DG9{KDE_=4UKY`DN~lR^YO`At(9wgq3dh z!XvNAUGJ|Pxcvs=Q@jfY|4-il-2q`8)rdg1O;5ox_%^qaU-!@#fHTUhgxXE;$9_dL zEtBZ?pV+Q=8wZX>Lh9_Rkv2^&&VU&=V+Kzn_ogKv%F9_$sw2_@5dl|b@J}$4Z~9wt z_q23v#O^h>dV1M`G8rhMTFqd%uahy}n$pR7<&|xC;Dca8lqCuQ=TN6VNfwlm(phSt zJsuz~)}5$hs)$fMI|md;M5v=({Rw$LVhtjs+NCx%jtG~tfAdaUn$m1PJ@CwUSKUi~ z_k$^~R8<6!=+ZR{WE&r_s^43s#g=KHzlmtjUDNNelznET#Vzg1)x zvzkJk#^nq;ldRN!(Gqr|RTerOsC-Ez$_q%L*^5D4qK5qF>Q~TCtN0$ao#oGrAXsLN(VD4PIvW7hCygwAkIA z3`jVGf0%wuK*%HS@0}z7aHdI=hlF}#aI*49vJ;zy8P?W$pQil|@=MnH1enO&E|L}u z-rl!jhi`S_(h1c3QeYGQ2T}me{RF57YxX2Ul+0(~X=WK)Vbu@t?mI4YRCwcONt_ zs*CQ_npaedHn-xeZ+4}!Rl&c<{t*s#8`h_Zn|*Z;FQuJ8<}}N%&m4dIJ;y@HZYQU; z4vGWngR}_rAfS<=1a#c6orY)uV{unkCzixOK>a0bv%e6c-H^hn-svuL^rEuPb5vi) zq4W37ihL%F%dV#nk^H8~gq~^f5J1vcmGuifQraW4(=o@2|Du3X-OLT0hS`zkku9e0 z|0M+d7wXS{_(0GWP3XV;_Wu%s_7QuDu=Wp@oWUzV+eB1whTw5xjD(8;=quPgXps;Y zVDL!c*ugX+&uB@DRU07?MMHA0Y6zFuP0bOgXM9f9ijf-5dn?(tY-~vLYd~fRLsxqa^<(T z=N8*$z_tR8lLMkOI$-FIfkF^&`V2!ghK5(2#d7LPNqN|0yci6i1qdYzDrKZ3E9xBb zIxN4!HE+V5Usd#@vcq-BZ*K*p zwdT929P}QRsKfdpxR9Za2=vgMmrl_!33>f(h)Oj>p)xq=>iQQ)$kF9=ss@;OXqOVj zd-K!Br(!0=Wpk1cM9G<^hl&6agx~bIcN-+w0_Y3$sq`Y;CPUv9)C@W@KM#_zpBCTf zm+Hqli`^`*7ix+TqNz>49r}-;Z9(LyG3T(c8uAe>-mG0XUOrVzwdMD1v?p*`Rwh=c zR_E-yUo8ri=6dbAg8FR5`EEOCr4cpqVQR2W!J$<4`QszaJX9LSaUft06t1(Z2ocQ# zl@_M4-KLxv0X9Ea4F`-TCNXc6B`MWvtdHec8kv%W!~II!{K(mprOs_va*tf3J?7Ho znOfC<&#C3YF)nMDfRqDP=l5!4yAL}Z0q(&C5%|N<>)cS_XK+uU8Q<#~U&cdz*G3Ou zO>%D24+~4XmsOd!mR^qMlU4-huN?9LBaUdmYnJr>cor9R($r-%V798N&Q;4x@6>m- zl&Yx1$E;8x2J8tt6S>@0w7@h@ieejGbL`BHz}MF{g;P@}j&~%?B~9svOuUGRgU;#l z^EQ$JlfD3=lz}MJ$clLqsAm`-#3{FJ`YkE!V;IJ2Yt)SI3^8wfl%*C`$pyF&Mg{8d z+aW-^BA~B<6wi~Rj|A_h1ceHH^O@B9^8~)mD+@QuM z_{Nuq&+PUJR_>@&I^MeL(0gJUM5)67NhI5QB>F||G{U$W4?_+*3wCrKDM>#$Ib8KE zsA!yNAr*xC%w`_=XsYa_A~RfriEk|L ziPzRG0HOb{NbyD&9uo%6AU=7bDxEBUWLj;#`+NvN5%+= z(c2*dbz<)oh{UuRZhY`B>6`gm0o9aK@vky>=)XMZd8N#*vSGjLN6X7>k+Xbsm;VKa zxMJ!4fVllIDJva74pzHY&(;ewv6!uA_N zsNR_-C53j|ISyY{dn)NJvL3RP7Dv;eF!VBkA?w0^XC%u|K0{BdEx~DDq(@Z5@D_Ol z1b9f*Z5U>L5WLr`92TrB+{!Ck`J_Dky`XKB!qJw-v~74P>W96L+IMSZ1m#@0f%^8OfRx3f~G()@6;G5Z@Ze-SL5B=7%fn0$?D*<R z(!-YC(!&OUU2IO%zq-8dp|vv}b* zTwpsQU*~cRV_Ti>Y0r2yJCQY1E%vWaZ3U>>A>x?pq`zQ4CQYv&TKosK>0$ug4q*^z zpz6-cjC@b3t?xeO(gUPe%)ngPQa7)xvrSbae>8QpUT`URnE3_Cd3`qa9)Fvzk_86w zSS*8mraDLmp19{72z<|oI{egTDU<^xY#X`=JF^^I=*(tUurk zvIBZK;Zk-`a^zZ}N1XI5sx)O^idnpn6zB}(U>zP^wO~n4|ASKQq3r=(_>I*G*?3;yok3?H6CR@t3Mgi z$h>TLsp8iQ#e5kY0N_Cb;FcGsRa;F-SgtqlUJ^#W_&Q+~CiR9TPqB(@6&TrJbqndU{Nb$*}v8(|K#SM>XFYCyc5OybJcPzl|A*e%lpEcdiNamp$QeAe&eBA=yFBL(W6t z0mq)5_7IeecPWAeJ7nT&%NHf@oJlV-*>K3sy_xW|E|J%1GC%e#FBm>wXcl>U8fqIx zS4t8h44Uv4hQ42OeXc1LYv81OWxH;fCcWcJns5Xk6U4@C&c>X8g6KV}iHPFn(L1oX zP!FurB`4eB$?nvz$9FEI4g5%56o}~#HbGkqKtOBU&P|wk8)`cpDFFWZV6*BAP;xf% z3Y>XpFdpN7IZ(cAv_AJFr&G?hi{JM(b24v7f8_rlu_nSY^Ja>I-D@|3q2C`S{Z)tp zo-n2an&X2cNlR-WP8{sZhIWUrr(7D)7VnUm&|L*eHC#69ghpyFuiKpaIxqO@-IJ5oIaq%10w9c4*+9s4b#{ur z{BG(p{xnthjZaQyS^P7R|v)Qb9jb$F3@ft?T1U#rRJDh(MHX1h@+5w;)glDS2;FZJ^? z_Fwg%H%}d=x}U#2>Q794f966=Uzp3m*{q>29Ds0=YaYz04RNplm~LBweq{y5W7kh> zfNb_s$>{7m=mw!)l_joD31sa|KJn%EKd6)`&Pv7|TtH~UpqBAy(B-hJuD2UCyZ$}8 zK051;@5+#Vz3&AcxBxKAqyzoiYz0HF8ilq7nHWZ5ppOsGz^0dvkT&lxO2AIoiFlgL z3>vyi;>TRk-f;~rUPIWn0CD}c5&DMp$(3B<&G);KSgVwz&*g*SkWS$hk_tRT?3|_% zVWoaHC;s?9sEDm(VA@$O45cUVuR2tNVMpuRvgO0D_p;g9P@7l?dW>^zJ#<8B{dlrf zJ-uY{dI&C;29ZH0Y)tU_*MNla4MXTjQ{<|IEnud|W-w)tFObBaRasm37oMj?rhb2UU40+5m%oPGB zuH(Xhwz2xegsYF?xv@UY^teCjO+E?-)lLUnaH%9%&V0$f9L+DWgna{|)Z%pjB%)JT zrb9-m^ygGgKQ~Ba&_8lwGh{wLnvkFV`kEjT0{4hnH;)(yC0gNnSUJfye-h;`4!A&gx)I8Uq#*d!Z$s1-XAnG-rbWp zV)^~-!a}GzQ2g59ePV(Jo~-Q_X-;Ii#)6&bK-%0!q<$umL_~3sJzaSc^SLEX#Z-Ezw0<>PhY9ID|N>$e|))*;FwHC9y zVEv{9YNnTnJ;|^eB(CwkWry!ap&<;8e$k3a&8;MK3 z>i`=?+p##n_ZzN$4ro4Og)#WdMyHY(!dR`Sm3?hzynMy)wdbwVkC*KQY*oDTW-)$P zSV?5Lm>GokZiZtS>N92qm9xU;+aOBwkj9@>Q)|BgX(fdVH3ce}CJk7f2X8N!X`Fb4 zLU$Jf8~YwM04;5bu%&C+YYv^$;KGzi?6)85*J09Ac(onk^Y>?2&m?DrzwC#IzIPrU zT#GvK;rpeD4}!YlZ>^Q~EtWNSojaDb{uA8s13Ylx(E+fY^=RY)+(&4-<6z@(|Je1* z);@t+E8m1FEtTx=mu&bqnVZ}0jyg6%!eA&rZT}wk?w;QGjb>wh4(9@rjT6#|`F$ZI zouy(Bh*%ISjDi@|l%5{Uw+?$oeZn8oHaLcoo1o9HZJ>Dqv7Gx_QS-x;)Hm6Ez)~$t zxLXe# zx=DT9>f{&Kle20IOC9_94oehe>6pYp|3i@r`)YVClqCTn{9{GHu*gp07ZlTIUm>6>Y+(F_+=*if`n3Z$#) z*GI2!+O*VO4%)+R=F3h4T4z_?%qhGSj#gE!F^G~4wv3N?EmIO9yNSWC8IT|~E}421 z!#N@BaC-2!y4BOn@bAhW*WB#V_uaH~vqIhASL9l!g7+oTjfIcpB6fd(e#l2TXw&~4 zasD=`x*ygIWQk?=A_6~|DjgI(>V6|_;=++lT?^1>@*IoS*F7L1LYioPuIe5j?=s00#O{!MX`7?PbqDC@UG#4RtC3RvKlAZ3kT@ z987~nM2gVaNA2H3@=zxC<%WG{ia<*oN@BgXUMQLKDi@^w9t>7k z{i`FAN?2F&BW(s2yu10vSRH?85YYi@qpJ5pVs7XYBzzp4kNv#6eoC<+u`tU+)WpBy z#*G`{GALwQ=_(L197WET)Ag2!&?4rw!`KvXura$^l+aR(1bRCDI^FS0Qa)s-qDI#@y{FUt$o~5Q74ZV&n_I;#UUS@{wt|l^wLjHZG`GV# zRMyw0=;!Ee2CYqYWH%_?-_+#@03C~ekdM$WKawzM*?ti7J5z17RK2xDq#6#Rq(bP9 zP3ZA;{iD+QOb_*d`&~QzWC9NDI(osA66{>_FZBBt{;kCT7}pG5-w;a#yuNcA|DddS zAdBI@AWGw*fz2%dWOaq{QbZioeIJMEE)Y($=8iZaNw96K&S>uDIUDH26?J?U$bC!q zp&%6l=DQifU%SCh2)iz6%RWt80w4;E3)O_cw9ruSapV1T;xJ7d*%m^XJc?}V31%wv zQvH2?hcC$w`jv?H%cPGxoE0zeP&nIUX6*F6TVREI1%>W87WoydVESd=kBR zqplnaN2^jt*^>S75UdFvL4XY+!{}{L8wwEBplFk&DSd`_BHkYi7nH$O4BrCj{c3ld zKGRy_)`K}uS+acwpAo9$NX{V?TLY9px&HctSKbHBX%v81y(;J>^z-&}Y-TMR^djK^ zOQshPm~Fe$g;3Y3`+J^r9hu(JwEt#MJ`lkmLvkJ4;Bf#1NmQui8VwL(Q zl85bBpcy+69E>Dx{C+xZa=q|5j6Q91WNMxsbF7>H>TNPg_R&MiE$;0>dB4fII_g|u z5WI>A{$Z?jqpQ&@F_Br@R3k1T7KiP+6F~6t_bzd|Uw_aq__ee1rvV+l-uoT;cRep1 z5!)qDRVp$Yk4Rc1kQR=WLzL?z7U&el&|Lz@OuI4ld@Qk1>W>a|{Q8K)lnz5a_C=mo z(eTG{i?BWA?Aoz=meaeNleujN)=QKGVNiALnhigX|;BNdnhKei% zZ!ZU1LrM>U$ZOQl4(n+N-;F7r20FfF?-8@1Dg2H_JGXD+nxDdd4OGT?n}0L8uMyrM zJv;sv5H)yI*d;B>_+WOEv6^3j8YAajsjnTJgR~!(`S~0$lUG!dq%$CK-`%@+N0p#v zhqAClh6l7 zZ0I!)RM~QCG(|%VLaA}~+@kxqdP$TVY^`l)U^Iq)mUItvqp-Y)KB2i&>VvoPbEGuY zE|D)ynhDY0r@xg~FjS1=aF;(|Tr=Fa;*n|qYd+HJ|+$`ZfBP##XpLD#0qJu{$9 z6Oflg@Ky}*4h%i<_>6ifB=mD|u`!1b#xutr`CDFSe&2*KiK=?#B_dn8tSR-712!f> zV$CO0c7?F7={}CuE&}-r1>%CsI*I+x_y87adqLQa?EWy@=ZAsr0p`yO-?MhLa6~mHZC3yv*@0DI)Xr z|INfr2y`z#n(S;wMojngP9v!@-1h#S)R#Xu&pfIi_j;6C ztsiXC?b}mULfm4K40&JpMbJO@Ooi$1LJm4 zLKF6P z>#h#f{PSn-^K<K~SqQ%$BWp&4C8oR{N3TyaIyA-0 zkm|9)oUNuKkBAy|p*i`#e=dKuet&A0=5W!C6WQ8tB*#Afp!atJNt!@%Q=5Y3u)Ygv zj5Hh4HIfmk`QYnML~tmOwg)!>J=ZrBJ;|`yrI?**=o!+XE_z%M4bG|8P%XMs8yIoh zWfniBNn6U#dIpsd|8HvnG((Z%)sm;0%PM#KeB>%9oR|is7p?FL~ly5{> zrJp8D0bSX}_aar{-WhBPaIzq^`|^3vt4X-Yym7$o&#L}X(?0i_G~94;7JhvE+L(;_ z_#>xhRzR-rP&&dd&op1c(l3f6v>c+7+O_3jkZUE0R_q@B{m!)ASz@YI&hm|!pc!V2 z>LFD4T-ERH{o~NNjzcKc5bVLU0{@#b+Np`|a+7AnC)7Y7bfsxu*LH8oifDdyvy4Lp zed9RyRvMYqh>nR=Y+?YpR)NL5do2;`MbF8umwIcoAF14bu6NmlOT(%aC4lHe0vOQ- zzCc)$jNSo8Dw2_Ytn{tqR9~8=7}JFwS6Br79LY};%9`jC=-KJ3Ypt_z=tC=J4BR(C z=n_tOCN%E1OLy$b?+CS#k5?0NQVR}cOTShW(=Fb`l{Umd0{Nk>91$M_U@x9B_G@G} z{xWbSC9JTlPPYT2AtJCjKBx@#JC#n`yRU|Qq#L15k;(ctB_$b8!GP}MCP^stsXC{px(n4XO&=`_H(g*{%_xO!JL(&U@2!V!R8Mr4YwS3X<|$HT zNf0V~k^27q>fChY25W}}v|a*}m}hXR$3geuPQ1GONI7@4FS3vR4oQ9SlX0@$X-~mC z?&6LU%A&lkpCu$7Wn1$7d%%@4E#ajOv^25yhaJ0zg-%t>D{_7ULpN9&={uk=;UE%g9fRN!B$`BlR;S^;qum5@Jk_CnIuR{w z#77lLaMntG(K|ys(ch7y=j&Ybf@YNUBGpeoKTh(T#lxqum*d9Y8b*S2Lq{>>8@PFp zK<4@fB|J|A#OmK1%rI>ljR7D_TY);`JR%NGDoH&vFsz5cko=B(VWL?o7N>ji>y5p~ zFrhynsJy?Q1odG*um1PD%LU*w+L8r~C*cAZtc`R4mo^iiNm`#7Febn4U!NI{?3wrQ zCgTZ{w9oy9bJO)dh^++p)+!A|9p-~f6KJM+7zk7W4NqqhFe1f2eGs$*0=p$mpPaVqF6_swm0bcuLYNgoYiT!vmPY1FBZhhw(b#do@c9$X;COD}<{p zl5Cf?#herkkA%OOakUj|cj398QP5?p&%tsXz!qZb99Xi#Oqa+Hc@;$m^XbSeRk1jB zc?_S5ZL2WguaI~ci+1Ou-EBp^aeN|WbacDl4{R<}z!{%=wt%y4>zfu-*CG#ItrX-J zm}IP7{pI)cSkLU9a@v}InhmzS{?4(y0@ATQvKhC>Zr?)A|-`Q>z8wU>YgLg|=DzS@EcK^V&KezHdo*_IYV@0oc*NU9A-bB^PVpv5aP z{0uX=^B&aKulEHjoz_oNy1OGZQ<>N1?&-_`7Qh&>8B|DnQv_tmWVQc6Nnqch>7n(2 z{5u;@Q(~S0oy$b}^cFbYjq}9_cFi2^1#oX|F|U9SDltj>66Sw-3-0yDW}EKAja@3d z*4*DS{lA)PDG64@_K>6_2O+v86y2iFSa}ytg(Q6Mgg|reR$k*wFHH&hn=g*+&5n}& zzQa`7M^@{*q@{jE&(pJ$1KxN9W9Luv~7$yGcqz4;8C49 zzd65r!V{ZOVDQB5qomE^xYMXmWbuDEbfjD~e6(jhkFow5*qK>YYZb2AF}>YbVa{xb?~;#umcY#R$9ICRRx}kqPRBGi1SOxcJL?B4y*_m)CYN_&^wI;={;Q_ z6KAAKrAdDN%#YvyVq!^3QqQ?lIYaAm^g&$k{r8>z?q_6gRz-<)_leA+lbiL*ofe%# zlfXDGAf8p!SN=&RWuP}u?SSVkfX1MI;h4kv05Fh#m`D{#Bhf(v_+3Pw;r5hVpV)kn z;;(#hIM7)?wPNLJb&{2*+T9D1S5*a2KZ+o9Bft>i>tzgux8B9$NFPC7xqy5h#$QtC*I)HAm1^k!k!aBjQNN)BfF3HTT zzYUSg6}dGKm2qmIM`pMG$&;X8@T|m%C=~G|SUbbWA+TJL!cdzn!aD&6b{|>K-}`RX zM!AEW$$rr*@h5ifx9FvD5b-#<86Gpg7+rP$V;M-W`4iyXM4BC)01)PUnlTTAKg+nJ zUN&)Q93+hBT}X8iwnOicN2csUB~SN@-4|xQ%sE}W%hdudX$EH0LO%WCU|U36^QVk0Tp=y3S6pX=*6hW95*JSocXP7;MI=Ghbvu?HW>`8SlsC zwJ#o?VMx9e`qF@#)Sk09=TnSn0(`8AboeMpJ;pKyLG(j=P5}%G(so1+eh#vkt45lE zGat^kZ+dl?M-)%GOt_vty-t6A0S^FYXJ$y*Mx2O8~ETdi&>Gxb3Zrv5!*J6L@2H<(T&2=wJNm0rsW=_ji;rg1mx9rvjX3c3+JXPF9(NKbm`>VS6u z{jaXTAB^f+C1ZL3nNA#3@h}O^Hm{d}Jig{sV%SqBH>)3z*9VCGGT>f72hK`{y!Y3} zb^EE0T$bi@q?_uDYD9Eb!SZCpcWLu27VyMVFHU9nKleE@T$AJ6GI^9~ zVxvD&rPi#|Wavd$aOT%Tx1^PFQ8QJb7l~3)csc0Bbe~^{(8#F$*CYDpP1Sa0s-BOtKmT4a zdYe(S`rgYoLJ(d1u$x=gmB0JbCb0`NZ4~7mXhIogA5WI84xFo2*Lroe;od zGt6^cO!3GbXos=*IOBEH|%8l&gfK@??jW0EFRNU>;`Z`JA>D|e^L;dk{!7^Fx zIZTt!e7jovB0=7xt&u>#vD&!`!TSj`ZK`%GR-#5*xb8)sej}1wvXIl~EvM7Yuu#$M z{LD<=m7RT8x{do5asCR)38M7*AE839x@2XDd~?5lAVuD4Pxje|)T?i+$-5dkZ`W}w z#r!l;7zeu-15C3DF~V=5VQ=@&)A5mH`<``wkM!f3sDRJ^~cKJ5de3PJk>s(Dm z$w$WnJU7N~1U*g%FI+wHNl+1Zx(h(n*K(zNzCNR6ADrn9Y--q{K> z4ZrsY@6S;gh}2EoMkc%Tg#-tcJH0sD9{W8b7ySgI;C`fUVr) z?7K$cjOfvmSkYw;VWSOfo-JpJSECxPf z4bsjM%){V;Y`_%>EI_iLF1on%-C#^ygQ|5qV?RFgUBA9;f=HYN1p@9o#1za|0y(gl z01)#vZRx{Cm*Zl3rUc<9g$p-u`R%temJX6DQu%A^3WmQsHs{7ZN(k|2IB;FmvbD8; z@ec%gtvubH0Np}tsubj%zg#*ad+qeNNRHf;l3>$ID#xLvD8J2<|Kf?Pt85KOG^UX1uw(2|EpEEuC9QNKK0m|!*ui;oQ7&aA4d_ph|v@HaXUNgA=vg zF})8Br#P&GymSFZ0HQz6{Rfq)-T-cXv}Os=I$n^317)VXY6vq$H9DWpCVam&Y_NM# zw$N+)y}R$Xzg<*kbVos$1}wbSf%bw#KfC(v{LFN>?Xup?TfC2lM!ihnlOLL8^hAcVm}5OIc%PWXU?CaK)JW{dWJJ_x-%@^St-x)42a&=9=$vF2{ME$8qyl`tBpx zhQ^Tt%m*F@p;Svi$=}T`@;!U*Ld4OWmEFO(@JVoLmksGP)po5B{M43zJJ0(d_jQQW z+~iMCVgMICs!>F38iJfX_CCCj-d=C2Z9!|fyyAbaIdeaoenN;rb`MudkhL21RK*&T z2DfH0_6@eeLWEcB56-Qay13{R(zA$dv1D`2D#KD))Z;FczMaH_CK{5e4i{v@%Gb$r z9ZJT^TpfEdnLC1ZA!A$eo!)it?=R;?UZgkV<Q+l935)j@2OnD*6H;1@!b6KARI}@VuJUS|FTu*7Nz>TjAyvHrh*+pPK)LY7{ zE8Yo7;u9+qYTeMMSzva9XdFIYlmW^QK+Ec-6c>NxoyY_qrjpOL&7)$&n+E6S7w?tF zvcLJXoy~Qal1CLa+rM@&4-19xfnln@SH(t(r1aJQmvO!c2 zC#yV0161`8c<05eSY!>f{^u=K2(Ee?aA|A2`+AL&*_f)fdRfRB(#QyjLjCH2?2k7k zxXd%zd;B<(u;h8(J>$iT7taY{rwmHJD8p){mummUR7+I;WRbuR@dIj;d0>6$kF#CY zhrj|k3_eTgvNf@iW7va1r36gM{^wumk{gW`q;bpL{pua{b7 z90k?kBM;7Mw@lvsWdgt|on3A{;syfO*<4>NCmU#7#}0DD;B}%Gd_#*191HG%lbGM( z*FTL^)+u_qM$pdPdG+CbNp5)Pwa95eVUREkW;Bn`*b3(7U}^q;5TQDflAG!LTS{S zwS)8hiQj;g&G_Hu08nsFKoRQjWUGv;w8IYjjCZqmbx?1oS2Lx@)Z}z@b8<(acK~`b zQ7P`|J>|f2+vyYcjddXg89XzJbQ(SNixwDs0)S}`&@|IhXbcbQ1YN4dgB3Hm(J1ii zVrsMQ$YMs$xi-Fi-Ia|ydMg_)LOL#?8w!6Mv$nV-pOdh z1g4Gz9@SB1_jxXH_V}Ss4u%gE-<_A@)b|~_g2t!aCenGOC92VJK{l^I&Rdq&RywWS zKINH0x!U3=LRrkpnGtC?>WeWrRr%t1;Th4w7n)+njjAH;ZRq3XbyIU_J~zqU@Stf-8;9me9s)6IStan1Ued?*5OA5rjMk|pUh zpu-AK4VO764qSt6K)zE!@M{0=(~-WmxO5p`J8Co7@H~{Qj*%Wvjnw`4<-qlC3+VvN zurD64h&~dm0-F=?OmG;+wE}Bs@rh?9P|?7dqX)4r)TJ_-8^| zkc#Ach-ZX1y!hdi&hZ^Qwr+%~;Nlge4kaA!em-vOLA=qQog6Kcbr+1+ezspib3v_( z_;Syx(uv6#*ni$J#mnFO*qJ$pWdS8G=hzk@XCBAK2yWgE<9L|)ZdnT|86x|ksUK>( zvv!z||AG*&@yMP+e=NV3Idf)!PbVjnQtnJHiP*maCR&W))0 z@kqNyfunuEhs8D~aaz#(vhn)+P~1O=(s>&ir!-L28Kpx1AU^#@?HOV9k@aX03J@sD zlNta~vk=s;8^tLGTt9wiL)-Z4c9TG_uD0dueW?s$V-oc3lxd&5I4DW~l}Lz_ zWlck6rbwZE{x^5mr|sQmPW6_5A`~!kn=lH~=rQ~K2briJp#dVa&V8-}nfOp`*7&W# zg6o4D{+zasNACxk62Z6On7^yp>cuYqKbfZn?i|rC2j>R>A%H!bwe-9sSkmJAMclOmelV~b74#c8JsEJ>+xH)1L5UJkV-p=Kt zTS8Z$*AmD>JKAL5@hBiSiEN)eG`v}2vCvtPa#zf3QS9W-MmK`fl7G1p`rllo;HP21 z-RpZG?+6U^<|xiB!m9r2CutIm7%Nmn`Slr-0}$7~cHxHSw!HLoKo#EqGHAc`kYo6T zAwg78*?k$RCLC3?tUl${S;}vi^S#HtzWLd^Nwd7WBAo1>RL7G(4<~E78h!r3&TUx? zwd&vrKk$>?5RKJ@vWSYDJ>mr@=R5tst*gIcj#*PGn*Syg7=WD>c@=(vPheu&Tu$5n z)?;eu_!OS(nMeIlNZ>k%k_As70wgDT{#FUeC|E9y%J-TGT6MtTv!Tl3`Kruo?`@pC zK4dZSdpSAes_h%X%XN~Q{vz$PeL&fg00_88{Y`Ls^-*9ts9(u$cfKR*CNG||E z>n8h`gU=p>LayKKyPLJfim1wjES4#G#rqu}(G2(0Q@@mGn07Fljx#K0C;h1az|(tN zFZ5f6>IdAZz^i`{eZrd%VI%90Au1Zb<_0jg-+`q+wg7e0CWxk~{G)_TD*v+xR{ zehcmf0Mn~Bk6=ba@-B~gKK!cUXdaq76LzM9E@dp!QVj-yfsgR;_<~;Vh9(l9qnK_Q3DQ5<{4GW>`~d3<3tU zOyGz!i_-=nvIRQ6n}~mw)rC`_xD=Pq+`W9gneS6|qr=f!CW+d>5^*+#6I^Bf+_~o1 znL5fQ$k53F-D|HY4Bb%yPo@M^&X`cC9F!>qSF%<0K@F zj(JK2+I~Bgdx*4kK7l;<|GP^+hSH!x1_5Y>c?}#fCN)@j-1 zbyfHlluug{>%{cO1;)#M{JGjCWOn(6R->0t@>xkgSmP?l2ZtaRHjWD-e&5%@xt}i9}JzrPr)*W1HN)|)IF|!&Q;Ja>pr4DE(b|s`4Bi# zRKYZYGEi6P5?+d=iOvMPHlSRJLNw62D7xb4EEDP{{;26P0mR0+!8_84B zXi^-I)p12_ObZ!VEY^sxs$$Hq%Wz4TikI<~tHTajR~N7>;H)dbN`Aq+h`i39hYnL8 z)R-MpFqa9>IgWB}w&)J-kJ0Smi>F|WpUL& zhn2n5gZhWA*8Ygh*KxQ*mzj4dZh@RR9{in+WkX(Chb5EM*^^HQv9k~$&@^)G#UF5` z4;4_sK%5NtZpVTJeMrZhE`KXg{f@pms^rr6vX^Xcs5kMB@mG}~2viP^{ig!+H}-#r z$A_DjrLU~G7q1r&=cd};l=7B{1)dgx;58$#rs7q>7OW*a6$Vzw3znRVeov;A<-2cT zM8!rQHjG1+FFVH7%PTXT6-}f?+&^yu>a#wYa1=4?fX@;%DiQX^!Bm3HRic;RPrb9l zqgRI2EVOTo{uAm5#6enfnjop8efb-jDY6U89{pOru%VzsVECr;3EK7EoMZz2Y5@SC1Dte?_@} zzq@N5Pf~BJsUqL>8y#Y~^U?JLpCuzBAXj9VC=w!9 zC+$F7uyV^Iwd$l>HU9UUK9yHR+W)FXmoE_Te9KP2IhHV9?r@(a< z*AXJlML$vlb(CgZjH3*BA!EuS|EGF0pM$!CG5=Y2qwZ;`)1J#W#Kn%p9^=`MBYvkJ zZOcTKjr@jjLrx#2yD?Y+((Fz5w^tr?03B85WQ&`BS!vU{`vIf==88}%Njh2GRx z^{a_3zr{ui2}7_thI9!}v7xb)k%O*NWXkd}>FFF+9S)BDuCG7Wd_74BJ2TL8n9n_2 zmp*2HFqCyc$AU7~LSy~*VbnKEDB!Bu78Q+8jtdg zGc>OKswci|r#nA&(MjXFjolMafoldFBjl5q!1qzY{U?S6wmcA(P9JF`8DMiS3`dlQYXB_$dh|F{4!bbE>hjr|GPE4o=6qe0_`w~GVZ+H2At zSmgcGaO86^cF+VV-me#x0o1Hi&X6m{xCfN0j08N;v`i(p$5$z^rt^gvMD3za}Ap`Gi_A&9cjnsw?vE3Zuf;A7Yevdp({Rf4i-Fk9qE&h)DjkM8W>ZN0amk_+gpG-fJ*r z9VL#JMv=IOfOG#HD zCSu1=e&~N2y>fL%w1EKR1x3(8!LV^xo$(r7%-5-Q75&TaiZy5S`7 z^|aYaNFxu^pJLgo{g2a0s~n*(fPO{- z=fj?L6y7z213#DqkvK4BU)C0$>!+*;dQWw1ao-Sg6B7`>F<-*BX9J>u|8711fOI!} zqFEXaTiRKN(S$#0aR!YFE(AH>m#U>_n|qG0O+`@8+zbrS>56Ab@m{)%&|acE2u0N< z0Mdpv6^M`p%hjwsB_%}Abn6>NRUIy=H9WJb&gxAutBX?RAD2>KnNCjFD*tipz6r^O zdQON8(GoDAO-H;O%>QQi%~e{iOV6)v!YVez&8IS;kxhzns7&V(KhPbG_bgenv2Cmt z*p%@<6UbvCdU)Y71A3?m0o_KE4N9Ff`ze!zO~2h95RLUH&u@ zY>O?L#HiozzkdAE(aNCJmm}88tum1sLd%~|&;HwNRR*X#vRa3x{dX$r;X)ru?>GS; z9c=5FP&B!sPM&Ko9O8X%?!%qFa_AY)M@jjObJnJ;{+@qE3D7Q8n2W6E72}9>u-~a3 z33mE$P%r88_Gh|FM(%9!(OY&b3-*8P)V;fiB0QOgFa#8Q7Edz9&nLi&o1K6J2RMQY z?d=-sk8mkoI-R7^;(>_D51P-(ADxpX9*>okMu1L~fvYW+UvaQh|23tQa>ySCzmE zyLBbr#6PP#vEewi>d;gdqnTG{WGpek7|9o}*~9wj!@T4xsgz&nzj=_z@3X!lkrfB; zeeHvclVW)+2+<44sy$Ohq(6qIjxvSi%f8`!u|EvL15o~J6@~~iT!80E3a7l91#!Qp zEEZ5CyZm>Qa)0hV@W4;1?>Q8{P?3vpdwyZVTF?KU+)XCNk$x^JPBai88|2g zIG1@FUO+7&vT*)&k6IksB!q4Mu4KW0WRI^mU#ahrt`zOSNXc5s%Ga!2R+?bS%{`l| z12s8EV}DN;Y*&t(6{ZNrHLGDDM=f&Kvqv>)v&b6q#PIa>!>OirKRtg0%%40GXf%8I zOrddyDZWZniBcU$I4--sw^N+ET&>-KcE{J7yZgT(`k!TxWscn`zCZ&Q24UCosW25O zL}P_7={jAuWFLAvdpG&#v(p0Q4s4H)zDjv}KTA$F8&uay;k(F4==T*!2jG(7t7zI; zhf4p%KKB|nMXYDf;$a5*2=9!^pRqpaG`iJ2K62^0ewT|+Zy<{BX%)?DhQbGRLFTWR_y#7E9gKH>;%lg zaD#cJuvC5VjQ97+m7N&pR5_hjCF0h}8p+0-amgis7W7lD9_uJryDwSc<>;1j+*Gqr zt#Oi}kwBDlxv5HT0c}K6Hd|9?M^~X2Px$?>=>z^+k_I2yXz zf2+?s4K&UqU=w?S!3_^{^MKf?vShzsmgH|$^HmLFo{fe|Zv=l$wcn-4Ch-aK^l)vm zL)jJOWpwuuviGSEL2Wb#?>V%068*y`m0Uy??EERVOvi83u?SAFCdfP`zBolN5PmHe`xBfkYH*I$wNN7pN-8?56=-@ch^#C($@ zCaY_(Ni=`;y)C9BQP0xManK9~ah0bwl&6_Vhq4IBzmr>>RN3rR$i~h%>)lj9konjX zGpvH)qRy^g`Gu{s=eiTK-!LIAGjHn}K}w*RF9Ft)+^PKvVWG%9q<^Z&at((4$Du7m5L>-TmW^7EE*@%K^WBqoJc{ikoN*o zTJ-2J8WihH=q;vXa_S~NRqY72@wKmK%!bHh!2BRQh2bTpX#qH~d(< zwk`k8{a2cU%&Pz^Y7)0Zt?_zr8I zw+|Y-&e$m)GjjHvrifYGF!|EAt6>O~FL>m|QdnlG!xBEBye6KrjAzHL`&u3WEqxt; z=v4KU6pwkKvC`7xcTXpt*fM9-dWs!?@9d^VrT(S6Tmut1)p)15_zK(@svO!ZxgcVb3EE!fb&p`=$TzbNBl5IHV30pnbMXEno2?YmCam>(heQ#K+15SROTG`HLgWxI$7*Rj=wr0deV=khO#E7pWF0GzP)?>h@VkN@@}i0~0j!jZHau6A zMdkBppJT?I4tQn>Kh35ZRyLWN*QvffF2AY|)mD_%HT{xQzkj|$s*u6>r{?^V!XVVD&JAwS|4VQuR&;;Ygz2$R8e;sA;j77VqjGz(MZDK7@%hMrc+m7d(@Dl2opmAbJNRKeo3 z8;kR6YY)4Y)LL4ouz-gZch@^fT(B(A#{=y1ou#Zl>v07nWv)&S3?LY{oFrv#Y=~TY zHTHm{qJHp7q#7&SyCnJN!~f4j9;BTk(rZ98Ky_Jy1ah8nNU{}DkZMe3?y@knq`2k~ zgIKDjxwyNXyz*xLc*(bYD>uaQ-7gRGFO0Y@bjzQ<0aAtT4#8R#I~^3+b-{VI1SWXj z5)LFfkLA{k>Y&_+0}7WDEF)DG@-+mnms==QfA8;a5pbQ#Z@8kbl>kfSQAIMKed_=a z{V0(WIvsW3AdU7pR-kh#?Eg%S!}3~F)hlUpwpvfzfUu!Lh4~5na6mPBBP)%0?|JDW z0Ej_C%dCm}Ity-8SOis$?A0+O^b-h)=^!a=9qG`MP1WfPW{XkLFLU3G?xiJJxM4;9 zBvwRDEF_ITO3Y0?zn4aXII-Z5DZ#-9VR#J0NtK3Gialt|VRHuI=rP}fwLXhWt5%q( zbhU7onKLGQ9zRlhOBNONZXIG_-U#8RhEG7PBXc!I8B`TBN{L85Daj!_t+k#d{ubwj zGJENJqEQ%z@K)w@K$b+vKv(hat{3ieQcuIhA3gl)Pv*{NK&;0vt@KV^2mLMf;$Z?p zQx}ryelWzoaSjbK`d8EtklG9aks$DtjJyXqcn>6yEPqNn31mckg3p6~C5-uDJ6*jM zwxBoxj5eFSOpHtBc@F$%UIlTn$Xogfsjz<=Q0GIf7myP8f2QC2TM@7jJ$`OO!9#h| z*PIT*IF}GFmOsyUeTSj0y6_l7s3rd{P+Tlpdqw}}DWa`T%3|%)poo{_KG{Iv&^`;D(pkh$})8X956YA&4 z_FPXPFdTG5kdW758S%}|D>B%DGBr3z@mjjE%>WTAlY2$FZ{(z^@C^nJDe63}L|p1m zs-c9_B;pPlP|J1D!Yd*m!7|9^n#_VF!NV1jqSk(ahr0lOC3|&`dNd1BYP#($+UI*>w;gF{0s4N&`veMJ^oGHAi2p=A-%r zTb9sysL#UwcOb9kj{e3m<5|1)-VN5eDU{$}mK^5GxS6TJrjS4{O3aX1O?_>$dl6RW z_&c?xIuRp>415h000+%@R4-1VPrM5zKCbRoe2Ow z`SE+W(7e8}Yz%p!d}O2240}5;zSV=#O#VU+Mlm$;9RDo_g+E*nRUnGSdF`M;j}v^p z5lu1}1=|4rRpQVSn&`>{_>qWt2-l^rQe(~$(Lr@CjWsXA>vSZrM%Gs%v4bthkz9W2 zbLvrGqDuk(y%`1ONK}>FI*~lU>LUaEEgNjrkX(*1#+YreYxRsf0J!`km4{u=h+OK?;6Pj%tND#CZI zv-cD?Gg2p)(icUXJ=BByqeq$LbIqT~3qdNifZ)sxbXFf!(}7@^nI!oS!uvmQ@wPpU z!xp@i5*RU$KzTr-)EjUUag40nG#4pDPR+`>Ib-A<^yrs^t%KU&ZU6I6neq)%>e3Bc z*ixwjDDp%9^$AOAnqm;tCPbv6wv6#s5MkU4KIz}B49oei^Uvd-`~x9`)htk5DIThi>Kf|J-}Emh9AOKK;;6F~3_~jX0b&3D zodH1q^;L&xoX@~l{Ska0y!r<}0z8ZOu)Pl%gr02+Ivtw=IHGbNdtu|Jb8UT9{D4G# zjHy$cMT6;3RmNlvqAQ<-?gck2Mtf%sx?X_3D( z&K;NGq&M1mR%nuby`@k46#GKV$CG0~Nx9<#Yn{`F+lE6*HWMB9dtl~gP{BJ{NM(>td=((QjYg2d|u zsmFEqeLRG}XMFAMGIy|={q```T)ULn-2}rlRY-rwhd5<7&XqHRsCnG(o|f)-2N&DVS+VrC6S2dSO#}58oovJeSkZ_?phNU~rN)1S{y06f z*8(cMuk?h2rOEin#Z2vjM$~&dkMfH#)~@ciP5Tgmh!=yptM5u6qDH(3$*T>|4kr~y zybcJzP&GDC3!Tz;jgXpbVDQs$(iG}8aA5i`d=cFb=zBmmbpN1GhZ(p~cDR5r6sdP= zlqM2@0>z+GsOb>wE($H!s5lx_II%7En*nXR3;;!OKN}fUuG~5 zEvBae&J@Uj$wS>)EDtrQw>2W}APj1@-~ zWj8<}@!Cm6CLf=vr;Poby|n?0;d*YEbS*2b-lYMygyJJJaz=N00Q>P>I`g<%Ck z!dh2pEIE|UuHUxaK=4Dj5)SXmdzHlSxJQ}^=Xtk1hj&y(uZJh8lU zoF)qgAQlhWzvtj!A_Q}lwn^RpV5X+4eb;$jR*;bp#7gT11!!>a!*1jH?7G)1_vwNa zfSYEURs#JQj?t+$qmxUYJ!cn2S^#en z4@xp;Pk!4Df;c?J$D^eajDw={d;%nHS=mb7xYPlq_?)=4@`~^pYg(qE zB|qF!wyploVBLpMnRBonTWft&cN`9}O8NOHeA%yBxGHukY`Ux}hWMlU;nVkBU)J*1 z_}(PG-%*1V&=UXw6rr&K)<)XlCUQ=Dy<4XjeJjQ4SC@^9gLYfWq=Ak~Hy!%S!a$qO z1<8}4&H~brr+Z5Kk?_;$9S|Di@B{3SP1TZSl+Bq25oBBOf?!H66xZufUhUqAR##AALUk>jnT>mZ#P%!QuvZv%5YQLRXBU_DOZi64qR zODZiRGLfTn4VCdN2RRqhHO#$lCCRbgyhv$#eg8@Mn~y&dK7*wE|9u7sJcJ~T{SDc( z)EDSvSy9*@Ilh$%cOvgOXw6@-_gDXPbD-dc86Ss?dQp7h#oqS|PcG?;UqUGZ0<45X zD>S-;8a*YCM~i9s?t9PmPJR_Nx1aV4bGB(n`ttl13wu!!=ULD_ngbC)_a@nv3kb?z zVWa@%LBU|PYFLHD1#zX6SP}~{A_3TKk?UKxf);|@N6xG)%Dp+oksue=F0Oa!01ph# z0(w%m6{_-&p`!9}16rsqtYLc%YSPA0u6gA5`tF1FEuFibw*oI1w`5C&`jG*lUpoqb zHQ~Mho@~``X8k!OnwF0d4iA_%ePeHHU%BSGF*f#r zUKh9!?O-P$oG>uy!B>e8+iE8ATcg)J`UpS`KzF*+&ynKNiHBQT9cgDQBwqYkj5&Yp zX37For>*bMf_!uZ9k~ij1wQz!q6!(=mF<>t(1h)#8J13zPI~R6krUSA_E7;gB(KLn z4_W8LJezpXZPhQ8XCBuj4SzWpH{%(Ol9an|+A4N@oA0XIE9~1qDQtmos?##F9l{rw zT}ld#%L>*epknM6pAoYnnom(agnw|Tf|@ex1adi^KYFJu`oKs+@R}^>_Dl)`(`En+ zIEyE&LSW-+hStsAQWYA>vQL{uS8(zum*G+`FUjVxEpk%%ygk@(X5eGzq5 z#C82yM(yBmW>{UwksWRrNi239wHn2-Q%{U;bLctORn=FMUp%>Le<4;#%H+mlKsXc0 z$gM9CdI}<>(T;r8VFHc<`gV}@P}k>D^YC^tl8BEg@>)~A<$m&BUV=r9nMATWUDDUb zRz-A-+?ekQiy*F)*P!yvx8TVapviZZ(ajn)V`?hC0rjp4?uITMa|>zhjnN`Adv zy(0qYq^&^JKO>qsMobx#FXb*DmI_sX?=)1Vwyl`T#+$lE^gl3Ou}*s2$|!2U492$} zgKsepj$d!E0(gCf`|KzMH{`zFJp7Z!5KpocxN%qBCB?w>j>lwQSM3XSxtn~9WMh?B znsqSn;o;C*a5TR=ku0N?QN%b^6n{wJ)w)8jLG=3@vy=XxTwVK) zB?o46Jq2|2C?<@kEcmops@&Ma?+RD&qtvq=#j`6l>N}fKTKBke9H-w`oC%XOPIq@= zpI}-rT4_P9zQfCbv+;C}>}WYzy3>2`zO-2&0OBGvXOR^;XJsn)FTF{%^L^tT@O-LU zb>!S8h8G*~miP0!iv?&6MsaJE-zQ^!MuGl4D4EBrWia4#vDY~R$B>1cy`Vl^Y%hkxtJ>~^TzOg-^~ zHojH+%*dpFF_H?!nj~SD2p-jOfVwk>|i`;A*}NDlCBFTt983O)i}Luh?^$T6C9fFm+S06HQC=p$Hl zpZm0z7#$AtkWCpfek%FIjDflW-Dv3Ie^d@|@~;P1pw>6d9ABv?z>X%5om8_>zC2pI zy(zk47ynv%py7PY&YVkG)L`@{y7^z?FEmaXFVDEi>w_s03BQ8`Xu0$6L`v8I`1B;q zcK6>7z_3a{vIPQm#$K<4J=Pu~(1m3l&78h5Viey+?nq5CX>Nri+vEhca65h_Z^G6S#2m0-j3ZGJae||lENA&_y|&uX@Cwd z);S8&k?i<_nBm{jd}erUFxTtDic0awih7K-^9$*?a|Ol^ne3U_S%$MOhnCE_|)h)}C0ylE%dS;z^SN=M`tK7M!Efw}yVE zOHkNX2Yb1QWX7pdy~%S=sAlA$&Ss{OV1s~aD@?@-585!Vf+C`GCHp}{Z_#VF^TvD>RS?hRFoqssLqo!|M%?D_>8FSJZg?Fctf%KHL**JwE_JTio>Ws0-YNuXVN|w&_XxR>n_eP#F_R zt_Z3T%U?^1aZ<~GO2T%!ag(E}I)`!h$fwJu z@p4YdCQfbdRCp0G%`<;3O<%A`;ZHpIS~1(r+L3J*RfnrBI|4iYTlVGC4rYA2j(1JO zMQl_0En0Yux54z%NL0J>BygmymfSssum%wy8Z238#$d0z@r<}{&KC7Froz}cM(xh~ zkI8kd^14iS0X!o@$clf2EzXK-L=wG@PEPc?R)^mlxiuzPlNbvL8ofHIz|&<6H1%Mm z^B%|K!}_}sbUkVZVA`8oN4ziq@X^SU+04dOrw@?VD#$NBWe)Bf3JWk(Y=3YH^vH>t zoKqMr!ruaQQQsdL0^nNFc7y>9ht9VhHQ* zRs1XL+QVfjsJ`xb*!e5(QeDNI$=R7_v)LZn04C)jSQjE9pU}isgo8yV_om0uM(d%rI-MgV%c%Mtyht3c z{c7!I_E?_L$uF*IhtwyET56a*gpN5gP|!5aj}ZBK88HD?+U?9Ru^Ooo;Xrlx)gRw@ z;p6Q|X)n-ESBq=tW8-Fwt}swFN)yw$)O79fpLsVo^&Efxrba6Zoejx&&9Ms84gyfY zN$FyO0sz2HiUmF38&lQc8UB!+)CK}KVI(%-TQAUhLZ%!BMmI#MG4t)mQKTw=n%P_TbZi}D^wOh86M zjvK|p0|?w{aG~&cmSCGA@ITe4s_nSE$g0)CAt3sb5sYHI&fAzy#L{c~3Vak)admO; z>R%Wuy%IyEOBbwpk&|S1R#Mvb#;tI(3>RyMNDHlSkoMa@G^q!v3#1cH!`a#J=8EW*at zOwoO(7*ZVR{&7Swt%4|@Fa#?ycmMdN+GJ>sHR-H*6p`Oh<8CofS=@Xg<-SOculS27 z#$(~=iL|z?P*Db$hyOQ@nQZwaSVZ*0l~TcT!|T6^epmh4#Ucc8-v{C*yXNCR;6W@< z^OZ_}pAH%sMknH{O{k*Gm>Ykt=3L<9p%&RiCta-#~YL>N_>0`a9gz7-7*A#WYmn!bYZ7pRzNFF^OGvdO4d zMyIs&XD^)pY)Tgi31Rjp;J0M2Hl)aJmWNrwAkxPvkF0M9Y$)h@U5vV16LYKRowb?u zb4?px_Cxx9Lal0GhuWK(>7ZdQ7zKnIJu^To=g!aqntVXCO9SPcCN*G2^rnx*gPS5;kujuAt~+$SnUpXtxk znH(jA<;C90%5L=87osBJEK*@pP{$lAP8fc6bBwj>TTTIdr*~uJ$HymDTRl8}Z6EV{ zG+CJq^pV{lljp8zt;fefgc2~-x2C0+mVY;SB4E^V?4;Gm_YasL$lcWz=pBVuYYyAK zN?@gvdW^mRwax;^@!Jws*eNb_^mr6el2#nxbpfllh4C|0escd_V#g0`NC|7kdqS)z zW-kbK*gqM321b?)0I4P7&|@8B!Oa$|Ys0WAn}uHBZN(xdw@9@+5lNgQ-Y=Lxl@0+c z8%Y&Bvg3OYNrO%=qd0I9!N@Y0NP4YsHGjw{FmKA7D{AD{hclupvP=usJv{OBC6@va z$li-rRJV$}3$^Q5Kjuh3T}G6yFVPO8h)P#m(BzBg$^G-d9jqg7jNp*&scF?3pHzXV zl8(JV=@CKn_8Ksmf~Ajh1R-X4_TNBSrAydi2%?_0jQWkLm^GB$i7R=Lqq>J1zq({B z@cl-;)BSda7e^(Fah0^`Ir0rlkX)51uL-`ReG!i6+>NFx&&w$x+fu7ftNOrU_W8mj zma%%0*d3b(gNMEvwrX0@ZDRwY)@z904LiYKMs$M zayldB{VvRQGMY|bDj8Aw84bsQe!1}jZZFW$L!eFngNPenhNwp@X>2Ow8XBu5ptX8U zN=>Yx9jPG;_tO+cefiJyyp!^7oV^_|&YNQ>qA91c^=Ufs3?23Mif~9*3Zzy%reg14 zuWh7g_15I+7@?5ivsf%Q=C=Zk!wv#whJ=Qdzp(U_Qt4OZtRsZ_j4uN$kyN6sk zX#1jyQ;f!Os$aTPO+GhYV zpZ$wr=6?6LLVne~DxXOXEG@tjVj!|38l)552Ol+~3~^keYqjpdnONXn*p}0!m!A4Qb3k3HtZl|OzqnOhT@%}*xOKYio&5jq z;ogAuNqKA2_P zWhR*`xf#NSWITxTCeQDrebeJFtEsC>d>HV8+ zfSG#4gvP1@n&jm**2+EryDmIuH`ViC&$al2qs&QF>^arAg=)X9$y4P}*UHbt9*L~y z3pR7$LXn(kY|?~crxu6|iegh*qc=yYj*G5rlJuGi4EWa@T7KEhtQ-MTK9Pu*3z4Qj zOpXEBHYsk*zFTSM94v21J?651f_n6zjA}-3fOZ~@G@r%HGC5_h;KZ6o6}cuo38@CU zu~=V*&pqrBt&(hwP){Y#FCY{5nrsg2R0k^7k^jr@|(6X?^v|Z34}W>Q0?ITG!J@mLnneDP@%d(eJJk(>rGk6XUx% zb}eV-$Tv4(1*iq=hpg=5Mr9(H8Oh>Wrzx%A6yMBFz;>kG!-N}mdaI_Z0BmRtWW3pq z4}9@O&`_8?dU6c_3C%$6V!k@11*1DS3NNqV?S>sk+4@>vaLwKN749(6V@XJwu)#BPT0LU<}HWz#vHXtg8;Y3qR$zBmu zZQ^km+o_*xl@976Z2m983Qe7!TARPHW-3eO<7@*Tj0W7Y^>}c2r+i3<5=uQDMLsLM z0Y~m>zgGC0|2%H~vcw+CaH5qtX|@UQ-pGPrs2fGThMe3OG8%!lIqWvLq95`9PMn;s z+%oYx^pYMV3xC8jTf&9k3Aq*-yC*ia!U%USLff zpkIU?sg7+{?|~L9KIE86$ze?&-vD1B2m5P#pM`;06v^6t8w&~IC+P1hK?HRB3-t)W z$CSWx#6jc~Oa07(MaidzDhbuSCoeqmx}Sc>`-y?y^8z)&uC~;U#%%V0*h8u6o0sq_ zOUTX#jC;)OEcGGM>Aar?YA;J-!%apS?&fi)#9Anbh*VtkxMqwkr+!z-eu~OxVdIl` z3qd{azkmSf|3LygwwdU{r2+4=XY(#rev_$R!J@E-L(NE(#`{2C%?WQ`l{%-3g|SC@ zNUM_LCQhdNPJN%3ii936%@ANkFs>;FhEewv(y_zh)9CPN*h?j3Rcy+Z`oz}Hu{}R7 zGwaO$_?9?6<)25h=h(vYv!l#>Ytj@x_+_4p-my2+frtG9R7gF9T*-ogZND+_52|Q? zE+(ReRfUxSkLQKur~?1_k73!s4Mn%fajvtJ%@sALUa~4H)3a9y9`RwfJa?R{%#(7m z$&@lP29RAe2sYh?wp4Sa2vj3WaIV!*a1g=|;FqVF+nuD9ZlDrW=%hu^Cj2WB2>rh* zf!9Eoe-?rjK#&b9J59v|TBEwb+)nVDb{L8-? zeSI&xx$+!u3c^{>4=mY9Wo^GUY>l^Kh?8VSa!nIR1X^|qL1_*!aYjuOvd56)OB*g^5b&a-pbG2dfqeL0>%|3QA=sOI z`~d4^em$?Ji$_S;AB#gd_cBsuZ_QTe9990t{o2Oq5n>XQyG{gyLc6!yAX>p6ag*CR zsvSyz9U;#t=KYW1E#&Rd1jm@Z^Id0UJ3y96#OPy*4AMTHP zbn`IFC}{r?j^Rqf&PHWbfAJ%GFQESV{$Pg?fdEzxft*XI#--CvS`IY@Q2-e;R^srp zCd)vteP!AAQflW!cog^BNbIyPm#X9<^PpJDubU{8jg=_C0D0NaovE=JvvV5z$8s^uLZ9Yx_wZtPk zkugFAQUlyZVnaGU3M-XgxmYrDRC}fz5_oB2Oi`MVjyqmODk`5a^8Byj|oOn|gvH$Lmxr?Dnu`B@lhh;)U1>@}k z-6e$UmoX1STgk}F2ToDxKw2xQBSz8moyYA{UK$rN&EwlO{9XgS+LzIvxK3SXV_qKl zoy!!VAU1-mV#2Z!Kq40Qr(wL3( z#u&om2~yk-tQ5#q0BX-U%*w^>bHLsj`MoQkvG+lfsY^-Eo0hR_V+vKL&a_3?@FmWT zkJ-K0)|Gsf7CE8>8cqSfDp-P^5j}ua2YFiW!df>C}jqmk4vt?{=fT;)O$v5 zd6s>)_Eo*Q@76fOUR~c&>Uj((#<-*5!|Zf-n2lD|+N|3^<8H%Ct3>$>uOD{8t#wx~ zb&KyuAH7kVFls}uognL6*ej|mEEEorH{>>t^@U^Tna2g#?I028b7BgRqjmfT6?6#m zM342sGnWc>I<#3Hyu$lq$k$q9>8eK9ASl1I^kcO+*U1C!e}-wdxc9s0{AqhN+ZALt zKgBdfGmb#t&uydap(Jps6)tdh5vZ>2%pC0wh_M?y<=+NFB(-E#4$Q$ZcUc0v$P_WK zpov3F^`9f_y^q@YOCFak4V>2d<5;%SQ8o6+_nZ8D*Y{f-OAAf91ml4f*w4yAIAA(B zKh1!>gId^v+u#40uUFlUW|jVOt|-9HiDn`1_F1>QYxdw_ww0%L$=irPSO}AkRnX*t zHK49i0BeABaH#j>@k zxHgB{m#=;SF`>+c$T!g440`QBvx1zfuKuFCu87ztqxP^BU2|UnzI^{5N*_Z!5HJI! zHl-nQ+ze3c`QD8c0LvZQjsa_yz)WMaTCh0%HEiE7{{(w_njjl&a1F931yHI*bKHgU zj~5(NT|~^TN^$5Ovg9CWm3DRx;X@q}W$O(Cs)d(U5vumsQw8 zga4eUvl2Yv?>%f3?d;0;g7dzzp!zS+L>a`~HAR<%xfvHND0pyJBsGwu!Sk)Q&@IO^ z`)KE%m&kDAE=(xf`sMMP9*|i>xy0`_k$eAld!EqlIPLPxNJKLiEW&7)JxmjenMMmN z+%A1ke0!r}bd(f%{Z&(?a7l(de@?z#4D|o=dD|{8#5nPFg|RgAEa0yY;#ip6N6o3v zhtpac+7xFxq00kW{fqr;hrUx%cx^Fvy#RQOsDpl(skKZ3Nge5Yc@i+CNFswJYutb? z4BR&tRya-vH)1z=2(G7n?jyS=Hrd?HihNVhI@7CuRJ=T^ddTnRTKd3SbW&}d!v6YiXY(eufw>s?W>8Sh?jlzzyxA8h}|OZ_vb=IMN&()Zt(q$5e60dqNIx zbBjD4MpRCc7R5&RG)vp~8#At?SlFH@R0!=j`bt1eeO?Ki{!rG_D`cb}!(NGAybpld zm4~^XKpAklbH#Fctk-24u*E4*R@ANQ88SmtQR^Ohd%fS1@N2EO(E|h9$E$$UnPN>jdW_t0cEK?{6Y!65je`T|DbNIlKSXjH1W3m zBS4Kr_D{RGz@x`+;fKF_H3WK6`@fXvHN7uY68XaJ-L1@)b`j~GqI!xik1^DlIO=Mf z(8^Tw%nPjpJP{|NxOObEDm+^c@{|%gjPU+F) zn7bf3gm;dihqPfh)_YIVgK2?1e zX<76>P{qA0H#$PbC$7u!!L5|S77QPpX6`3S&nylu``o|Q`#FkF*;?Lh-_`Rx(z{G# zr!n0GrVNM+RdZ-|thg)#BsCVvZsx*)WRU~yIVEldBAY0*I8F*)y;fK2H5$O;Zo8iq z@#U_%8kjSE`ApvZg5zG!s8TG*AvjNi$9aHo0nciKsCnX!&?4sJ!2D^C6sE%IwzJfS zx=1Tq(N8h%SM$>CJ9I^#auJEjC=@qsemksi(%!Tg+a@4~S^z~2lrT9mASuU`aAX`_ zcsTTOsIN^Zo|OKvX+fhl?Y87Y^MKe-&qm)=+3ox!e73TjJ4{gE@kg)(Zm`M0NZ*t! zh$k;$HQ|@CndGDhS)R%;D}zgxUDb2chorWfkuwqnU;H-xGOS;Ss(pFSqYNZ9>@W=w z)oY6R+MNCiETUYr6KJgzPyJ`WhAW4N2hijkbQOLX&k%JE#*jPm-h#0Vxg^|oR>h2j z9*-UdKAYauCF7PMtPpTTTdeKED<8_f7!-2Q>pv9cmbnwq~>l0Aw6|DDeU!c z8tV{Wk&paN^AB`|~8z2Jp+gSt0r)}pW!aM{~ zv?YQYCWlr68VMrfV`mSWDjvIM;ZuH6Sj0l=S&#mVaQQW^J0YOh>x`i-(I;R7hQ^Y< zVNe?ZARbuucenix(=#SP-(SFjj|+4;>{n$&#)aT6ie3*ahwMye=rPaHaM|EI2m-$Q z<7Hl1ZA)G*g+Jhij+vipwnywK32p$jF=5*uf>uJ4H?=o+L$?T9ESfkH`qu8x1ZH%D zDHcxWW933V=uv-VR^GAGUSt{W7K4fj`{4F<%g0*YTo(C>`loZqd+$SiPZCm3U+=tu zGJGQhj3VqO&fLw7Bt1kkkKh1YI{Of{m4@(xc=D+&OhGF0|sEF-;!V_IAYGV?08JwIC zNROfZ8$b6449E<-yQZNgB`-??>t0yW2FPizyKS>k8$V_HdmIubP;Gtxy9Vj`+&#|U}pZ>5rq_=~%4+$Ws@<0B(PLY)B+`C}YK#-ZxM zXyXIUwhpp|)#FVCZckq{_lrIg8>@d-&b{`WjNX(D zz-%GFte9!y1_6r?dMyym4j99(fJ@L%#Q5Ss=0bHZT<)P7QP5FXr$;k#F*!%VTSvSe zv7+$KSM{@Ti-JT$VDu{0S03km@_u#DO`~uEP5ywPp0^%Ng6KbBfMa^mqiCgtEfv!} zgi6mGO~awLX}U4Gz3xAq)K4wW`jD<9+sP;TvoV84wv$V)*C18eg-C#~%^IZi0|mV} zz=+riNQ>(rGm zcVI>{EDN)b+*7!fl6z!d3}C;;pT<%^*stg}HL?vlDZ2f7=^a?8VYG{+H=q+ zwvtrbb=k-=;uc#)^4`Rs1a9>$ki~>Q2k1LSgg`FZLbvAb#;P~<`!rdQbpmf_#l1UU z8~1>-=Txum`A!d49igY8$(`Iv+-us1!ai0t^vfEMONiR$GHp<}ZxFFy3tHcHxg4(4 zq%ph}-nbKgd$|0;T6NATRll{4dX?K+=l9$_gegd0^JVTbLU=bpK12kR7CFdn*lFgt z^ME~#vuQ)JET3K!tyx$P>5$F(zV80LO7Y{M?8U!-hXNo!eQw0J$&%rTb@(`l*Q(*` zZn{w%qf(zZRd(#bd+f2OmdS(Er0Br7 zm(HCRa_z#;XN!we_wH_0a=R+=c~(SFSLiU?CNL2kDs0 zg9BCb^{bS-cb=4Tt&R^364?=EeO-toN4+P+6gHn30X(Uc)lftYt62X%(A(0I%Eav% z{Hu9lK?6E>u>D?8pv?Haqa;Sw0>$-+dJWIGL~RIkNRRQIaSdL&1`b%buCd^_#rxZp z*PIvivjW=^f)pOCq+M=475dfxdvT}a6ZC;y@7Q4~ez5NeR`$)YT=;D1 z5x@XdLdVb(!8sKT&g{fcO(1^c^F!_U!(#*qC|LGDjgj{W%X^|ZMUSNajMvm!eDSNg zW~dR5j<8gljX)vi)i?SOe*Y2Qf)U80R|l_^ZB-X;+7SEga|aJlinU9*Y;>}+BcxnH zVUKKB^D%I9EFf>~t=-!wbc7Aa>P0aiiK5==WA1u1W55(CAreINR%7iN&dKhFp_qN? zmA@LWJpoydKR*AUs&u~4>u}_Bv(4TMC0{L2?Cbj5aPBOaOc@=)*Py}dWjw%swaPRD zCb-RG`z74m6Neo63I>#;WU$&4K0vWh5*uat!v@*r;t7`{^V_gqp z#TuG%wzp@|Lfyvqh5iLyU{aFvY?D{gl`l3bah|zucF-D-M1c(lan7BD%gz-wP1Mz} zzl}z5(K9(~*1H|gWTaG~`qNKb4GVMJ$$R7z$2aZ~IqF^W#Pwc4tzN~7`>j=zVEdEO zX6r1TW}D$yFSD+1Eb}1lyv3LN&+K!5{-7qzyr{Z__n;n|#N5RWc(ZB>nIhcN+Mft0 zIJJY(j$!vc{B*z>QScIq?#*~{N8)PTE_TC;WY=;IP8IIEOt1f8gH7Z>C{3)AJu+Mk zL{p%Z_wov4sr{>ez+b1LWj8>An zOymQk8d5feW~Si7mh`soSeqiyaViwH{!+jHx4m~D57WYT=hUgP6X$ky1qi;v)_RQw1EIpmsx{HeRJcAH}A@*8}eTh4!htc|{CX_e8gH zV=nRRon@!GpC9x&{P@J=@BRm+KBcZ#^RmQMm9+}hKDQ=+3Ne)qQ)hgJ`5x%Mp&lXz ziHmd}Z`%@$Pql!Q#2)M&%hRx!1x!aHYa3LsdR{O!U`69IA7EO%4Mw*ba!!!U-`#$L zscX^~-u(Z!8H74B_GPpFICulgvZVotj18&AGul>KnKb-`4<-sH&a4(sEfq06n#R)$@W z4f<`(l$-;ivQY?6DD}SgiM7Qo4~%_H!tZ^al1FY1tt!PHw11(fSX`VadW8*01AQ~x zEJS#D8T^ZAsuj4ZD!HvwPb534yzySLQjtF&F&cy8@3LMmE_&4~oqJKKbnb!<@%k&W zJNv?Ilsc>huar#%=Tdy*^<>VjEnOJQl#yc>=Ug=dQcsyaa_qMtLY*4o4Td?;)Z0@l z3IIc90Gd>M%y-zmMRqqFG$SP^+Uw_4+*)+axyR+6Xq#<}w0FwSrW=+OY~p^K(Kfyd zY9TwB+2ADg1TL+BTut-!7|Gemfmn<^HK6IM6lr_9@yXRik?2{}^f!w=zTl*qMHPpk zud2`Z!rCLC$Zy$}OT7YU7S^Xiuu{c87MeA_1wK1wo!=U)KekxqNc0mQl{|+mJsj5` zkq`NtSDp_DJEhVY;CgTc1>>*Ou+q_YO3tino-lIKw(4JfKkXQ`i*(Jxy@WeU@2iNJ za0{On?=JJSwRkL8rEn@j!u(a)?vUA8Vh!lO2^feBBo-7^k_hA%=*p1}KA~92=4|(w zzIa-zi~jRM-AVJjF(B5leq4@0Z-4GlS7B_&0Y zSKp@hbgNhN9MS=#KHEe}3@=3B(+IxW8rDRAs;O*hsB&Df{W&~)Qr%2TLEnzw1;WZb z=e5~)-t5XlyIL$j^}WuSIPb2!u-=qJg9tVCP{^VDwB~dlGDp)y%`99|ZDw+*v)F<4xRLrCk#sc!CsE^ra&DbWl^Ly7G z8LT*R8fX*TMWOsKq|=ZghH4Lnc$VnVET9_jiXd#P#8k%!03t!Oa1F2C+Q`EfhIh8^ z{L#^M)IBq0+^;KhUcx@$D2koBbZU4N`>&pv_uO#qyM=8zvmp>?4fns{ve-2k`M-Pp zAwm!cxi=o;HUs5GtE|N9T+V-}82(*Qv)dp|x$BOotLxYP#WJ#v1lteyG>kqLWE9~h zfXYCec^N41?Hz?JiHKNZ zQ&b(`V*Km)h&qx@Ure(!tE{^k@;>2KZFORD`ICdq%fum%-UjA-vq2GFYSK@0Qx-E^H z2Gq8NKUcD=vnQDxio9iW2lYv=yB*gK5uf7Z3ANy@f%0iTR{M| z*Fym{vF7Ji+)rm+e{Xi>Ecv=lUuqpw(vsl;aze$Z@IC@_pY00(@D6POkejOyoE#2g zO+aOV7za~kAbk5+skp`9y0uXEwAG;#1HV(@Uj91$%u7z)x-vzPHD`vnEPed%M}Qfa z%761EkTc;tz#k!Hth|6Q~OP9GxX{)4Kq3vX6m zBKd}`U{xsEydTeV$W>oKj~-me!pO`9v-!%F6+iohYO?whE2|(Fv49 z9^BE^dheRsY3Thv$yeR1|FlMrbV_Qzn~X&IQz|j~C>Nnd4e+V@dMo?tbp8&{&*%q*=g$((&`_UM5qrREI9H}k>b^4;Rn!D;Kio+g|)WAJ4Q zq!oHPct73LeKnIN^+Xp^Fr0T1kibcX&}OJ!i)%J z%o2G5N_ilAxAFW3)o!;E#gy^`>rJM0GDWiKZnQvrn>TdQ3KGYf3@H%m`T^FDay&s!^WfS_}h!_zWzb3@|~AFn5r7Wb;3 z(R?i#SQ7nh_uAX}`=o=qUmm7m1nj4*-a5D5^FCa=A@;knUE}=4nZTp``Jg&0Y<`vn zfatJAFcVNMCkGTi;RAMY#+ulD|HgQdDs60>D-$1ozY)vUPdRqsw9icz<>qO(=p%0H z@}VKtpK}Ev@;3r$sSS6mU*ni*K5W#QB{MPZhgr;}COb|%4ihokJazHq*DJ0O(x6)y zcs=3-qB-P?zA5zM^Q5=+g^Vei0c~yHn~$I<*QO7a0vgKwJMOM&ztobbG|z(3I(_6h z$YzRq*N(YMs55t4*@;1>M^Op(1HMgHskZtR;pYd>j&_!f^@-+-oY}8otE$9MYaZ@$ zbC;^)6xamt(0`8uLX5WoievDfYamMww}NgL4)=e=na#SCnSVN&@BfQg$t4kQMW^JpyU%p9H@ZmvQ3!ypK&} z{}c8w9+hMiI%A9Vhe}9X2ay~gDmD$)dwIpF^X}oAXHMNeHS=h$Vn4?r{d;qJufA5@ zxG(a7@P%DVYk39`;5!YV~uIn<~W0lk)ned9>eZ^6yGLm^arZ^7Q81H=N z=57_I{N{m#eD=+Q_%R_^*W9T&iQWqn-)heFFH{9<85L=Al+GS2dc5O(#G&ejk`Ffg zW+wM#da>Jc;Q#%|5U!xfqj*@s7tlfN2Z*;k))cM@e6!Fn5_5Qn%1VqX950~Di?kbE zYx;FRzLMjL#4(*CoBnKDn`ssH5X3z*#gqY$&UqB>X6$hQG;-#ENYOc1R!;Kt(`A9Vp9y}C_c%LFmA=n$usYDqWqZx%Mu_#1Zaw-OE*PAT zGBjtC9q63m*(AR-YTpvf13kYsgEay-he>p_mZzrH{waC#s87ic8a!GNqL(^OgGNWEbB-zKENG z{r+4gO23Pl0IUcv-hNC<5K}~`jo!Qq@M8!&c^D2w)<5QaigPW$YT|mxx~@cxi+{{i z!QU_B3=sZ>rs-2PBP?n6_CDqTaNA}|=zl=3jTX2*FPS%4)l6<$S_@$h^Ku{sg6)cK z6nQ$7z3bJ_#|`^5b-6vuoiuwpady_nY57o{2j(Y^@G1$`4JSi!)z%zX9IfVO=sSnt z%k6g;9^OhS`jV|MSf-h<@KRBE?g=j7dbw1P&b2vvl+YI;&=dq>Ptp=as9p0O)HX$J zWZs*O7%n4CAAZSwIiu5RAemZD+*$WI_7z*IJw@YcjLHHUR>)+EEzz@v!7VGq@L`o^ z`eK_^Xd98($0hC46P8xAeao5D{LAI6zoywbU!LqJ6M83KB#*c8qGZvYee=HTg2_|ug(>i>bc>OF)W_usGL|l`XHb8<3!B9Z1&q~ z#haPSN#G`|vr3I6aQ)4E!&=RUfyfa3?qYfSy*I&e7c)S+(V$|=6JP0yL;PW7cTM>V z`STBQJXi&h6&Vl3B0Lb$4rfe)t4s$Et2UjalzO#2bSJsAEJ_Cc&htB7Gk8uHFV#Uw zKEGF2Uq-?_1?iLn$Cq&uH<-ggjS35pIoXz@JXDkI=XJwfc0rkE#@yuv=c&`~?l(nk zuEwB1seK}0n@t8-Ibsm{4q5`&A_oLfV+bNm?0V2keB9yu%8K&(`lzpthk_n)zByNt z%pYBKX#aBA+_gvv?oAg^JRHM~VE+2$ileEoK@=5>UjV95XJO45Ledn~Y$`5ChUy&t zE;!9;?CqlZ4HvoRe#&AE2{m7qV}?Gr@2(x{k@-WpMlr)5X?}Lm$+ea>V z(`kt4$(wyQqe*XOp??LsLy&2~qGf?o4q4y$Q~wvD?$wvBB_mlfBo($8Q6F zAKN+_@yT~89=LJ&NrE}*k-tSZ9#0@ouELt;D=HKMtftaB6@?sz;5;xNEGr5~4oNnw z%ouIuW-;RR47aQuz0c+Z_DV<#w+RP#&aSo*B6H+n1Mo#ud~Efj74|lqT*D(7^Yb60 z+mGjW-R=7PD5)!cSF$Po14JIhgiWf$1>jgl!8vUwM z%IRNs{7xByfDQ#!P!NU2Z*6O4fuCb|`Y(8@82K_)Vcp&rO_CNfa2N?;N|nHoEmCdn zbs3K=kGdzg?)==&C(psf5x)lH=>+iG_Xj-t>MpOkh9|kz2JL(gv%tV5 z?ce+5?TkJYQjEG7g4#hSL&K^ntxoqSXaJ)0Pzs+M(<$sO2~KO5MU5POsn(%y^Qfoj z@8(RbynO`c%cri%M`vRZ*+A9~keFqy{LU=4pY&|V$u=TQkep6@>+{mpu}eI4T|&lQ z_)3T=2dMmmcOC8w%uac3wMr)l0VfD8ebKqQ^!lQm#b_BR5Bv_$1bQaNJqI8MlRswk zew{Cm%56r{uRoBmQQ9)V&s|0Yc(#u>$XSW@hgT&$9}4C@n8tAa%8Ffj_}V1dgE ze(Z$ISFTMtg$r)cwnthjYt5 zjVoPv)%2uxrDB;0g2Ml>G~cG+<8kE7I86Z8<0Iunq)Y~_Lw%2l3q?|fhmQ}3Uy5a1 z_~2!CV?+7im&82U$5Yus5B$BE=YbsIAIb<*Or5eXmnp?NjUj1$-7qhQOK0Y+cCKEj zty&uvGfe1(!Y5Df^)50}W?SQG54!p8l^T8;%`_gPHvlq`co>|ch&`a4%m()f_kF6u zuH9Q5;oqd9Lw+CQp5eiIO=nmOFE&V-0vb&|KDiaI##ZIz`G=Kby;9Y|e}qLOu9UDb zSH5xo1(^*OD9;ej8|xrrhEO}6gL#GKHlL^fx&iMS<~%Q6OdMGBq8Gk&jrgfDAOAr9 zosyb;JOzdOZbvp?N{CY(+aHQEf!CLH{$&_}4`-VCzV&=~t3>0GJfXfWb*}JC%p9Hf zvIf_;^M{g6SeyUX>V1GnzBAq%MC;MI8S(h)AKpHJ+B~q=rQy;W^|l^vPzGt(BITMY z|CLJAT~p!_rP3hLd33ExSfF(bRB6zT7WBBE00Eud7RlP|!ZGz?sqC2dZxfHS>g+Ty zI?yf8zqV}&!MI*7)oGQ)=$HiU4>4#Dp%_DKR>63kw^!lqH3tVr@tb3cF+BE^YtuF+ zC=}=8A+R4q5Yg=)2<6ci2SC&*Zl)G(qYW>NJwZ)s^KvpHtqJX#uy_mAUz-Ic|KeJo-7~iW}3PRv4}aMHy=k9Yf<6>PTAkubnJi-q}01d>g-NEwv|AbytYm z;-@g4E=9sAUs;W1uzD-=2m4 zQWG})^Ml)+ZUxV?VRv`DL-D8L^k85VITu-YV`=!$nUPG&NSl>gb@qqf&BT)1{gd{S z-q7`P_fBG-IQ2!_DwGlWPq)qX10@aM-@i;~Vslo)Tl<6LwA-1rqc7Nj=` z(vEZ)xhAA%uBrkm;N8@uR(zDhM*Eb^f~+{TBkeIhILf&pvGe@E@@w63=X0}aiU(#> ztx%8!C{{)>MI9;P%Z&_SCJHUkWD545$GC`=Gihb*Fck2TIz@^L)^HbgwEXbw%>%9T zYr1nTHshi%&@)2+)19b-h~wKWvx(636zGus*^nbS`1U&@F>3WT`0Z~|k)CHs zF}{>>*|PzAuunvX`wre)JF}7N^v<62fLq9ebxnPW*`SZcs^-t?rfQ&H!23q0o3BVA z<>}h{e>E{e(0~DV2vqxWJNrnM^k2ixqRg{2dIgQU^jpNB04vN)g@U1Vp(N;jGcfT2To{pH%2k?HI{j(O zZA0%BRvUff!iUdIJ_$n>q|d*`oMf)BSF)a?$pomTt_32ow;0RkQtT42PQHbU7awdH zMAxYL{_Q=hV(i#{49xVWyL2f$odz1i2e&yC!t1~dd`FN(yI6}cHGxQQRfjmjSwY*M z>j#zHHq1y-a$LOJbZOH<=!KxIhDiGrfldi!6!)JEh9*;thFRIoI0YLJaXi2miNJ-< zLkf|d=PQ!}s?v?d={uuifB$Y%Q#_=zGe3z0!N(iPb+M}mn5%+Axx(LBk$cf+8ov9~ z1XCt_y@LO27V$svkMH7j`ou*tQ4T{v+;bcM)6u2|yCZzOv>34P;g6Q|^wdA*nKK>P zK8Me&t5kitGw&E@A)X++^44!pB<8n-$-bR)LTekC??F%gLEY?&9RQ|2+px2>hN>Mc z)fx0XNtpPG2R(B`GnpoBm`D>0^3jj(kEZ>U_J(%aN65=15~vJc(h`z*Xl5SSa~aTj z+c0*6!-p|y73n<6UI~$zSUgKjl^aO6%)(@8M|1(|ZVSCg6hTG$@ zy)y6XFEwTd{VBiz`CV5$h?EOgcxm@$q&y6t;COunRiXy$gXyFmoX@KlV{hb|mqei7 z-t{C21DA)h>|5H~-IANzctLnMfn17~MY~OnskF%L#7P9N+QQH@1HPN9K`|?wyDjl6j|#^l zS%GcK&;6PyN^=HRo|BylVcK~A%9w%~s$kuLQ`N&9k4jxXhAq;sdiMx5$|rjK;OZb- zTwA$=J3?}>gaW07&6Nkv%^aa5SRRf)j{@@@$F2kM9dP|Q3RrK!?2sEoGz61p=V|^w zv}2kudHW_`-^n{jR3=X9E7W_O3NaR?QG`Kv$!cCt0B1!8}+Iko6s&~&R+^$)5*bj#a>6C`{8{$SU^7M}!5jDR13c@>6i za5(6Rjt=OV@C?{gsu-wV@=eF%Z*@iL<5KeEGqBRAyS$&!Cr0M_m@+Id&lEU_UcTFp zeo@UvLo>~H%|KfQXdwXlVzfbZ!fwHxDLjJkg}#A^E=*1em+`LbPsl;4#y9A;XHkH! zrsO2KoBrVDi`QZ*-O0M?oM$%xS=n)~%!?9id!P_>p6;uh@U?=S;x$0{Va@<}SVYiw z0LTnO)kB1qTmhzmF`6cm5FbREt!v2ktQ|fPa8*8WzkR-L#fhtlsB7;yUQlLz!E%T+ z$-%7k*hNCs*G!;Hu`8}y^O*hGHD#m7Kaw?-B}c9$OFZ2jaz=Wj^Z)7#L!AFdV0bwS z*{KKisg7#C4Dcw9Uw~|wAs~K<=7tBYk-S*Bx8?e|K3+YTn)dm6j+WP`FH@@W@hB@C zNH5y_aG*x;Ch!}YTAm50fmDHgM6Z>aFM|zomK6i@H#g5R6f2vZ}8XoXnAHj^mou$m*}}qEl(&O-k;(FRQ_Pv2~@0pk?&0D zSodE##AAFZV z^Pa#EP7J6N>;n(x1FrR(>M2+izv7+w`~@&h6dTTY(OuA5H{Nej(9T` z+(_2TNtw=$t4_M~aiHn!Mv!gt?#R?5SrhE((kmK=Vp(g{y66?G%k)blyNX+nG%pRS z_uO|Q^-4#}SjZ&u0L?y0m6ZI!(=}6MUMJ6uUTzaJCm75*e-BYocCo($e?hSFc6XE ztmrrzfnQ#IUGw4oS{6q9Y}sYB>UGUC+i6?GcuziX&x}yi!;#jX zRKP~OaNptp9ZlB-3~8JGA@#cC*KEnPm6Ji80iy^h0g+k0p&gJp?yBSqjI z>K|rfolobkk-s2hx_RlliQG$2!39Ig+Hb*~ARNk#(kKOON zE?2yd(>bNk7R9;quYA~p&UQ?u+-dd{PF6X7MkrG_olO+F0N5JAvK-Om0z?iRxQn<> zhfDw>nm5rz$1_uYkz~_-ChZ6HB;KUYSSSR#bHFtF+>Q>M|KA%y#pa{#_fCOj#kL{QJoTMgA{ zT3&s8h;@`#1nLyg(Dg~cU2IB8Yh>T6LcdLjor1Ltm<6l!Sa3q;FMziY8G^Pm24e)o z-xW(!d;S_}%U}iSKo#qWPuZ~M<;w%dPHl|X3fFO62`&9*QB6)C=S5LeP)GFngvH>@)s4CZ8=fzWg5`qZi zjoeL1!Yzow=|$L*eQCn|pZ0yxoCw$(R$K8P2BKOxS<@?v`8ilZDUk6`Cz0UwDb2IhpM~^p_-)E!E zgZ443_d#`|uE&ZaKE$5(m+&$l3cHd`(63ZrT&wU}Il)xQ8fma(`Z-d4R9Ca_x%`k@i0iJV|xA z`}vBn`Q3XM?8u*wASf{UKibpK{}!J{;XhBons!X-_P5}P)Ih#8CKu5x1N&p48ILOYAMPyo>4rMOs{u-I_1h&%4lP_a++<-u?tIO+L`qsQCs~ z2EX*Ty4@S7Jvx6j&dT$f)A71E zxS1!YW7p78Oz$_Or7X`6(e0{rA7b2+{Bs+|>|O8cfyp%b5ZKdu;NDx~dZ*}C)p9KP z$PnXH@Os(wRgqnrmmosEwzSiJ=1tpEtnP1I7|OPS zV)>pcS)?8~(XNopGke0G>{fm#9CIAI4MR_9>B-6dq9dv>w@U%+@AYBeQqutQ+=iWP zxd?l=w3ScjsJW3PKcy50mO|_ztY!Nq<8jcbmWc?)g- zUUhZ0&2Zp&h*Z+rqTP-@deC>*Y7v9vf#BNHDQW-uIe~cB9VW+n-9d{W5FNbKH^z>% z0+0*IIvl%w-WHb-o&=DNb1N-?#Lzm?aHQ4y-t-ty3CU=9v`D&MJ?MC`y16d8xcB21 zzh@JcZ@GBn*Sp$-fhvs)BFL%$gQ|DXlGuY(bjx0*uNztOgg4)%p}^yAe>*ZS)}_xh zoOx~U`o%(VS+Gq^txcfO0v1B}fsK1gfW}iwpkMcgXb@8r%0@2bGAyCdD>O9nvg{o@ z{UjX(GvOnpKvjm&YN(xL%rgf1`y5IaNw*DaGx5!NfrW48Y;AyLJR|x!gDrHd5%q$7 zLx?GEgz)vf0Xrg7%9MLUg0(O*@IVLOzF9HdNhv9#M@Pn7y8dw7YidTB6xJ*I<<7=^ zg4T|?{vxcixRH1V);sX6lhEuUjvK^MMF2S{2n#V3`j4x-U5iz*Tj`mef)C6uhXP8= zX7S4Afq<#qrxm;1f$Zrjg%bKaZ=J_pkH;LGKf;DS3$&2eBV$ zQdP#hZ0FgO+2JL?bKDNUK@vtxfMsa`mIV(20rFbX=BbP{Ko-CU%x=Hq7IGCRm|2N~ zzby^+#|w1$Y2>!}iepqFe2$4zAMeZ zpm6HyS8w0t7{=$c1LM2k@sEPscLlJ(xd=z&ItYJv&^8cJ5s>-nZ9O`Ty1NZ4El-pC zM%Bq3QQ(+NBT=5yt9-!^**FXG`9>K1Mu89Zu6+R=wSNC=|1Lt{7d!A^$d?>mR(Uh; zQmzo77qxAcrxK#&4~~zWGxq4C_DBU<2(OhL<)(>FftuJe42>sh+3O#aiMAXxeGxXN7a*2g~5I^S4*?+}X-zbx{l^`6>}6Zq5J6y>zl z>h1Ud`gbh(0t72;Yzvog9;QvE%_PN4iZ}a;c#3?etz+`o@nkrQGEaoN|`sql9=V*SOGnsl7@{@>wihP@wUp zzh{LS??FbVs$8)9Xd_SlGWs~$_dq)KB!^cu-b&pUD)dSYuzC~|O6pKrW;*RytN(eI zgL9X?fua2aTQazDef`)&aL}})ksP1%gTjl6RKEI}d-7^O7shc6wfi?djCJU|Kyf937OLtz`3PCoc@Fz8Uh#_&H+U1P$*WD9zt}&|@V>$}VQww#zJGVrPWfmxI~LADU}q+AF@J-;fzd_Xg+V5q^VQym!LzZ0sGbL*bI z%7cd+Rs54Qi=9vI=Dg)<;To+!K`RjzK_N?wKnV1(-U1EWPzM^eZk7+Wy9`CO>pGTx z9Qqmet~MtSMF{-KbMQBJ&xb!D+(+>97(@VU?$br4(3aQKR??yk3kL)T!C;ddtY<(s zf}KD+Hcgc9q{+JEnn#`I1JSodc#A-HBZaQ3H|4pnQPhI<=$R`PjB8j;Rv5w$wRqA_ zwmX=IV&y#jxUvP>TZ|y?taH)i$i3KVHx`CqA>cm0fFX^jWsuZ3 z9lOk|eu!d$crW@HbYp5uThblb-2g6>qTP@&KdrD8Ggd|St{Fc3=p*-xN~?CXgUbVd zwV35~BTwDL#CSGaTmHmEQA&W@^|E}SX-Ptn5PV1qxp*9!v1X_m<|4_S)odh?@w~xQ zl&&uiKrKd;XV`i7wHeys&oOB)4s@ad*!!E4!Cl?%g^)A`MK$6V*TY+aMI33J#a(lJ7W|NVqD7AMjTd(g=m#L~z+qm}@kU1$7*j?@ zb{&8)BFVlLGqzHenJf)v-aVJk?{`1H=e|GR<9Ysh{%~|0GtGOsuIt>-^YsFr z*pE^i0R}$UG(FUE4oz*%gEm3@{q55znHwB~~d>642>M(&i#fZTh%ktN# zhdwqmRQGFyzboT^s#ShA7l5N*51qX)!8AYw3WU_vx221T0d384lK@)cfe+)*71kC^ z*)&}|az_bnLrXo#j+8=DQp%ppJYhNRTJD}+{F8Ug_>)(3#H?54rRO@B#Jt3#_YA&O zf;^62&9cXiERJ<^hK4MmYtfQlPSS!~w&!q(@>@<-&zv3G_>t*Hp1vq1$Ru>hqKsYz zppOz$%`BKFA-}(fWv!iS{-+qRdc4!LwHe>2%4t@bsR?c-9bS9PEK@xcD|6me*86_Q zd8a1%{Vego*PW3=jrqdS9Uu%2St;A~(y69&DX>p+ofK8LBN~9)$X(O?xm?M&mj>y8s7wN&JUg`zG z+(%gUFbvIJWSM>iIiv5U%>y^ohN3K9cEH-~t)&_49P&TQNUzKwO!QqD^hc9zb*>t&Ke=Y4Wc^G_XUtR^|H?=dw9R>jc5Fx{G&vXU02`Pkc zvPK5rC4S(Asq#(P{4PO?HZIq0-uNXv5T9;zo!+Z>wCDJp^9fLBDl;vR@N*XC26>KW zCtRUTEjKVMEqIOT7C6hV&}D__-~VF`H$Ymj224>vVENg7kP+f!beN~ohN^&C^s<$C zoz_Hf%lk>s@^{_(@3Dv9@2x%2GP~G)7_l1(fzGzWsdRpM5|%C`+y_(-rCRs+sm)w+ zD)yaQokxG!kW6IYH_yStJ!h}4j*T;gT~Hg}x(^@VWyC^1_Rg--6+qr-#(jM=3IXy% z6Iz?DG*;5|nD|WqU$e~G0_eI0$i3Vz>nhXEe~3$P(Np})nBGF}EBA0zPF9Egq8UG2Y zk#7&BXthj7TZ1(6jYD}g(7VO3?%y5m&u5#Hx_2%eixj=KayX=V_+iKJ9lg8YSHI|7 zi~#=|RAh5q&9eQBIF=39_;cXep!}d?eZ2XQ$CbdHjmju1mxOY@xhuy%T+7Tuez$_z z^1;~tJ;n>-(o0wLwt4b;K3%d2n!P^-BDAvm7l6M8yp$U>)F!lh52fB&TS;YD!Cl0P z`!QRb+BEY{^OpPGsA7pE$k4~!U?NVyNy5v#eYPk1JN7R7sVj*qUi6T^VFJUe?atdOf|QZgE6xexyVy)9>C*Kd;ZuRdLEOW2~Qiu6{|U0~H^f z0L^PoOMxtuq9aK4^qi#O0m(WvnG483APPv-DpD?tHlYO?RC#h7$lc7+9_ms4k%3=0@@2&{)YEXi?T; z`U-12fEjpa1!$|2HZ8+R#86%F*EVz=;Y4fO#Tq{Lj^c$@onkt8tp zj|gI--8cZ2bpzLA19m;pK$Wt<%n=>Mxfs8|@TBdx@(Tz6 zgsxlZf=C6bT7}ONzD|!1<9XoEP+3j*DjtQwS)BpbWw5wC6iY7YmP-@PDoe9od?W9= z*H@>jQ+7r6y9Ql#bmdQ&DgEB!jOq?YKK$n4I@31TdcLD@D{jDzlEjf#QeiDD@0-pE2Rfh z3Cds~BN+2zx0B<}L12$tCIDL*(FJ6QS(lP_l>O~S8dztuW45IdX8L%#l{XakKlrt? zTGmr3AZ~w#dZSzS-y8=%h80#I{HK8h8O z3+!%odtx_^_)R!vhJ9ztD^fkrLIQ1~KB->qd)!lBmh54JNN88gL5!%)E)a{4aWJ0WzTt&sR zPM*Si7fZ8=DG*1eB$V{dUEQmfcp>)ZUi=ZL`4O5_WCyYkLC^7%jfc}m?b6rCR>4&# z*8BEdZU}N^0#SyuB^SZWp|0Vf8;moVR>*&gkA_pCMi#+M{WQMVf&_d00MR zfr@GY9L<1OZ29`3ulH8NR@x&EoZI}Z(8H%=#5CmKtxX_jeVB#Wdw_^AgP{fi=gpU2 z@G>}6v${|Z6`A-OgXSjhem&Fy)It-dy@_hy-FYu+?td`Hyc6>3ot_~5Qa`NjuS%JY z(be%2&V7k1Hm)8~pM1YozA!O!DCLOAJyx!`JO}2!i_u{7D~rQ>ht$3IZTXt{0HVY#Xi{%8F5+%1oTaXE2@9%rsB zu#K<{=|1;kb5`8%#SekDtVz(W@Ppug1z3-won2t1RRGt*33HfHqk;byd>elhu(9dh z@G8PVtK@U(DWPPB(+0~QGn;1YoiAH&zxK#NuJkxn+H^a2F|5+ZUR+xWC zpTWkU1SZ_WT_xfL?Mkn|ND7o=Jr#fKT(;2s-dRGG{c&5&@y0- zU&T1_4iwVgK8NMaFyrv9O&RstE;)DxCVpM+XkYx zDprP|Y5sc!G|DLiy$_hQrk`?Y}Lzn8y!d=@<}0<#~WUhRMp!`Wx3=Zh9Mg-59` zU6svO;HeSION>#z?j4(=7gEFYB&gMs?g9?c2P$w6If_?d9iVbBba5bVESywT;7OZP z`=T4AX`T91WUMa6GyRgKos=KsFtDtYFk-tbOKDg9G{AN1V3ERx_q6;^Zv0;GG8w4q z)ppfbv|yu>zYaA%W}{Dt=4{+g9$gx=s(U;cZSyAcz?~#R;}D0AtJ+bpK`MO`6h2r|Di^u{+W@4 z%CCg44uB$Lt7{jrcAqKzQ57w+#DGMMB(f120HSTB$~&-izcgvx1&We3QKK2%y(5$_YS}1Sn)o4ULugs z;B*lJVFge#QV>UIORx9^ZLe>-UC73I_n&7A8}(L-73<{i|Bd)G88^gocn5g$AQ*(t zgT%y#X^I&(c1jLh=`G!^e{jY*M%@F|d=hX(RzBuNS!7?Xsj3X@I#Vuk<+IVD%yOes zUI7SnX%u}5D7-KtDQp|eEFha;mLb54vMk?K=A(0K4oNbK$m3t6*ZM z!gCpiTW|9`&G(Uh(R+92euFaTcp4wo^ei}{p zY(!JCLhDmg{a)yvuz*H;h?>?^qK5s(!2tC0>HnzS!5An5EX;Q(9%gsn;l^Rf#<)Pb zNs*0-W??agjb8uo!~2PWJkEd?A!&uwtqk$FugjSHAOF)DyBpttn4VM$QA~l`Qvkgi zjnWf%X@O`d@ps z#7dVa+cCpUZyJGDHpTU{5s@2(4W|fhVxe(SSQ3gDG_`Oxam!;^g1{6mqigj$6&#Iv zYAzi0ryWUGAOPe&1au{vf|aN9)?>gm&^Sz|bxoYEZ7waTQ|graWB2GM?0HY*ItIqN z`aKd|a#M_MyoS8niVDO}v+j*w=<7t$)S%h(HSN7m@M3fm8s;$xT~*tg z+-p9syp$Kwm3G*6zv42o{M3{+g9(VDD}Ewdpiysm;++GSPXO{RdB>lA zivo4h1wS-Oy6EXqwZi;m$dWuA3mrY@iK(XFl!wk5IV8Vwblv|H@zoH4(D}l05=j@; z-!Z!Bp5pa0o4=8De6jp;wC}0XoT8h3Ea$`i#L1P#I^3&0!1?kHLaDY328le!zleZG z*rk#kZ5)!E74P??U{R0h%iT&&o#Fwa%X|K)3Bgc1lb0$pyaECHOfSN6>Z!NyGFw8Y zKoaJ^qUcs408yZe)lkJZDU4d(Gt^+RX?%l#Ju;48G1mPu)!~n2ucNP^S(D!9&5=RY z$lO>X9--vG`Cx9M8eJ1epFFJh4F~4t;yX|Rc;p!EM(M0rNk{zSf^R?hY&@lI2OYWl zDt2x%GiT1C@9IPY;LyGL7Xpo6{w}~Y4^vUin`pj<8!vJ0P5jo2>8HYC3w!g=b9r0a zeAUqoP5tyR(Q+$89>H@A!}kv~dMw?Kc4M|zrY+sHeKn%hykBLup?b$O!gJ-eN(isi z30Z(m67-f0$Yu>8ypgkth9yZNTdUP>nmE5)q_~d!zN~+d3bl%LypBK!AGmN2(RbTM zsN0{7>hV1uW~FJ_4OwdqX_Rnu!;1+Ii;b1*4Ci=S?@Q@eS)JZ1vrN1OSPI zX!58IEDCqw_bGJSln|xr8=T=yjbsr}@F!GKyk3CDomuCrjQU0`x9#NiZ=D^;$UZM0 zMH3mMz!X&AWS@9X7UBH}u@?b77_wRe8@VqEKZO?2y$>9F){#bWTy$ra3>6*Xv&!}D z=csUc?|RL$P;dQN8C;g7Q+(upULK@`7iPc%A+!ZjpR7Dk^Im@-+XcyUO#CENNH!fmNl~(6Nk9VMiQ|lAEBWvF`5amJ2rPKZ$!GP=%Ys&ZIV@B>iP* zk4e*_?s!4ULe-m_zfynY*`CU5AO3XTC&>q%4|y|&}>Wf$h}sQ;w*Hr9ZJ2oiMJ zhUmOgzc#hNJbHr1+0M@yZIN!-^+WJf`ct2fjcO4?sCYhw=T6s}P#l#GQWpPmZZKaA zLW!g|Xi)%bbYK{-1w37wStByf{6@-dYt0RKJMZt4EY<~q%5KGnFY?1s$quHSuJ<_*gS znJR&kBz2Rx@)wadP4k$JMXg&k9)`#kAdu~J->iVSCK&Ja6hLPVuDE6M z(H>gVi1MA~Rp+YuolCs@iCxAaE|bqbtGy&i%rVC@O3|cGm>4@{MxHL_Ed+=Wk|_hT z%_lo@m0=JGu%Dk_E-hwzwiPSbgLPKAWH$cn^mlTW}WN_Oujvu!wN;5tESz=Nq3mT zp89K63(yKLLl!i`?CoNk;#jwZy)}ax#TlH&xK|0U;L^W70wt*HS~yP2=n%6C+t5+Av3ftG?72k1-mb|MJCd3DaP-i1gvB%7 zC#TV6>8WH2^J54da1x{rG<3yDneH=Mksc2)NLR5pAKqewg*Q+76o0$AQKiQwwpOU_jo21<_AWI?nZotq8%r0YEw`l#9MxiY6f|)N8{<361y?e zg0HU(H!Vl{=JPbaySNP3I(?K)^kYWg+rF?;ih=s)(x6tr5r5iM>4!l1gEow!v|_~3 zKjQt(!HEKYfDy;)4*lf9&U=Cw|t*2eqG8>_~@sin?=0LYG6EzRf9f7 zBf>}wBe#@pQ)K+oU!5kO|HIt2d)ucp=XT+g?WB!}+~I{Z)mKIi*Cs?q4i7))=VO{c zpx+e1Ak{~jji<>wcl3Zc1=TjFPxywvCe=3rsISR_swYyw=N$MxbUU37D|ktwrKGI2 zqpA4TENeQ04;)VsVCVZ$ylB!ZRP|;i1vAqZK$pzFiR&Jir|T5qE^=l#RKBmO&RTI< znXRpT-r%jzxr`eVd|+|-*Ad*if8tUfsEs)0yE#u)Z9#{?Ob|u+TXx{X8$Y+hqdiJX z^BU`M12-p2s)i0~CrcXi)jppipGE}WrRm4O?cM$H0F0T_fbuzk0?nxXw4NWhF_Rfn9rsX9tl2I zHsqlps$8+Q#ozlx$z56^mRg>8)ayn3qGB`tNDZ|wxH8jiEq-Fth{j8D*V;KFYoJbMf4a z!efm5O^~JkmgP&|c~ZQvrn&mNcD#O$(1oPMVm9-@t3nh|J7uW@S5Ycp{y_jp_?uF+ zLW=23YI}S{mKw#bmW+nD-O_bW?oDzRCOCTeJgCU)D;Gt0M?LSE6_w^%i=S4Ye*iCn zLz5cu^57iQ{j@s)X2Vojf9Rp8`60@KmQjCN3ze*Gza`bR@x97&bu>!Re-ZBJ;~8P~ zopJW%9%9BcO=obElU7gV-26kCZXk~Gc{xBcAYZnCbuIt)&OZ9Rq1Bw~aMEf!;2l06b<|(k|7C>)&!y8m zrdQh`d%&e-gbkQF#CVP-WuW%?GtY{v57N!Z5Vy_vPs?KIr#8Jkf?u^|JrT2w?i2$c z!oed%*DeCdoe&N&EmYkQ?3J*-Tr8Lh6?`TfI*H%12@+{|i~y>EL6;Iy3ZzqBil06ExrcM$nA0m$RAp-P>aiuamX zQuTE;&iX<2Dcap9vvQt+N#qKkb-t?7Rmhn(*91pi#;7 z?$p6yT^sCu?^IurhhJ=4QW7R_wwP;^)3kI$QhgoSqWAmZ&r^KqQpBAL*0_4()HW2i z9{W!*E?+Or*Y5daZqP&DRL$n*F4zIN%oA9X12u^+Sl zqa1AeziLzjD+n&zY*k7GB!Agu4e*H(|KnIPB#d*ne8OxO)RmO~_2yf%OLA2_%LhC9 zKV{2?lvk5$o4D{p;7BJBXsTGcK>TcY7+4q}=pt1Guqdv_roX|Xv~NZdlsGO7I1hQO zc*uKSNe$hv7{umqm)GD8J6!AX5=mYofmdPO45TfCnmI(lhf~65Sgi9SR7M7-ObpJ} zl~smh8TY7{6)Ae**w}Y{YZP*bciFJs0ES8eKeH|KsZ_aDd<3T zK(9$NHlWJK;iY@-YvgUubZwi2mK26ua)=YPPcgdQC-#!3NY6=jnXL(xcg<=IXa>Gw zcqkiMH-&*AC8b({hQwq%@CBPufM*woU9&lK8;pQhLdcphALGb30~_u_klHt_$L3oC zvqMc{TNud80@c0Jv4Ggd|96zNCyJg6gd`l`g2oH)Cb#5qisZgl%wC$W9W^yVy^1QA zQ;18G%eYki&ec?Gxzvnl5wTlvk#0e|*F^(px(8(W>$RA0kiu7y>fsvVuu9;Q|sk)N(w zSS21M%HD00p95^?|9-9wh%Q>ZBj|sHk-ZG7;eK^WAxCuOb7k0<_O86mXqp`7WvWN0X=lpa@hUno4{Gn)FUeggTNmOJMr}OHexl(x(={gx3w3 z0c?qi_4#Or`^K_m-6aBHd&dnY&WI|YFS;NI3(hc47bBqY7_J!RxH-*Ni42G3Kp53v zn=@`bHszA;Ul%s+afh!*qL{zpLetX=Bqmnrjn4I6Oe~RtLDEh<2E#a)puUO5J~FC> z$V@Y^7=F;JDQRFkM7QE@Z*L&FT%ojkNLIBeX_gu{#CTsvyi{30!2S==W303J5gWy) z^!iBC?d&S6-;HrX4XTgDD(d%Mxsl6}E9-5@Tr0$Qv5U;d{fk&pv&2flG~o{iiDUS{ z5TW(u*uqx9SJ$EzI1QoM1bv?U`^?o&j;^hO>Ou1_Vr5lht+QMqWt!x%kRkD7^q%!% zT}kCy%YLlky!)20mf};14zdwTu|4ZXF3fQosE7V4Ml6w(l(mQPszHXdxZ!bihA!rf zvoaZBvw5QDK6ylyZ?&P2SS}cuT-d z*el2g(K`eX+owZhzW5taoW!HE`So8GYMy*?1gkM{D&0i!tYVd58X-8 zURGZIcC)2qz7F~*;9n9P#P->J24rX{Ce!iRf^@$;9M*`YLzUSvwyBHzJoXLuaAkyd z<@Db9T|e)HQXBB|6ye?BCRQ`LAtMCs2_|__G*c3ac}k!3AZprr$lnI*^I){byj@~; zX%S%58m62Ogo4R((j zz(H|4YM{O6TAa(sc`hcIdt6M-GN*d)USaJ6o}Ps% z6o{8$yn#qBvPB!1$>N5qbdwnZr|p<)b)R&M>2dk)jCf}4roKn8mIiuEel{hc;H&jb ze+k;Fiixf-UAZlzZ<)a;T113y+{*;aH%VZiC!pq6)9*meX4e}$Q$>ihlck76>MD~Z zSzEOpUHW4^98&k#_`dg-=L53I7Es=drfgtfr|#%_>Td5W2~=`Tz-0JU+UzVL1mbNF zA@yS0vn2b~dop?Gi^7D*_uai@r`$Zlj$-T&qqX;UHMir}trk#piBsUP{@g8*NH%bi zcSjt*(Di4rqUTlKjAk7$avWG%dIc#b^#r-R_Dkz(o+>KJUlj<25U>M84r|ne`lWW! zNf1Zu~FZXtDhlu`AO)>reDyHmuK%eILoz_+hsFTrc_k%2Rd?bPe<-ozLl%a_nHS zU#M|J`NQMfAMnQ#FEPh(|{sJ@RCyUpn|e)%@gW;e5t~CYc@GI7W}3u1*nHXS>tdZ z;}vnq5JX^K6n}RA`$GWiOi2BCm`MXdf==0>1i<>BgV%$`Pf(F?hKeHXE@8Fr#>ic| zPM$5OUkq2phQ_FR-M{a3I7%FsV60PL#$<7g-$)w4u0aQjcY{%YCdFuBgmg^~fwa#D zW3$F2i#qp=4^o|0;g@TlernIFHY`ol2+cSo#3y*RYGx zLmQl3C8b*r5na+=a?_@l{!YvUtc(4|D)^cT#SQSBGb|}s|6sCbjiUT zae#}7-24+S_uDq9iB%m*_A}8sndeK@B^B7-q*}zb2qgjrAKCk|+AXzHd~r?sLgLUd z*5%_sb6EA!!C?~uqN-N|$N(#uJWaRE&J3V;IH8pVcS;zs&NMKAF3os@w*SD<*|cdV zL=|ZU{j{9#uH+a16&EHSY^_ZJ=D=Iiv>9FUup){}i!gQ*144AG*(6d1dAcLNx z;=#SfGO(cIs(*An*-n2)3g3)jgKnv^{M3`Ap-Hxi%_l9(d^acOsuNuo>Yu3$tC<{c zcq{0jkTOx9f$%)Bh29JX*UBL}V8=UWRfwb~Htku`BO0dMUo&*-OOOWczl5JLeXviv zuDE>XTJo&-3;q+(OXQcIFo20|1t`F}20e(Mm^@QO zefU1eJyy88Lp)8xc)xhKujWAK!Lv&*5zAF+-utYi0h}F*u*?aw9RUj9J|hcSG6QS; zC5tGxj-0^4Y!y=V4*hf)s-T+E4@=_=6IZhS`&;kbTV5NOvq{IyNhR+vD8=o1Vjux# z!fHwJm`Zv=*7>#HaX@~MgAu#@5UJWd`8$L&0MT(QhU-qiNshw>tEvYEOAft%pQ5cy z(uZUd4V%q1m=8tjB3Nb95s6XsNg7Z_tT*wU5SO0xFDc3Fb~}aPH~pSjpN)G}&m{ir zCO;)rnujqBp4!hPwt*ZohItT-bcpl{y}vI2kD}#X3kGu?RlOxyHTjyma~*!tN`}77 z3koNiWS#c~pfj_X9kta?xot=M2x@2LN=e!63c zi)M+vBXp>@H}liOr5$=o;?(ttb38YaZ@$x)4ZUZ(HjfA4XYhUK4hTvJmqvh795992 zd6?iUv|KY$WpeR1Qca7eo}5m8mF&IG{Jg{Mr%gd%5YY;1?g1t(6p9}+g`2E>;pZia zlyYMI^n~-KHSNic1N|V8f@)jkSZq8>$;zYlXoN1z?>Xo3@!(0JL@n0bT*X@nFZhw& z*;H=%B{ay)ql|zegh4*<;{&GO@pMrwu%BzYPLd-(hM_q&y43Es+PoT!;s(sW(*eS; z7$q(#hsF$%SA*})zc?E|H!*g$J}<@9F(DG&wkjKAG*@=Yoixo0H3O>C>R^K1SUewX zf_9@%cF)3KtM^B&S5?$%Y&3V*SeLda%e}+Q>jP0+pmk;$ICia!EMW8-b%q;MDLrkY z-`mnn-q)Y>EW7gnOf%Jfr6>w4urz&Tl|HK70T}{Q?m?hwhi~3y1U8 zg0Ao0rczuU*}L&Au8cE2mFDezhG4&J$aOsUDMD|C@I4uC0IT1{KJ@^_H00PKwJ=N~JwIe3cnragb_X2V(go+ZZgshR<3 zB(5SXSx^X@6q>zAh@LV|%a~dde!I??ro57FI5FkO!hSGC&PYaEmOn;Ik=L6Yc+QGe z8(zTO3<6eAqLR2HUIj5Y6gCL|rBrrg$f_+EZhuyq>nogA^MrDExW6j7=Un;VgUQ46 z_a1lbqFBvMWy?O^F#;gfh#$N7^t-Jf@ExECHi%nBHLZjlIbmqqXV{%OMXbe69*5Z% zDfTc2A30ORh}KDxl<(NYOV&O9aIDlJeQKt|z&4R>A#lLLfXP>G8H%%yDNhm;+FFGV zHGetqt-lZh=)yZh0UNqS z8Kp6S;%wB9E?<36SYocjuMQ@25znJ+bf9L!o&$jaDXM-H)fr~~ z1<^B33qug86T77dRiYU|>ZS}yKTJQjzZp>Er)jM_G9U)a(kMoctC|b~>hVc#lh-5a zDN)y}v{xiDyLp37HG`osePlFEu3?3_79)VG45k5$(K1jq#Z~ocSZ%8 zlnRP201|&YEZUD6O>T^zHD2ZEls)-ms76FMmG{D&aLh-ntmnssr*AJ7DxKaN2}z?` zH}I=a^8@0%6?QC@0mv5rB7SN;#^g_+=;D4m{4`(diXmlq&ktoU`;#}%sc1a8Jae$C zIgB<}>yY4m!pi)uwsg?>Cl=J%O)rY-@(GHw1(>0j=5oc$QS7SrOxiA5nI&+(dV1B> zTuk$Vg+iCb(*lJv2vA_VkEw(HAB=YYMQB&N4(4|Mbrc#4zyyt>DQNW#Z&6fbqN`F$ zGf1G`u=x9W=(hT1C?okGmNXY zLomcldr~ENXT@eR*Q3Zy^In?6o#pcVp8SVLWyyRNfN?;=Djx>>Mgcktz~!?uW|nQ* zP;B`9KVamjM!iK@J<&L$xF;+CbTS;Vo4FyT?!hh?Qi}%J=Css9lpX4*d4xLD_5Py7h%fe6o&>7k&d~l5s3uN>-l@%<0B$`%)V4Qx{ z;Jf*R(TL}V2chyjv{-}Utg-?W4ml*QteMw7JHtqtS6O^jXL%y*`d>t(c-9$Ft~|O~ zA+P~or^}$~jKZ5Xe-T*P&)KEl>iD~QV#&FwEV13bT!lZ8uMzkc{TR@P)2s`TDv1ap zDZdV`$_4SA#@xh0uvnj7TJenx&W|~|qIWLHe>$(tWpMVwqfzEvsH9Fz`RI}s_4_p% zb`~l0ODsG}UBj%YQzRzY+AbE?4&5Qz{KRK zr8XCscru%&g6+VLt&&HZ z!jgAP!>jSaqm5j&;4@+#VHqBGC&pALd^2nNvr!$nVe*q$6N?~*oosXk3S<3a>B%di6x(G zVfXFE5^_&`x!p~dM>?2s_5ksIIZKZD%hzUOn_{x6N_I33fX)4gDkm-%Ql|}Q_sOP0 zzaBx6x~DHaa{HBj)phevl2^(4QhcdtD)d- z4fkIz!g7@C$P()w^Fh_C544WGkQ!6yZoXk_=-_B+{`O^jJTxO_ItIZJdSH2j85jUn z^@Dkh0CNEwoB^Ve+N*yN_bql9cFsHUa3_X~c$*ONkpzXn$U$8ZvQK@=*4w2|iRyp& z?fuJpWO&sRYdMN*$0M$t-*0i{l-hqY;{DUHfPRamsk_5hK9aL{M18Uuhr!4U{Uf;P z1>!tcmMDLP6Kz71;V!K3JTF%%Da>CFVR zSbUAJp!pAo?Jkfl2jd6$UL1`4;23$tU!ASf+sDlb&5sz+fA}1@SF1puGD{4clAMFyW*>~Q()+9t^O31Y|r)J8( zGI?ZCXpTMLQ_;((hF7!^C^m3xN`P0Ii46o8)X-mq8RH_6&`pPu!FvLXfl|QkA)vEa zRYPl@!A4F~-faL{&SQvxb2 z8J{nR_(=yUE_eZ&?cBT#$d9v*z}Bbm+a;eo$8PXupKfV92<}6InuXR;G^O}v&{GY7 z>Wh9gLv}|gl3&+;JbG87l~OF(m#C7HKJAU=d3VktSW)r9356Smtkh0m_$bElTi0|p zi-1G8QUO3m(oMi|jF4zB-bIkl0ck;Tut{=OKSh>+fnYfU{A@Mw%8FI1Lk@ zJ9%#!&&1lIme%a1ZyN9$tFw27ihP734m!S%@l-PAXQF$I{sghJXbmtx*dgLl_Rs9? ztM-_);6Baq6a9@DOVgDPBT`^{KV#Jp+q`RiOc4NI!bh6tF~<%Bvw=0;H|Mfz`7kkf zXOaWt$&v^IS$ReBFZ=I~!Q7o`7`Tqn#Tb!-bH$x4@lOU10Z3cW6MP#Cq$TJf+-1j2(Cb5$24vBcu5=n zPPlSqGOOnoqZ6{s)FhZ)2W*T92nUynXkbq+Z&q!7r%RmyA9|Mo1;GkV!a(%IP7f$M zfH%Wmhr6b$kwJKQ+T`pW+C)#&$!B7lnXg^z#1~^#ay=AG?*8r`{0KcyJjZhdalI($ z;R@?l;x1qz#Zv`DAA4PBy<~90CJfpx&>qh$qfKp5S00z>{?ts}EURcZW6$4VpI~jU z70j#et;PgM3t?iVDor|{p3}GK2Oj4Fc!;x)=sAPH?eAQ6z_e}>CUnjxPJY5@+@VCj zjY$&^N_u!&8~V)7;#@}f7C&yYd5;48_TZh9I4yE~sL3ptjXmjLZJT6kym0A9@T!jQ z9V5LLUszvSiS8Ic9~)?EaQio)JdE;6KNsMNTOT$ar!>CKomM;*SK_Xm*;4G`6nE}$ zlT;_m+G$i%@{?k)C_kZYx7~iFF>i=*pq4O@9qFD;ao|cPv`OCi-U3N2KY+Mz+BT$n z#O08~8|o?pUcB|bSaVp`QNokscwF)-XdofdG-OP{$?$_mw^iHwgz48nvi|o@i`$yZ zLrpOqSFhFwKl;<0oYM6VMtln}t8|$Az9#^EYc~w4iA`a~FhMUvJ;;WmD;+roU#>9! zukdI$L^}MYA)2>GtzbhQOmMXSMVylPi)ai*;DK}UuOVRw3pVO1dW^Z1xcd*7`p9PY zH!8rmY;M=a{p`6?b>gvj*X19Miv2@cQ~EJ6zj?Uv$DlSIyF)-ub7Q`MU&p2blJdiQ zkhEiy@Wtt65Ix_D6#&}A&Wd~px|L1Q2k0Fsz#C<`N6C&}fcBft#*5-64DE+Bgeo;P z6DCjJY!$pG;?1UZ!b!L- zRTG&w2$i(>1ztXV=jAfzix-JZTZ-Tgz}P%{kN=F>yrQ1XIC2plRiuW)>_E=<#axF$ zD^Rt32&{k`oSC65pqRgw7Qc3V^e}3|&WCN>TJ%st{?kgAGj{^G+f**AA(!S7-(vj! zOC>8D^ASb=`ezjldn+9%toK1nErgnw`HRQ`(e?D%tWrtuyrUa}h|1mco&@07Ql-C{ zoc+!Sy-Qplcg7$`H<*AZcmk2V2GSq2!^Gt$#s0RG>~>Y!X&+S~1My$gR@?e+2{&(c zik(Yrv2^)#t0RgdvsE!6<^}x^1+WPPWSCGF61--z<5TJg;nMZ7(1xN8lJi5!@DL)1 z+NpT+ee83u_r>;rTyR%2)Dq{qL7VFDt)xGx?Pn#>4?nsFuT1 z5+YaV+@=1?ZFZu=QZkRR-1T{UCfsd0PrJmlWm;J+txkKNaYUzq-hAfO2=tFRe>6Cz z{@L`j(~uCF^c4DC=)4>VVvKL#IGBi?#|v9+8te!`&AhM_+zluZ;`!iDHP#)38D+&8 z7OSf?f|b>dZq1*XE9be&n*tKJ+@78-Nb{808-4s3`gwXPeg9&qUhci!D5XSB=eW#% zVJw^BZw2Rhu9l@eV;zPFfw3#By=aPaI6ZYJgXtjDKT7A5hq-G8K)o0(p+Sfg``&g! zPa0SL%k+u-C3jnm%1L8!zl)WVxoTYw4-Y>-fnXYx*O)<5%m_4f)Fse)u#2#CnPS?( zM(1>%{dU&F*L=X|lu(ssyxIHY`P#%I=cow+UF|JL4;dc?I`k##GdcjcKYsntuWcj) z+F!K;!gf}WbpAL*M*@9i{^u{kc<-b9CE$SXyLtmB1G-9lvL*e!YpA9n>-EF+L+Y`r zt*iOp+&I!4Bb~MQ{cgRtZ9Agli@5f*bp=gRv7+;048R7pPtwqq)zrKx768jVVyvw? zw`-Mzy*2+tl6b;#{=DSyduO0o1ggsdx{0E=snT;$PFKTcR5%MiFRgE1fg>%d9g5ei~}lhZsWFAYe{0suPM_ zR13adugv$4{2m?gr}X!vcP(dtk!}`9@q-oH=MR>vymR?Jm|K!kr+pNO1!A)2=h0s& z)|b;>X{N7XY`POC)f3a^7!!s$Q30GD%-UVjsG1}%1Aik3@nBc%u1`~3M(H_8t>{g) zU+8T!%$wn@Byj0}qq(90Ht?hG^jv*%C*#?qDMQ08ncgwFKA(CllUeA*KX5F&kSszw z{ovPkGcBh76A$&V#zpfd1r(@7( zm76ASJ*+-ib^c*hZBUErou`U>@`{UI9F?AZdLH6xRHWJ!&JqRPtQTquFmBtQ%Y2kK z7#nlFr`ng>O6`ms5+a>_os>ScB^t@6!@tn(sE|>BS`YG;@saqw11n0)8m3~ntJl8V z{a&X8Ra1I>T$R{<3vo3w8?raaBS-@65K2!8>^d(z3iARG?7WXVdi0BpNyc?4{VPfj zQtPT-bj)+MD>%x=mL>bh9+eI{^Ah<0XpzgX7OsLkO&7Lmk`n0mS=KkDjJAoJSoXD9 zJ7D4HVgrx7Y1_%I2yZO|-HY9`JTL2lp=txmS)gSW(6Leq|IL?CmPZS=9n!D`9Io#c zPfF$u-~nI8j=I|OnKJ>kzE1MR0w1;aZ-biUYw(aq;eKd$atLr`7gCwYpKB?rEokm# z4fd6Js!+waaiNi4zU}yV?jpW433b?BO33_p%Ga)v9;mr_7kwC-xY`M@?^Pr0&=jjy z6rcZT3Zd=D%e;HgwwTfHp^TLez zo-_v>U)sB)`-~v@z$jQpZ$EH$_YmFtBC~lbA!K}@p57N?)ylxw&yyW#hcZK-+6OBQ zylXiaXS7KY2*iv{w@h=IlCWWRlKm=qQ8t+<3uWzpkLnwu0T7Ma(&a)D=ZlYDx}rXAY2k!e{#GL@gGhM6m5zX=R|#*1 zR-I~V(`}-o!(AOytW0`5V zgGMI|(lTzWkkW?_X1nf`D|G*=y=N#b5payE459+w+074wt8Z$so zU`|BL^Zwg9FG!97q9>YI=i>ojD>Q2m)u^>g|AV&0Re)1B-hiI{;NDtyyEe&t%yq;% zv`u)V>~IT6(FOjWk{DNeg zXMTlDuA16Wh3-#zyoQ8&B8iYqIXgWO2S``Y9VU-kf!Aa)OXR`*g4)ynFWTNb9P0P) z8y+EL$&w|cNtDW#vV{yOOA;yjGD))2Vj0_fMD{I&D27zZ(%5&#PGw(;`Sbv zqt!ziSDjd&86FQjrD=WZQHBhqj(IqS<_L{6&~1P z!agmQe*05D+q8~r{CVy>B)l=N0%`a&^o+oF4uU~C%2%hj zK1OvAX9{=BfWZcImx2Es0&+9Zuxjo7?me^o2Wn(4XZJ7&Dp(LvJD&gwsv$s=oq&Rm zKBt2z${3S79UfUI!6o7jVUeHSuL{5lGZ!Q6BMpPp>>%A>A9LYc7Fyy6y{wL=y)tIfqVjocfXH;Ovn5&)AQyHsV9u`e5F$|`4A5M z`rox!qosaxhI2BVfIs_b3jEn5_($OpF|x#D(_3TWEJufGa9MKd_uG?P4c{A#T&J~v zB|;U)Hocy(up(akgD@4M1H?@GBt$lxPeE5NZ&Z;0Q!O{0qXc-R&DW$R@4eQ_C&gBO zo7UH*1?|r{dREH+nYUAF#!=$pQDj7b$j_C);aS@Om6n0#1Ud7rZ_;t&-)yY^CCPYn zirNBB)C;a!FipmhjiE<#@{|C)KZM7m+mmaWfA2<(?DcHe`)zKCB~w!0)I8|CbPn_S zI*J$#4tldNjHi(~2u8+I?n~C4p9(%5_Au(PiS2mwH4lMb1KiqSG*vi~3bMV3P`Whi zy6{f!4h0z*dI5+@JwJ7|T&`2@W2w$;(&xc5hU$Io`Mg13G4C%!67(;cBv5AZUIEVt zA48}>cbf9J1#cF-{()7c2z9qMxqHiGHoh`ztL#4f>VX+*piW(Sk8WG<&kN`YG+p98 z_*R?nYg#b256jUA+vI~;mtxHH*XgLu6wl0Y1ybJeSe0fBgQSnBj1cEIdiXGeLX($J z(?6L>LkmNog8&8_oCwBu;7L^fUh9P-`a#ofF?8(Cte$NJ*);L>NVWRrtDmSUuhwqQ zGj;k}e{LJyIsNKMLHQX3>rMwjnLwPxedbuOS_iEdwto|-V4k(n^JdAvTGIq;S`U$w$S^rdlxBuL#im6246hnmCPvRRUI4>B!r!BS z{^KnE^C*~w1mZj{$x)V301;DAY|S={-XUfbwK#nB+Kgw}HKwWR>WcS{c#~UV%EVVa zZV!jbeyfMVw~c7WTG7A&&R{q(hb2<`LyX5ykmOzbmbaKc!9iwPuM!<67LWo z`pfS5yTYe$580fE6OACAVZU}W1C`o{r1RBLTc>JDG(nhm#l=y#9S5Jt)oCa{(G$JP zDZ1nZojy5etOV3*?#cc>$K;X$W|RWAPy(6zB*-WdZ{#l;uMp-Z(Mv;qu%yU+yXix5 z!iCczrzZP__8Rb>xmQ+QS?DYeG%Wfhpeyra7uTK?Aq(l#9PAMGAG)g-bbko&r=3%uG*s%RVhfIO$lO$MF`Q-x%Lu6ew*lKH;E(U?mu>!xN z?PY$<(;nA8?IwURHG^YH?Pfo_O=X+tPl~%O#kO=|7;{vw;`I8c4+x)8;J$0Kv_lbn zQP7KE=1T%`1B{)PyYVx1M466xzR*+Tr8%W~*!S&xlBs_AUXyDWeAG+n@=F|Bma37< zh)tGG_S5gJ$iDv`gNA{4(;rG*H%=6w#_J~MPEXbF1S_(^)4w}|MVr(6!%%>sxYIO8 z*24Mcp~aB6fW0!eGcI==D`vj6TK1)j^U~NuF9EX-1Fl)+4(JtiZK%rgcIBO;m`o+y zFjBBxGp@}W{Wz!oUjIxwuCnSW+q%F>FZDa>u{7xA2q zZc&S--!FxIuh5LYDQ~ELgqRClp4LyD00RC;zR%r2ucj*GhCbmbFD$1>=PRMU1iV`} zC<%__`3^`BL+iw9fvkLCR@Ra{@ic3)%o_?)>9{P($^Edmxv-EWqQRlBZ=L1OKM0xv z-Hl?kPNC_+nmu{@r~G+&Fon7kl6k$?E7R|k3k00`b;s|nzCOh6Ad&IN^5{Q^z#aO1 zWse=XVPom}t2@b-$;zzZZ)1y#i}w`2 zML-DI5x`W26w|T*BQ4GpGH}TqAOz&kdOE6&0Bw@8JJX6@ii-IW-={C1w@O+BA&nU2 z6m20ji2}7heAH^gh5VCRM45czMT z#)j{5AZVxo=~dGWW6&~W7lzO3mcWrDWIUaV1S&YV#Dt<!gCn7YxJqO^HQ(1n7gAnKjaj|8_wA5rnT1I2EvzhC}06PEvzvCDrycS&Nr zWGWiaw3$P#8kF@+`C6@`QZ6eoXz1)kIe_9z946S_DiAxq6$cS+ z`6i3`Ja_ef>me2suCnH7XXln<^WG&)g9NS?dv}0#8aBDSS}++UD9msV9J} zg$o=Rgd#3M$@8^XcT5smMF- z7T%w|(~Pj=@uJE67EJU0Qe2S*4B9CiAAX4Z);`F{sWDvll^$J(Ht7bQ_c7%abrJ42 zjNJ+YdnN*{@1B13?eR7Ji!TtNngBKHiJgXiorgikXumOvd4d4E8Zn$eHpdG9_XdF1 zL|mq$HYGF5xd>CGJ1Il%DY=k6<6Au7%hOEow5G%+lyz+CT#2@njgqv2XYi|@Ozx>+ z-)ab!ep3cr_?9tSs!`M>e{FTpsK9Mk{L1aBPyoKCw|>(Nlh5x)?J5NqT6OQ!0x^VcKV z@29>&HWK3R55xqMJb2CGZNP8a%&a3B*|FkmT6<`>i+8CP!8;)kVE_ zBPCc?r`hZ?yB8bZ-;c2{zlKnuKOW}T;RkpA6ZpmZ%v0b`Q~GrZR~tdtyE@F+o?P}c z_*-6slUJr4F-Z8J7y0P)z21BGBR{Ym*~jq1FLN-t6NY`CQ{TZ8#ayykkE&V4b${%J zbJ9(KNu@zVqG=+Id<{Qc*h$#Ws%<85k52cx&-|JO116tnYQKauDHF3Vix~WY_`@KB2h5|HeJQDme4?+fWq>PfgkD zQhDXyXIr1&dna}*qQAcjkuPA>4exu6OMe}w3l~=Kn0#KN)KEEq<}5_{Axq< zn7^#}p}6J42z(Xg~5$9>|Jn7hJNB3H<(;1B;zJe*`@>s zHJ!=h%{eD+7HjmY{VY?%JNB#>@9(h*U%iaspo;_o$sW)b5ayshPG)}G>@iL`H0N-s zj<5Qz^&M|YUqSfCvrnLpoe}$_;t{(L5y&X*(5x`1g}j(|m=0P8_6)h^^>pjI!V@vR zw|1il2AA?wX2pr$toN|8Jm~}h@MdN@qj>VXQeD5!=U;sT&>oE=F(C_K9z!N$UYj*M z24=bPW!~N|-QP51d_Juon-9pl5GUEvbc7f?D;_tl)>*QhiGy@G>h^@o;Mu{s;ILoR z&}7r0&$0`qJzNsP^IcXh9jEs7-3aGhfj<6E(v>veP-^GNW)7ar{n;R!q*AQ9d3jb;gL`A0 z>gz`RCK^L^JuiHEHW_Fix`1k_O6nC=AB6Hu;RHMC*guGZp$-&$1%v~N3JrK!4)-5w z)4%EbxNe+NWm7Zl2;tSNeGp%D-X&SofD}>U?b)-vk8+q-A>h+rfirXg?_PrzAQ0b8 zBfB7;(3{nxK6E3Zq-f$k@Xczl*Oq6X zyFZuSh0?xm$MgK*F8;T4%~{y^+*qkw%>BTCI4(0sa1tnDKH)&?kx=ujPOKtmP|BMl z4?HXLtM~izYh_dVX7Y-39M(H&;t=1*esy&xh#_u8u>;4gIqcgE&W$~gk-PBM7ru_{ z)$ksV^X0lS(kb{!I*l)%#4nWnj9mQc1xpTd{4D|x`b0z$P)6oRKoK*b@-lp4DA$|M zj+nac9J;pXn9r6$_TIE@yRY`*>HLSDM%B6OGZcSu@ChfbSr&)~@Ln7Ct?`AS6Q++8 z%DmohdRG%-Cj&*&y1BFhQfqN-T@<~q3o^(c<7S9#Gy6CFGY))1uD5;RYm5c#tLg2Fg8y0ZX4fz?m2(xc2m~T zgGrqiE+ep{f9=!&)!aYN!AUDZ?M_;MYo0f-5RuC_T2%wAEH9WazTWp#qsaFMUlK;& z4Z7IY>6N0GR}4b29{qKEB@;|HZ9&?=IDiFlq{Dm&V8+#wToU$eNgD)T^?Cr}Hnj*=_(ID&|SE(A%3(Vxrno z0!J)gO_E!>JAT{Y+|{^Ok6E$>jkx=49?qQgKqL+&0-s_wKRsp( zHqe7F@H%>y35WS!omlrO99t@ilVQKIb9A8lhp!RWebxcRf z%$B>Z03m^4QSAsLo!`n~nnO!!mk<5HKo@hug5L$*Lt5E084+Aliu5h4zH z@7<*_IY;~TZy{ zeJ>$zwL|+i{E|?Pl(150x_hz#^2y4(%5U^7EcfXBI=SBa57WaDtZ7g-@8d?E-|^T} zu&2rB`sR+jvDt)t*@m~blY3D^#%qh4m&|Ok;}D4b(o-7-0OprJ&pWL~#J;1-64ocl zvpa8_w(a`Papm`1fM*Eq96k>ta}a;h{~xIk$c3=OKyZ*NcfptdtL4zB&ClB#@8q2s zE8CklZPwsYnL}TVd(D;n{IR1~gfrjyX-*E&uc&k}#xF7PH$Lv+rcf+;S0aBxe;EmcnOqT+K-q*Ol;vL0qD6&^C zT|{r8jxM;?JI|+mgl=0mvSZ@h8SDW^^xBu#8Y91o-hNc($R;k%O`7Q0_vg`U#tgk} zY9od&DwRKPM7ta6G$H(Pqxy5Gh(u}DZQ*UH?5%v4#8%Z97XwM75Bqv6fVlM8tkgrS zo>@eYPGl+Yut+#5fu}N(4aU{P#5r-Zk2aykr9Oyl)TrjvN;l$inDc=mSNWEHC40NU z*V6HqxzkP&BY4@6dT11ATeV9=F1eOW&l3EXPsFh9f&L`9$`8cc^LYl&nYXW=2cYs3 ztzzSV$4jcBchpnyC7>7#lQTK&zp-P4Wlcw7~;Nd1iIv}dFD9-x&{{& zWVVIgun0IFZQx~kJbyHvWL-9e{Y2J5Ox2$uD+2!NFfcVtoN9(q?TrSVTRb*D8V_U@ z{b~QRY0D5YMm6)6Pz(O!IaIKzz#RLvv-Vxy0VlvK=G; zsYeT`OV$eepct3dA4{&OPfsa@8&|SYk7HTEOctz2T^~V4Qlj{pGqK`FGa<5aUr|ZV< zK*@goO-Np?^UQd?J<@;f%5U%d(bc*?fTe~Wgxg8D6i`h98Y9!a9s$$6QK81p64b09 zAEm=&^^Z@~r6l%A9gWbmB3q%SMd*Atm}bi~Er5k81Q(~^0wT=cVkgOQN@vsuzvY@d zIoKmopXYvHJYJbSJ8m7phI?)O{v1S22AH8B$}%INF@~9Lq=Sr|uSU^D0542gap--@ z)!m}?cYyx*diwRO4egVrEI+Kc-m9yt?J{9y`91=58zsS+@Qb{+Ld zo0~3F0gEiA9JF~>CIfM~2b)rv{S7rXUb7!=<=0!aNnSg98It#>;_2eI-vP8+@n13u zrw;%T82vpQRT&JR4%+^}CbFP*Q!IEi?Xp-lgGGGY!PSEXke^(2natHVmvNP#dbuYh zl|_9GckjBGao9iMUB=V(J82rwh!Q*jR-1BM|Kca5vM#jl5rw7+x98#0gm$n`kwsM< zZ&kg&q+jlNAtgbU&7uSJ<#mvx8f&Q`M$6GkEP3LUA9Pcsx}IIj$NqzoIhG( z&B_W{(M^a`Q1D&ObZ}@r0*B^a&(S}zlfzYP#iMd?bWmwExbnRgA!Qk>pD@QmI#fR7bfM5bJXvH{%#AV(#alovQf2c#dTU{ey9> z6i_u$a&{CSA*&QSabF&G*5#l&(JU~kO~bO_A@x%Uir1abz1-fU=0?9?)Xj4n7UHpk zf(ItP0a}JM>CpMzMpf6|K&NLA_wQlT9kw8+bP)E?_dAmyc8sqcIV<{CTgTXyk#*g>+bS5S$c=tbsw+lTZQRtwq5+w3DHVj7%>TE_5_>fPRq7oUu%GX-Vj8lyA++ z@+ssT-FID&Pn~U9D1kzta=;QaW4H&!A9@K!k%xM1=>Z?^<|Ma}N-kVnxplF>V*l}^ z=eHEP6z*{$ZFhyrMbW{nh_(!c5#^@GA!lU{9p)w2J$8(`le+Q|KOx3{JsF6bF z!xdc;Mg}hGQj)PhJePsAsR&&#lIp^U1*}BAmly@12VE&vOaWs;%}T3)xA2%{#$y}H zW~Eu7IHCPB_2sv%!+F(dm4p{a_`)*tS`iUq#Agw9txvM%gR#&y@C6{R`q5hkk>@a; z{~%JP*7c!C5Oa>e`yT>p10c{ShHsgB?L3>jA6jW0GOJ$wf*T==TimY89cF#S z2H~HA6tJGDIMK`jK41z=D{%|XV5St?Vy-RrHCN{5=SlR(&o~%5W!yf%do)!CnHoGr z7Z;(4Pp_%n2M~gLn7h57vZvNZS9fKyeWRtE?=Bt2>peMOaD_I;+=m9QY0Jn1xK&6L zE?nU4M8;1-;#x-j98{%AZxt>Zh%}NPmUf|a#zsP|@A{#v^BwOIOSq->XHCMlZJsTq z&k^jv=j-1(v^pI~7n14{lW2YSR=3b2kDXB8T%JstoL;ql@6Bc^eQV>cuJm&ww)fPY zvu>=LzVK;Yy7+6F4E;8vf(~%8{MZp&({>&paH|S>6xifbDIBHZ)1%GthL%Ww`zR8;9pV0N}4=CGYP0;X*?}Moz%9 zml<>Id#@BeD3r#k00^j-a|vdi$yrA;aPYK!Q&8eyOE;qI#c|hV@oIK^`@S7wZ$53E zCKZ2ZF-h>=+xUDyE`gV3Jf}OCgrXSvp$E&{z;C@=6Z2_sRe8OW)H(CjPCfg~hhuyn zJI6tvgZ3KEr9d4JTeEs&Zz3RHo8Py>dv->iTJuiN$!oOyxU;~Ok4>?bk`L1acdc;X z>2yILWbP{9I3RNM7zFQN4+0b2bX1ynPQ}4W`H%_m#|9S1o&-N_?<+xD8dVXlKq#Da{Y znsnUZgct8lp9SjJQiO$l$Y_uNI5Yz*iI7Dz6MiF_vZSbCBvf=KBVZQ{YLu_8^)H5SO{_>*u(zlAo-zoB#IaXGE-m*DU*38o5^!^ zrEq6=p4^OYVA-ltS(Aa=>3(Gw)Y9k)N&Vnpn_UGR;ADEyGSPAxp#4+dv zk*=-yNi-YX*q4KI71~mJ0O!S_0L&CXCPMVJ7P*d;9c;M(#Zif~~fY9lTMF`x#Io!PNxWk%RmadXA_V z(`b+`m{k|zG`3E>I3zb_cu(-8tLtsV5?L13u{zoQG{faQTL69@4WIs(tcI`$itLVB zpAWge;EfF#5?r-C9$lrCwOyT4(>T3)cAw9oT9-sEKGmtdS#DR+>*2i9rZ6UD{V#9L z=HOfW8c{Z**O@$(KKkTa7a3e5ZWXqP%^0 z!z<#|y~_~W4h>|zdjsj0DeQIY8MIsO6>B-w!BEV)@~o+Gja>tqhW~hN&f84P4fSLL zVJaPS z3Ak;MErJ#Ke&L!Dy~A^Q=Xbn+(}3}f=EiXqiHolFjg6pjkh7qFJbU(YT0dBSh8`H< z>rliQ^r6)`Pz*;5(* z#~FUgWdHbn?>hIQx)3c8bnza5;{;ciIO7{dAYZ{J2SW1#a&)T&UL>aaOHFdd!?$gj zsVn?_qOPY?D-I04Qm~5>FF*T09fD8R(Zy28S=8T*cPjJonRyx%ROhU5w+XfE_Q#%4 z&6$u-qo&yp-dM78B{f`6ePY;i4|VfxJ})S-Bhh4+n06e3eq(qCe=yXPl9Hq>udv~x zsC=p#hP)Oggo60BDjNJR-f!~RFN#Q9!nTL%!r3G!@`yl~Yj>XHaIx-aG<NZgEArq{_fQLTqM$Txq^66~bwQyDKC5 zHjnQc_Ujx@Pvm5R%9L1$RhnoHq=nJl#{FAvm`$qerM$=QZ)m9x`hH4csiLRpS$fTN z3!NW(!qtUtQ1?)h%ZR9%7?N-iRIr*8o$|JGk>`{E;I<%U3vDhKu#88r%AQTl)0HX! zO3Oepx&oN$0~jO-?A+*mn8k&nt`c;l_R(Rb0^J4A=QPidF#U<|Ctb4tSHqAA>2e??mjCP=1q@gER9-Cm7;qz2}Dp2H&Fx0_zu9$^m(8N zl<`#PPDQ-?wEOdi)oDgjxaRFE+lF1Yd>)1uB7%kfZZGbg2Y#F~PX}Ca+686c8&_bJ zXK1<)L+)iI&$5a^vi8>7=dwB7i;q2kq{?ejob6qyE-5bmU zKY=BbflJVi-U&PL^X9lhb&E)+t@#Qy-Cnc(wsd4@gtUx@&jafXR_P_MqcysjLJJGs zPdQNLS+oMU&sqiM(det0)X7Pv;@G#G1p`S7evr3i$0AtL429M#=zF?l^^P^3lYPfs zTExcx)rv|1mXt@i3o!Zc37K_TnP*N0wE`;jAn^Jbm&~kvnGQ*4$m=-~B%K;}_pWpr zz~$%j=D&T}OrUs%*j#I4o+IND%l!U;MIkramexXvs?;a%hnv`MpGOc=Z3u#p$9xolwOLO%brBt> z%|gbOU1*%`g)KMkn)X18TKHje)lX@jQ7Xd^oB}pYk0(>Vcu!Xc%=6iG8 zKcNI2yGzaGo~q;3?DY12Ys%it`JX8=-#zGGIdVSldPp0x))%x}5{}4MgPC>|Ak9{x zD6~L?^+unJg%H_KO}Vk7+I@u?xwS8=4zos?n9dx$C2 zyjrcDvb3VA@%lb=KmGCbo#acB8IP9ubcJZ9AWLkj*nK}x(>XCzQ8HuW&lvM> zI<1Cu-@p7fo%R+FbXH$f&)<()5Ji2SVWyOH;9xcIU;dmR-11X|rKj38P2uA|vHjPl z?d|C@&7+SW*mEsH%R{HKeY)Xihl6l#7BBd>@32HPBNyHi@{wGjPLw+k`A%(1sbU;m zr?Fa2*?LRAKyg~2yZHutp>51;{YCHW0;Og5q&)-1-#xv;6!b zQo5eHKWrOihFyDlyx<`X(v`|k|(#yMUN#Qyi75YP~*cIJx>=v?H&>Ftz2T)y@!DraFqIaHVX31o{$~WGJF| zA+YoAkNm&??C*~tbhR1BHaoD{dgd(dmmakJ6PP~eYhVBOA6f0t{+n0+UwdKb|F>8E zUwPqO2%?LP(EJ?Ho!mHf$?9YqA+exPm;7fWpoz%Va<^?Y;GO#6JsAAw6!;jDE=GF^ z+=muD2t?^bF+vRUXy}y+(NTG2_)eHq=TLffVNix+P;8Gxp-J}9Qz#W|J?Jz7yE|;U zAy1AXxS_s$THpx3Vdik0exKB0pkp@~0}Tq_-JCSk8hxIHoj(^a+o|4TV9PWQbpRf& z6#Lat0<4e~eWpuQOs$vte5AARrOztU zHtwdW7n({dh$bjGZ@)#8}a_QTBZ@pETkSK& zi#woqpvxX95*jR8;2Pq0w@nP->k92THaEFy^eSMc;^s;A?7*`M_K=mb8OI_-mq3Et z!W;%soc$;zs}mQC+*{2+b?2l+`~6Yo0er30nR~`;It9K+&=_foCpK_T({U8TnQ$jvwZgJ z>N)FoG_8lTvjYL-0v`wGbI*PgeZMq$EVHhP{fCs&9s)N2F3!V8mwO2_Q`B0J$KxY4 z6mW~NKFWubc?|xZE;Oc4J?8V%uxhI*NBvt8eWBj+>uZCLx_yj6up#;+48`PeV>Bu% zlA=u`o=h9a#V2XCo*lk=FXEmqK;MrCU;vs65UwwAF(vOa znovJL=jPioXcRNA;lH3^)!%@Mk8*>#pGG@LcaL0HTxk`fnD#u&A6pzQTTU>`(v6tZ z{A}+2Jz787;OT5t>_Px%Year85tC;&hw_xugb_`urL>GM6juynqW9&>} ze-bVx&G?;SZt2SItKF_4?k8o>p_Tc0!0LDl2Uj8 z7d}OJj6DQDCod8EUetLk`}lri>(3% z&?iE_xJV0tMnGuz^jDOeo{JvA(fI&K>(2rnK7pejeBOyVpmjf_Xcj1d&E)!CkLcTq zG09vi5l5d9fR4LS!v_hcyf*q+Qoh^-DgmIozdZv?6r4z;1LT=xOdW@lO6eEK+Bk}M zhqla=&&~|b?XfN~$LAT1*~@2|MFOw)RqETnJ1V5kru1P?-8u_6lRE5_8TRu$ufHM^ zMw$YS9Te}HW#eE$nFAbYJpR}na7TQT7S1wyJp}|4a`Lwjv zLy>QR|9&1*8-gr9@pJ)K>KQPlBjypv*Fc9F#j(S1hD@hwhB!W7YmE99p)uY3Tjb$v zU#f5CS*dHz2ilDvb6;2ULSFtm9-(K3f=*PVDda_nK?~Pxh;#s$B|uC#ijHkgUpUtl z;3z%n2iLf%pHlp?`F_+5&B2}cN;aBvu*rc?rU5h>3xS^mBrzZzQS12It>{Dp2pAax z?eX~z6t9Y#mhpv+o`xciG!Fs0E6JJnx?La2ay$Y2_Zyci9%kOEAb?Iz`v(G9M{)Wf zb&Ap81UHdAb;rOU3QjSq9ozafF(LP>sZ7kyy36U=?2y2fG}b3(ndXPiKB{^CyB4@y zJ&?w!#8AIa?0nKRPKgX?bZ8l0fj~FUQE7OZ>+)U1r=#(drIY!alKjsRBx@paHJy5z zfm>!gMcqM0qqu!)PWqSNVwfUh%_w=g*)wy>M%n1WV+GQ0Z)YRY6Q_zmAbWtUMs`fX z-^d4hkLg~T)9Mu1%$`1CaBOWpPr*O5d>!u)vIpT`#e$+8F$&A5t@38y)nd0u!=#1Q( z5mxGh=Ml?!8T6XEnN3z*7S%d!P%T6p!BA(IJnaNVnkz*uh6KmGvrRJH2$86Uef7(Z z6w7^ztbg;PYws=f5BDDP6zvFP-CJc9)^-8*WDgPJATa{*1yD>n7OD>qyjfxLw^9yd zwn~iAwX&$wN#m(;FY7()a;9x0FMVXY`PDvm=`l+nv%jjsMUSnj&E+O5^e}k7JC~!g zRbz~PJ}9)@qw%G0gz811CYGi*))9yyY3H#E$L~Sa9PP$XB9r&bDZ8-Sz31Gg(0<)A zzl{!Wcp<_L@Lt!E_;w#Uj8z1{w+x_FSe-==IoUrkA)w_bTV?-MuJfYpP#`I~A@(t$j_qcP^g1hKrh>>cS`1IdW7{ z>SF!N#5&IG0?vwUBf)D$$1jSU?ou5-N7qdu4|7aX)hawJ&A#faNvrLv)L$$-CCWkB z`x^gZXDGs$ynJK$*LgtR(KzUB(K=8xqvLW~m52~Resk10?7WSK?!ec{yIUkeOi@Mi zR1J#XNTvOSJ^H-66uU*h9&37`JIYd-W&FD14T>q%dt6%z9$d7nqr9moG|)JfVCU~E zulm-Wa&#^6*ZV-`p&?5kx0AY{Nc#qsQ8o}E-9n(DL+%N#V+f}3L-XK#dOcslHZ;{$ z*kX_iey6{iyU%j8%o@YwZ-Mq%JHrrLy*TVYq}Nso9SRMIY0Kj!PgcBhw=(No*ncbDOxj%eAB2)#=yI#b1rBmv z#=?B{eBzGpVGk@9nWuZ;+2otF*&Y4MUHr3GF6zH}*byzAczBaT;vMA9%Mp5Im?`+x zc)?T-G0nVaO9yTLK~xt- z(v|$lG^?qd0fo8UwX5Z+@~=|AT&cQqEMnj4I&|SrKF<)0*4__5C{-W?!bl`Mjy(-C zz>{NJ-Eb=Q%%o;{BuFNu&U|yqIL~Qt^n3GfYB1wjvtkfAM!$JV**Ijw*eqe=fsw5C z#fFdEu8Ak+dsPZ3u`2<9Aw&dZ^d09&xdt>$bKpa?`zpk#{|2?S4=-8P)t)G2(A;x>`Vc59?#p*9x zhU%P_h*~sZyFcXXEcoDri}}5e`HA5M!#O*I7+bdyEYi*LmbY+YShM=C8L^ZV9dTu= zRJe+IemE4p;K?u#^XwvT&b=ViRX*LH@+(_H6AZ?mAAq=vss< ziu!=;-TMf8z^&FQMQYB7zX8{O{*LdM{J z{jL|z22u0(5l980yZ{Dl8S@X~k+A_?S@98=CwPEn0{N8ZW5!EuRt&&ku-DKSs7}B@ z{jHRfVj`AtM{}xRzy=nlp)lv{VO`S9r=bHYzpbn+jg&|GxRet+rPFaYU^9-e0=g?; z+^ezn0`)APyH!>e#1>l+PqO(MZq&BB2jW~QNhkkqBkY53SVx+Tg`$eS%*_bg&QX?p z@y5Lhs7UT#YYW*-jk`m$wq7K#PK?eSpR#bWKzds>NG=2rxYO zQ3k2hcYX~hWdS2tR0E~NK7%s%xw9i18?Q1J_CNF#_>cjqYss-;YX+IaM?~PPA_t5k z@*pPNXhOPi+(Mb+D}YbezPZ=yV#DDS^lL=e{mU3%y6jdm;g=RB5P(6_!QyW5{QAry zPGl9ZI{;D9604Y-&o5eM*dCHfKGRiecskzn(CSInFKZcOgB~8m1(~=VZ#AQM>h}wx z^oFh4lTeL2sy?YnF8G)I{i)PEUurd+q1K6eY=w3XFfp*c@8o5*b%JPI{+n|jN5!#5 zBzPJ;1@H~>(A80XAs+!H`56{wWq{~I_y`9m4%ciV6igBRC!Py_(C|shn3}kDIg})y z^m_wM0n{Jb1@GoQW2`#14{3k!n&Ih`efr5CVoNK1pxf4l@PCU{43$o&-mI#k_(rOj zTs(BU(a_h+=lHzf>X#mjD953rOFy6JpJZJL{)UyNdw|TxNNx&E@I29a;MSfwqrj-1 zt@@%vUp028bYFTTymy-ntSFHSeDZ3Q*GyE8ciB;jc9W6C6!GoYs%7BTqp7-zxD-WS zD0^?9YhsuiYZ1um`eFZn5V~Jbo-=nU~Qb**C%`xcO;GmgaZuD$YR4Mq$ZHnx#@9RW_Bu~^i zo}=xDJ^J@8&U2iuO*@WBlo2_d0v z-VNb-{vA_RJ2<^t&A9OMSI5^i?^ArfafhVoA0Qsdm!Sv*{b-S0Q#^1O4(?>OfX_XA z8Damjxfg7qm%Rk!{b4;dK_!^z(S>id&=wmFd!?+*IOetW#jvAxl(EG24-YP`|!s59FlyM-um&|M{Qe zAu0%r@5bL&kF={P0D$zPcTWUU(Vzn9YXZ+;sv6qehZkcI0QFEJ=)zM)6F@I%$ng`2 z@J9q(+Ffc2ctnCM0nNPB>hU*l!r~AoU85p3(!+`?F!Uy)uSaKSLp570mB1dH5p19&DoFH@nV5sX-Iq@vhn z-38y76LX7QIah-fUMkn=dd1w^$49D4e~}g(rfD@MM4BROV})^4Hq5i zrzIDBEnM76!?3RO(dFnO@FP5I(3PheV?e$%cQV<*!!^zC*(FKMXKdfZwOcSlEe`2D z8oh}q4``23DaI1bIl$Rp(X;&r=VqQVc=0Wfk{nDM=I}0rK2SX7sq%PcVtfG%z$G$q zK((Sp9>62^755}T;!vmW>Oc+H+Q9P1Q6cgEs+nyHR#JK^>Z-dL^6i(7&U*r8I7 zC?{$EKTD|uBY|*H%G6@*YYOZ2Rx&z#(l{Y@F`ahf`VFX7(``y~!mjR~9oarTbs5F# z#pJaQ)qd1`d*pq?_rBvi$EWE{t>_&{@v0yGEuJp<2IJg0cl5bwMP1Ev^JLHGMM?L@ z`pmvP1HSgd4*mt?s+iCSsLVw>odbzs&UOvPabm_z4hou&LtlhjzgXfv5cZb!A?+L$ z_>#OQ;Q7yQ=+QlKX}mGu2#~Eve7KD*s_yP@PkS}ZEZ5z67&%K1vC4-!FJe5IaVc>G zvVjOZB@U+6?pRGmeyc=nE-6>9GkW@09Q$m%ZB37jQW62feWXP#WSC>@$VmztpheiA zzSep;z>_Gi`AR8U`c|PjA`mYMA~|4BpkE!QTP%o|xX~?2+P|;tI&0r4uh2R*^SQjY zb*u?qo0aW^yk=QV$kZt3g*8pUyD98m)hqG1oO1m0SC)&K8|HR0i7uu*@rVI97h4C3 zouF_$+Q6&uJk-epwX3jM&&9b;nUDLNdUV;1h!Od}u8z6FAw}2$NQ}-ER0J7FYdBU` ztpxg>+Z(poogQ@9RiRKjm8nuf<6YhLWAaVP=NUnylb$OIDsG!1UfppWFZ`#^4;2dU zh)ytSUXNxGUwwkqvcjm(=1n*u%vdD-0+ZyG_hqLWah>c5tO_vtV-n1-Ucg8BjgMu- zpjw>!==+~xg*Gfm^W`lNHCHj*->~`59m+wp)^zcpX>R=s$@5FeyfV4nC12hC(vJL# zdgK{A*^T9a%aiwwyeAv@er*YDBI9+*weN`X+F{0R1iqS*W!~>|?ju*IRk!t7oeppy z5SUe-FcZpyMUxa#ZPfw%TZ8{m-L=Lu`S*V!Q%;GTS%rv@^LbMu^3yud9JgvYx8zjA z+C>UEhN2Wxh;q*Pv^gJ=RJ4U0wxPtBVusoMzwZ10?0)cn{(rFR!Cu$vdhNQl@AvcB z=Y1-6Lr*yRsE?+0L5=^p${_1}1=+(h&5XYPJhrlHYDMh)X6q-|2n;%p!o}SXw(W)>imWyK|hv` zPs(>)D3pGS74kl%>y$B|CT_0gB2i&8Q)lo4==$6=(clR*k_JCB;Z49~Uejv+bI@b0 z6(KoX#A!X#G#a6oCbZJ`Yj3t}wR0uEpdZBAra;r&#M<2*MgT8oOOH>?oh~l7X7s7DuL^TjZH@tw3rftF~(b z9bX?{EN1Dtrj>C=m$DR8NASe@Fj5QCy(rL!c#YR!&Jch;SRcOl)HD3bg-em{u7A^CX zaM2$38Kj9zF+o4@`Hn|boLVvu-`A7r}9 zQ1H^n+l85kK=6BfV%F5gA)BZ>rarc2m@f(52xvKoUoYN?hy!a;__1Ii+G!t z_74Bz3+LasQrSRcXI(l+j<}H@r|}qJKcFFI1GPbFuMug*RlQXS)Lg)LzW+NTBh$5cLUBN| zv+?R;}OwqH{0#%NEXsC@ee;-$xvb=$wi0;h{h$_EJVUtwSDAxA>9D4fN zGu%KWCiR1(Y8()krnQ8#?L{AmpE-oT^fi3P)I7)+HDV zTDvJ%Navq`R}ql;n|;Oe-uTU*8fsa;gQWV1Vt4^oKYdOCk%(7hL&X9wz8eoLd+Tkm zMebc7{M~OZplokAV8s{CJ6X#jE|0ScFMY$yA4@k)1di%!d5u_jdpWhZAXkYkj$N}M zFOg+jtF_^hWUhX(^X@B52oT|OsOL+!L9LKxG3i365Q2u%*m2|yPw!*Nc6wiS{}N8D zo$-7oYA7;T#Ah^65+7Qk)!!Jk4v!%UwCT^LN}Q$EO_^701WoB4FB? zbX)NQNqqD0V6z)dEJ#9OJVGnL_}MpTh<6Gm3!<74H(dpEwQ#^cHFlxk65sn6NWF`e zhbe%KrxgB(dOQ31ZiV{upckH)aLSNcr<XxaUaZY@m^^S3 zm*t!rSykyZ0TemPe3#vC2KpF^jl9twts5B^=&-Oa489v&_%TQ3{fRG)lfIUKyTjqx z28$%V6)w7Pf|JnIlA5|c91JEiuOYMsRP|q7Zd$Pt75zIw!h!~GK{Qm(Y&$qMaav3I z@tb`o)5$;1>pn8m?vm^2*sQER2>7+#I~5tA^2$%AZQ#AFh1{I&mnSKboTM$XD+oxp zdD5yL4VU1~I@Z^+$`5w>lmd3GAG5<#ldpzvxrgP?R`6lC8k|!svIk4%d1;V}j+3(Q zit=ViUZ9-2T+M)1q;ivS1w-!vn+N7vhZ7~ne{W~!1(k3DB@a!NlgT&IL#swsga=nS z-_U%Vpsrkrc%DLk84lspTEFIDWoq%HH|0@CR*l0a*maw2zpt%utN@b=$3i8CZk4f} zA@*rK2XLbJ!u0N+P?iE8jY$we5AU4u;+VGu(9^5C-z0Cm^lr=}KEYZs-fEb!H>--R z)WO}4V`pWEd9(YDzPWs%9EE4LxQB56iOxb@yax8W!_lB)F4*9I&TP-QcytdM_zR+i z(AnPpKCo&8Y9#uAyt9uV zx?CzSRv!(bmEafv^p$!6l!>|G-C4U-`!)4vAG51tm1KWeyhXRjF}XAvZ9Ap>Mq_il$?n{du{X3Pd{JCpiJBk8n0nDh1;lYg#c1ShaZVNwTq7XA z6NKmg>m6k4f1e2aA7?7*rNX2FbE{(3y& z#zmSI@_KoPoN6K}y@OEdZL@VbRBRkaSKRuM@zU4@NFwhmeBXw(9hV2iww+kfhCAO@RTf zDJzGL%6J#R3|kXYEE93Z7)^72CZt} ze1F`>JF_U^aM*ws!I%QxzBsWKomTICwwpYOI#~^jg|AAWPDO~3$M0YzGdhl(+9_1Z zLb3coRixhqaMpTh+WZwFL9&ip9;WlBE^>J?Y+yAd>Bnw1Pc{%O!c%rusgErWq^92s zZ|#osiQ#utl6pAUFONYzwc`tctH7kZxc)4UU)4*{@2t8dBPJUS|0I$rrAQUHBLTE% z1Enl-LIqcyHl(aOd}m^jrQU|weaaWM5|Z0)$9FbrROe;i6ecL)+`6;s=n`1O#S*^X?`LtU1X6b`kA>trAtw? zRCJwGx|qVha?kO%<6Q3DM6M}@&Xi#Q=f^6pwY_iges6d9LpAR~gkY!`3KYo!WX=&r zdW=U(&W!km(6_osC_dMiAv8%`nTwgZ;MxR@YVZ1^T}IyTtn{UyJ&xT!kfBud?DFGg zX8!Yv{h9(F!4!uoNMODo$1=F&H#_wEwEEbm>sg918i&`GP|ra%&;nVS>%9hOck{IU zx%#v`J&&&nhkSC~v&|2lw>>W8I$}duCDdo_7=dZH_Mjh?H0MQcHk)mckXUrO;fq66 z^`4&HuN~qp{FylOs;jR-?I<{X5WfoMLabi;yx@~sS@`jwB=4Q8x&8JH(e3+op}_mi z9>r8-_d$-Q2M7yjJPZj45d{(!GS%tkkkUS(Xcmx&e0%wfrutu#O5Tf_k)Jjy7uWRI zZgD-+dcl&MFpoA?W$W=NBTC1!pVotXZv{6yJ6HA%mRG>p_U#}EEI5*pE*q$#d4IUk z&ex@161IiuT#?>$FKDu~V1Lw;O#f}AVT>R9xz5#NB5oi07Y;$S_5inK%>+Ipvv?oj zVjB;r#;b~)cT>y_X4I7=Gd1rDtnjsrv_f?k{@pQ^`sF7V_)*8{dP>|Id3xnou2fOu zbCO&og}LXcIe*u49OI<|8`LSs?B>E`r$IwUS8VZbXGv0IQPXn_c1ZsE$>!AhIZ{)4mH*<&t;{^98P#^v~F=MpNQuq|4!)t~8*D4Q&t5)#na|H+c##T z`Mr-n6eaogPMe9Pon@Ra*pE_=U1>8fwNt1aRIf{VXHlbNbEuOrm@hrnc8G-nMiAf1 z7i!wl=)V3wa#uw%CFt5;OAADyj}#W_X)~hMl(?eG#!|JUy@ltFP8ZES?oUapXDk)r zmKe~!nSH-=M;-z+rO`H1)sHJwZQVb&aVw5mg?u4{!_gbk{AYU_H>1!AtG8}~KchN5 z9a_MVhQC&1>zD!W?`9+C{y-lBGmwax#A(9>u>Ky1*>QoJ